diff --git a/packages/skin-database/addSkin.ts b/packages/skin-database/addSkin.ts index 80ed24f1..14dea685 100644 --- a/packages/skin-database/addSkin.ts +++ b/packages/skin-database/addSkin.ts @@ -76,6 +76,7 @@ async function addClassicSkinFromBuffer( await Shooter.withShooter((shooter) => shooter.takeScreenshot(tempFile, tempScreenshotPath, { minify: true, + md5, }) ); diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index c818dbce..b40998e6 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -13,6 +13,8 @@ import * as SkinHash from "./skinHash"; import * as Analyser from "./analyser"; import { searchIndex } from "./algolia"; import { scrapeLikeData } from "./tasks/scrapeLikes"; +import { screenshot } from "./tasks/screenshotSkin"; +import Shooter from "./shooter"; async function main() { const client = new Discord.Client(); @@ -21,6 +23,30 @@ async function main() { try { switch (argv._[0]) { + case "screenshot": { + const md5 = argv._[1] || (await Skins.getSkinToShoot()); + if (md5 == null) { + return; + } + await Shooter.withShooter(async (shooter: Shooter) => { + await screenshot(md5, shooter); + }); + console.log("Screenshot update complete."); + break; + } + case "screenshots": { + let count = 1000; + await Shooter.withShooter(async (shooter: Shooter) => { + while (count--) { + const md5 = await Skins.getSkinToShoot(); + if (md5 == null) { + break; + } + await screenshot(md5, shooter); + } + }); + break; + } case "readme": { const rows = await knex.raw( 'SELECT md5 FROM files LEFT JOIN skins on skins.md5 = files.skin_md5 WHERE source_attribution = "Web API" AND readme_text IS NULL;' diff --git a/packages/skin-database/data/skins.ts b/packages/skin-database/data/skins.ts index 5b9c7c09..d471efc4 100644 --- a/packages/skin-database/data/skins.ts +++ b/packages/skin-database/data/skins.ts @@ -382,6 +382,41 @@ export async function deleteSkin(md5: string): Promise { console.log(`Done deleting skin ${md5}.`); } +export async function recordScreenshotUpdate( + md5: string, + errorMessage?: string +) { + // TODO: Could this invalidate the CloudFlare cache? + const update_timestamp = Math.floor(Date.now() / 1000); + + await knex("screenshot_updates").insert([ + { + skin_md5: md5, + update_timestamp, + success: errorMessage == null, + error_message: errorMessage, + }, + ]); +} + +export async function getSkinToShoot(): Promise { + // TODO: Once return an ordered list here of skins that have no record, + // followed by skins that have not been shot in a long time. + const result = await knex("skins") + .leftJoin( + "screenshot_updates", + "skins.md5", + "=", + "screenshot_updates.skin_md5" + ) + .where("screenshot_updates.id", null) + .first(["md5"]); + if (result == null) { + return null; + } + return result.md5; +} + export async function recordSearchIndexUpdates( md5: string, fields: string[] @@ -662,6 +697,29 @@ LIMIT ? offset ?`, }); } +export async function getAllClassicSkins(): Promise< + Array<{ fileName: string; url: string }> +> { + const skins = await knex.raw( + ` +SELECT skins.md5, + files.file_path +FROM skins + LEFT JOIN files ON files.skin_md5 = skins.md5 +WHERE skin_type = 1 +GROUP BY skins.md5 +LIMIT 100`, + [] + ); + + return skins.map(({ md5, file_path }) => { + return { + fileName: path.basename(file_path), + md5: md5, + }; + }); +} + export async function getAllClassicScreenshotUrls(): Promise< Array<{ fileName: string; url: string }> > { diff --git a/packages/skin-database/discord-bot/commands/screenshot.js b/packages/skin-database/discord-bot/commands/screenshot.js index e1d12d82..3fb06f6a 100644 --- a/packages/skin-database/discord-bot/commands/screenshot.js +++ b/packages/skin-database/discord-bot/commands/screenshot.js @@ -18,6 +18,9 @@ async function handler(message) { attachments.map(async (attachment) => { const { filename, url } = attachment; const response = await fetch(url); + if (!response.ok) { + throw new Error("Failed to download skin."); + } console.log("got response"); const buffer = await response.buffer(); console.log("got buffer"); @@ -40,6 +43,7 @@ async function handler(message) { await withShooter(async (shooter) => { await shooter.takeScreenshot(tempFile, tempScreenshotPath, { minify: true, + md5: file.md5, }); }); console.log("Completed screenshot"); diff --git a/packages/skin-database/index.js b/packages/skin-database/index.js index 1dc3668d..a852f611 100644 --- a/packages/skin-database/index.js +++ b/packages/skin-database/index.js @@ -12,6 +12,7 @@ var bodyParser = require("body-parser"); var LRU = require("lru-cache"); const S3 = require("./s3"); const asyncHandler = require("express-async-handler"); +const expressSitemapXml = require("express-sitemap-xml"); const Sentry = require("@sentry/node"); // or use es6 import statements @@ -47,6 +48,14 @@ const corsOptions = { }, }; +async function getUrls() { + const md5s = await Skins.getAllClassicSkins(); + const skinUrls = md5s.map(({ md5, fileName }) => `skin/${md5}/${fileName}`); + return ["/about", "/", "/upload", ...skinUrls]; +} + +app.use(expressSitemapXml(getUrls, "https://skins.webamp.org")); + // parse application/json app.use(bodyParser.json()); diff --git a/packages/skin-database/package.json b/packages/skin-database/package.json index 94b0715b..1c41094b 100644 --- a/packages/skin-database/package.json +++ b/packages/skin-database/package.json @@ -14,6 +14,7 @@ "express": "^4.17.1", "express-async-handler": "^1.1.4", "express-fileupload": "^1.1.7-alpha.3", + "express-sitemap-xml": "^2.0.0", "imagemin": "^7.0.0", "imagemin-optipng": "^7.0.0", "knex": "^0.21.1", diff --git a/packages/skin-database/shooter.js b/packages/skin-database/shooter.js index 213623fd..7a78dd68 100644 --- a/packages/skin-database/shooter.js +++ b/packages/skin-database/shooter.js @@ -2,6 +2,7 @@ const path = require("path"); const puppeteer = require("puppeteer"); const imagemin = require("imagemin"); const imageminOptipng = require("imagemin-optipng"); +const Skins = require("./data/skins"); function min(imgPath) { return imagemin([imgPath], path.dirname(imgPath), { @@ -9,7 +10,7 @@ function min(imgPath) { }); } -class Shooter { +export default class Shooter { constructor(url) { this._initialized = false; this._url = url; @@ -18,7 +19,7 @@ class Shooter { static async withShooter(cb) { const shooter = new Shooter("https://webamp.org"); try { - await cb(shooter); + return await cb(shooter); } finally { shooter.dispose(); } @@ -58,8 +59,22 @@ class Shooter { } } - async takeScreenshot(skin, screenshotPath, { minify = false }) { - console.log("start!", this._page); + async takeScreenshot(skin, screenshotPath, { minify = false, md5 }) { + try { + await this._takeScreenshot(skin, screenshotPath, { minify }); + } catch (e) { + await Skins.recordScreenshotUpdate( + md5, + e.message || String(e) || "Unkown error" + ); + return false; + } + await Skins.recordScreenshotUpdate(md5); + return true; + } + + async _takeScreenshot(skin, screenshotPath, { minify = false }) { + console.log("start!"); await this._ensureInitialized(); console.log("Going to try", screenshotPath, skin); try { @@ -68,27 +83,31 @@ class Shooter { console.log("uploading skin"); await new Promise(async (resolve, reject) => { - console.log("start promise"); - const dialogHandler = (dialog) => { - reject(dialog.message()); - }; - this._page.on("dialog", dialogHandler); - await handle.uploadFile(skin); - console.log("waiting for skin to load..."); - await this._page.evaluate(() => { - return window.__webamp.skinIsLoaded(); - }); - console.log("waiting for screenshot"); - await this._page.screenshot({ - path: screenshotPath, - omitBackground: true, // Make screenshot transparent - // https://github.com/GoogleChrome/puppeteer/issues/703#issuecomment-366041479 - clip: { x: 0, y: 0, width: 275, height: 116 * 3 }, - }); + try { + console.log("start promise"); + const dialogHandler = (dialog) => { + reject(new Error(`Dialog message: ${dialog.message()}`)); + }; + this._page.on("dialog", dialogHandler); + await handle.uploadFile(skin); + console.log("waiting for skin to load..."); + await this._page.evaluate(() => { + return window.__webamp.skinIsLoaded(); + }); + console.log("waiting for screenshot"); + await this._page.screenshot({ + path: screenshotPath, + omitBackground: true, // Make screenshot transparent + // https://github.com/GoogleChrome/puppeteer/issues/703#issuecomment-366041479 + clip: { x: 0, y: 0, width: 275, height: 116 * 3 }, + }); - this._page.off("dialog", dialogHandler); + this._page.off("dialog", dialogHandler); - resolve(); + resolve(); + } catch (e) { + reject(e); + } }); console.log("Wrote screenshot to", screenshotPath); @@ -111,5 +130,3 @@ class Shooter { this._initialized = false; } } - -module.exports = Shooter; diff --git a/packages/skin-database/tasks/screenshotSkin.ts b/packages/skin-database/tasks/screenshotSkin.ts new file mode 100644 index 00000000..2f621806 --- /dev/null +++ b/packages/skin-database/tasks/screenshotSkin.ts @@ -0,0 +1,42 @@ +// eslint-disable-next-line +import _temp from "temp"; +import fs from "fs"; +import fetch from "node-fetch"; +import md5Buffer from "md5"; +import * as S3 from "../s3"; +import * as Skins from "../data/skins"; + +const Shooter = require("../shooter"); +const temp = _temp.track(); + +export async function screenshot(md5: string, shooter: typeof Shooter) { + const url = Skins.getSkinUrl(md5); + const response = await fetch(url); + if (!response.ok) { + await Skins.recordScreenshotUpdate(md5, `Failed to download from ${url}.`); + console.error(`Failed to download skin from "${url}".`); + return; + } + const buffer = await response.buffer(); + const actualMd5 = md5Buffer(buffer); + if (md5 !== actualMd5) { + throw new Error("Downloaded skin had a different md5."); + } + + const tempFile = temp.path({ suffix: ".wsz" }); + const tempScreenshotPath = temp.path({ suffix: ".png" }); + + fs.writeFileSync(tempFile, buffer); + + console.log("Starting screenshot"); + const success = await shooter.takeScreenshot(tempFile, tempScreenshotPath, { + minify: true, + md5, + }); + if (success) { + console.log("Completed screenshot"); + await S3.putScreenshot(md5, fs.readFileSync(tempScreenshotPath)); + } else { + console.log(`Screenshot failed ${md5}`); + } +} diff --git a/yarn.lock b/yarn.lock index aaed9f04..c14fd386 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4977,6 +4977,13 @@ express-fileupload@^1.1.7-alpha.3: dependencies: busboy "^0.3.1" +express-sitemap-xml@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/express-sitemap-xml/-/express-sitemap-xml-2.0.0.tgz#190b4993458007a61b4747a66e3a723121a207f6" + dependencies: + p-memoize "^4.0.1" + xmlbuilder "^15.1.1" + express@^4.16.3, express@^4.17.1: version "4.17.1" resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" @@ -8143,7 +8150,7 @@ makeerror@1.0.x: dependencies: tmpl "1.0.x" -map-age-cleaner@^0.1.1: +map-age-cleaner@^0.1.1, map-age-cleaner@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a" dependencies: @@ -8207,6 +8214,13 @@ mem@^4.0.0: mimic-fn "^2.0.0" p-is-promise "^2.0.0" +mem@^6.0.1: + version "6.1.1" + resolved "https://registry.yarnpkg.com/mem/-/mem-6.1.1.tgz#ea110c2ebc079eca3022e6b08c85a795e77f6318" + dependencies: + map-age-cleaner "^0.1.3" + mimic-fn "^3.0.0" + memory-fs@^0.4.0, memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" @@ -8356,6 +8370,10 @@ mimic-fn@^2.0.0, mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" +mimic-fn@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-3.1.0.tgz#65755145bbf3e36954b949c16450427451d5ca74" + mimic-response@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" @@ -9161,6 +9179,13 @@ p-map@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" +p-memoize@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/p-memoize/-/p-memoize-4.0.1.tgz#6f4231857fec10de2504611fe820c808fa8c5f8b" + dependencies: + mem "^6.0.1" + mimic-fn "^3.0.0" + p-pipe@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/p-pipe/-/p-pipe-1.2.0.tgz#4b1a11399a11520a67790ee5a0c1d5881d6befe9" @@ -12728,6 +12753,10 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" +xmlbuilder@^15.1.1: + version "15.1.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5" + xmlbuilder@~9.0.1: version "9.0.7" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"