Mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp.git synced 2026-07-21 09:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Screenshot updates and sitemap

Browse source
This commit is contained in:
Jordan Eldredge 2020-11-11 23:25:46 -05:00
parent 89c4f6dd9c
commit e0da87a570
9 changed files with 213 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

1
packages/skin-database/addSkin.ts
View file

@ -76,6 +76,7 @@ async function addClassicSkinFromBuffer(
await Shooter.withShooter((shooter) =>
shooter.takeScreenshot(tempFile, tempScreenshotPath, {
minify: true,
md5,
})
);

26
packages/skin-database/cli.ts
View file

@ -13,6 +13,8 @@ import * as SkinHash from "./skinHash";
import * as Analyser from "./analyser";
import { searchIndex } from "./algolia";
import { scrapeLikeData } from "./tasks/scrapeLikes";
import { screenshot } from "./tasks/screenshotSkin";
import Shooter from "./shooter";
async function main() {
const client = new Discord.Client();
@ -21,6 +23,30 @@ async function main() {
try {
switch (argv._[0]) {
case "screenshot": {
const md5 = argv._[1] || (await Skins.getSkinToShoot());
if (md5 == null) {
return;
}
await Shooter.withShooter(async (shooter: Shooter) => {
await screenshot(md5, shooter);
});
console.log("Screenshot update complete.");
break;
}
case "screenshots": {
let count = 1000;
await Shooter.withShooter(async (shooter: Shooter) => {
while (count--) {
const md5 = await Skins.getSkinToShoot();
if (md5 == null) {
break;
}
await screenshot(md5, shooter);
}
});
break;
}
case "readme": {
const rows = await knex.raw(
'SELECT md5 FROM files LEFT JOIN skins on skins.md5 = files.skin_md5 WHERE source_attribution = "Web API" AND readme_text IS NULL;'

58
packages/skin-database/data/skins.ts
View file

@ -382,6 +382,41 @@ export async function deleteSkin(md5: string): Promise<void> {
console.log(`Done deleting skin ${md5}.`);
}
export async function recordScreenshotUpdate(
md5: string,
errorMessage?: string
) {
// TODO: Could this invalidate the CloudFlare cache?
const update_timestamp = Math.floor(Date.now() / 1000);
await knex("screenshot_updates").insert([
{
skin_md5: md5,
update_timestamp,
success: errorMessage == null,
error_message: errorMessage,
},
]);
}
export async function getSkinToShoot(): Promise<string | null> {
// TODO: Once return an ordered list here of skins that have no record,
// followed by skins that have not been shot in a long time.
const result = await knex("skins")
.leftJoin(
"screenshot_updates",
"skins.md5",
"=",
"screenshot_updates.skin_md5"
)
.where("screenshot_updates.id", null)
.first(["md5"]);
if (result == null) {
return null;
}
return result.md5;
}
export async function recordSearchIndexUpdates(
md5: string,
fields: string[]
@ -662,6 +697,29 @@ LIMIT ? offset ?`,
});
}
export async function getAllClassicSkins(): Promise<
Array<{ fileName: string; url: string }>
> {
const skins = await knex.raw(
`
SELECT skins.md5,
files.file_path
FROM skins
LEFT JOIN files ON files.skin_md5 = skins.md5
WHERE skin_type = 1
GROUP BY skins.md5
LIMIT 100`,
[]
);
return skins.map(({ md5, file_path }) => {
return {
fileName: path.basename(file_path),
md5: md5,
};
});
}
export async function getAllClassicScreenshotUrls(): Promise<
Array<{ fileName: string; url: string }>
> {

4
packages/skin-database/discord-bot/commands/screenshot.js
View file

@ -18,6 +18,9 @@ async function handler(message) {
attachments.map(async (attachment) => {
const { filename, url } = attachment;
const response = await fetch(url);
if (!response.ok) {
throw new Error("Failed to download skin.");
}
console.log("got response");
const buffer = await response.buffer();
console.log("got buffer");
@ -40,6 +43,7 @@ async function handler(message) {
await withShooter(async (shooter) => {
await shooter.takeScreenshot(tempFile, tempScreenshotPath, {
minify: true,
md5: file.md5,
});
});
console.log("Completed screenshot");

9
packages/skin-database/index.js
View file

@ -12,6 +12,7 @@ var bodyParser = require("body-parser");
var LRU = require("lru-cache");
const S3 = require("./s3");
const asyncHandler = require("express-async-handler");
const expressSitemapXml = require("express-sitemap-xml");
const Sentry = require("@sentry/node");
// or use es6 import statements
@ -47,6 +48,14 @@ const corsOptions = {
},
};
async function getUrls() {
const md5s = await Skins.getAllClassicSkins();
const skinUrls = md5s.map(({ md5, fileName }) => `skin/${md5}/${fileName}`);
return ["/about", "/", "/upload", ...skinUrls];
}
app.use(expressSitemapXml(getUrls, "https://skins.webamp.org"));
// parse application/json
app.use(bodyParser.json());

1
packages/skin-database/package.json
View file

@ -14,6 +14,7 @@
"express": "^4.17.1",
"express-async-handler": "^1.1.4",
"express-fileupload": "^1.1.7-alpha.3",
"express-sitemap-xml": "^2.0.0",
"imagemin": "^7.0.0",
"imagemin-optipng": "^7.0.0",
"knex": "^0.21.1",

67
packages/skin-database/shooter.js
View file

@ -2,6 +2,7 @@ const path = require("path");
const puppeteer = require("puppeteer");
const imagemin = require("imagemin");
const imageminOptipng = require("imagemin-optipng");
const Skins = require("./data/skins");
function min(imgPath) {
return imagemin([imgPath], path.dirname(imgPath), {
@ -9,7 +10,7 @@ function min(imgPath) {
});
}
class Shooter {
export default class Shooter {
constructor(url) {
this._initialized = false;
this._url = url;
@ -18,7 +19,7 @@ class Shooter {
static async withShooter(cb) {
const shooter = new Shooter("https://webamp.org");
try {
await cb(shooter);
return await cb(shooter);
} finally {
shooter.dispose();
}
@ -58,8 +59,22 @@ class Shooter {
}
}
async takeScreenshot(skin, screenshotPath, { minify = false }) {
console.log("start!", this._page);
async takeScreenshot(skin, screenshotPath, { minify = false, md5 }) {
try {
await this._takeScreenshot(skin, screenshotPath, { minify });
} catch (e) {
await Skins.recordScreenshotUpdate(
md5,
e.message || String(e) || "Unkown error"
);
return false;
}
await Skins.recordScreenshotUpdate(md5);
return true;
}
async _takeScreenshot(skin, screenshotPath, { minify = false }) {
console.log("start!");
await this._ensureInitialized();
console.log("Going to try", screenshotPath, skin);
try {
@ -68,27 +83,31 @@ class Shooter {
console.log("uploading skin");
await new Promise(async (resolve, reject) => {
console.log("start promise");
const dialogHandler = (dialog) => {
reject(dialog.message());
};
this._page.on("dialog", dialogHandler);
await handle.uploadFile(skin);
console.log("waiting for skin to load...");
await this._page.evaluate(() => {
return window.__webamp.skinIsLoaded();
});
console.log("waiting for screenshot");
await this._page.screenshot({
path: screenshotPath,
omitBackground: true, // Make screenshot transparent
// https://github.com/GoogleChrome/puppeteer/issues/703#issuecomment-366041479
clip: { x: 0, y: 0, width: 275, height: 116 * 3 },
});
try {
console.log("start promise");
const dialogHandler = (dialog) => {
reject(new Error(`Dialog message: ${dialog.message()}`));
};
this._page.on("dialog", dialogHandler);
await handle.uploadFile(skin);
console.log("waiting for skin to load...");
await this._page.evaluate(() => {
return window.__webamp.skinIsLoaded();
});
console.log("waiting for screenshot");
await this._page.screenshot({
path: screenshotPath,
omitBackground: true, // Make screenshot transparent
// https://github.com/GoogleChrome/puppeteer/issues/703#issuecomment-366041479
clip: { x: 0, y: 0, width: 275, height: 116 * 3 },
});
this._page.off("dialog", dialogHandler);
this._page.off("dialog", dialogHandler);
resolve();
resolve();
} catch (e) {
reject(e);
}
});
console.log("Wrote screenshot to", screenshotPath);
@ -111,5 +130,3 @@ class Shooter {
this._initialized = false;
}
}
module.exports = Shooter;

42
packages/skin-database/tasks/screenshotSkin.ts Normal file
View file

@ -0,0 +1,42 @@
// eslint-disable-next-line
import _temp from "temp";
import fs from "fs";
import fetch from "node-fetch";
import md5Buffer from "md5";
import * as S3 from "../s3";
import * as Skins from "../data/skins";
const Shooter = require("../shooter");
const temp = _temp.track();
export async function screenshot(md5: string, shooter: typeof Shooter) {
const url = Skins.getSkinUrl(md5);
const response = await fetch(url);
if (!response.ok) {
await Skins.recordScreenshotUpdate(md5, `Failed to download from ${url}.`);
console.error(`Failed to download skin from "${url}".`);
return;
}
const buffer = await response.buffer();
const actualMd5 = md5Buffer(buffer);
if (md5 !== actualMd5) {
throw new Error("Downloaded skin had a different md5.");
}
const tempFile = temp.path({ suffix: ".wsz" });
const tempScreenshotPath = temp.path({ suffix: ".png" });
fs.writeFileSync(tempFile, buffer);
console.log("Starting screenshot");
const success = await shooter.takeScreenshot(tempFile, tempScreenshotPath, {
minify: true,
md5,
});
if (success) {
console.log("Completed screenshot");
await S3.putScreenshot(md5, fs.readFileSync(tempScreenshotPath));
} else {
console.log(`Screenshot failed ${md5}`);
}
}

31
yarn.lock
View file

@ -4977,6 +4977,13 @@ express-fileupload@^1.1.7-alpha.3:
dependencies:
busboy "^0.3.1"
express-sitemap-xml@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/express-sitemap-xml/-/express-sitemap-xml-2.0.0.tgz#190b4993458007a61b4747a66e3a723121a207f6"
dependencies:
p-memoize "^4.0.1"
xmlbuilder "^15.1.1"
express@^4.16.3, express@^4.17.1:
version "4.17.1"
resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134"
@ -8143,7 +8150,7 @@ makeerror@1.0.x:
dependencies:
tmpl "1.0.x"
map-age-cleaner@^0.1.1:
map-age-cleaner@^0.1.1, map-age-cleaner@^0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a"
dependencies:
@ -8207,6 +8214,13 @@ mem@^4.0.0:
mimic-fn "^2.0.0"
p-is-promise "^2.0.0"
mem@^6.0.1:
version "6.1.1"
resolved "https://registry.yarnpkg.com/mem/-/mem-6.1.1.tgz#ea110c2ebc079eca3022e6b08c85a795e77f6318"
dependencies:
map-age-cleaner "^0.1.3"
mimic-fn "^3.0.0"
memory-fs@^0.4.0, memory-fs@^0.4.1:
version "0.4.1"
resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552"
@ -8356,6 +8370,10 @@ mimic-fn@^2.0.0, mimic-fn@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
mimic-fn@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-3.1.0.tgz#65755145bbf3e36954b949c16450427451d5ca74"
mimic-response@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
@ -9161,6 +9179,13 @@ p-map@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
p-memoize@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/p-memoize/-/p-memoize-4.0.1.tgz#6f4231857fec10de2504611fe820c808fa8c5f8b"
dependencies:
mem "^6.0.1"
mimic-fn "^3.0.0"
p-pipe@^1.1.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/p-pipe/-/p-pipe-1.2.0.tgz#4b1a11399a11520a67790ee5a0c1d5881d6befe9"
@ -12728,6 +12753,10 @@ xml2js@0.4.19:
sax ">=0.6.0"
xmlbuilder "~9.0.1"
xmlbuilder@^15.1.1:
version "15.1.1"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5"
xmlbuilder@~9.0.1:
version "9.0.7"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"