diff --git a/packages/skin-database/api/processUserUploads.ts b/packages/skin-database/api/processUserUploads.ts index 0c361003..d4e990c2 100644 --- a/packages/skin-database/api/processUserUploads.ts +++ b/packages/skin-database/api/processUserUploads.ts @@ -2,6 +2,7 @@ import * as Skins from "../data/skins"; import S3 from "../s3"; import { addSkinFromBuffer } from "../addSkin"; import { EventHandler } from "./app"; +import DiscordEventHandler from "./DiscordEventHandler"; async function* reportedUploads(): AsyncGenerator<{ skin_md5: string; id: string; filename: string; }, void, unknown> { const seen = new Set(); @@ -37,7 +38,8 @@ function timeout(p: Promise, duration: number): Promise { ), ]); } -export async function processGivenUserUploads(eventHandler: EventHandler, uploads: AsyncGenerator<{ skin_md5: string; id: string; filename: string; }>) { + +async function processGivenUserUploads(eventHandler: EventHandler, uploads: AsyncGenerator<{ skin_md5: string; id: string; filename: string; }>) { log("Uploads to process..."); for await (const upload of uploads) { log("Going to try: ", upload); @@ -79,6 +81,38 @@ export async function processGivenUserUploads(eventHandler: EventHandler, upload log("Done processing uploads."); } +export async function reprocessFailedUploads(handler: DiscordEventHandler) { + // eslint-disable-next-line no-inner-declarations + async function* erroredUploads(): AsyncGenerator< + { skin_md5: string; id: string; filename: string }, + void, + unknown + > { + const seen = new Set(); + while (true) { + const upload = await Skins.getErroredUpload(); + console.log("Found one", { upload }); + if (upload == null) { + return; + } + if (seen.has(upload.id)) { + console.error( + "Saw the same upload twice. It didn't get handled?" + ); + return; + } + seen.add(upload.id); + yield upload; + } + } + const uploads = erroredUploads(); + + await processGivenUserUploads( + (event) => handler.handle(event), + uploads + ); +} + export async function processUserUploads(eventHandler: EventHandler) { log("process user uploads"); diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index 3cda6987..ddee79de 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -1,234 +1,282 @@ #!/usr/bin/env node import fs from "fs"; import { knex } from "./db"; -import { argv } from "yargs"; import logger from "./logger"; -import DiscordWinstonTransport from "./DiscordWinstonTransport"; import * as Skins from "./data/skins"; import Discord from "discord.js"; import { tweet } from "./tasks/tweet"; -import { insta, } from "./tasks/insta"; +import { insta } from "./tasks/insta"; import md5Buffer from "md5"; import { addSkinFromBuffer } from "./addSkin"; -import { searchIndex } from "./algolia"; import { scrapeLikeData } from "./tasks/scrapeLikes"; -import {followerCount, popularTweets} from "./tasks/tweetMilestones" +import { followerCount, popularTweets } from "./tasks/tweetMilestones"; import UserContext from "./data/UserContext"; import { integrityCheck } from "./tasks/integrityCheck"; import { ensureWebampLinks, syncWithArchive } from "./tasks/syncWithArchive"; import { fillMissingMetadata, syncFromArchive } from "./tasks/syncFromArchive"; -import { getSkinsToRefresh, refreshSkins } from "./tasks/refresh"; -import { processGivenUserUploads, processUserUploads } from "./api/processUserUploads"; +import { refreshSkins } from "./tasks/refresh"; +import { + reprocessFailedUploads, + processUserUploads, +} from "./api/processUserUploads"; import DiscordEventHandler from "./api/DiscordEventHandler"; import SkinModel from "./data/SkinModel"; -import { chunk } from "./utils"; import _temp from "temp"; import Shooter from "./shooter"; -import rl from "readline"; +import { program } from "commander"; -const temp = _temp.track(); - -async function main() { - const client = new Discord.Client(); - // The Winston transport logs in the client. - await DiscordWinstonTransport.addToLogger(client, logger); - const ctx = new UserContext("CLI"); +async function withHandler( + cb: (handler: DiscordEventHandler) => Promise +) { const handler = new DiscordEventHandler(); - try { - switch (argv._[0]) { - case "ensure-webamp-links": - await ensureWebampLinks(); - break; - case "sync-from-ia": - await syncFromArchive(); - break; - case "sync-ia": - await syncWithArchive(handler); - break; - case "ia-metadata": - await fillMissingMetadata(); - break; - case "integity-check": - await integrityCheck(); - break; - case "reject": { - const md5 = argv._[1]; - if (md5 == null) { - return; - } - await Skins.reject(new UserContext("CLI"), md5); - break; - } - case "refresh": { - const md5 = argv._[1]; - if (md5 != null) { - const skin = await SkinModel.fromMd5(ctx, md5); - if (skin == null) { - throw new Error(`Could not find skin ${md5}`); - } - refreshSkins([skin]); - } else { - const toRefresh = await getSkinsToRefresh(ctx, 100); - - const chunks = chunk(toRefresh, toRefresh.length / 3); - - await Promise.all(chunks.map(refreshSkins)); - } - break; - } - case "nested": { - const nested = await knex("archive_files") - .select("archive_files.skin_md5", "file_name") - .leftJoin("skins", "skins.md5", "=", "file_md5") - .where(function () { - //this.where("file_name", "like", "%.wsz"); - this.orWhere("file_name", "like", "%.zip"); - }) - .where("skins.md5", "IS", null); - - for (const row of nested) { - const url = `https://zip-worker.jordan1320.workers.dev/zip/${ - row.skin_md5 - }/${encodeURI(row.file_name)}`; - console.log(url); - } - /* - const query = `SELECT skin_md5, error - FROM refreshes - WHERE - error LIKE "Not a skin%";`; - const rows = await knex.raw(query); - for (const row of rows) { - const files = await knex("archive_files") - .where("skin_md5", row.skin_md5) - .select(); - console.log("Download:", Skins.getSkinUrl(row.skin_md5)); - // const url = `; - console.table( - files.map((f) => ({ - file_name: f.file_name, - url: `https://zip-worker.jordan1320.workers.dev/zip/${ - row.skin_md5 - }/${encodeURI(f.file_name)}`, - })), - ["file_name", "url"] - ); - const answer = await ask("skip (s), delete (d)"); - switch (answer) { - case "s": - break; - case "d": - await Skins.deleteSkin(row.skin_md5); - } - } - */ - break; - } - case "tweet": { - console.log("tweet & insta"); - const hash = argv._[1]; - await tweet(client, hash); - await insta(client, hash); - break; - } - case "insta": { - console.log("insta"); - const hash = argv._[1]; - await insta(client, hash); - break; - } - case "stats": { - console.log(await Skins.getStats()); - break; - } - case "add": { - const filePath = argv._[1]; - const buffer = fs.readFileSync(filePath); - console.log(await addSkinFromBuffer(buffer, filePath, "cli-user")); - break; - } - case "screenshot": { - const filePath = argv._[1]; - const buffer = fs.readFileSync(filePath); - const md5 = md5Buffer(buffer); - const tempPath = temp.path({ suffix: ".png" }); - await Shooter.withShooter( - (shooter) => { - shooter.takeScreenshot(buffer, tempPath, { md5 }); - }, - (message) => console.log(message) - ); - console.log("Screenshot complete", tempPath); - break; - } - case "delete": { - const md5 = argv._[1]; - await Skins.deleteSkin(md5); - break; - } - case "index": { - console.log(await Skins.updateSearchIndex(argv._[1])); - break; - } - case "process": { - const handler = new DiscordEventHandler(); - await processUserUploads((event) => handler.handle(event)); - handler.dispose(); - break; - } - case "add-missing-indexes": { - // eslint-disable-next-line no-inner-declarations - async function indexPage(pageNumber: number): Promise { - const page = await Skins.getMuseumPage({ - offset: pageNumber * 500, - first: 500, - }); - const toCheck = page.map(({ md5 }) => md5); - const found = await searchIndex.getObjects(toCheck, { - attributesToRetrieve: ["objectId"], - }); - // console.log({ toCheck, found }); - const missing = toCheck.filter((_md5, i) => { - return found.results[i] == null; - }); - - console.log( - `Found ${missing.length} missing skins on page ${pageNumber}` - ); - console.log(await Skins.updateSearchIndexs(missing)); - } - - for (let i = 0; i < 140; i++) { - await indexPage(i); - } - break; - } - case "tweet-data": { - await scrapeLikeData(); - break; - } - case "popular-tweets": { - await popularTweets(handler); - break; - } - - case "follower-count": { - await followerCount(handler); - break; - } - - default: - console.log(`Unknown command ${argv._[0]}`); - } + await cb(handler); } finally { - knex.destroy(); - logger.close(); - client.destroy(); await handler.dispose(); } } +async function withDiscordClient( + cb: (handler: Discord.Client) => Promise +) { + const client = new Discord.Client(); + try { + await cb(client); + } finally { + client.destroy(); + } +} + +const temp = _temp.track(); + +/** + * CLI starts here. + */ +program + .name("skins-database") + .description("CLI for interacting with the skins database"); + +/** + * Social Media Commands + */ + +program + .command("share") + .description( + "Share a skin on Twitter and Instagram. If no md5 is " + + "given, random approved skins are shared." + ) + .argument("[md5]", "md5 of the skin to share") + .option("-t, --twitter", "Share on Twitter") + .option("-i, --instagram", "Share on Instagram") + .action(async (md5, { twitter, instagram }) => { + if (!twitter && !instagram) { + throw new Error("Expected at least one of --twitter or --instagram"); + } + await withDiscordClient(async (client) => { + if (twitter) { + await tweet(client, md5); + } + if (instagram) { + await insta(client, md5); + } + }); + }); + +/** + * Operate on individual skins. + */ +program + .command("skin") + .description("Operate on a skngle skin from the database.") + .argument("", "md5 of the skin to operate on") + .option( + "--delete", + "Delete a skin from the database, including its S3 files " + + "CloudFlare cache and seach index entries." + ) + .option("--index", "Update the seach index for a skin.") + .option( + "--refresh", + "Retake the screenshot of a skin and update the database." + ) + .option("--reject", 'Give a skin a "rejected" review.') + .action(async (md5, { delete: del, index, refresh, reject }) => { + const ctx = new UserContext("CLI"); + if (del) { + await Skins.deleteSkin(md5); + } + if (index) { + console.log(await Skins.updateSearchIndex(md5)); + } + if (refresh) { + const skin = await SkinModel.fromMd5(ctx, md5); + if (skin == null) { + throw new Error(`Could not find skin ${md5}`); + } + await refreshSkins([skin]); + } + if (reject) { + await Skins.reject(ctx, md5); + } + }); + +program + .command("file") + .description("Operate on a skin file.") + .argument("", "Path to the skin to add to the database.") + .option("--add", "Add this skin to the database.") + .option( + "--screenshot", + "Take (or retake) a screenshot of the given skin file." + ) + .action(async (filePath, { add, screenshot }) => { + if (add) { + const buffer = fs.readFileSync(filePath); + console.log(await addSkinFromBuffer(buffer, filePath, "cli-user")); + } + if (screenshot) { + const buffer = fs.readFileSync(filePath); + const md5 = md5Buffer(buffer); + const tempPath = temp.path({ suffix: ".png" }); + await Shooter.withShooter( + async (shooter: Shooter) => { + await shooter.takeScreenshot(buffer, tempPath, { md5 }); + }, + (message: string) => console.log(message) + ); + console.log("Screenshot complete", tempPath); + } + }); + +/** + * Internet Archive Commands + */ +program + .command("ia") + .description("Interact with the Internet Archive API.") + .option( + "--fetch-metadata ", + "Fetch missing metadata for items from the Internet " + + "Archive. Currently it only fetches missing metadata. In the " + + "future it could refresh stale metadata." + ) + .option( + "--fetch-items", + "Seach the Internet Archive for items that we don't know about" + + "and add them to our database." + ) + .option( + "--webamp-links", + "Seach the Internet Archive for items missing a Webamp link" + + "and add them." + ) + .option( + "--upload", + "Find newly uploaded skins, and publish them to the Internet Archive." + ) + .action(async ({ metadata, webampLinks, fetchItems, upload }) => { + if (metadata) { + await fillMissingMetadata(Number(metadata || 1000)); + } + if (webampLinks) { + await ensureWebampLinks(); + } + if (fetchItems) { + await syncFromArchive(); + } + if (upload) { + await withHandler(async (handler) => { + await syncWithArchive(handler); + }); + } + }); + +/** + * Investigation and recovery commands + */ + +program + .command("stats") + .description( + "Report information about skins in the database. " + + "Identical to `!stats` in Discord." + ) + .action(async () => { + console.table([await Skins.getStats()]); + }); + +program + .command("process-uploads") + .description("Process any unprocessed user uploads.") + .option("--errored", "Reprocess errored uploads.") + .action(async ({ errored }) => { + await withHandler(async (handler) => { + if (!errored) { + await processUserUploads((event) => handler.handle(event)); + } else { + await reprocessFailedUploads(handler); + } + }); + }); + +program + .command("integrity-check") + .description("Perfrom a non-exhaustive list of database consistency checks") + .action(async () => { + await integrityCheck(); + }); + +/** + * Scrape Twitter Commands + */ + +program + .command("scrape-twitter") + .description("Scrape Twitter in various ways.") + .option( + "--likes", + "Scrape @winampskins tweets for like and retweet counts, " + + "and update the database." + ) + .option( + "--milestones", + "Check the most recent @winampskins tweets to see if they have " + + "passed a milestone. If so, notify the Discord channel." + ) + .option( + "--followers", + "Check if @winampskins has passed a follower count milestone. " + + "If so, notify the Discord channel." + ) + .action(async ({ likes, milestones, followers }) => { + if (likes) { + await scrapeLikeData(); + } + if (milestones) { + await withHandler(async (handler) => { + await popularTweets(handler); + }); + } + if (followers) { + await withHandler(async (handler) => { + await followerCount(handler); + }); + } + }); + +async function main() { + try { + await program.parseAsync(process.argv); + } finally { + knex.destroy(); + logger.close(); + } +} + +/* +import rl from "readline"; + function ask(question): Promise { return new Promise((resolve) => { const r = rl.createInterface({ @@ -241,5 +289,6 @@ function ask(question): Promise { }); }); } +*/ main(); diff --git a/packages/skin-database/package.json b/packages/skin-database/package.json index a3df4b97..016f4a24 100644 --- a/packages/skin-database/package.json +++ b/packages/skin-database/package.json @@ -16,6 +16,7 @@ "algoliasearch": "^4.3.0", "async-parallel": "^1.2.3", "aws-sdk": "^2.663.0", + "commander": "^9.0.0", "cookie-session": "^1.4.0", "cors": "^2.8.5", "dataloader": "^2.0.0", diff --git a/packages/skin-database/tasks/syncFromArchive.ts b/packages/skin-database/tasks/syncFromArchive.ts index d789a82f..91a5adc5 100644 --- a/packages/skin-database/tasks/syncFromArchive.ts +++ b/packages/skin-database/tasks/syncFromArchive.ts @@ -109,12 +109,12 @@ async function ensureIaRecord( console.log(`Inserted "${identifier}".`); } -export async function fillMissingMetadata() { +export async function fillMissingMetadata(count: number) { const skins = await knex("skins") .leftJoin("ia_items", "skins.md5", "ia_items.skin_md5") .where("ia_items.metadata", null) .whereNot("ia_items.identifier", null) - .limit(4000) + .limit(count) .select("skins.md5", "ia_items.identifier"); for (const { md5, identifier } of skins) { await updateMetadata(identifier); diff --git a/yarn.lock b/yarn.lock index 75ff104f..761f3aea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5478,6 +5478,11 @@ commander@^6.2.0: resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== +commander@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-9.0.0.tgz#86d58f24ee98126568936bd1d3574e0308a99a40" + integrity sha512-JJfP2saEKbQqvW+FI93OYUB4ByV5cizMpFMiiJI8xDbBvQvSkIk0VvQdn1CZ8mqAO8Loq2h0gYTYtDFUZUeERw== + commander@~2.19.0: version "2.19.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"