mirror of
https://github.com/captbaritone/webamp.git
synced 2026-07-22 09:37:17 +00:00
Use Commander for CLI and organize commands
This commit is contained in:
Jordan Eldredge
parent
a2723c5579
commit
06243ffffc
5 changed files with 297 additions and 208 deletions
|
|
@ -2,6 +2,7 @@ import * as Skins from "../data/skins";
|
|||
import S3 from "../s3";
|
||||
import { addSkinFromBuffer } from "../addSkin";
|
||||
import { EventHandler } from "./app";
|
||||
import DiscordEventHandler from "./DiscordEventHandler";
|
||||
|
||||
async function* reportedUploads(): AsyncGenerator<{ skin_md5: string; id: string; filename: string; }, void, unknown> {
|
||||
const seen = new Set();
|
||||
|
|
@ -37,7 +38,8 @@ function timeout<T>(p: Promise<T>, duration: number): Promise<T> {
|
|||
),
|
||||
]);
|
||||
}
|
||||
export async function processGivenUserUploads(eventHandler: EventHandler, uploads: AsyncGenerator<{ skin_md5: string; id: string; filename: string; }>) {
|
||||
|
||||
async function processGivenUserUploads(eventHandler: EventHandler, uploads: AsyncGenerator<{ skin_md5: string; id: string; filename: string; }>) {
|
||||
log("Uploads to process...");
|
||||
for await (const upload of uploads) {
|
||||
log("Going to try: ", upload);
|
||||
|
|
@ -79,6 +81,38 @@ export async function processGivenUserUploads(eventHandler: EventHandler, upload
|
|||
log("Done processing uploads.");
|
||||
}
|
||||
|
||||
export async function reprocessFailedUploads(handler: DiscordEventHandler) {
|
||||
// eslint-disable-next-line no-inner-declarations
|
||||
async function* erroredUploads(): AsyncGenerator<
|
||||
{ skin_md5: string; id: string; filename: string },
|
||||
void,
|
||||
unknown
|
||||
> {
|
||||
const seen = new Set();
|
||||
while (true) {
|
||||
const upload = await Skins.getErroredUpload();
|
||||
console.log("Found one", { upload });
|
||||
if (upload == null) {
|
||||
return;
|
||||
}
|
||||
if (seen.has(upload.id)) {
|
||||
console.error(
|
||||
"Saw the same upload twice. It didn't get handled?"
|
||||
);
|
||||
return;
|
||||
}
|
||||
seen.add(upload.id);
|
||||
yield upload;
|
||||
}
|
||||
}
|
||||
const uploads = erroredUploads();
|
||||
|
||||
await processGivenUserUploads(
|
||||
(event) => handler.handle(event),
|
||||
uploads
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
export async function processUserUploads(eventHandler: EventHandler) {
|
||||
log("process user uploads");
|
||||
|
|
|
|||
|
|
@ -1,234 +1,282 @@
|
|||
#!/usr/bin/env node
|
||||
import fs from "fs";
|
||||
import { knex } from "./db";
|
||||
import { argv } from "yargs";
|
||||
import logger from "./logger";
|
||||
import DiscordWinstonTransport from "./DiscordWinstonTransport";
|
||||
import * as Skins from "./data/skins";
|
||||
import Discord from "discord.js";
|
||||
import { tweet } from "./tasks/tweet";
|
||||
import { insta, } from "./tasks/insta";
|
||||
import { insta } from "./tasks/insta";
|
||||
import md5Buffer from "md5";
|
||||
import { addSkinFromBuffer } from "./addSkin";
|
||||
import { searchIndex } from "./algolia";
|
||||
import { scrapeLikeData } from "./tasks/scrapeLikes";
|
||||
import {followerCount, popularTweets} from "./tasks/tweetMilestones"
|
||||
import { followerCount, popularTweets } from "./tasks/tweetMilestones";
|
||||
import UserContext from "./data/UserContext";
|
||||
import { integrityCheck } from "./tasks/integrityCheck";
|
||||
import { ensureWebampLinks, syncWithArchive } from "./tasks/syncWithArchive";
|
||||
import { fillMissingMetadata, syncFromArchive } from "./tasks/syncFromArchive";
|
||||
import { getSkinsToRefresh, refreshSkins } from "./tasks/refresh";
|
||||
import { processGivenUserUploads, processUserUploads } from "./api/processUserUploads";
|
||||
import { refreshSkins } from "./tasks/refresh";
|
||||
import {
|
||||
reprocessFailedUploads,
|
||||
processUserUploads,
|
||||
} from "./api/processUserUploads";
|
||||
import DiscordEventHandler from "./api/DiscordEventHandler";
|
||||
import SkinModel from "./data/SkinModel";
|
||||
import { chunk } from "./utils";
|
||||
import _temp from "temp";
|
||||
import Shooter from "./shooter";
|
||||
import rl from "readline";
|
||||
import { program } from "commander";
|
||||
|
||||
const temp = _temp.track();
|
||||
|
||||
async function main() {
|
||||
const client = new Discord.Client();
|
||||
// The Winston transport logs in the client.
|
||||
await DiscordWinstonTransport.addToLogger(client, logger);
|
||||
const ctx = new UserContext("CLI");
|
||||
async function withHandler(
|
||||
cb: (handler: DiscordEventHandler) => Promise<void>
|
||||
) {
|
||||
const handler = new DiscordEventHandler();
|
||||
|
||||
try {
|
||||
switch (argv._[0]) {
|
||||
case "ensure-webamp-links":
|
||||
await ensureWebampLinks();
|
||||
break;
|
||||
case "sync-from-ia":
|
||||
await syncFromArchive();
|
||||
break;
|
||||
case "sync-ia":
|
||||
await syncWithArchive(handler);
|
||||
break;
|
||||
case "ia-metadata":
|
||||
await fillMissingMetadata();
|
||||
break;
|
||||
case "integity-check":
|
||||
await integrityCheck();
|
||||
break;
|
||||
case "reject": {
|
||||
const md5 = argv._[1];
|
||||
if (md5 == null) {
|
||||
return;
|
||||
}
|
||||
await Skins.reject(new UserContext("CLI"), md5);
|
||||
break;
|
||||
}
|
||||
case "refresh": {
|
||||
const md5 = argv._[1];
|
||||
if (md5 != null) {
|
||||
const skin = await SkinModel.fromMd5(ctx, md5);
|
||||
if (skin == null) {
|
||||
throw new Error(`Could not find skin ${md5}`);
|
||||
}
|
||||
refreshSkins([skin]);
|
||||
} else {
|
||||
const toRefresh = await getSkinsToRefresh(ctx, 100);
|
||||
|
||||
const chunks = chunk(toRefresh, toRefresh.length / 3);
|
||||
|
||||
await Promise.all(chunks.map(refreshSkins));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "nested": {
|
||||
const nested = await knex("archive_files")
|
||||
.select("archive_files.skin_md5", "file_name")
|
||||
.leftJoin("skins", "skins.md5", "=", "file_md5")
|
||||
.where(function () {
|
||||
//this.where("file_name", "like", "%.wsz");
|
||||
this.orWhere("file_name", "like", "%.zip");
|
||||
})
|
||||
.where("skins.md5", "IS", null);
|
||||
|
||||
for (const row of nested) {
|
||||
const url = `https://zip-worker.jordan1320.workers.dev/zip/${
|
||||
row.skin_md5
|
||||
}/${encodeURI(row.file_name)}`;
|
||||
console.log(url);
|
||||
}
|
||||
/*
|
||||
const query = `SELECT skin_md5, error
|
||||
FROM refreshes
|
||||
WHERE
|
||||
error LIKE "Not a skin%";`;
|
||||
const rows = await knex.raw(query);
|
||||
for (const row of rows) {
|
||||
const files = await knex("archive_files")
|
||||
.where("skin_md5", row.skin_md5)
|
||||
.select();
|
||||
console.log("Download:", Skins.getSkinUrl(row.skin_md5));
|
||||
// const url = `;
|
||||
console.table(
|
||||
files.map((f) => ({
|
||||
file_name: f.file_name,
|
||||
url: `https://zip-worker.jordan1320.workers.dev/zip/${
|
||||
row.skin_md5
|
||||
}/${encodeURI(f.file_name)}`,
|
||||
})),
|
||||
["file_name", "url"]
|
||||
);
|
||||
const answer = await ask("skip (s), delete (d)");
|
||||
switch (answer) {
|
||||
case "s":
|
||||
break;
|
||||
case "d":
|
||||
await Skins.deleteSkin(row.skin_md5);
|
||||
}
|
||||
}
|
||||
*/
|
||||
break;
|
||||
}
|
||||
case "tweet": {
|
||||
console.log("tweet & insta");
|
||||
const hash = argv._[1];
|
||||
await tweet(client, hash);
|
||||
await insta(client, hash);
|
||||
break;
|
||||
}
|
||||
case "insta": {
|
||||
console.log("insta");
|
||||
const hash = argv._[1];
|
||||
await insta(client, hash);
|
||||
break;
|
||||
}
|
||||
case "stats": {
|
||||
console.log(await Skins.getStats());
|
||||
break;
|
||||
}
|
||||
case "add": {
|
||||
const filePath = argv._[1];
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
console.log(await addSkinFromBuffer(buffer, filePath, "cli-user"));
|
||||
break;
|
||||
}
|
||||
case "screenshot": {
|
||||
const filePath = argv._[1];
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
const md5 = md5Buffer(buffer);
|
||||
const tempPath = temp.path({ suffix: ".png" });
|
||||
await Shooter.withShooter(
|
||||
(shooter) => {
|
||||
shooter.takeScreenshot(buffer, tempPath, { md5 });
|
||||
},
|
||||
(message) => console.log(message)
|
||||
);
|
||||
console.log("Screenshot complete", tempPath);
|
||||
break;
|
||||
}
|
||||
case "delete": {
|
||||
const md5 = argv._[1];
|
||||
await Skins.deleteSkin(md5);
|
||||
break;
|
||||
}
|
||||
case "index": {
|
||||
console.log(await Skins.updateSearchIndex(argv._[1]));
|
||||
break;
|
||||
}
|
||||
case "process": {
|
||||
const handler = new DiscordEventHandler();
|
||||
await processUserUploads((event) => handler.handle(event));
|
||||
handler.dispose();
|
||||
break;
|
||||
}
|
||||
case "add-missing-indexes": {
|
||||
// eslint-disable-next-line no-inner-declarations
|
||||
async function indexPage(pageNumber: number): Promise<void> {
|
||||
const page = await Skins.getMuseumPage({
|
||||
offset: pageNumber * 500,
|
||||
first: 500,
|
||||
});
|
||||
const toCheck = page.map(({ md5 }) => md5);
|
||||
const found = await searchIndex.getObjects(toCheck, {
|
||||
attributesToRetrieve: ["objectId"],
|
||||
});
|
||||
// console.log({ toCheck, found });
|
||||
const missing = toCheck.filter((_md5, i) => {
|
||||
return found.results[i] == null;
|
||||
});
|
||||
|
||||
console.log(
|
||||
`Found ${missing.length} missing skins on page ${pageNumber}`
|
||||
);
|
||||
console.log(await Skins.updateSearchIndexs(missing));
|
||||
}
|
||||
|
||||
for (let i = 0; i < 140; i++) {
|
||||
await indexPage(i);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "tweet-data": {
|
||||
await scrapeLikeData();
|
||||
break;
|
||||
}
|
||||
case "popular-tweets": {
|
||||
await popularTweets(handler);
|
||||
break;
|
||||
}
|
||||
|
||||
case "follower-count": {
|
||||
await followerCount(handler);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
console.log(`Unknown command ${argv._[0]}`);
|
||||
}
|
||||
await cb(handler);
|
||||
} finally {
|
||||
knex.destroy();
|
||||
logger.close();
|
||||
client.destroy();
|
||||
await handler.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
async function withDiscordClient(
|
||||
cb: (handler: Discord.Client) => Promise<void>
|
||||
) {
|
||||
const client = new Discord.Client();
|
||||
try {
|
||||
await cb(client);
|
||||
} finally {
|
||||
client.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
const temp = _temp.track();
|
||||
|
||||
/**
|
||||
* CLI starts here.
|
||||
*/
|
||||
program
|
||||
.name("skins-database")
|
||||
.description("CLI for interacting with the skins database");
|
||||
|
||||
/**
|
||||
* Social Media Commands
|
||||
*/
|
||||
|
||||
program
|
||||
.command("share")
|
||||
.description(
|
||||
"Share a skin on Twitter and Instagram. If no md5 is " +
|
||||
"given, random approved skins are shared."
|
||||
)
|
||||
.argument("[md5]", "md5 of the skin to share")
|
||||
.option("-t, --twitter", "Share on Twitter")
|
||||
.option("-i, --instagram", "Share on Instagram")
|
||||
.action(async (md5, { twitter, instagram }) => {
|
||||
if (!twitter && !instagram) {
|
||||
throw new Error("Expected at least one of --twitter or --instagram");
|
||||
}
|
||||
await withDiscordClient(async (client) => {
|
||||
if (twitter) {
|
||||
await tweet(client, md5);
|
||||
}
|
||||
if (instagram) {
|
||||
await insta(client, md5);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Operate on individual skins.
|
||||
*/
|
||||
program
|
||||
.command("skin")
|
||||
.description("Operate on a skngle skin from the database.")
|
||||
.argument("<md5>", "md5 of the skin to operate on")
|
||||
.option(
|
||||
"--delete",
|
||||
"Delete a skin from the database, including its S3 files " +
|
||||
"CloudFlare cache and seach index entries."
|
||||
)
|
||||
.option("--index", "Update the seach index for a skin.")
|
||||
.option(
|
||||
"--refresh",
|
||||
"Retake the screenshot of a skin and update the database."
|
||||
)
|
||||
.option("--reject", 'Give a skin a "rejected" review.')
|
||||
.action(async (md5, { delete: del, index, refresh, reject }) => {
|
||||
const ctx = new UserContext("CLI");
|
||||
if (del) {
|
||||
await Skins.deleteSkin(md5);
|
||||
}
|
||||
if (index) {
|
||||
console.log(await Skins.updateSearchIndex(md5));
|
||||
}
|
||||
if (refresh) {
|
||||
const skin = await SkinModel.fromMd5(ctx, md5);
|
||||
if (skin == null) {
|
||||
throw new Error(`Could not find skin ${md5}`);
|
||||
}
|
||||
await refreshSkins([skin]);
|
||||
}
|
||||
if (reject) {
|
||||
await Skins.reject(ctx, md5);
|
||||
}
|
||||
});
|
||||
|
||||
program
|
||||
.command("file")
|
||||
.description("Operate on a skin file.")
|
||||
.argument("<file-path>", "Path to the skin to add to the database.")
|
||||
.option("--add", "Add this skin to the database.")
|
||||
.option(
|
||||
"--screenshot",
|
||||
"Take (or retake) a screenshot of the given skin file."
|
||||
)
|
||||
.action(async (filePath, { add, screenshot }) => {
|
||||
if (add) {
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
console.log(await addSkinFromBuffer(buffer, filePath, "cli-user"));
|
||||
}
|
||||
if (screenshot) {
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
const md5 = md5Buffer(buffer);
|
||||
const tempPath = temp.path({ suffix: ".png" });
|
||||
await Shooter.withShooter(
|
||||
async (shooter: Shooter) => {
|
||||
await shooter.takeScreenshot(buffer, tempPath, { md5 });
|
||||
},
|
||||
(message: string) => console.log(message)
|
||||
);
|
||||
console.log("Screenshot complete", tempPath);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Internet Archive Commands
|
||||
*/
|
||||
program
|
||||
.command("ia")
|
||||
.description("Interact with the Internet Archive API.")
|
||||
.option(
|
||||
"--fetch-metadata <count>",
|
||||
"Fetch missing metadata for <count> items from the Internet " +
|
||||
"Archive. Currently it only fetches missing metadata. In the " +
|
||||
"future it could refresh stale metadata."
|
||||
)
|
||||
.option(
|
||||
"--fetch-items",
|
||||
"Seach the Internet Archive for items that we don't know about" +
|
||||
"and add them to our database."
|
||||
)
|
||||
.option(
|
||||
"--webamp-links",
|
||||
"Seach the Internet Archive for items missing a Webamp link" +
|
||||
"and add them."
|
||||
)
|
||||
.option(
|
||||
"--upload",
|
||||
"Find newly uploaded skins, and publish them to the Internet Archive."
|
||||
)
|
||||
.action(async ({ metadata, webampLinks, fetchItems, upload }) => {
|
||||
if (metadata) {
|
||||
await fillMissingMetadata(Number(metadata || 1000));
|
||||
}
|
||||
if (webampLinks) {
|
||||
await ensureWebampLinks();
|
||||
}
|
||||
if (fetchItems) {
|
||||
await syncFromArchive();
|
||||
}
|
||||
if (upload) {
|
||||
await withHandler(async (handler) => {
|
||||
await syncWithArchive(handler);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Investigation and recovery commands
|
||||
*/
|
||||
|
||||
program
|
||||
.command("stats")
|
||||
.description(
|
||||
"Report information about skins in the database. " +
|
||||
"Identical to `!stats` in Discord."
|
||||
)
|
||||
.action(async () => {
|
||||
console.table([await Skins.getStats()]);
|
||||
});
|
||||
|
||||
program
|
||||
.command("process-uploads")
|
||||
.description("Process any unprocessed user uploads.")
|
||||
.option("--errored", "Reprocess errored uploads.")
|
||||
.action(async ({ errored }) => {
|
||||
await withHandler(async (handler) => {
|
||||
if (!errored) {
|
||||
await processUserUploads((event) => handler.handle(event));
|
||||
} else {
|
||||
await reprocessFailedUploads(handler);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
program
|
||||
.command("integrity-check")
|
||||
.description("Perfrom a non-exhaustive list of database consistency checks")
|
||||
.action(async () => {
|
||||
await integrityCheck();
|
||||
});
|
||||
|
||||
/**
|
||||
* Scrape Twitter Commands
|
||||
*/
|
||||
|
||||
program
|
||||
.command("scrape-twitter")
|
||||
.description("Scrape Twitter in various ways.")
|
||||
.option(
|
||||
"--likes",
|
||||
"Scrape @winampskins tweets for like and retweet counts, " +
|
||||
"and update the database."
|
||||
)
|
||||
.option(
|
||||
"--milestones",
|
||||
"Check the most recent @winampskins tweets to see if they have " +
|
||||
"passed a milestone. If so, notify the Discord channel."
|
||||
)
|
||||
.option(
|
||||
"--followers",
|
||||
"Check if @winampskins has passed a follower count milestone. " +
|
||||
"If so, notify the Discord channel."
|
||||
)
|
||||
.action(async ({ likes, milestones, followers }) => {
|
||||
if (likes) {
|
||||
await scrapeLikeData();
|
||||
}
|
||||
if (milestones) {
|
||||
await withHandler(async (handler) => {
|
||||
await popularTweets(handler);
|
||||
});
|
||||
}
|
||||
if (followers) {
|
||||
await withHandler(async (handler) => {
|
||||
await followerCount(handler);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
await program.parseAsync(process.argv);
|
||||
} finally {
|
||||
knex.destroy();
|
||||
logger.close();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
import rl from "readline";
|
||||
|
||||
function ask(question): Promise<string> {
|
||||
return new Promise((resolve) => {
|
||||
const r = rl.createInterface({
|
||||
|
|
@ -241,5 +289,6 @@ function ask(question): Promise<string> {
|
|||
});
|
||||
});
|
||||
}
|
||||
*/
|
||||
|
||||
main();
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
"algoliasearch": "^4.3.0",
|
||||
"async-parallel": "^1.2.3",
|
||||
"aws-sdk": "^2.663.0",
|
||||
"commander": "^9.0.0",
|
||||
"cookie-session": "^1.4.0",
|
||||
"cors": "^2.8.5",
|
||||
"dataloader": "^2.0.0",
|
||||
|
|
|
|||
|
|
@ -109,12 +109,12 @@ async function ensureIaRecord(
|
|||
console.log(`Inserted "${identifier}".`);
|
||||
}
|
||||
|
||||
export async function fillMissingMetadata() {
|
||||
export async function fillMissingMetadata(count: number) {
|
||||
const skins = await knex("skins")
|
||||
.leftJoin("ia_items", "skins.md5", "ia_items.skin_md5")
|
||||
.where("ia_items.metadata", null)
|
||||
.whereNot("ia_items.identifier", null)
|
||||
.limit(4000)
|
||||
.limit(count)
|
||||
.select("skins.md5", "ia_items.identifier");
|
||||
for (const { md5, identifier } of skins) {
|
||||
await updateMetadata(identifier);
|
||||
|
|
|
|||
|
|
@ -5478,6 +5478,11 @@ commander@^6.2.0:
|
|||
resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
|
||||
integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==
|
||||
|
||||
commander@^9.0.0:
|
||||
version "9.0.0"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-9.0.0.tgz#86d58f24ee98126568936bd1d3574e0308a99a40"
|
||||
integrity sha512-JJfP2saEKbQqvW+FI93OYUB4ByV5cizMpFMiiJI8xDbBvQvSkIk0VvQdn1CZ8mqAO8Loq2h0gYTYtDFUZUeERw==
|
||||
|
||||
commander@~2.19.0:
|
||||
version "2.19.0"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue