From 25a88a4ec51f115859aa028776dbca992375b268 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sat, 19 Feb 2022 21:25:39 -0800 Subject: [PATCH] Cleanup/refactor --- packages/skin-database/cli.ts | 38 ++++----- packages/skin-database/data/IaItemModel.ts | 36 +++------ .../skin-database/services/internetArchive.ts | 46 +++++++++++ .../tasks/__tests__/refresh.test.ts | 10 +-- packages/skin-database/tasks/syncToArchive.ts | 79 +++++++++++++++---- 5 files changed, 143 insertions(+), 66 deletions(-) create mode 100644 packages/skin-database/services/internetArchive.ts diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index 6a2e0abf..467ae80e 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -15,12 +15,7 @@ import { integrityCheck, checkInternetArchiveMetadata, } from "./tasks/integrityCheck"; -import { - ensureWebampLinks, - findItemsMissingImages, - syncWithArchive, - uploadScreenshotIfSafe, -} from "./tasks/syncToArchive"; +import * as SyncToArchive from "./tasks/syncToArchive"; import { fillMissingMetadata, syncFromArchive } from "./tasks/syncFromArchive"; import { refreshSkins } from "./tasks/refresh"; import { @@ -109,7 +104,8 @@ program "Retake the screenshot of a skin and update the database." ) .option("--reject", 'Give a skin a "rejected" review.') - .action(async (md5, { delete: del, index, refresh, reject }) => { + .option("--metadata", "Push metadata to the archive.") + .action(async (md5, { delete: del, index, refresh, reject, metadata }) => { const ctx = new UserContext("CLI"); if (del) { await Skins.deleteSkin(md5); @@ -124,6 +120,9 @@ program if (reject) { await Skins.reject(ctx, md5); } + if (metadata) { + await SyncToArchive.updateMetadata(ctx, md5); + } }); program @@ -172,29 +171,32 @@ program "and add them to our database." ) .option( - "--webamp-links", - "Seach the Internet Archive for items missing a Webamp link" + - "and add them." + "--update-metadata ", + "Find items in our database that have incorrect or incomplete " + + "metadata, and update the Internet Archive" ) .option( "--upload-new", "Find newly uploaded skins, and publish them to the Internet Archive." ) - .action(async ({ fetchMetadata, webampLinks, fetchItems, upload }) => { + .action(async ({ fetchMetadata, updateMetadata, fetchItems, upload }) => { if (fetchMetadata) { await fillMissingMetadata(Number(fetchMetadata || 1000)); } - if (webampLinks) { - await ensureWebampLinks(); - } if (fetchItems) { await syncFromArchive(); } if (upload) { await withHandler(async (handler) => { - await syncWithArchive(handler); + await SyncToArchive.syncToArchive(handler); }); } + if (updateMetadata) { + await SyncToArchive.updateMissingMetadata( + new UserContext(), + Number(updateMetadata || 1000) + ); + } }); /** @@ -297,14 +299,14 @@ program .action(async ({ uploadIaScreenshot, uploadMissingScreenshots }) => { if (uploadIaScreenshot) { const md5 = uploadIaScreenshot; - if (!(await uploadScreenshotIfSafe(md5))) { + if (!(await SyncToArchive.uploadScreenshotIfSafe(md5))) { console.log("Did not upload screenshot"); } } if (uploadMissingScreenshots) { - const md5s = await findItemsMissingImages(); + const md5s = await SyncToArchive.findItemsMissingImages(); for (const md5 of md5s) { - if (await uploadScreenshotIfSafe(md5)) { + if (await SyncToArchive.uploadScreenshotIfSafe(md5)) { console.log("Upladed screenshot for ", md5); } else { console.log("Did not upload screenshot for ", md5); diff --git a/packages/skin-database/data/IaItemModel.ts b/packages/skin-database/data/IaItemModel.ts index bd619cc0..0565da42 100644 --- a/packages/skin-database/data/IaItemModel.ts +++ b/packages/skin-database/data/IaItemModel.ts @@ -3,8 +3,7 @@ import { IaItemRow } from "../types"; import SkinModel from "./SkinModel"; import DataLoader from "dataloader"; import { knex } from "../db"; -import { exec } from "../utils"; -import fetch from "node-fetch"; +import { fetchMetadata, fetchTasks } from "../services/internetArchive"; const IA_URL = /^(https:\/\/)?archive.org\/details\/([^/]+)\/?/; @@ -77,10 +76,10 @@ export default class IaItemModel { return []; } try { - return JSON.parse(this.row.metadata).files; - } catch(e) { - console.warn("Could not parse", this.row.metadata); - return []; + return JSON.parse(this.row.metadata).files; + } catch (e) { + console.warn("Could not parse", this.row.metadata); + return []; } } @@ -91,20 +90,15 @@ export default class IaItemModel { // There should be exactly one, but in error cases there can be more or none. getSkinFiles(): any[] { return this.getUploadedFiles().filter((file) => { - return file.name.toLowerCase().endsWith(".wsz") - || file.name.toLowerCase().endsWith(".zip"); + return ( + file.name.toLowerCase().endsWith(".wsz") || + file.name.toLowerCase().endsWith(".zip") + ); }); } async getTasks(): Promise { - const command = `ia tasks ${this.getIdentifier()}`; - const result = await exec(command, { - encoding: "utf8", - }); - return result.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)) + return fetchTasks(this.getIdentifier()); } async hasRunningTasks(): Promise { @@ -132,15 +126,11 @@ export default class IaItemModel { async updateMetadataUnsafe(): Promise { // TODO: Move some of this into a IA service. const identifier = this.getIdentifier(); - const r = await fetch(`https://archive.org/metadata/${identifier}`); - if (!r.ok) { - console.error(await r.json()); - throw new Error(`Could not fetch metadata for ${identifier}`); - } - const response = await r.json(); + const metadata = await fetchMetadata(identifier); + await knex("ia_items") .where("identifier", identifier) - .update({ metadata: JSON.stringify(response, null, 2) }) + .update({ metadata: JSON.stringify(metadata, null, 2) }) .update("metadata_timestamp", knex.fn.now()); } diff --git a/packages/skin-database/services/internetArchive.ts b/packages/skin-database/services/internetArchive.ts new file mode 100644 index 00000000..da89100e --- /dev/null +++ b/packages/skin-database/services/internetArchive.ts @@ -0,0 +1,46 @@ +import fetch from "node-fetch"; +import { exec } from "../utils"; + +export async function fetchMetadata(identifier: string): Promise { + const r = await fetch(`https://archive.org/metadata/${identifier}`); + if (!r.ok) { + console.error(await r.json()); + throw new Error(`Could not fetch metadata for ${identifier}`); + } + return r.json(); +} + +export async function fetchTasks(identifier: string): Promise { + const command = `ia tasks ${identifier}`; + const result = await exec(command, { + encoding: "utf8", + }); + return result.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); +} + +export async function uploadFile( + identifier: string, + filepath: string +): Promise { + const command = `ia upload ${identifier} "${filepath}"`; + await exec(command, { encoding: "utf8" }); +} + +export async function identifierExists(identifier: string): Promise { + const result = await exec(`ia metadata ${identifier}`); + const data = JSON.parse(result.stdout); + return Object.keys(data).length > 0; +} + +export async function setMetadata( + identifier: string, + data: { [key: string]: string } +) { + const pairs = Object.entries(data).map(([key, value]) => `${key}:${value}`); + const args = pairs.map((pair) => `--modify="${pair}"`); + const command = `ia metadata ${identifier} ${args.join(" ")}`; + await exec(command); +} diff --git a/packages/skin-database/tasks/__tests__/refresh.test.ts b/packages/skin-database/tasks/__tests__/refresh.test.ts index 6dd4961d..23f5f9f9 100644 --- a/packages/skin-database/tasks/__tests__/refresh.test.ts +++ b/packages/skin-database/tasks/__tests__/refresh.test.ts @@ -33,10 +33,7 @@ test("refresh", async () => { test("can't extract", async () => { const ctx = new UserContext(); - const skin = await SkinModel.fromMd5(ctx, "a_fake_md5"); - if (skin == null) { - throw new Error("Could not find skin"); - } + const skin = await SkinModel.fromMd5Assert(ctx, "a_fake_md5"); skin.getBuffer = async () => Buffer.from(""); await refresh(skin, shooter); @@ -54,10 +51,7 @@ test("can't extract", async () => { test("valid skin (TopazAmp)", async () => { const ctx = new UserContext(); - const skin = await SkinModel.fromMd5(ctx, "a_fake_md5"); - if (skin == null) { - throw new Error("Could not find skin"); - } + const skin = await SkinModel.fromMd5Assert(ctx, "a_fake_md5"); skin.getBuffer = async () => { return fs.readFileSync( diff --git a/packages/skin-database/tasks/syncToArchive.ts b/packages/skin-database/tasks/syncToArchive.ts index a38e476d..cd41184f 100644 --- a/packages/skin-database/tasks/syncToArchive.ts +++ b/packages/skin-database/tasks/syncToArchive.ts @@ -9,6 +9,7 @@ import * as Parallel from "async-parallel"; import IaItemModel from "../data/IaItemModel"; import DiscordEventHandler from "../api/DiscordEventHandler"; import { exec } from "../utils"; +import * as IAService from "../services/internetArchive"; export async function findItemsMissingImages(): Promise { const ctx = new UserContext(); @@ -21,18 +22,16 @@ export async function findItemsMissingImages(): Promise { if (iaItem == null) { throw new Error("Expected to find IA item"); } - if(iaItem.getSkinFiles().length > 1) { - console.warn("Too many skin files", row.skin_md5, row.identifier); - continue; + if (iaItem.getSkinFiles().length > 1) { + console.warn("Too many skin files", row.skin_md5, row.identifier); + continue; } - if(iaItem.getSkinFiles().length < 1) { - console.log(iaItem.getAllFiles()); - console.warn("Missing skin file", row.skin_md5, row.identifier); - continue; + if (iaItem.getSkinFiles().length < 1) { + console.log(iaItem.getAllFiles()); + console.warn("Missing skin file", row.skin_md5, row.identifier); + continue; } - if ( - iaItem.getUploadedFiles().length >= 2 - ) { + if (iaItem.getUploadedFiles().length >= 2) { continue; } @@ -66,18 +65,66 @@ export async function uploadScreenshotIfSafe(md5: string): Promise { } const uploadedFiles = iaItem.getUploadedFiles(); if (uploadedFiles.length !== skinFiles.length) { - console.warn(`Has ${skinFiles.length} skins and ${uploadedFiles.length} uploaded files.`); + console.warn( + `Has ${skinFiles.length} skins and ${uploadedFiles.length} uploaded files.` + ); return false; } await skin.withScreenshotTempFile(async (screenshotFile) => { - const command = `ia upload ${iaItem.getIdentifier()} "${screenshotFile}"`; - await exec(command, { encoding: "utf8" }); + await IAService.uploadFile(iaItem.getIdentifier(), screenshotFile); }); await iaItem.invalidateMetadata(); return true; } +export async function updateMissingMetadata( + ctx: UserContext, + count: number +): Promise { + const results = await knex("ia_items").limit(count).select(); + + for (const row of results) { + const _iaItem = new IaItemModel(ctx, row); + throw new Error("Not implemented yet"); + } + // +} + +export async function updateMetadata( + ctx: UserContext, + md5: string +): Promise { + const skin = await SkinModel.fromMd5Assert(ctx, md5); + if (skin.getSkinType() !== "CLASSIC") { + throw new Error("Only classic skins can be updated"); + } + + const filename = await skin.getFileName(); + + const title = `Winamp Skin: ${filename}`; + + const metadata: { [key: string]: string } = { + title, + skintype: "wsz", + mediatype: "software", + webamp: skin.getWebampUrl(), + museum: skin.getMuseumUrl(), + }; + + if (await skin.getIsNsfw()) { + metadata.review = "NSFW"; + } + + const iaItem = await skin.getIaItem(); + if (iaItem == null) { + throw new Error("Expected IA item to exist"); + } + + await IAService.setMetadata(iaItem.getIdentifier(), metadata); + await iaItem.invalidateMetadata(); +} + /** LEGACY BELOW HERE */ const CONCURRENT = 1; @@ -123,9 +170,7 @@ export async function identifierExists(identifier: string): Promise { if (existing.length > 0) { return true; } - const result = await exec(`ia metadata ${identifier}`); - const data = JSON.parse(result.stdout); - return Object.keys(data).length > 0; + return IAService.identifierExists(identifier); } async function getNewIdentifier(filename: string): Promise { @@ -162,7 +207,7 @@ export async function archive(skin: SkinModel): Promise { return identifier; } -export async function syncWithArchive(handler: DiscordEventHandler) { +export async function syncToArchive(handler: DiscordEventHandler) { const ctx = new UserContext(); console.log("Checking which new skins we have..."); const unarchived = await knex("skins")