Mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp.git synced 2026-08-02 06:52:19 +00:00
Code Issues Projects Releases Wiki Activity

Cleanup/refactor

Browse source
This commit is contained in:
Jordan Eldredge 2022-02-19 21:25:39 -08:00
parent a4a3508d41
commit 25a88a4ec5
5 changed files with 143 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

38
packages/skin-database/cli.ts
View file

@ -15,12 +15,7 @@ import {
integrityCheck,
checkInternetArchiveMetadata,
} from "./tasks/integrityCheck";
import {
ensureWebampLinks,
findItemsMissingImages,
syncWithArchive,
uploadScreenshotIfSafe,
} from "./tasks/syncToArchive";
import * as SyncToArchive from "./tasks/syncToArchive";
import { fillMissingMetadata, syncFromArchive } from "./tasks/syncFromArchive";
import { refreshSkins } from "./tasks/refresh";
import {
@ -109,7 +104,8 @@ program
"Retake the screenshot of a skin and update the database."
)
.option("--reject", 'Give a skin a "rejected" review.')
.action(async (md5, { delete: del, index, refresh, reject }) => {
.option("--metadata", "Push metadata to the archive.")
.action(async (md5, { delete: del, index, refresh, reject, metadata }) => {
const ctx = new UserContext("CLI");
if (del) {
await Skins.deleteSkin(md5);
@ -124,6 +120,9 @@ program
if (reject) {
await Skins.reject(ctx, md5);
}
if (metadata) {
await SyncToArchive.updateMetadata(ctx, md5);
}
});
program
@ -172,29 +171,32 @@ program
"and add them to our database."
)
.option(
"--webamp-links",
"Seach the Internet Archive for items missing a Webamp link" +
"and add them."
"--update-metadata <count>",
"Find <count> items in our database that have incorrect or incomplete " +
"metadata, and update the Internet Archive"
)
.option(
"--upload-new",
"Find newly uploaded skins, and publish them to the Internet Archive."
)
.action(async ({ fetchMetadata, webampLinks, fetchItems, upload }) => {
.action(async ({ fetchMetadata, updateMetadata, fetchItems, upload }) => {
if (fetchMetadata) {
await fillMissingMetadata(Number(fetchMetadata || 1000));
}
if (webampLinks) {
await ensureWebampLinks();
}
if (fetchItems) {
await syncFromArchive();
}
if (upload) {
await withHandler(async (handler) => {
await syncWithArchive(handler);
await SyncToArchive.syncToArchive(handler);
});
}
if (updateMetadata) {
await SyncToArchive.updateMissingMetadata(
new UserContext(),
Number(updateMetadata || 1000)
);
}
});
/**
@ -297,14 +299,14 @@ program
.action(async ({ uploadIaScreenshot, uploadMissingScreenshots }) => {
if (uploadIaScreenshot) {
const md5 = uploadIaScreenshot;
if (!(await uploadScreenshotIfSafe(md5))) {
if (!(await SyncToArchive.uploadScreenshotIfSafe(md5))) {
console.log("Did not upload screenshot");
}
}
if (uploadMissingScreenshots) {
const md5s = await findItemsMissingImages();
const md5s = await SyncToArchive.findItemsMissingImages();
for (const md5 of md5s) {
if (await uploadScreenshotIfSafe(md5)) {
if (await SyncToArchive.uploadScreenshotIfSafe(md5)) {
console.log("Upladed screenshot for ", md5);
} else {
console.log("Did not upload screenshot for ", md5);

36
packages/skin-database/data/IaItemModel.ts
View file

@ -3,8 +3,7 @@ import { IaItemRow } from "../types";
import SkinModel from "./SkinModel";
import DataLoader from "dataloader";
import { knex } from "../db";
import { exec } from "../utils";
import fetch from "node-fetch";
import { fetchMetadata, fetchTasks } from "../services/internetArchive";
const IA_URL = /^(https:\/\/)?archive.org\/details\/([^/]+)\/?/;
@ -77,10 +76,10 @@ export default class IaItemModel {
return [];
}
try {
return JSON.parse(this.row.metadata).files;
} catch(e) {
console.warn("Could not parse", this.row.metadata);
return [];
return JSON.parse(this.row.metadata).files;
} catch (e) {
console.warn("Could not parse", this.row.metadata);
return [];
}
}
@ -91,20 +90,15 @@ export default class IaItemModel {
// There should be exactly one, but in error cases there can be more or none.
getSkinFiles(): any[] {
return this.getUploadedFiles().filter((file) => {
return file.name.toLowerCase().endsWith(".wsz")
|| file.name.toLowerCase().endsWith(".zip");
return (
file.name.toLowerCase().endsWith(".wsz") ||
file.name.toLowerCase().endsWith(".zip")
);
});
}
async getTasks(): Promise<any[]> {
const command = `ia tasks ${this.getIdentifier()}`;
const result = await exec(command, {
encoding: "utf8",
});
return result.stdout
.trim()
.split("\n")
.map((line) => JSON.parse(line))
return fetchTasks(this.getIdentifier());
}
async hasRunningTasks(): Promise<boolean> {
@ -132,15 +126,11 @@ export default class IaItemModel {
async updateMetadataUnsafe(): Promise<void> {
// TODO: Move some of this into a IA service.
const identifier = this.getIdentifier();
const r = await fetch(`https://archive.org/metadata/${identifier}`);
if (!r.ok) {
console.error(await r.json());
throw new Error(`Could not fetch metadata for ${identifier}`);
}
const response = await r.json();
const metadata = await fetchMetadata(identifier);
await knex("ia_items")
.where("identifier", identifier)
.update({ metadata: JSON.stringify(response, null, 2) })
.update({ metadata: JSON.stringify(metadata, null, 2) })
.update("metadata_timestamp", knex.fn.now());
}

46
packages/skin-database/services/internetArchive.ts Normal file
View file

@ -0,0 +1,46 @@
import fetch from "node-fetch";
import { exec } from "../utils";
export async function fetchMetadata(identifier: string): Promise<any> {
const r = await fetch(`https://archive.org/metadata/${identifier}`);
if (!r.ok) {
console.error(await r.json());
throw new Error(`Could not fetch metadata for ${identifier}`);
}
return r.json();
}
export async function fetchTasks(identifier: string): Promise<any> {
const command = `ia tasks ${identifier}`;
const result = await exec(command, {
encoding: "utf8",
});
return result.stdout
.trim()
.split("\n")
.map((line) => JSON.parse(line));
}
export async function uploadFile(
identifier: string,
filepath: string
): Promise<any> {
const command = `ia upload ${identifier} "${filepath}"`;
await exec(command, { encoding: "utf8" });
}
export async function identifierExists(identifier: string): Promise<boolean> {
const result = await exec(`ia metadata ${identifier}`);
const data = JSON.parse(result.stdout);
return Object.keys(data).length > 0;
}
export async function setMetadata(
identifier: string,
data: { [key: string]: string }
) {
const pairs = Object.entries(data).map(([key, value]) => `${key}:${value}`);
const args = pairs.map((pair) => `--modify="${pair}"`);
const command = `ia metadata ${identifier} ${args.join(" ")}`;
await exec(command);
}

10
packages/skin-database/tasks/__tests__/refresh.test.ts
View file

@ -33,10 +33,7 @@ test("refresh", async () => {
test("can't extract", async () => {
const ctx = new UserContext();
const skin = await SkinModel.fromMd5(ctx, "a_fake_md5");
if (skin == null) {
throw new Error("Could not find skin");
}
const skin = await SkinModel.fromMd5Assert(ctx, "a_fake_md5");
skin.getBuffer = async () => Buffer.from("");
await refresh(skin, shooter);
@ -54,10 +51,7 @@ test("can't extract", async () => {
test("valid skin (TopazAmp)", async () => {
const ctx = new UserContext();
const skin = await SkinModel.fromMd5(ctx, "a_fake_md5");
if (skin == null) {
throw new Error("Could not find skin");
}
const skin = await SkinModel.fromMd5Assert(ctx, "a_fake_md5");
skin.getBuffer = async () => {
return fs.readFileSync(

79
packages/skin-database/tasks/syncToArchive.ts
View file

@ -9,6 +9,7 @@ import * as Parallel from "async-parallel";
import IaItemModel from "../data/IaItemModel";
import DiscordEventHandler from "../api/DiscordEventHandler";
import { exec } from "../utils";
import * as IAService from "../services/internetArchive";
export async function findItemsMissingImages(): Promise<string[]> {
const ctx = new UserContext();
@ -21,18 +22,16 @@ export async function findItemsMissingImages(): Promise<string[]> {
if (iaItem == null) {
throw new Error("Expected to find IA item");
}
if(iaItem.getSkinFiles().length > 1) {
console.warn("Too many skin files", row.skin_md5, row.identifier);
continue;
if (iaItem.getSkinFiles().length > 1) {
console.warn("Too many skin files", row.skin_md5, row.identifier);
continue;
}
if(iaItem.getSkinFiles().length < 1) {
console.log(iaItem.getAllFiles());
console.warn("Missing skin file", row.skin_md5, row.identifier);
continue;
if (iaItem.getSkinFiles().length < 1) {
console.log(iaItem.getAllFiles());
console.warn("Missing skin file", row.skin_md5, row.identifier);
continue;
}
if (
iaItem.getUploadedFiles().length >= 2
) {
if (iaItem.getUploadedFiles().length >= 2) {
continue;
}
@ -66,18 +65,66 @@ export async function uploadScreenshotIfSafe(md5: string): Promise<boolean> {
}
const uploadedFiles = iaItem.getUploadedFiles();
if (uploadedFiles.length !== skinFiles.length) {
console.warn(`Has ${skinFiles.length} skins and ${uploadedFiles.length} uploaded files.`);
console.warn(
`Has ${skinFiles.length} skins and ${uploadedFiles.length} uploaded files.`
);
return false;
}
await skin.withScreenshotTempFile(async (screenshotFile) => {
const command = `ia upload ${iaItem.getIdentifier()} "${screenshotFile}"`;
await exec(command, { encoding: "utf8" });
await IAService.uploadFile(iaItem.getIdentifier(), screenshotFile);
});
await iaItem.invalidateMetadata();
return true;
}
export async function updateMissingMetadata(
ctx: UserContext,
count: number
): Promise<void> {
const results = await knex("ia_items").limit(count).select();
for (const row of results) {
const _iaItem = new IaItemModel(ctx, row);
throw new Error("Not implemented yet");
}
//
}
export async function updateMetadata(
ctx: UserContext,
md5: string
): Promise<void> {
const skin = await SkinModel.fromMd5Assert(ctx, md5);
if (skin.getSkinType() !== "CLASSIC") {
throw new Error("Only classic skins can be updated");
}
const filename = await skin.getFileName();
const title = `Winamp Skin: ${filename}`;
const metadata: { [key: string]: string } = {
title,
skintype: "wsz",
mediatype: "software",
webamp: skin.getWebampUrl(),
museum: skin.getMuseumUrl(),
};
if (await skin.getIsNsfw()) {
metadata.review = "NSFW";
}
const iaItem = await skin.getIaItem();
if (iaItem == null) {
throw new Error("Expected IA item to exist");
}
await IAService.setMetadata(iaItem.getIdentifier(), metadata);
await iaItem.invalidateMetadata();
}
/** LEGACY BELOW HERE */
const CONCURRENT = 1;
@ -123,9 +170,7 @@ export async function identifierExists(identifier: string): Promise<boolean> {
if (existing.length > 0) {
return true;
}
const result = await exec(`ia metadata ${identifier}`);
const data = JSON.parse(result.stdout);
return Object.keys(data).length > 0;
return IAService.identifierExists(identifier);
}
async function getNewIdentifier(filename: string): Promise<string> {
@ -162,7 +207,7 @@ export async function archive(skin: SkinModel): Promise<string> {
return identifier;
}
export async function syncWithArchive(handler: DiscordEventHandler) {
export async function syncToArchive(handler: DiscordEventHandler) {
const ctx = new UserContext();
console.log("Checking which new skins we have...");
const unarchived = await knex("skins")