Mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp.git synced 2026-07-20 16:49:52 +00:00
Code Issues Projects Releases Wiki Activity

Scrape Internet Archive metadata

Browse source
This commit is contained in:
Jordan Eldredge 2022-02-18 20:09:16 -08:00
parent 42cac02806
commit 11ad87a95c
3 changed files with 57 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

14
packages/skin-database/migrations/20220218192912_ia_metadata.ts Normal file
View file

@ -0,0 +1,14 @@
import * as Knex from "knex";
export async function up(knex: Knex): Promise<any> {
await knex.raw(
`ALTER TABLE ia_items ADD COLUMN metadata;`
);
}
export async function down(knex: Knex): Promise<any> {
throw new Error("I never implemented a down migration for adding metadata.");
}

14
packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts Normal file
View file

@ -0,0 +1,14 @@
import * as Knex from "knex";
export async function up(knex: Knex): Promise<any> {
await knex.raw(
`ALTER TABLE ia_items ADD COLUMN metadata_timestamp;`
);
}
export async function down(knex: Knex): Promise<any> {
throw new Error("I never implemented a down migration for adding metadata_timestamp.");
}

31
packages/skin-database/tasks/syncFromArchive.ts
View file

@ -73,8 +73,7 @@ async function ensureIaRecord(
ctx: UserContext,
identifier: string
): Promise<void> {
const r = await fetch(`https://archive.org/metadata/${identifier}`);
const response = await r.json();
const response = await updateMetadata(identifier);
const files = response.files;
const skins = files.filter((file) => file.name.endsWith(".wsz"));
if (skins.length === 0) {
@ -110,6 +109,34 @@ async function ensureIaRecord(
console.log(`Inserted "${identifier}".`);
}
export async function fillMissingMetadata() {
const skins = await knex("skins")
.leftJoin("ia_items", "skins.md5", "ia_items.skin_md5")
.where("ia_items.metadata", null)
.whereNot("ia_items.identifier", null)
.limit(10)
.select("skins.md5", "ia_items.identifier");
for (const { md5, identifier } of skins) {
await updateMetadata(identifier);
console.log(`Updated metadata for ${identifier} ${md5}`);
}
console.log("Done updating metadata.");
}
async function updateMetadata(identifier: string): Promise<any> {
const r = await fetch(`https://archive.org/metadata/${identifier}`);
if (!r.ok) {
console.error(await r.json())
throw new Error(`Could not fetch metadata for ${identifier}`);
}
const response = await r.json();
await knex("ia_items")
.where("identifier", identifier)
.update({ metadata: JSON.stringify(response, null, 2) })
.update("metadata_timestamp", knex.fn.now());
return response;
}
export async function syncFromArchive() {
const ctx = new UserContext();
// Ensure we know about all items in the `winampskins` collection.