diff --git a/packages/skin-database/migrations/20220218192912_ia_metadata.ts b/packages/skin-database/migrations/20220218192912_ia_metadata.ts new file mode 100644 index 00000000..8fe96eaf --- /dev/null +++ b/packages/skin-database/migrations/20220218192912_ia_metadata.ts @@ -0,0 +1,14 @@ +import * as Knex from "knex"; + + +export async function up(knex: Knex): Promise { + await knex.raw( + `ALTER TABLE ia_items ADD COLUMN metadata;` + ); +} + + +export async function down(knex: Knex): Promise { + throw new Error("I never implemented a down migration for adding metadata."); +} + diff --git a/packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts b/packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts new file mode 100644 index 00000000..2996257e --- /dev/null +++ b/packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts @@ -0,0 +1,14 @@ +import * as Knex from "knex"; + + +export async function up(knex: Knex): Promise { + await knex.raw( + `ALTER TABLE ia_items ADD COLUMN metadata_timestamp;` + ); +} + + +export async function down(knex: Knex): Promise { + throw new Error("I never implemented a down migration for adding metadata_timestamp."); +} + diff --git a/packages/skin-database/tasks/syncFromArchive.ts b/packages/skin-database/tasks/syncFromArchive.ts index 00cbbb56..7bc83fcf 100644 --- a/packages/skin-database/tasks/syncFromArchive.ts +++ b/packages/skin-database/tasks/syncFromArchive.ts @@ -73,8 +73,7 @@ async function ensureIaRecord( ctx: UserContext, identifier: string ): Promise { - const r = await fetch(`https://archive.org/metadata/${identifier}`); - const response = await r.json(); + const response = await updateMetadata(identifier); const files = response.files; const skins = files.filter((file) => file.name.endsWith(".wsz")); if (skins.length === 0) { @@ -110,6 +109,34 @@ async function ensureIaRecord( console.log(`Inserted "${identifier}".`); } +export async function fillMissingMetadata() { + const skins = await knex("skins") + .leftJoin("ia_items", "skins.md5", "ia_items.skin_md5") + .where("ia_items.metadata", null) + .whereNot("ia_items.identifier", null) + .limit(10) + .select("skins.md5", "ia_items.identifier"); + for (const { md5, identifier } of skins) { + await updateMetadata(identifier); + console.log(`Updated metadata for ${identifier} ${md5}`); + } + console.log("Done updating metadata."); +} + +async function updateMetadata(identifier: string): Promise { + const r = await fetch(`https://archive.org/metadata/${identifier}`); + if (!r.ok) { + console.error(await r.json()) + throw new Error(`Could not fetch metadata for ${identifier}`); + } + const response = await r.json(); + await knex("ia_items") + .where("identifier", identifier) + .update({ metadata: JSON.stringify(response, null, 2) }) + .update("metadata_timestamp", knex.fn.now()); + return response; +} + export async function syncFromArchive() { const ctx = new UserContext(); // Ensure we know about all items in the `winampskins` collection.