Fixes for screenshot upload to ia

packages/skin-database/data/IaItemModel.ts

@@ -4,6 +4,7 @@ import SkinModel from "./SkinModel";
 import DataLoader from "dataloader";
 import { knex } from "../db";
 import { exec } from "../utils";
+import fetch from "node-fetch";
 
 const IA_URL = /^(https:\/\/)?archive.org\/details\/([^/]+)\/?/;
 
@@ -72,10 +73,15 @@ export default class IaItemModel {
   }
 
   getAllFiles(): any[] {
-    if (this.row.metadata == null) {
+    if (!this.row.metadata) {
       return [];
     }
-    return JSON.parse(this.row.metadata).files;
+    try {
+        return JSON.parse(this.row.metadata).files;
+    } catch(e) {
+        console.warn("Could not parse", this.row.metadata);
+        return [];
+    }
   }
 
   getUploadedFiles(): any {
@@ -84,17 +90,21 @@ export default class IaItemModel {
 
   // There should be exactly one, but in error cases there can be more or none.
   getSkinFiles(): any[] {
-    return this.getUploadedFiles().filter((file) => file.name.endsWith(".wsz"));
+    return this.getUploadedFiles().filter((file) => {
+        return file.name.toLowerCase().endsWith(".wsz")
+            || file.name.toLowerCase().endsWith(".zip");
+    });
   }
 
   async getTasks(): Promise<any[]> {
-    const result = await exec(`ia tasks ${this.getIdentifier()}`, {
+    const command = `ia tasks ${this.getIdentifier()}`;
+    const result = await exec(command, {
       encoding: "utf8",
     });
     return result.stdout
       .trim()
       .split("\n")
-      .map((line) => JSON.parse(line));
+      .map((line) => JSON.parse(line))
   }
 
   async hasRunningTasks(): Promise<boolean> {

packages/skin-database/data/SkinModel.ts

@@ -34,7 +34,11 @@ export default class SkinModel {
     ctx: UserContext,
     md5: string
   ): Promise<SkinModel> {
-    return SkinModel.fromMd5Assert(ctx, md5);
+    const skin = await SkinModel.fromMd5(ctx, md5);
+    if(skin == null) {
+        throw new Error(`Expected to find skin with md5 "${md5}".`);
+    }
+    return skin;
   }
 
   static async fromAnything(

packages/skin-database/tasks/syncFromArchive.ts

@@ -60,20 +60,22 @@ async function allItems(): Promise<string[]> {
 
 export async function fillMissingMetadata(count: number) {
   const ctx = new UserContext();
-  const skins = await knex("skins")
-    .leftJoin("ia_items", "skins.md5", "ia_items.skin_md5")
-    .where("ia_items.metadata", null)
+  const skins = await knex("ia_items")
+    .where("ia_items.metadata", "")
     .whereNot("ia_items.identifier", null)
-    .limit(count)
-    .select("skins.md5", "ia_items.identifier");
-  for (const { md5, identifier } of skins) {
+    .select("ia_items.skin_md5", "ia_items.identifier");
+    console.log(`Found ${skins.length} items to fetch metadata for`);
+
+    const items = skins.slice(0, count);
+
+  for (const { skin_md5, identifier } of items) {
     const iaItem = await IaItemModel.fromIdentifier(ctx, identifier);
     if (iaItem == null) {
       console.error(`Could not find IA item for ${identifier}`);
       break;
     }
     await iaItem.updateMetadata();
-    console.log(`Updated metadata for ${identifier} ${md5}`);
+    console.log(`Updated metadata for ${identifier} ${skin_md5}`);
   }
   console.log("Done updating metadata.");
 }

packages/skin-database/tasks/syncToArchive.ts

@@ -21,9 +21,17 @@ export async function findItemsMissingImages(): Promise<string[]> {
     if (iaItem == null) {
       throw new Error("Expected to find IA item");
     }
+    if(iaItem.getSkinFiles().length > 1) {
+        console.warn("Too many skin files", row.skin_md5, row.identifier);
+        continue;
+    }
+    if(iaItem.getSkinFiles().length < 1) {
+        console.log(iaItem.getAllFiles());
+        console.warn("Missing skin file", row.skin_md5, row.identifier);
+        continue;
+    }
     if (
-      iaItem.getUploadedFiles().length >= 2 ||
-      iaItem.getSkinFiles().length !== 1
+      iaItem.getUploadedFiles().length >= 2
     ) {
       continue;
     }
@@ -44,17 +52,21 @@ export async function uploadScreenshotIfSafe(md5: string): Promise<boolean> {
     throw new Error("Expected ia item to exist");
   }
   if (!iaItem.row.metadata) {
+    console.warn("No metadata found for row");
     return false;
   }
   if (await iaItem.hasRunningTasks()) {
+    console.warn("Has running tasks");
     return false;
   }
   const skinFiles = iaItem.getSkinFiles();
   if (skinFiles.length != 1) {
+    console.warn(`Has ${skinFiles.length} skins`);
     return false;
   }
   const uploadedFiles = iaItem.getUploadedFiles();
   if (uploadedFiles.length !== skinFiles.length) {
+    console.warn(`Has ${skinFiles.length} skins and ${uploadedFiles.length} uploaded files.`);
     return false;
   }