Improve upload flow robustness

packages/skin-database/api/processUserUploads.ts

@@ -82,6 +82,7 @@ async function processGivenUserUploads(
         message: e.message,
       } as const;
       eventHandler(action);
+      throw e
       console.error(e);
     }
   }

packages/skin-database/cli.ts

@@ -74,7 +74,7 @@ program
   .command("share")
   .description(
     "Share a skin on Twitter and Instagram. If no md5 is " +
-      "given, random approved skins are shared."
+    "given, random approved skins are shared."
   )
   .argument("[md5]", "md5 of the skin to share")
   .option("-t, --twitter", "Share on Twitter")
@@ -107,7 +107,7 @@ program
   .option(
     "--delete",
     "Delete a skin from the database, including its S3 files " +
-      "CloudFlare cache and seach index entries."
+    "CloudFlare cache and seach index entries."
   )
   .option(
     "--hide",
@@ -116,7 +116,7 @@ program
   .option(
     "--delete-local",
     "Delete a skin from the database only, NOT including its S3 files " +
-      "CloudFlare cache and seach index entries."
+    "CloudFlare cache and seach index entries."
   )
   .option("--index", "Update the seach index for a skin.")
   .option(
@@ -195,18 +195,18 @@ program
   .option(
     "--fetch-metadata <count>",
     "Fetch missing metadata for <count> items from the Internet " +
-      "Archive. Currently it only fetches missing metadata. In the " +
-      "future it could refresh stale metadata."
+    "Archive. Currently it only fetches missing metadata. In the " +
+    "future it could refresh stale metadata."
   )
   .option(
     "--fetch-items",
     "Seach the Internet Archive for items that we don't know about" +
-      "and add them to our database."
+    "and add them to our database."
   )
   .option(
     "--update-metadata <count>",
     "Find <count> items in our database that have incorrect or incomplete " +
-      "metadata, and update the Internet Archive"
+    "metadata, and update the Internet Archive"
   )
   .option(
     "--upload-new",
@@ -240,7 +240,7 @@ program
   .command("stats")
   .description(
     "Report information about skins in the database. " +
-      "Identical to `!stats` in Discord."
+    "Identical to `!stats` in Discord."
   )
   .action(async () => {
     console.table([await Skins.getStats()]);
@@ -286,17 +286,17 @@ program
   .option(
     "--likes",
     "Scrape @winampskins tweets for like and retweet counts, " +
-      "and update the database."
+    "and update the database."
   )
   .option(
     "--milestones",
     "Check the most recent @winampskins tweets to see if they have " +
-      "passed a milestone. If so, notify the Discord channel."
+    "passed a milestone. If so, notify the Discord channel."
   )
   .option(
     "--followers",
     "Check if @winampskins has passed a follower count milestone. " +
-      "If so, notify the Discord channel."
+    "If so, notify the Discord channel."
   )
   .action(async ({ likes, milestones, followers }) => {
     if (likes) {
@@ -324,7 +324,7 @@ program
   .option(
     "--upload-ia-screenshot <md5>",
     "Upload a screenshot of a skin to the skin's Internet Archive itme. " +
-      "[[Warning!]] This might result in multiple screenshots on the item."
+    "[[Warning!]] This might result in multiple screenshots on the item."
   )
   .option(
     "--upload-missing-screenshots",
@@ -367,10 +367,12 @@ program
           LEFT JOIN algolia_field_updates ON skins.md5 = algolia_field_updates.skin_md5
           WHERE skin_type = 1
             AND md5 != "5470d71673a88254d3c197ba10bae16c"
+            AND md5 != "b23ee30b939d8c9c8664615fa2bb0b42" -- Readme too big
+            AND md5 != "e8bf0eb8c5a2c7950ccf3ed2b8211a96" -- region.txt fails
           GROUP BY md5
           ORDER BY update_timestamp
           LIMIT ?;`,
-        [5000]
+        [20000]
       );
       const md5s = rows.map((row) => row.md5);
       console.log(await Skins.updateSearchIndexs(ctx, md5s));
@@ -424,10 +426,15 @@ program
   });
 
 async function main() {
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error("Unhanded rejection")
+    console.error(reason, promise)
+  });
   try {
     await program.parseAsync(process.argv);
   } finally {
     knex.destroy();
+    console.log("CLOSING THE LOGGER")
     logger.close();
   }
 }

packages/skin-database/data/SkinModel.ts

@@ -350,7 +350,16 @@ export default class SkinModel {
       return 0;
     }
     const text = await region.getTextContent();
+    if(text == null) {
+      return 0;
+    }
+    try {
+
     return getTransparentAreaSize(text);
+    } catch(e) {
+      console.error(`Failed: ${this.getMd5()}`)
+      return 0
+    }
   }
 
   async hasAni(): Promise<boolean> {

packages/skin-database/data/skins.ts

@@ -219,7 +219,7 @@ async function getSearchIndexes(
         md5: skin.getMd5(),
         nsfw: await skin.getIsNsfw(),
         tweetStatus: await skin.getTweetStatus(),
-        readmeText: readmeText ? truncate(readmeText, 4800) : null,
+        readmeText: readmeText ? truncate(readmeText, 4000) : null,
         fileName: await skin.getFileName(),
         twitterLikes: likes,
         equalizer: await skin.hasEqualizer(),
@@ -391,6 +391,8 @@ export async function getErroredUpload(): Promise<{
 } | null> {
   const found = await knex("skin_uploads")
     .where("status", "ERRORED")
+    .where("skin_md5", "!=", "c7df44bde6eb3671bde5a03e6d03ce1e")
+    .where("skin_md5", "!=", "fedc564eb2ce0a4ec5518b93983240ef")
     .first(["skin_md5", "id", "filename"]);
   return found || null;
 }