Server improvements

packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts

@@ -95,7 +95,7 @@ export default class ClassicSkinResolver
   /**
    * URL of a screenshot of the skin
    * @gqlField */
-  screenshot_url(): string {
+  async screenshot_url(): Promise<string> {
     return this._model.getScreenshotUrl();
   }
   /**

packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts

@@ -90,10 +90,8 @@ export interface ISkin {
    */
   webamp_url(): string | null;
 
-  /**
-   * @gqlField
-   * @deprecated Needed for migration */
-  screenshot_url(): string | null;
+  /** @gqlField */
+  screenshot_url(): Promise<string | null>;
 
   /**
    * @gqlField

packages/skin-database/api/graphql/resolvers/ModernSkinResolver.ts

@@ -7,6 +7,7 @@ import ReviewResolver from "./ReviewResolver";
 import InternetArchiveItemResolver from "./InternetArchiveItemResolver";
 import ArchiveFileResolver from "./ArchiveFileResolver";
 import TweetResolver from "./TweetResolver";
+import { XMLParser } from "fast-xml-parser";
 
 /**
  * A "modern" Winamp skin. These skins use the `.wal` file extension and are free-form.
@@ -109,11 +110,45 @@ export default class ModernSkinResolver
     return null;
   }
 
-  /**
-   * @gqlField
-   * @deprecated Needed for migration */
-  screenshot_url(): string | null {
-    return null;
+  /** @gqlField */
+  async screenshot_url(): Promise<string | null> {
+    const archiveFiles = await this._model.getArchiveFiles();
+    const skinXml = archiveFiles.find((f) =>
+      f.getFileName().match(/skin\.xml$/i)
+    );
+
+    if (skinXml == null) {
+      return null;
+    }
+    const xmlContent = await skinXml.getTextContent();
+    if (xmlContent == null) {
+      return null;
+    }
+
+    const parser = new XMLParser();
+    const parsedXml = parser.parse(xmlContent);
+    const screenshotPath =
+      parsedXml?.WinampAbstractionLayer?.skininfo?.screenshot;
+    if (screenshotPath == null) {
+      return null;
+    }
+
+    const screenshotFile = archiveFiles.find((f) => {
+      const fileName = f.getFileName().toLowerCase();
+      const normalizedScreenshotPath = screenshotPath.toLowerCase();
+      return (
+        fileName === normalizedScreenshotPath ||
+        fileName === `/${normalizedScreenshotPath}` ||
+        fileName === `\\${normalizedScreenshotPath}`
+      );
+    });
+
+    console.log({ screenshotFile });
+    if (screenshotFile == null) {
+      return null;
+    }
+
+    return screenshotFile.getUrl();
   }
   /**
    * @gqlField

packages/skin-database/api/graphql/schema.graphql

@@ -192,7 +192,7 @@ type ModernSkin implements Node & Skin {
   reivew page, or via the Discord bot.
   """
   reviews: [Review]
-  screenshot_url: String @deprecated(reason: "Needed for migration")
+  screenshot_url: String
   """Has the skin been tweeted?"""
   tweeted: Boolean
   """List of @winampskins tweets that mentioned the skin."""
@@ -358,7 +358,7 @@ interface Skin {
   reivew page, or via the Discord bot.
   """
   reviews: [Review]
-  screenshot_url: String @deprecated(reason: "Needed for migration")
+  screenshot_url: String
   """Has the skin been tweeted?"""
   tweeted: Boolean
   """List of @winampskins tweets that mentioned the skin."""

packages/skin-database/cli.ts

@@ -74,7 +74,7 @@ program
   .command("share")
   .description(
     "Share a skin on Twitter and Instagram. If no md5 is " +
-    "given, random approved skins are shared."
+      "given, random approved skins are shared."
   )
   .argument("[md5]", "md5 of the skin to share")
   .option("-t, --twitter", "Share on Twitter")
@@ -107,7 +107,7 @@ program
   .option(
     "--delete",
     "Delete a skin from the database, including its S3 files " +
-    "CloudFlare cache and seach index entries."
+      "CloudFlare cache and seach index entries."
   )
   .option(
     "--hide",
@@ -116,7 +116,7 @@ program
   .option(
     "--delete-local",
     "Delete a skin from the database only, NOT including its S3 files " +
-    "CloudFlare cache and seach index entries."
+      "CloudFlare cache and seach index entries."
   )
   .option("--index", "Update the seach index for a skin.")
   .option(
@@ -195,18 +195,18 @@ program
   .option(
     "--fetch-metadata <count>",
     "Fetch missing metadata for <count> items from the Internet " +
-    "Archive. Currently it only fetches missing metadata. In the " +
-    "future it could refresh stale metadata."
+      "Archive. Currently it only fetches missing metadata. In the " +
+      "future it could refresh stale metadata."
   )
   .option(
     "--fetch-items",
     "Seach the Internet Archive for items that we don't know about" +
-    "and add them to our database."
+      "and add them to our database."
   )
   .option(
     "--update-metadata <count>",
     "Find <count> items in our database that have incorrect or incomplete " +
-    "metadata, and update the Internet Archive"
+      "metadata, and update the Internet Archive"
   )
   .option(
     "--upload-new",
@@ -240,7 +240,7 @@ program
   .command("stats")
   .description(
     "Report information about skins in the database. " +
-    "Identical to `!stats` in Discord."
+      "Identical to `!stats` in Discord."
   )
   .action(async () => {
     console.table([await Skins.getStats()]);
@@ -286,17 +286,17 @@ program
   .option(
     "--likes",
     "Scrape @winampskins tweets for like and retweet counts, " +
-    "and update the database."
+      "and update the database."
   )
   .option(
     "--milestones",
     "Check the most recent @winampskins tweets to see if they have " +
-    "passed a milestone. If so, notify the Discord channel."
+      "passed a milestone. If so, notify the Discord channel."
   )
   .option(
     "--followers",
     "Check if @winampskins has passed a follower count milestone. " +
-    "If so, notify the Discord channel."
+      "If so, notify the Discord channel."
   )
   .action(async ({ likes, milestones, followers }) => {
     if (likes) {
@@ -325,7 +325,7 @@ program
   .option(
     "--upload-ia-screenshot <md5>",
     "Upload a screenshot of a skin to the skin's Internet Archive itme. " +
-    "[[Warning!]] This might result in multiple screenshots on the item."
+      "[[Warning!]] This might result in multiple screenshots on the item."
   )
   .option(
     "--upload-missing-screenshots",
@@ -377,7 +377,7 @@ program
         [500]
       );
       const md5s = rows.map((row) => row.md5);
-      console.log(md5s.length)
+      console.log(md5s.length);
       console.log(await Skins.updateSearchIndexs(ctx, md5s));
     }
     if (refreshContentHash) {
@@ -401,17 +401,22 @@ program
       const skinRows = await knex("skins")
         .leftJoin("archive_files", "skins.md5", "archive_files.skin_md5")
         .leftJoin("file_info", "file_info.file_md5", "archive_files.file_md5")
-        .where("skin_type", 1)
+        .where("skin_type", "in", [1, 2])
         .where((builder) => {
           return builder.where("file_info.file_md5", null);
         })
-        .limit(90000)
+        .limit(1000)
         .groupBy("skins.md5")
         .select();
       console.log(`Found ${skinRows.length} skins to update`);
       const skins = skinRows.map((row) => new SkinModel(ctx, row));
       for (const skin of skins) {
-        await setHashesForSkin(skin);
+        console.log("Working on", skin.getMd5(), await skin.getFileName());
+        try {
+          await setHashesForSkin(skin);
+        } catch (e) {
+          console.error(e);
+        }
         // await Skins.setContentHash(skin.getMd5());
         process.stdout.write(".");
       }
@@ -429,15 +434,15 @@ program
   });
 
 async function main() {
-  process.on('unhandledRejection', (reason, promise) => {
-    console.error("Unhanded rejection")
-    console.error(reason, promise)
+  process.on("unhandledRejection", (reason, promise) => {
+    console.error("Unhanded rejection");
+    console.error(reason, promise);
   });
   try {
     await program.parseAsync(process.argv);
   } finally {
     knex.destroy();
-    console.log("CLOSING THE LOGGER")
+    console.log("CLOSING THE LOGGER");
     logger.close();
   }
 }

packages/skin-database/data/SkinModel.ts

@@ -350,15 +350,14 @@ export default class SkinModel {
       return 0;
     }
     const text = await region.getTextContent();
-    if(text == null) {
+    if (text == null) {
       return 0;
     }
     try {
-
-    return getTransparentAreaSize(text);
-    } catch(e) {
-      console.error(`Failed: ${this.getMd5()}`)
-      return 0
+      return getTransparentAreaSize(text);
+    } catch (e) {
+      console.error(`Failed: ${this.getMd5()}`);
+      return 0;
     }
   }
 

packages/skin-database/package.json

@@ -27,6 +27,7 @@
     "express-fileupload": "^1.1.7-alpha.3",
     "express-graphql": "^0.12.0",
     "express-sitemap-xml": "^2.0.0",
+    "fast-xml-parser": "^4.2.2",
     "graphql": "14.7.0",
     "grats": "^0.0.0-main-8de1c8ac",
     "imagemin": "^7.0.0",
@@ -49,7 +50,7 @@
   "scripts": {
     "lint": "eslint .",
     "start": "ts-node --transpile-only api/server.ts",
-    "dev": "NODE_ENV=development ts-node --transpile-only api/server.ts",
+    "dev": "NODE_ENV=production ts-node --transpile-only api/server.ts",
     "tweet": "ts-node --transpile-only ./cli.ts tweet",
     "fetch-metadata": "ts-node --transpile-only ./cli.ts fetch-metadata",
     "bot": "ts-node --transpile-only ./discord-bot/index.js",

packages/skin-database/skinHash.ts

@@ -13,7 +13,10 @@ type FileData = {
 };
 
 function mightBeText(filename: string): boolean {
-  return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm|ai|log|uue|)$/i.test(
+  if (/skin\.xml/i.test(filename)) {
+    return true;
+  }
+  return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm|ai|log|uue|xml)$/i.test(
     filename
   );
 }
@@ -21,7 +24,8 @@ function mightBeText(filename: string): boolean {
 async function getFileData(file: JSZip.JSZipObject): Promise<FileData | null> {
   try {
     let textContent: string | null = null;
-    if (mightBeText(file.name)) {
+    const extractText = mightBeText(file.name);
+    if (extractText) {
       textContent = await file.async("text");
     }
     const blob = await file.async("nodebuffer");
@@ -56,19 +60,24 @@ export async function setHashesForSkin(skin: SkinModel): Promise<void> {
   const hashes = await getSkinFileData(skin);
   const rows = hashes
     .filter(notEmpty)
-    .map(({ fileName, md5, date, isDirectory }) => {
-      return {
-        skin_md5: skin.getMd5(),
-        file_name: fileName,
-        file_md5: md5, // TODO: Consider using an id into file_info table
-        file_date: date,
-        // text_content: textContent,
-        // uncompressed_size: uncompressedSize,
-        is_directory: isDirectory,
-      };
-    });
+    .map(
+      ({ fileName, md5, date, isDirectory, textContent, uncompressedSize }) => {
+        return {
+          skin_md5: skin.getMd5(),
+          file_name: fileName,
+          file_md5: md5, // TODO: Consider using an id into file_info table
+          file_date: date,
+          text_content: textContent,
+          uncompressed_size: uncompressedSize,
+          is_directory: isDirectory,
+        };
+      }
+    );
   await knex("archive_files").where("skin_md5", skin.getMd5()).delete();
-  await knex("archive_files").insert(rows);
+  // Avoid bulk insert because size can be large
+  for (const row of rows) {
+    await knex("archive_files").insert(row);
+  }
 
   const fileInfoRows = hashes
     .filter(notEmpty)

packages/skin-database/tasks/syncToArchive.ts

@@ -169,7 +169,7 @@ export async function identifierExists(identifier: string): Promise<boolean> {
 }
 
 async function getNewIdentifier(filename: string): Promise<string> {
-  // The internet archvie has a max identifier length of 80 chars.
+  // The internet archive has a max identifier length of 80 chars.
   const identifierBase = `winampskins_${sanitize(
     path.parse(filename).name
   )}`.slice(0, 76);
@@ -222,6 +222,14 @@ export async function syncToArchive(handler: DiscordEventHandler) {
   await Parallel.map(
     unarchived,
     async ({ md5 }) => {
+      if (
+        md5 === "513fdd06bf39391e52f3ac5b233dd147" ||
+        md5 === "91477bec2b599bc5085f87f0fca3a4d5"
+      ) {
+        // The internet archive claims this one is corrupt for some reason.
+        console.warn(`Skipping this skin. It's known to not upload correctly.`);
+        return null;
+      }
       const skin = await SkinModel.fromMd5Assert(ctx, md5);
       try {
         console.log(`Attempting to upload ${md5}`);
@@ -231,18 +239,14 @@ export async function syncToArchive(handler: DiscordEventHandler) {
       } catch (e) {
         console.log("Archive failed...");
         errorCount++;
-        // The internet archive claims this one is corrupt for some reason.
-        if (md5 === "513fdd06bf39391e52f3ac5b233dd147") {
-          console.warn(`This skin is known to not upload correctly.`);
-        }
         if (/error checking archive/.test(e.message)) {
-          console.log(`Corrupt archvie: ${skin.getMd5()}`);
+          console.log(`Corrupt archive: ${skin.getMd5()}`);
         } else if (
           /archive files are not allowed to contain encrypted content/.test(
             e.message
           )
         ) {
-          console.log(`Corrupt archvie (encrypted): ${skin.getMd5()}`);
+          console.log(`Corrupt archive (encrypted): ${skin.getMd5()}`);
         } else if (/case alias may already exist/.test(e.message)) {
           console.log(
             `Invalid name (case alias): ${skin.getMd5()} with ${e.message}`

yarn.lock

@@ -10098,9 +10098,9 @@ comma-separated-tokens@^1.0.0:
   integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==
 
 commander@^10.0.0:
-  version "10.0.0"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.0.tgz#71797971162cd3cf65f0b9d24eb28f8d303acdf1"
-  integrity sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA==
+  version "10.0.1"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06"
+  integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==
 
 commander@^2.19.0, commander@^2.20.0, commander@^2.20.3, commander@^2.3.0:
   version "2.20.3"
@@ -13510,6 +13510,13 @@ fast-stringify@^1.1.0:
   resolved "https://registry.npmjs.org/fast-stringify/-/fast-stringify-1.1.2.tgz"
   integrity sha512-SfslXjiH8km0WnRiuPfpUKwlZjW5I878qsOm+2x8x3TgqmElOOLh1rgJFb+PolNdNRK3r8urEefqx0wt7vx1dA==
 
+fast-xml-parser@^4.2.2:
+  version "4.2.2"
+  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.2.tgz#cb7310d1e9cf42d22c687b0fae41f3c926629368"
+  integrity sha512-DLzIPtQqmvmdq3VUKR7T6omPK/VCRNqgFlGtbESfyhcH2R4I8EzK1/K6E8PkRCK2EabWrUHK32NjYRbEFnnz0Q==
+  dependencies:
+    strnum "^1.0.5"
+
 fastest-levenshtein@^1.0.7, fastest-levenshtein@^1.0.9:
   version "1.0.16"
   resolved "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz"
@@ -14771,22 +14778,15 @@ graphql-ws@^5.4.1:
   resolved "https://registry.npmjs.org/graphql-ws/-/graphql-ws-5.11.1.tgz"
   integrity sha512-AlOO/Gt0fXuSHXe/Weo6o3rIQVnH5MW7ophzeYzL+vYNlkf0NbWRJ6IIFgtSLcv9JpTlQdxSpB3t0SnM47/BHA==
 
-graphql@14.7.0:
-  version "14.7.0"
-  resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.7.0.tgz#7fa79a80a69be4a31c27dda824dc04dac2035a72"
-  integrity sha512-l0xWZpoPKpppFzMfvVyFmp9vLN7w/ZZJPefUicMCepfJeQ8sMcztloGYY9DfjVPo6tIUDzU5Hw3MUbIjj9AVVA==
-  dependencies:
-    iterall "^1.2.2"
+graphql@14.7.0, graphql@15.3.0, graphql@^16.6.0:
+  version "15.3.0"
+  resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.3.0.tgz#3ad2b0caab0d110e3be4a5a9b2aa281e362b5278"
+  integrity sha512-GTCJtzJmkFLWRfFJuoo9RWWa/FfamUHgiFosxi/X1Ani4AVWbeyBenZTNX6dM+7WSbbFfTo/25eh0LLkwHMw2w==
 
-graphql@^16.6.0:
-  version "16.6.0"
-  resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.6.0.tgz#c2dcffa4649db149f6282af726c8c83f1c7c5fdb"
-  integrity sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==
-
-grats@^0.0.0-main-44ac2e23:
-  version "0.0.0-main-44ac2e23"
-  resolved "https://registry.yarnpkg.com/grats/-/grats-0.0.0-main-44ac2e23.tgz#b788e44c3d3eaac28588741a7c34ff4c69832183"
-  integrity sha512-IFVtiWuOtcjw24+TsvuZvYZci4e8jPvs7WiPrEQtsvFkZ4HEJqn+PUsMPq+R5D7Ox4NQSSTZP/r7iFUwsi19TQ==
+grats@^0.0.0-main-8de1c8ac:
+  version "0.0.0-main-8de1c8ac"
+  resolved "https://registry.yarnpkg.com/grats/-/grats-0.0.0-main-8de1c8ac.tgz#ddd0eb60c290951e76307028143b24a3f8608ea4"
+  integrity sha512-lP/CpAU/6qE/Cn7+MUPbQT100kzuxh5rJ2fAiDbtAzxvG3MFhtXySy8Cz6sFt0eBjHv77a2G3J3Ua6+xoM6GnA==
   dependencies:
     "@graphql-tools/utils" "^9.2.1"
     commander "^10.0.0"
@@ -16506,11 +16506,6 @@ isurl@^1.0.0-alpha5:
     has-to-string-tag-x "^1.2.0"
     is-object "^1.0.1"
 
-iterall@^1.2.2:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea"
-  integrity sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg==
-
 jake@^10.8.5:
   version "10.8.5"
   resolved "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz"
@@ -25027,6 +25022,11 @@ strip-outer@^1.0.0:
   dependencies:
     escape-string-regexp "^1.0.2"
 
+strnum@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db"
+  integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==
+
 strtok3@^2.3.0:
   version "2.3.0"
   resolved "https://registry.npmjs.org/strtok3/-/strtok3-2.3.0.tgz"