Get archive tool working for huge (5k) collection

experiments/skinArchiveTools/lib/collectSkins.js

@@ -2,6 +2,7 @@ const path = require("path");
 const fs = require("fs");
 const fsPromises = require("fs").promises;
 const childProcess = require("child_process");
+const Bluebird = require("bluebird");
 const { memoize } = require("lodash");
 const md5File = require("md5-file/promise");
 const Filehound = require("filehound");
@@ -15,30 +16,41 @@ module.exports = async function collectSkins({
   outputDir,
   filenamesPath
 }) {
+  const flatSkinDir = path.join(outputDir, "md5skins");
   const files = await Filehound.create()
     .ext(["zip", "wsz"])
     .paths(inputDir)
     .find();
 
-  const uniqueFiles = await Utils.unique(files, memoizedMd5File);
+  const seen = new Set();
 
-  const collectionInfo = await Promise.all(
-    uniqueFiles.map(async filePath => {
+  // We use Bluebird so that we can limit the concurrency.
+  const collectionInfo = (await Bluebird.map(
+    files,
+    async filePath => {
       const md5 = await memoizedMd5File(filePath);
-      const skinType = await Utils.skinType(filePath);
-      const md5Path = path.join(outputDir, `${md5}.wsz`);
-      const info = { filePath, md5, skinType, moved: false, md5Path };
-      if (info.skinType !== "CLASSIC") {
+      if (seen.has(md5)) {
+        return null;
+      }
+      seen.add(md5);
+      const md5Path = path.join(flatSkinDir, `${md5}.wsz`);
+      const info = { filePath, md5, moved: false, md5Path };
+      if (fs.existsSync(md5Path)) {
+        info.skinType = FILE_TYPES.CLASSIC;
         return info;
       }
-      const exists = fs.existsSync(info.md5Path);
-      if (!exists) {
-        await fsPromises.link(filePath, info.md5Path);
-        info.moved = true;
+      info.skinType = await Utils.skinType(filePath);
+      if (info.skinType !== FILE_TYPES.CLASSIC) {
+        return info;
       }
+      await fsPromises.link(filePath, info.md5Path);
+      info.moved = true;
       return info;
-    })
-  );
+    },
+    { concurrency: 50 }
+  )).filter(Boolean);
+
+  console.log("moved all files", "---------------------------");
 
   const pathList = collectionInfo
     .map(info => `${info.md5} ${path.relative(inputDir, info.filePath)}`)
@@ -48,7 +60,7 @@ module.exports = async function collectSkins({
   collectionInfo
     .filter(info => info.skinType === FILE_TYPES.PACK)
     .forEach(pack => {
-      const packPath = path.resolve(`./assets/md5Packs/${pack.md5}`);
+      const packPath = path.join(outputDir, `md5Packs/${pack.md5}`);
       if (fs.existsSync(packPath)) {
         return;
       }
@@ -61,7 +73,7 @@ module.exports = async function collectSkins({
   collectionInfo
     .filter(info => info.skinType === FILE_TYPES.INVALID)
     .forEach(async invalid => {
-      const filePath = path.resolve(`./assets/md5Invalids/${invalid.md5}.wsz`);
+      const filePath = path.join(outputDir, `md5Invalids/${invalid.md5}.wsz`);
       if (fs.existsSync(filePath)) {
         return;
       }
@@ -69,7 +81,7 @@ module.exports = async function collectSkins({
     });
   return {
     total: files.length,
-    unique: uniqueFiles.length,
+    unique: collectionInfo.length,
     moved: collectionInfo.filter(info => info.moved).length,
     newScreenshots: collectionInfo.filter(info => info.screenshotCreated)
       .length,

experiments/skinArchiveTools/lib/index.js

@@ -15,13 +15,12 @@ const [
   _thisFile,
   rawInputDir,
   rawOutputDir,
-  rawScreenshotDir,
   rawFilenamesPath
 ] = process.argv;
 
 const inputDir = path.resolve(rawInputDir);
 const outputDir = path.resolve(rawOutputDir);
-const screenshotDir = path.resolve(rawScreenshotDir);
+const screenshotDir = path.join(outputDir, "md5Screenshots");
 const filenamesPath = path.resolve(rawFilenamesPath);
 
 // It's nice to be able to fiddle these manually
@@ -36,15 +35,17 @@ async function main() {
       outputDir
     });
     console.log(collectionInfo);
+    /*
     const packCollectionInfo = await collectSkins({
       filenamesPath,
-      inputDir: path.resolve(path.join("assets", "md5Packs")),
+      inputDir: path.join(outputDir, "md5Packs"),
       outputDir
     });
     console.log(packCollectionInfo);
+    */
   }
   if (screenshots) {
-    await takeScreenshots(outputDir, screenshotDir);
+    await takeScreenshots(path.join(outputDir, "md5Skins"), screenshotDir);
   }
 }
 

experiments/skinArchiveTools/lib/shooter.js

@@ -39,7 +39,7 @@ class Shooter {
 
   async takeScreenshot(skin, screenshotPath, { minify = false }) {
     await this._ensureInitialized();
-    console.log("Going to try", screenshotPath);
+    console.log("Going to try", screenshotPath, skin);
     try {
       console.log("geting input");
       const handle = await this._page.$("#webamp-file-input");

experiments/skinArchiveTools/lib/takeScreenshots.js

@@ -11,6 +11,9 @@ module.exports = async function takeScreenshots(skinDir, screenshotDir) {
     .find();
   const shooter = new Shooter();
   for (const filePath of files) {
+    if (!fs.existsSync(filePath)) {
+      throw new Error("wat", filePath);
+    }
     const md5 = await md5File(filePath);
     const screenshotPath = path.join(screenshotDir, `${md5}.png`);
     if (fs.existsSync(screenshotPath)) {

experiments/skinArchiveTools/package.json

@@ -5,12 +5,13 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "node lib/index.js ./assets/skins ./assets/md5Skins ./assets/md5Screenshots ./assets/filenames.txt",
+    "start": "node lib/index.js \"/Volumes/Mobile Backup/skins/skins/\" \"/Volumes/Mobile Backup/skins/\" ./assets/filenames.txt",
     "clean": "cd md5Skins && find . -delete"
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "bluebird": "^3.5.3",
     "filehound": "^1.17.0",
     "imagemin": "^6.1.0",
     "imagemin-optipng": "^6.0.0",