Move skin extraction code from the skin museum into this package

Record the readme path

experiments/skinArchiveTools/lib/color.js

@@ -0,0 +1,18 @@
+const exec = require("child_process").exec;
+const shellescape = require("shell-escape");
+
+const getColor = imgPath => {
+  return new Promise((resolve, reject) => {
+    const excapedImgPath = shellescape([imgPath]);
+    const command = `convert ${excapedImgPath} -scale 1x1\! -format '%[pixel:u]' info:-`;
+    exec(command, (error, stdout, stderr) => {
+      if (error !== null) {
+        reject(error);
+        return;
+      }
+      resolve(stdout.slice(1));
+    });
+  });
+};
+
+module.exports = { getColor };

experiments/skinArchiveTools/lib/index.js

@@ -4,6 +4,8 @@ const Bluebird = require("bluebird");
 const collectSkins = require("./collectSkins");
 const Utils = require("./utils");
 const { FILE_TYPES } = require("./constants");
+const { getColor } = require("./color");
+const { extractTextData } = require("./readme");
 const Shooter = require("./shooter");
 
 /**
@@ -32,8 +34,11 @@ const getTypes = false;
 const moveHome = false;
 const screenshots = false;
 const detectGenerated = false;
-const archive = true;
+const archive = false;
 const force = false;
+const color = false;
+const favorites = false;
+const readme = true;
 const cacheFilePath = path.join(CACHE_PATH, "info.json");
 
 function getPath(skin) {
@@ -48,9 +53,16 @@ async function main() {
   // Run this if new files are added
   const start = Date.now();
   let cache = JSON.parse(fs.readFileSync(cacheFilePath, "utf8"));
-
   console.log("parsed cache in ", (Date.now() - start) / 1000, "seconds");
 
+  // Writing a backup
+  const cacheBackupFilePath = path.join(
+    CACHE_PATH,
+    `backup-${Date.now()}.json`
+  );
+  fs.writeFileSync(cacheBackupFilePath, JSON.stringify(cache, null, 2));
+  console.log(`Wrote a backup of the cache to ${cacheBackupFilePath}`);
+
   function report() {
     let classic = 0;
     let modern = 0;
@@ -89,7 +101,7 @@ async function main() {
   }
 
   function save() {
-    fs.writeFileSync(cacheFilePath, JSON.stringify(cache));
+    fs.writeFileSync(cacheFilePath, JSON.stringify(cache, null, 2));
   }
 
   console.log("Starting with...");
@@ -107,7 +119,9 @@ async function main() {
       if (md5Path === skin.md5Path) {
         return;
       }
-      fs.linkSync(getPath(skin), md5Path);
+      if (!fs.existsSync(md5Path)) {
+        fs.linkSync(getPath(skin), md5Path);
+      }
       skin.md5Path = md5Path;
     });
     save();
@@ -159,8 +173,10 @@ async function main() {
     console.log("Taking screenshots");
     const skins = Object.values(cache);
     let i = 0;
+    let j = 0;
     const interval = setInterval(() => {
       console.log(`Took ${i} screenshots`);
+      console.log(`Found ${j} screenshots`);
       console.log("Saving...");
 
       save();
@@ -181,16 +197,21 @@ async function main() {
           "md5Screenshots",
           `${skin.md5}.png`
         );
-        await shooter.takeScreenshot(getPath(skin), screenshotPath, {
-          minify: true,
-        });
+        if (!fs.existsSync(screenshotPath) || force) {
+          await shooter.takeScreenshot(getPath(skin), screenshotPath, {
+            minify: true,
+          });
+          i++;
+        } else {
+          j++;
+        }
         skin.screenshotPath = screenshotPath;
-        i++;
       }
       shooter.dispose();
     }
     clearInterval(interval);
     console.log(`Took ${i} screenshots`);
+    console.log(`Found ${j} screenshots`);
     save();
   }
 
@@ -261,6 +282,100 @@ async function main() {
     );
     save();
   }
+
+  if (color) {
+    const classicSkins = Object.values(cache).filter(
+      skin => skin.type === FILE_TYPES.CLASSIC
+    );
+    let i = 0;
+    const interval = setInterval(() => {
+      console.log(`Found the average color of ${i} skins`);
+      console.log("Saving...");
+
+      save();
+    }, 10000);
+
+    await Bluebird.map(
+      classicSkins,
+      async skin => {
+        if (skin.screenshotPath == null) {
+          console.log("Could not extract color from skin");
+          return;
+        }
+        if (skin.averageColor != null && !force) {
+          return;
+        }
+        const averageColor = await getColor(skin.screenshotPath);
+        skin.averageColor = averageColor;
+        i++;
+      },
+      { concurrency: 10 }
+    );
+    clearInterval(interval);
+    save();
+  }
+
+  if (favorites) {
+    const content = fs.readFileSync(
+      path.join("/Volumes/Mobile Backup/skins/", "likes.txt"),
+      "utf8"
+    );
+    content.split("\n").forEach(line => {
+      if (!line.length) {
+        return;
+      }
+      const [hash, likes] = line.split(" ");
+      const skin = cache[hash];
+      if (!skin) {
+        console.log(`Found like for unknown skin ${hash}`);
+        return;
+      }
+      skin.twitterLikes = likes;
+    });
+    save();
+  }
+
+  if (readme) {
+    const classicSkins = Object.values(cache).filter(
+      skin => skin.type === FILE_TYPES.CLASSIC
+    );
+    let i = 0;
+    const interval = setInterval(() => {
+      console.log(`Found readme text for ${i} skins`);
+      console.log("Saving...");
+
+      save();
+    }, 10000);
+
+    await Bluebird.map(
+      classicSkins,
+      async skin => {
+        const readmePath = path.join(outputDir, "readmes", `${skin.md5}.txt`);
+        // if (skin.readmePath || skin.emails) {
+        if (skin.readmePath !== undefined) {
+          return;
+        }
+        if (fs.existsSync(readmePath)) {
+          skin.readmePath = readmePath;
+          i++;
+          return;
+        }
+        const { emails, raw } = await extractTextData(skin.md5Path);
+        skin.emails = emails;
+        if (raw) {
+          fs.writeFileSync(readmePath, raw);
+          skin.readmePath = readmePath;
+        } else {
+          // Note that we extracted it, but found nothing
+          skin.readmePath = null;
+          i++;
+        }
+      },
+      { concurrency: 10 }
+    );
+    clearInterval(interval);
+    save();
+  }
   console.log("Ended with...");
   report();
   save();

experiments/skinArchiveTools/lib/readme.js

@@ -0,0 +1,46 @@
+const { exec } = require("child_process");
+
+const reg = /[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/g;
+const extract = value => (value && value.match(reg)) || [];
+
+async function extractTextData(skinPath) {
+  const ignoreFiles = [
+    "genex.txt",
+    "genexinfo.txt",
+    "gen_gslyrics.txt",
+    "region.txt",
+    "pledit.txt",
+    "viscolor.txt",
+    "winampmb.txt",
+    "gen_ex help.txt",
+    "mbinner.txt"
+    // Skinning Updates.txt ?
+  ];
+
+  const ignoreArgs = ignoreFiles
+    .map(file => `-x "**/${file}" ${file}`)
+    .join(" ");
+
+  // Change -p to -c to get context of which file is missing
+  const debug = false;
+  const listFlag = debug ? "-c" : "-p";
+
+  // TODO: Escape path
+  const cmd = `unzip ${listFlag} -C "${skinPath}" "file_id.diz" "*.txt" ${ignoreArgs}`;
+
+  const raw = await new Promise((resolve, reject) => {
+    exec(cmd, (error, stdout, stderr) => {
+      if (error != null) {
+        // reject(error);
+        // return;
+      }
+      resolve(stdout);
+    });
+  });
+
+  return { raw, emails: extract(raw) };
+}
+
+module.exports = {
+  extractTextData
+};

experiments/skinArchiveTools/package.json

@@ -22,6 +22,7 @@
     "lodash": "^4.17.11",
     "md5-file": "^4.0.0",
     "node-fetch": "^2.3.0",
-    "puppeteer": "^1.12.2"
+    "puppeteer": "^1.12.2",
+    "shell-escape": "^0.2.0"
   }
 }

experiments/skinArchiveTools/yarn.lock

@@ -1850,6 +1850,10 @@ shebang-regex@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
 
+shell-escape@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/shell-escape/-/shell-escape-0.2.0.tgz#68fd025eb0490b4f567a027f0bf22480b5f84133"
+
 signal-exit@^3.0.0:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"