Progress

Also, add Algolia logo

scripts/createSearchIndex.js

@@ -1,3 +1,5 @@
+const path = require("path");
+const fs = require("fs");
 const skins = require("../src/skins.json");
 const algoliasearch = require("algoliasearch");
 const { getSkinMetadata } = require("./utils");
@@ -5,6 +7,11 @@ const { getSkinMetadata } = require("./utils");
 const client = algoliasearch("HQ9I5Z6IM5", "f5357f4070cdb6ed652d9c3feeede89f");
 const index = client.initIndex("Skins");
 
+const CACHE_PATH = "/Volumes/Mobile Backup/skins/cache/";
+const info = JSON.parse(
+  fs.readFileSync(path.join(CACHE_PATH, "info.json"), "utf8")
+);
+
 function tuncate(str, len) {
   const overflow = str.length - len;
   if (overflow < 0) {
@@ -18,38 +25,46 @@ function tuncate(str, len) {
   return `${start} ########### ${end}`;
 }
 
-async function buildSkinIndex(hash) {
-  const textMetadata = await getSkinMetadata(hash, "extracted-data");
+async function buildSkinIndex(skin) {
+  const { md5, filePaths } = skin;
+  if (!filePaths || filePaths.length === 0) {
+    console.warn("no file name for ", md5);
+    return;
+  }
+  const fileName = path.basename(filePaths[0]);
+  let readmeText = null;
+  if (skin.readmePath) {
+    readmeText = tuncate(fs.readFileSync(skin.readmePath, "utf8"), 4800);
+  }
   return {
-    objectID: hash,
-    md5: hash,
-    fileName: skins[hash].fileName,
-    emails: textMetadata.emails,
-    readmeText: tuncate(textMetadata.raw, 4800)
+    objectID: skin.md5,
+    md5,
+    fileName,
+    emails: skin.emails || null,
+    readmeText
   };
 }
 
 const indexesPromise = Promise.all(
-  Object.keys(skins).map(hash => {
-    return buildSkinIndex(hash);
-  })
+  Object.values(info)
+    .filter(skin => skin.type === "CLASSIC")
+    .map(skin => {
+      return buildSkinIndex(skin);
+    })
 );
 
 async function go() {
+  console.log("Building index");
   const indexes = await indexesPromise;
-  const large = indexes.filter(index => {
-    return index.readmeText.length > 4790;
-  });
-  large.map(l => {
-    return l.fileName;
-  });
 
-  return new Promise((resolve, reject) => {
+  console.log("Writing index");
+  const results = await new Promise((resolve, reject) => {
     index.saveObjects(indexes, function(err, content) {
       if (err != null) reject(err);
       resolve(content);
     });
   });
+  console.log("done!", results);
 }
 
 go(); // .then(content => console.log("Updated index for:", content.length));

scripts/extractSkinText.js

@@ -1,78 +0,0 @@
-const path = require("path");
-const fs = require("fs");
-const Bluebird = require("bluebird");
-var { exec } = require("child_process");
-const Utils = require("./utils");
-
-const reg = /[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/g;
-const extract = value => (value && value.match(reg)) || [];
-
-async function extractTextData(path) {
-  const ignoreFiles = [
-    "genex.txt",
-    "genexinfo.txt",
-    "gen_gslyrics.txt",
-    "region.txt",
-    "pledit.txt",
-    "viscolor.txt",
-    "winampmb.txt",
-    "gen_ex help.txt",
-    "mbinner.txt"
-    // Skinning Updates.txt ?
-  ];
-
-  const ignoreArgs = ignoreFiles
-    .map(file => `-x "**/${file}" ${file}`)
-    .join(" ");
-
-  // Change -p to -c to get context of which file is missing
-  const debug = false;
-  const listFlag = debug ? "-c" : "-p";
-
-  // TODO: Escape path
-  const cmd = `unzip ${listFlag} -C "${path}" "file_id.diz" "*.txt" ${ignoreArgs}`;
-
-  const raw = await new Promise((resolve, reject) => {
-    exec(cmd, function(error, stdout, stderr) {
-      if (error != null) {
-        // reject(error);
-        // return;
-      }
-      resolve(stdout);
-    });
-  });
-
-  return { raw, emails: extract(raw) };
-}
-
-async function writeTextData(skinPath, md5) {
-  try {
-    const data = await extractTextData(skinPath);
-    const skinMetadataPath = await Utils.writeSkinMetadata(
-      md5,
-      "extracted-data",
-      data
-    );
-    console.log("Done", skinMetadataPath);
-  } catch (e) {
-    console.error(e);
-  }
-}
-
-function main() {
-  const dir = process.argv[2];
-  console.log("Reading dir");
-  const files = fs.readdirSync(dir);
-  console.log(`Found ${files.length} files`);
-  return Bluebird.map(
-    files,
-    file => {
-      const filePath = path.join(dir, file);
-      const md5 = path.basename(file, ".wsz");
-      return writeTextData(filePath, md5);
-    },
-    { concurrency: 10 }
-  );
-}
-
-main();

scripts/findSkins.js

@@ -1,76 +1,29 @@
 const fs = require("fs");
 const path = require("path");
-const exec = require("child_process").exec;
 const Bluebird = require("bluebird");
-const shellescape = require("shell-escape");
 
-const getFileNames = async () => {
-  const content = fs.readFileSync(
-    "/Volumes/Mobile Backup/skins/pathnames.txt",
-    "utf8"
-  );
-  const fileNames = {};
-  content.split("\n").forEach(line => {
-    if (!line.length) {
-      return;
-    }
-    const [hash, pathName] = line.split(" ");
-    fileNames[hash] = path.basename(pathName);
-  });
-  return fileNames;
-};
-
-const getFavoriteCounts = () => {
-  const content = fs.readFileSync(path.join(__dirname, "../likes.txt"), "utf8");
-  const favorites = {};
-  content.split("\n").forEach(line => {
-    if (!line.length) {
-      return;
-    }
-    const [hash, fileName] = line.split(" ");
-    favorites[hash] = fileName;
-  });
-  return favorites;
-};
-const testFolder = path.resolve(
-  __dirname,
-  "/Volumes/Mobile Backup/skins/md5Screenshots"
+const info = JSON.parse(
+  fs.readFileSync("/Volumes/Mobile Backup/skins/cache/info.json", "utf8")
 );
-const files = fs
-  .readdirSync(testFolder)
-  .filter(skinPath => skinPath.endsWith(".png"));
 
-const genAverage = img => {
-  return new Promise((resolve, reject) => {
-    const imgPath = shellescape([path.join(testFolder, img)]);
-    const command = `convert ${imgPath} -scale 1x1\! -format '%[pixel:u]' info:-`;
-    exec(command, (error, stdout, stderr) => {
-      if (error !== null) {
-        reject(error);
-        return;
-      }
-      resolve(stdout);
-    });
-  });
-};
-
-const getFileData = async files => {
-  const fileNames = await getFileNames();
-  const favortes = getFavoriteCounts();
-  console.log(favortes);
+const getFileData = async () => {
   const fileData = {};
   await Bluebird.map(
-    files,
-    async file => {
-      const md5 = path.basename(file, ".png");
-      const fileName = fileNames[md5];
-      if (!fileName) {
+    Object.values(info).filter(skin => skin.type === "CLASSIC"),
+    async skin => {
+      const md5 = skin.md5;
+      const filePaths = skin.filePaths;
+      if (!filePaths || filePaths.length === 0) {
         console.warn("no file name for ", md5);
         return;
       }
+      if (skin.twitterLikes) {
+        console.log({ skin });
+      }
+      const fileName = path.basename(filePaths[0]);
       fileData[md5] = {
-        color: (await genAverage(file)).slice(1),
-        favorites: favortes[md5],
+        color: skin.averageColor,
+        favorites: skin.twitterLikes,
         fileName
       };
     },
@@ -79,9 +32,8 @@ const getFileData = async files => {
   return fileData;
 };
 
-console.log("Finding the average color of screenshots...");
-console.log("This might take a moment");
-getFileData(files).then(data => {
+console.log("Extracting skin data from cache");
+getFileData().then(data => {
   const json = JSON.stringify(data);
   fs.writeFileSync("src/skins.json", json, "utf8");
 });