diff --git a/experiments/skinArchiveTools/lib/collectSkins.js b/experiments/skinArchiveTools/lib/collectSkins.js index b77092da..f5cb7f2f 100644 --- a/experiments/skinArchiveTools/lib/collectSkins.js +++ b/experiments/skinArchiveTools/lib/collectSkins.js @@ -1,13 +1,6 @@ -const path = require("path"); -const fs = require("fs"); -const fsPromises = require("fs").promises; const childProcess = require("child_process"); const Bluebird = require("bluebird"); -const { memoize } = require("lodash"); -// const md5File = require("md5-file/promise"); const Filehound = require("filehound"); -const Utils = require("./utils"); -const { FILE_TYPES } = require("./constants"); function md5File(filePath) { return new Promise((resolve, reject) => { @@ -22,110 +15,46 @@ function md5File(filePath) { }); } -const memoizedMd5File = memoize(md5File); - -async function collectFile(filePath, outputDir) { - const flatSkinDir = path.join(outputDir, "md5skins"); - - const md5 = await memoizedMd5File(filePath); - - const md5Path = path.join(flatSkinDir, `${md5}.wsz`); - // It's faster to check if the file exists, than to actually look at the skin file - // This is just an optimization - const skinType = fs.existsSync(md5Path) - ? FILE_TYPES.CLASSIC - : await Utils.skinType(filePath); - - let destination = null; - switch (skinType) { - case [FILE_TYPES.CLASSIC]: - destination = md5Path; - break; - /* - case [FILE_TYPES.INVALID]: - destination = path.join(outputDir, `md5Invalids/${md5}.wsz`); - break; - case [FILE_TYPES.PACK]: - const packPath = path.join(outputDir, `md5Packs/${md5}`); - if (fs.existsSync(packPath)) { - break; - } - try { - childProcess.execSync(`unzip "${filePath}" -d "${packPath}"`); - } catch (e) { - console.log(e); - } - break; - */ - default: - // - break; - } - if (destination != null && !fs.existsSync(destination)) { - await fsPromises.link(filePath, destination); - } - return { filePath, md5, md5Path, skinType }; -} - -module.exports = async function collectSkins({ - inputDir, - outputDir, - filenamesPath -}) { +module.exports = async function collectSkins({ inputDir, cache }) { + console.log("Searching for files in", inputDir); + const paths = new Set(); + Object.values(cache).forEach(skin => { + skin.filePaths.forEach(filePath => { + paths.add(filePath); + }); + }); const files = await Filehound.create() - .ext(["zip", "wsz"]) + .ext(["zip", "wsz", "wal"]) .paths(inputDir) .find(); - const seen = new Set(); - let tried = 0; + console.log(`Found ${files.length} potential files`); + let i = 0; const interval = setInterval(() => { - console.log({ seen: seen.size, tried, total: files.length }); + console.log(`Checked ${i} files...`); }, 10000); - // We use Bluebird so that we can limit the concurrency. - const collectionInfo = (await Bluebird.map( + await Bluebird.map( files, async filePath => { - const md5 = await memoizedMd5File(filePath); - tried++; - if (seen.has(md5)) { - return null; + if (paths.has(filePath)) { + return; + } + i++; + const md5 = await md5File(filePath); + if (cache[md5]) { + const filePathsSet = new Set(cache[md5].filePaths); + filePathsSet.add(filePath); + cache[md5].filePaths = Array.from(filePathsSet); + } else { + cache[md5] = { + md5, + filePaths: [filePath] + }; } - seen.add(md5); - return collectFile(filePath, outputDir); }, - { concurrency: 50 } - )).filter(Boolean); - clearInterval(interval); - - console.log("moved all files", "---------------------------"); - - const classics = collectionInfo.filter( - info => info.skinType === FILE_TYPES.CLASSIC + { concurrency: 10 } ); - const pathList = classics - .map(info => `${info.md5} ${path.relative(inputDir, info.filePath)}`) - .join("\n"); - - console.log(`Writing ${classics.length} files to ${filenamesPath}`); - - await fsPromises.writeFile(filenamesPath, pathList); - console.log("done"); - return { - total: files.length, - unique: collectionInfo.length, - newScreenshots: collectionInfo.filter(info => info.screenshotCreated) - .length, - classic: collectionInfo.filter(info => info.skinType === FILE_TYPES.CLASSIC) - .length, - packs: collectionInfo.filter(info => info.skinType === FILE_TYPES.PACK) - .length, - invalid: collectionInfo.filter(info => info.skinType === FILE_TYPES.INVALID) - .length, - modern: collectionInfo.filter(info => info.skinType === FILE_TYPES.MODERN) - .length, - // TODO: Tell modern skins from packs - nonClassic: collectionInfo.filter(info => !info.classic).length - }; + clearInterval(interval); + return cache; }; diff --git a/experiments/skinArchiveTools/lib/index.js b/experiments/skinArchiveTools/lib/index.js index 2b5fea47..dab8ab4b 100755 --- a/experiments/skinArchiveTools/lib/index.js +++ b/experiments/skinArchiveTools/lib/index.js @@ -1,6 +1,10 @@ const path = require("path"); +const fs = require("fs"); +const Bluebird = require("bluebird"); const collectSkins = require("./collectSkins"); -const takeScreenshots = require("./takeScreenshots"); +const Utils = require("./utils"); +const { FILE_TYPES } = require("./constants"); +const Shooter = require("./shooter"); /** * Usage @@ -8,45 +12,258 @@ const takeScreenshots = require("./takeScreenshots"); * command */ +const CACHE_PATH = "/Volumes/Mobile Backup/skins/cache/"; + const [ // eslint-disable-next-line no-unused-vars _node, // eslint-disable-next-line no-unused-vars _thisFile, rawInputDir, - rawOutputDir, - rawFilenamesPath + rawOutputDir ] = process.argv; const inputDir = path.resolve(rawInputDir); const outputDir = path.resolve(rawOutputDir); -const screenshotDir = path.join(outputDir, "md5Screenshots"); -const filenamesPath = path.resolve(rawFilenamesPath); // It's nice to be able to fiddle these manually -const collect = true; +const collect = false; +const getTypes = false; +const moveHome = false; const screenshots = false; +const detectGenerated = false; +const archive = true; +const force = false; +const cacheFilePath = path.join(CACHE_PATH, "info.json"); + +function getPath(skin) { + const filePath = skin.filePaths[0]; + if (filePath == null) { + throw new Error(`Missing file path for ${skin.md5}`); + } + return filePath; +} async function main() { + // Run this if new files are added + const start = Date.now(); + let cache = JSON.parse(fs.readFileSync(cacheFilePath, "utf8")); + + console.log("parsed cache in ", (Date.now() - start) / 1000, "seconds"); + + function report() { + let classic = 0; + let modern = 0; + let invalid = 0; + let screenshotCount = 0; + let archived = 0; + let generated = 0; + Object.values(cache).forEach(skin => { + switch (skin.type) { + case FILE_TYPES.CLASSIC: + classic++; + break; + case FILE_TYPES.MODERN: + modern++; + break; + case FILE_TYPES.INVALID: + invalid++; + break; + } + if (skin.screenshotPath) { + screenshotCount++; + } + if (skin.iaItemName) { + archived++; + } + if (skin.generated === true) { + generated++; + } + }); + console.log(`${classic} classic skins`); + console.log(`${modern} modern skins`); + console.log(`${invalid} invalid skins`); + console.log(`${screenshotCount} screenshots`); + console.log(`${generated} generated`); + console.log(`${archived} archived`); + } + + function save() { + fs.writeFileSync(cacheFilePath, JSON.stringify(cache)); + } + + console.log("Starting with..."); + report(); + if (collect) { - const collectionInfo = await collectSkins({ - filenamesPath, - inputDir, - outputDir + console.log("Collecting skins"); + cache = await collectSkins({ inputDir, cache }); + save(); + } + if (moveHome) { + console.log("Moving skins to md5AllSkins"); + Object.values(cache).forEach(skin => { + const md5Path = path.join(outputDir, "md5AllSkins", `${skin.md5}`); + if (md5Path === skin.md5Path) { + return; + } + fs.linkSync(getPath(skin), md5Path); + skin.md5Path = md5Path; }); - console.log(collectionInfo); - /* - const packCollectionInfo = await collectSkins({ - filenamesPath, - inputDir: path.join(outputDir, "md5Packs"), - outputDir - }); - console.log(packCollectionInfo); - */ + save(); + console.log("Moving classic skins to md5Skins"); + Object.values(cache) + .filter(skin => skin.type === FILE_TYPES.CLASSIC) + .forEach(skin => { + const classicMd5Path = path.join( + outputDir, + "md5Skins", + `${skin.md5}.wsz` + ); + if (classicMd5Path === skin.classicMd5Path) { + return; + } + if (!fs.existsSync(classicMd5Path)) { + fs.linkSync(getPath(skin), classicMd5Path); + } + skin.classicMd5Path = classicMd5Path; + }); + save(); + } + if (getTypes) { + console.log("Computing skin types"); + const skins = Object.values(cache); + let i = 0; + console.log(`Ran on ${i} of ${skins.length} skins`); + const interval = setInterval(() => { + console.log(`Ran on ${i} of ${skins.length} skins`); + console.log("Saving..."); + + save(); + }, 10000); + await Bluebird.map( + skins, + async skin => { + if (skin.type && force === false) { + return; + } + skin.type = await Utils.skinType(getPath(skin)); + i++; + }, + { concurrency: 10 } + ); + clearInterval(interval); + save(); } if (screenshots) { - await takeScreenshots(path.join(outputDir, "md5Skins"), screenshotDir); + console.log("Taking screenshots"); + const skins = Object.values(cache); + let i = 0; + const interval = setInterval(() => { + console.log(`Took ${i} screenshots`); + console.log("Saving..."); + + save(); + }, 10000); + + const needScreenshot = skins.filter(skin => { + return skin.type === FILE_TYPES.CLASSIC && !skin.screenshotPath; + }); + + console.log(`Need to take ${needScreenshot.length} screenshots...`); + + if (needScreenshot.length > 0) { + const shooter = new Shooter(); + await shooter.init(); + for (const skin of needScreenshot) { + const screenshotPath = path.join( + outputDir, + "md5Screenshots", + `${skin.md5}.png` + ); + await shooter.takeScreenshot(getPath(skin), screenshotPath, { + minify: true + }); + skin.screenshotPath = screenshotPath; + i++; + } + shooter.dispose(); + } + clearInterval(interval); + console.log(`Took ${i} screenshots`); + save(); } + + if (detectGenerated) { + const unknownSkins = Object.values(cache).filter( + skin => skin.type === FILE_TYPES.CLASSIC && skin.generated == null + ); + console.log( + `Found ${unknownSkins.length} skins to check if they are generated` + ); + let i = 0; + let j = 0; + const interval = setInterval(() => { + console.log(`Found ${i} generated skins out of ${j}`); + console.log("Saving..."); + + save(); + }, 10000); + await Bluebird.map( + unknownSkins, + async skin => { + const generated = await Utils.skinWasGenerated(skin.classicMd5Path); + j++; + if (generated) { + i++; + } + skin.generated = generated; + }, + { concurrency: 10 } + ); + clearInterval(interval); + save(); + } + + if (archive) { + const classicSkins = Object.values(cache).filter( + skin => skin.type === FILE_TYPES.CLASSIC && skin.iaItemName == null + ); + + const seenItemNames = new Set(); + Object.values(cache).forEach(skin => { + if (skin.iaItemName != null) { + seenItemNames.add(skin.iaItemName.toLowerCase()); + } + }); + + await Bluebird.map( + classicSkins, + async skin => { + const itemName = Utils.getArchiveItemName(skin); + console.log("going to upload ", itemName); + if (seenItemNames.has(itemName.toLowerCase())) { + console.log(itemName, "already exists"); + return; + } + try { + await Utils.archive(skin, itemName); + } catch (e) { + console.error(e); + return; + } + console.log("Uploaded:", itemName); + skin.iaItemName = itemName; + seenItemNames.add(itemName.toLowerCase()); + save(); + }, + { concurrency: 4 } + ); + save(); + } + console.log("Ended with..."); + report(); + save(); } // Blastoff! diff --git a/experiments/skinArchiveTools/lib/utils.js b/experiments/skinArchiveTools/lib/utils.js index 6fbd8fb9..cb7dff61 100644 --- a/experiments/skinArchiveTools/lib/utils.js +++ b/experiments/skinArchiveTools/lib/utils.js @@ -1,4 +1,6 @@ const fsPromises = require("fs").promises; +const path = require("path"); +const childProcess = require("child_process"); const JSZip = require("jszip"); const { FILE_TYPES } = require("./constants"); @@ -19,7 +21,81 @@ async function unique(arr, hasher) { return items; } +function md5File(filePath) { + return new Promise((resolve, reject) => { + childProcess.execFile(`md5`, ["-q", filePath], (err, stdout) => { + if (err) { + // node couldn't execute the command + reject(err); + } + + resolve(stdout.trimRight()); + }); + }); +} + +function skinWasGenerated(filePath) { + return new Promise(resolve => { + childProcess.execFile( + `zipgrep`, + ["This file was created by Skinamp", filePath], + err => { + if (err) { + // node couldn't execute the command + resolve(false); + } + + resolve(true); + } + ); + }); +} + +function getArchiveItemName(skin) { + const { filePaths } = skin; + const filename = `${path.parse(filePaths[0]).name}`; + const token = filename.replace(/[^0-9a-zA-Z]+/g, "_"); + return `winampskin_${token}`; +} + +function getArchiveTitle(skin) { + const { filePaths } = skin; + const filename = `${path.parse(filePaths[0]).name}`; + return `Winamp Skin: ${filename}`; +} + +function archive(skin, itemName) { + return new Promise((resolve, reject) => { + const { classicMd5Path, filePaths, screenshotPath } = skin; + const filename = `${path.parse(filePaths[0]).name}`; + childProcess.execFile( + `python`, + [ + "/Users/jordaneldredge/projects/ia-skins/sync.py", + classicMd5Path, + screenshotPath, + `${filename}.wsz`, + getArchiveTitle(skin), + itemName + ], + (err, stdout) => { + if (err) { + // node couldn't execute the command + reject(err); + } + + resolve(stdout.trimRight()); + } + ); + }); +} + async function skinType(skinPath) { + if (skinPath.endsWith(".wsz")) { + return FILE_TYPES.CLASSIC; + } else if (skinPath.endsWith(".wal")) { + return FILE_TYPES.MODERN; + } const buffer = await fsPromises.readFile(skinPath); try { const zip = await JSZip.loadAsync(buffer); @@ -27,12 +103,19 @@ async function skinType(skinPath) { // This is a collection of skins return FILE_TYPES.PACK; } - return zip.file(/^main\.bmp$/i).length === 1 - ? FILE_TYPES.CLASSIC - : FILE_TYPES.MODERN; + const mains = zip.file(/^(.+\/)main\.bmp$/i); + return mains.length === 1 ? FILE_TYPES.CLASSIC : FILE_TYPES.MODERN; } catch (e) { return FILE_TYPES.INVALID; } } -module.exports = { unique, skinType }; +module.exports = { + unique, + skinType, + skinWasGenerated, + md5File, + archive, + getArchiveItemName, + getArchiveTitle +}; diff --git a/experiments/skinArchiveTools/package.json b/experiments/skinArchiveTools/package.json index c014670b..886f7989 100644 --- a/experiments/skinArchiveTools/package.json +++ b/experiments/skinArchiveTools/package.json @@ -6,8 +6,8 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "start": "node lib/index.js \"/Volumes/Mobile Backup/skins/skins/\" \"/Volumes/Mobile Backup/skins/\" \"/Volumes/Mobile Backup/skins/pathnames.txt\"", - "sync-skins": "aws s3 sync \"/Volumes/Mobile Backup/skins/md5Skins/\" s3://webamp-uploaded-skins/skins/", - "sync-screenshots": "aws s3 sync \"/Volumes/Mobile Backup/skins/md5Screenshots/\" s3://webamp-uploaded-skins/screenshots/", + "sync-skins": "aws s3 sync --acl=public-read \"/Volumes/Mobile Backup/skins/md5Skins/\" s3://webamp-uploaded-skins/skins/", + "sync-screenshots": "aws s3 sync --acl=public-read \"/Volumes/Mobile Backup/skins/md5Screenshots/\" s3://webamp-uploaded-skins/screenshots/", "sync": "npm run sync-skins && npm run sync-screenshots", "clean": "cd md5Skins && find . -delete" },