diff --git a/experiments/skinArchiveTools/lib/color.js b/experiments/skinArchiveTools/lib/color.js new file mode 100644 index 00000000..f0ac2cf2 --- /dev/null +++ b/experiments/skinArchiveTools/lib/color.js @@ -0,0 +1,18 @@ +const exec = require("child_process").exec; +const shellescape = require("shell-escape"); + +const getColor = imgPath => { + return new Promise((resolve, reject) => { + const excapedImgPath = shellescape([imgPath]); + const command = `convert ${excapedImgPath} -scale 1x1\! -format '%[pixel:u]' info:-`; + exec(command, (error, stdout, stderr) => { + if (error !== null) { + reject(error); + return; + } + resolve(stdout.slice(1)); + }); + }); +}; + +module.exports = { getColor }; diff --git a/experiments/skinArchiveTools/lib/index.js b/experiments/skinArchiveTools/lib/index.js index 453e7f8d..b736cd0e 100755 --- a/experiments/skinArchiveTools/lib/index.js +++ b/experiments/skinArchiveTools/lib/index.js @@ -4,6 +4,8 @@ const Bluebird = require("bluebird"); const collectSkins = require("./collectSkins"); const Utils = require("./utils"); const { FILE_TYPES } = require("./constants"); +const { getColor } = require("./color"); +const { extractTextData } = require("./readme"); const Shooter = require("./shooter"); /** @@ -32,8 +34,11 @@ const getTypes = false; const moveHome = false; const screenshots = false; const detectGenerated = false; -const archive = true; +const archive = false; const force = false; +const color = false; +const favorites = false; +const readme = true; const cacheFilePath = path.join(CACHE_PATH, "info.json"); function getPath(skin) { @@ -48,9 +53,16 @@ async function main() { // Run this if new files are added const start = Date.now(); let cache = JSON.parse(fs.readFileSync(cacheFilePath, "utf8")); - console.log("parsed cache in ", (Date.now() - start) / 1000, "seconds"); + // Writing a backup + const cacheBackupFilePath = path.join( + CACHE_PATH, + `backup-${Date.now()}.json` + ); + fs.writeFileSync(cacheBackupFilePath, JSON.stringify(cache, null, 2)); + console.log(`Wrote a backup of the cache to ${cacheBackupFilePath}`); + function report() { let classic = 0; let modern = 0; @@ -89,7 +101,7 @@ async function main() { } function save() { - fs.writeFileSync(cacheFilePath, JSON.stringify(cache)); + fs.writeFileSync(cacheFilePath, JSON.stringify(cache, null, 2)); } console.log("Starting with..."); @@ -107,7 +119,9 @@ async function main() { if (md5Path === skin.md5Path) { return; } - fs.linkSync(getPath(skin), md5Path); + if (!fs.existsSync(md5Path)) { + fs.linkSync(getPath(skin), md5Path); + } skin.md5Path = md5Path; }); save(); @@ -159,8 +173,10 @@ async function main() { console.log("Taking screenshots"); const skins = Object.values(cache); let i = 0; + let j = 0; const interval = setInterval(() => { console.log(`Took ${i} screenshots`); + console.log(`Found ${j} screenshots`); console.log("Saving..."); save(); @@ -181,16 +197,21 @@ async function main() { "md5Screenshots", `${skin.md5}.png` ); - await shooter.takeScreenshot(getPath(skin), screenshotPath, { - minify: true, - }); + if (!fs.existsSync(screenshotPath) || force) { + await shooter.takeScreenshot(getPath(skin), screenshotPath, { + minify: true, + }); + i++; + } else { + j++; + } skin.screenshotPath = screenshotPath; - i++; } shooter.dispose(); } clearInterval(interval); console.log(`Took ${i} screenshots`); + console.log(`Found ${j} screenshots`); save(); } @@ -261,6 +282,100 @@ async function main() { ); save(); } + + if (color) { + const classicSkins = Object.values(cache).filter( + skin => skin.type === FILE_TYPES.CLASSIC + ); + let i = 0; + const interval = setInterval(() => { + console.log(`Found the average color of ${i} skins`); + console.log("Saving..."); + + save(); + }, 10000); + + await Bluebird.map( + classicSkins, + async skin => { + if (skin.screenshotPath == null) { + console.log("Could not extract color from skin"); + return; + } + if (skin.averageColor != null && !force) { + return; + } + const averageColor = await getColor(skin.screenshotPath); + skin.averageColor = averageColor; + i++; + }, + { concurrency: 10 } + ); + clearInterval(interval); + save(); + } + + if (favorites) { + const content = fs.readFileSync( + path.join("/Volumes/Mobile Backup/skins/", "likes.txt"), + "utf8" + ); + content.split("\n").forEach(line => { + if (!line.length) { + return; + } + const [hash, likes] = line.split(" "); + const skin = cache[hash]; + if (!skin) { + console.log(`Found like for unknown skin ${hash}`); + return; + } + skin.twitterLikes = likes; + }); + save(); + } + + if (readme) { + const classicSkins = Object.values(cache).filter( + skin => skin.type === FILE_TYPES.CLASSIC + ); + let i = 0; + const interval = setInterval(() => { + console.log(`Found readme text for ${i} skins`); + console.log("Saving..."); + + save(); + }, 10000); + + await Bluebird.map( + classicSkins, + async skin => { + const readmePath = path.join(outputDir, "readmes", `${skin.md5}.txt`); + // if (skin.readmePath || skin.emails) { + if (skin.readmePath !== undefined) { + return; + } + if (fs.existsSync(readmePath)) { + skin.readmePath = readmePath; + i++; + return; + } + const { emails, raw } = await extractTextData(skin.md5Path); + skin.emails = emails; + if (raw) { + fs.writeFileSync(readmePath, raw); + skin.readmePath = readmePath; + } else { + // Note that we extracted it, but found nothing + skin.readmePath = null; + i++; + } + }, + { concurrency: 10 } + ); + clearInterval(interval); + save(); + } console.log("Ended with..."); report(); save(); diff --git a/experiments/skinArchiveTools/lib/readme.js b/experiments/skinArchiveTools/lib/readme.js new file mode 100644 index 00000000..3ad766b0 --- /dev/null +++ b/experiments/skinArchiveTools/lib/readme.js @@ -0,0 +1,46 @@ +const { exec } = require("child_process"); + +const reg = /[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/g; +const extract = value => (value && value.match(reg)) || []; + +async function extractTextData(skinPath) { + const ignoreFiles = [ + "genex.txt", + "genexinfo.txt", + "gen_gslyrics.txt", + "region.txt", + "pledit.txt", + "viscolor.txt", + "winampmb.txt", + "gen_ex help.txt", + "mbinner.txt" + // Skinning Updates.txt ? + ]; + + const ignoreArgs = ignoreFiles + .map(file => `-x "**/${file}" ${file}`) + .join(" "); + + // Change -p to -c to get context of which file is missing + const debug = false; + const listFlag = debug ? "-c" : "-p"; + + // TODO: Escape path + const cmd = `unzip ${listFlag} -C "${skinPath}" "file_id.diz" "*.txt" ${ignoreArgs}`; + + const raw = await new Promise((resolve, reject) => { + exec(cmd, (error, stdout, stderr) => { + if (error != null) { + // reject(error); + // return; + } + resolve(stdout); + }); + }); + + return { raw, emails: extract(raw) }; +} + +module.exports = { + extractTextData +}; diff --git a/experiments/skinArchiveTools/package.json b/experiments/skinArchiveTools/package.json index 886f7989..2b61105b 100644 --- a/experiments/skinArchiveTools/package.json +++ b/experiments/skinArchiveTools/package.json @@ -22,6 +22,7 @@ "lodash": "^4.17.11", "md5-file": "^4.0.0", "node-fetch": "^2.3.0", - "puppeteer": "^1.12.2" + "puppeteer": "^1.12.2", + "shell-escape": "^0.2.0" } } diff --git a/experiments/skinArchiveTools/yarn.lock b/experiments/skinArchiveTools/yarn.lock index 160e8ae6..0679547a 100644 --- a/experiments/skinArchiveTools/yarn.lock +++ b/experiments/skinArchiveTools/yarn.lock @@ -1850,6 +1850,10 @@ shebang-regex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" +shell-escape@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/shell-escape/-/shell-escape-0.2.0.tgz#68fd025eb0490b4f567a027f0bf22480b5f84133" + signal-exit@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"