mirror of
https://github.com/captbaritone/webamp.git
synced 2026-07-25 19:13:54 +00:00
Rewrite archive tools
This commit is contained in:
Jordan Eldredge
parent
2bcd69762d
commit
9efa7f2f58
4 changed files with 356 additions and 127 deletions
|
|
@ -1,13 +1,6 @@
|
|||
const path = require("path");
|
||||
const fs = require("fs");
|
||||
const fsPromises = require("fs").promises;
|
||||
const childProcess = require("child_process");
|
||||
const Bluebird = require("bluebird");
|
||||
const { memoize } = require("lodash");
|
||||
// const md5File = require("md5-file/promise");
|
||||
const Filehound = require("filehound");
|
||||
const Utils = require("./utils");
|
||||
const { FILE_TYPES } = require("./constants");
|
||||
|
||||
function md5File(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
|
@ -22,110 +15,46 @@ function md5File(filePath) {
|
|||
});
|
||||
}
|
||||
|
||||
const memoizedMd5File = memoize(md5File);
|
||||
|
||||
async function collectFile(filePath, outputDir) {
|
||||
const flatSkinDir = path.join(outputDir, "md5skins");
|
||||
|
||||
const md5 = await memoizedMd5File(filePath);
|
||||
|
||||
const md5Path = path.join(flatSkinDir, `${md5}.wsz`);
|
||||
// It's faster to check if the file exists, than to actually look at the skin file
|
||||
// This is just an optimization
|
||||
const skinType = fs.existsSync(md5Path)
|
||||
? FILE_TYPES.CLASSIC
|
||||
: await Utils.skinType(filePath);
|
||||
|
||||
let destination = null;
|
||||
switch (skinType) {
|
||||
case [FILE_TYPES.CLASSIC]:
|
||||
destination = md5Path;
|
||||
break;
|
||||
/*
|
||||
case [FILE_TYPES.INVALID]:
|
||||
destination = path.join(outputDir, `md5Invalids/${md5}.wsz`);
|
||||
break;
|
||||
case [FILE_TYPES.PACK]:
|
||||
const packPath = path.join(outputDir, `md5Packs/${md5}`);
|
||||
if (fs.existsSync(packPath)) {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
childProcess.execSync(`unzip "${filePath}" -d "${packPath}"`);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
break;
|
||||
*/
|
||||
default:
|
||||
//
|
||||
break;
|
||||
}
|
||||
if (destination != null && !fs.existsSync(destination)) {
|
||||
await fsPromises.link(filePath, destination);
|
||||
}
|
||||
return { filePath, md5, md5Path, skinType };
|
||||
}
|
||||
|
||||
module.exports = async function collectSkins({
|
||||
inputDir,
|
||||
outputDir,
|
||||
filenamesPath
|
||||
}) {
|
||||
module.exports = async function collectSkins({ inputDir, cache }) {
|
||||
console.log("Searching for files in", inputDir);
|
||||
const paths = new Set();
|
||||
Object.values(cache).forEach(skin => {
|
||||
skin.filePaths.forEach(filePath => {
|
||||
paths.add(filePath);
|
||||
});
|
||||
});
|
||||
const files = await Filehound.create()
|
||||
.ext(["zip", "wsz"])
|
||||
.ext(["zip", "wsz", "wal"])
|
||||
.paths(inputDir)
|
||||
.find();
|
||||
|
||||
const seen = new Set();
|
||||
let tried = 0;
|
||||
console.log(`Found ${files.length} potential files`);
|
||||
|
||||
let i = 0;
|
||||
const interval = setInterval(() => {
|
||||
console.log({ seen: seen.size, tried, total: files.length });
|
||||
console.log(`Checked ${i} files...`);
|
||||
}, 10000);
|
||||
// We use Bluebird so that we can limit the concurrency.
|
||||
const collectionInfo = (await Bluebird.map(
|
||||
await Bluebird.map(
|
||||
files,
|
||||
async filePath => {
|
||||
const md5 = await memoizedMd5File(filePath);
|
||||
tried++;
|
||||
if (seen.has(md5)) {
|
||||
return null;
|
||||
if (paths.has(filePath)) {
|
||||
return;
|
||||
}
|
||||
i++;
|
||||
const md5 = await md5File(filePath);
|
||||
if (cache[md5]) {
|
||||
const filePathsSet = new Set(cache[md5].filePaths);
|
||||
filePathsSet.add(filePath);
|
||||
cache[md5].filePaths = Array.from(filePathsSet);
|
||||
} else {
|
||||
cache[md5] = {
|
||||
md5,
|
||||
filePaths: [filePath]
|
||||
};
|
||||
}
|
||||
seen.add(md5);
|
||||
return collectFile(filePath, outputDir);
|
||||
},
|
||||
{ concurrency: 50 }
|
||||
)).filter(Boolean);
|
||||
clearInterval(interval);
|
||||
|
||||
console.log("moved all files", "---------------------------");
|
||||
|
||||
const classics = collectionInfo.filter(
|
||||
info => info.skinType === FILE_TYPES.CLASSIC
|
||||
{ concurrency: 10 }
|
||||
);
|
||||
const pathList = classics
|
||||
.map(info => `${info.md5} ${path.relative(inputDir, info.filePath)}`)
|
||||
.join("\n");
|
||||
|
||||
console.log(`Writing ${classics.length} files to ${filenamesPath}`);
|
||||
|
||||
await fsPromises.writeFile(filenamesPath, pathList);
|
||||
console.log("done");
|
||||
return {
|
||||
total: files.length,
|
||||
unique: collectionInfo.length,
|
||||
newScreenshots: collectionInfo.filter(info => info.screenshotCreated)
|
||||
.length,
|
||||
classic: collectionInfo.filter(info => info.skinType === FILE_TYPES.CLASSIC)
|
||||
.length,
|
||||
packs: collectionInfo.filter(info => info.skinType === FILE_TYPES.PACK)
|
||||
.length,
|
||||
invalid: collectionInfo.filter(info => info.skinType === FILE_TYPES.INVALID)
|
||||
.length,
|
||||
modern: collectionInfo.filter(info => info.skinType === FILE_TYPES.MODERN)
|
||||
.length,
|
||||
// TODO: Tell modern skins from packs
|
||||
nonClassic: collectionInfo.filter(info => !info.classic).length
|
||||
};
|
||||
clearInterval(interval);
|
||||
return cache;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
const path = require("path");
|
||||
const fs = require("fs");
|
||||
const Bluebird = require("bluebird");
|
||||
const collectSkins = require("./collectSkins");
|
||||
const takeScreenshots = require("./takeScreenshots");
|
||||
const Utils = require("./utils");
|
||||
const { FILE_TYPES } = require("./constants");
|
||||
const Shooter = require("./shooter");
|
||||
|
||||
/**
|
||||
* Usage
|
||||
|
|
@ -8,45 +12,258 @@ const takeScreenshots = require("./takeScreenshots");
|
|||
* command <input_dir> <output_dir> <screenshot_dir>
|
||||
*/
|
||||
|
||||
const CACHE_PATH = "/Volumes/Mobile Backup/skins/cache/";
|
||||
|
||||
const [
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
_node,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
_thisFile,
|
||||
rawInputDir,
|
||||
rawOutputDir,
|
||||
rawFilenamesPath
|
||||
rawOutputDir
|
||||
] = process.argv;
|
||||
|
||||
const inputDir = path.resolve(rawInputDir);
|
||||
const outputDir = path.resolve(rawOutputDir);
|
||||
const screenshotDir = path.join(outputDir, "md5Screenshots");
|
||||
const filenamesPath = path.resolve(rawFilenamesPath);
|
||||
|
||||
// It's nice to be able to fiddle these manually
|
||||
const collect = true;
|
||||
const collect = false;
|
||||
const getTypes = false;
|
||||
const moveHome = false;
|
||||
const screenshots = false;
|
||||
const detectGenerated = false;
|
||||
const archive = true;
|
||||
const force = false;
|
||||
const cacheFilePath = path.join(CACHE_PATH, "info.json");
|
||||
|
||||
function getPath(skin) {
|
||||
const filePath = skin.filePaths[0];
|
||||
if (filePath == null) {
|
||||
throw new Error(`Missing file path for ${skin.md5}`);
|
||||
}
|
||||
return filePath;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// Run this if new files are added
|
||||
const start = Date.now();
|
||||
let cache = JSON.parse(fs.readFileSync(cacheFilePath, "utf8"));
|
||||
|
||||
console.log("parsed cache in ", (Date.now() - start) / 1000, "seconds");
|
||||
|
||||
function report() {
|
||||
let classic = 0;
|
||||
let modern = 0;
|
||||
let invalid = 0;
|
||||
let screenshotCount = 0;
|
||||
let archived = 0;
|
||||
let generated = 0;
|
||||
Object.values(cache).forEach(skin => {
|
||||
switch (skin.type) {
|
||||
case FILE_TYPES.CLASSIC:
|
||||
classic++;
|
||||
break;
|
||||
case FILE_TYPES.MODERN:
|
||||
modern++;
|
||||
break;
|
||||
case FILE_TYPES.INVALID:
|
||||
invalid++;
|
||||
break;
|
||||
}
|
||||
if (skin.screenshotPath) {
|
||||
screenshotCount++;
|
||||
}
|
||||
if (skin.iaItemName) {
|
||||
archived++;
|
||||
}
|
||||
if (skin.generated === true) {
|
||||
generated++;
|
||||
}
|
||||
});
|
||||
console.log(`${classic} classic skins`);
|
||||
console.log(`${modern} modern skins`);
|
||||
console.log(`${invalid} invalid skins`);
|
||||
console.log(`${screenshotCount} screenshots`);
|
||||
console.log(`${generated} generated`);
|
||||
console.log(`${archived} archived`);
|
||||
}
|
||||
|
||||
function save() {
|
||||
fs.writeFileSync(cacheFilePath, JSON.stringify(cache));
|
||||
}
|
||||
|
||||
console.log("Starting with...");
|
||||
report();
|
||||
|
||||
if (collect) {
|
||||
const collectionInfo = await collectSkins({
|
||||
filenamesPath,
|
||||
inputDir,
|
||||
outputDir
|
||||
console.log("Collecting skins");
|
||||
cache = await collectSkins({ inputDir, cache });
|
||||
save();
|
||||
}
|
||||
if (moveHome) {
|
||||
console.log("Moving skins to md5AllSkins");
|
||||
Object.values(cache).forEach(skin => {
|
||||
const md5Path = path.join(outputDir, "md5AllSkins", `${skin.md5}`);
|
||||
if (md5Path === skin.md5Path) {
|
||||
return;
|
||||
}
|
||||
fs.linkSync(getPath(skin), md5Path);
|
||||
skin.md5Path = md5Path;
|
||||
});
|
||||
console.log(collectionInfo);
|
||||
/*
|
||||
const packCollectionInfo = await collectSkins({
|
||||
filenamesPath,
|
||||
inputDir: path.join(outputDir, "md5Packs"),
|
||||
outputDir
|
||||
});
|
||||
console.log(packCollectionInfo);
|
||||
*/
|
||||
save();
|
||||
console.log("Moving classic skins to md5Skins");
|
||||
Object.values(cache)
|
||||
.filter(skin => skin.type === FILE_TYPES.CLASSIC)
|
||||
.forEach(skin => {
|
||||
const classicMd5Path = path.join(
|
||||
outputDir,
|
||||
"md5Skins",
|
||||
`${skin.md5}.wsz`
|
||||
);
|
||||
if (classicMd5Path === skin.classicMd5Path) {
|
||||
return;
|
||||
}
|
||||
if (!fs.existsSync(classicMd5Path)) {
|
||||
fs.linkSync(getPath(skin), classicMd5Path);
|
||||
}
|
||||
skin.classicMd5Path = classicMd5Path;
|
||||
});
|
||||
save();
|
||||
}
|
||||
if (getTypes) {
|
||||
console.log("Computing skin types");
|
||||
const skins = Object.values(cache);
|
||||
let i = 0;
|
||||
console.log(`Ran on ${i} of ${skins.length} skins`);
|
||||
const interval = setInterval(() => {
|
||||
console.log(`Ran on ${i} of ${skins.length} skins`);
|
||||
console.log("Saving...");
|
||||
|
||||
save();
|
||||
}, 10000);
|
||||
await Bluebird.map(
|
||||
skins,
|
||||
async skin => {
|
||||
if (skin.type && force === false) {
|
||||
return;
|
||||
}
|
||||
skin.type = await Utils.skinType(getPath(skin));
|
||||
i++;
|
||||
},
|
||||
{ concurrency: 10 }
|
||||
);
|
||||
clearInterval(interval);
|
||||
save();
|
||||
}
|
||||
if (screenshots) {
|
||||
await takeScreenshots(path.join(outputDir, "md5Skins"), screenshotDir);
|
||||
console.log("Taking screenshots");
|
||||
const skins = Object.values(cache);
|
||||
let i = 0;
|
||||
const interval = setInterval(() => {
|
||||
console.log(`Took ${i} screenshots`);
|
||||
console.log("Saving...");
|
||||
|
||||
save();
|
||||
}, 10000);
|
||||
|
||||
const needScreenshot = skins.filter(skin => {
|
||||
return skin.type === FILE_TYPES.CLASSIC && !skin.screenshotPath;
|
||||
});
|
||||
|
||||
console.log(`Need to take ${needScreenshot.length} screenshots...`);
|
||||
|
||||
if (needScreenshot.length > 0) {
|
||||
const shooter = new Shooter();
|
||||
await shooter.init();
|
||||
for (const skin of needScreenshot) {
|
||||
const screenshotPath = path.join(
|
||||
outputDir,
|
||||
"md5Screenshots",
|
||||
`${skin.md5}.png`
|
||||
);
|
||||
await shooter.takeScreenshot(getPath(skin), screenshotPath, {
|
||||
minify: true
|
||||
});
|
||||
skin.screenshotPath = screenshotPath;
|
||||
i++;
|
||||
}
|
||||
shooter.dispose();
|
||||
}
|
||||
clearInterval(interval);
|
||||
console.log(`Took ${i} screenshots`);
|
||||
save();
|
||||
}
|
||||
|
||||
if (detectGenerated) {
|
||||
const unknownSkins = Object.values(cache).filter(
|
||||
skin => skin.type === FILE_TYPES.CLASSIC && skin.generated == null
|
||||
);
|
||||
console.log(
|
||||
`Found ${unknownSkins.length} skins to check if they are generated`
|
||||
);
|
||||
let i = 0;
|
||||
let j = 0;
|
||||
const interval = setInterval(() => {
|
||||
console.log(`Found ${i} generated skins out of ${j}`);
|
||||
console.log("Saving...");
|
||||
|
||||
save();
|
||||
}, 10000);
|
||||
await Bluebird.map(
|
||||
unknownSkins,
|
||||
async skin => {
|
||||
const generated = await Utils.skinWasGenerated(skin.classicMd5Path);
|
||||
j++;
|
||||
if (generated) {
|
||||
i++;
|
||||
}
|
||||
skin.generated = generated;
|
||||
},
|
||||
{ concurrency: 10 }
|
||||
);
|
||||
clearInterval(interval);
|
||||
save();
|
||||
}
|
||||
|
||||
if (archive) {
|
||||
const classicSkins = Object.values(cache).filter(
|
||||
skin => skin.type === FILE_TYPES.CLASSIC && skin.iaItemName == null
|
||||
);
|
||||
|
||||
const seenItemNames = new Set();
|
||||
Object.values(cache).forEach(skin => {
|
||||
if (skin.iaItemName != null) {
|
||||
seenItemNames.add(skin.iaItemName.toLowerCase());
|
||||
}
|
||||
});
|
||||
|
||||
await Bluebird.map(
|
||||
classicSkins,
|
||||
async skin => {
|
||||
const itemName = Utils.getArchiveItemName(skin);
|
||||
console.log("going to upload ", itemName);
|
||||
if (seenItemNames.has(itemName.toLowerCase())) {
|
||||
console.log(itemName, "already exists");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await Utils.archive(skin, itemName);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
return;
|
||||
}
|
||||
console.log("Uploaded:", itemName);
|
||||
skin.iaItemName = itemName;
|
||||
seenItemNames.add(itemName.toLowerCase());
|
||||
save();
|
||||
},
|
||||
{ concurrency: 4 }
|
||||
);
|
||||
save();
|
||||
}
|
||||
console.log("Ended with...");
|
||||
report();
|
||||
save();
|
||||
}
|
||||
|
||||
// Blastoff!
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
const fsPromises = require("fs").promises;
|
||||
const path = require("path");
|
||||
const childProcess = require("child_process");
|
||||
const JSZip = require("jszip");
|
||||
const { FILE_TYPES } = require("./constants");
|
||||
|
||||
|
|
@ -19,7 +21,81 @@ async function unique(arr, hasher) {
|
|||
return items;
|
||||
}
|
||||
|
||||
function md5File(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
childProcess.execFile(`md5`, ["-q", filePath], (err, stdout) => {
|
||||
if (err) {
|
||||
// node couldn't execute the command
|
||||
reject(err);
|
||||
}
|
||||
|
||||
resolve(stdout.trimRight());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function skinWasGenerated(filePath) {
|
||||
return new Promise(resolve => {
|
||||
childProcess.execFile(
|
||||
`zipgrep`,
|
||||
["This file was created by Skinamp", filePath],
|
||||
err => {
|
||||
if (err) {
|
||||
// node couldn't execute the command
|
||||
resolve(false);
|
||||
}
|
||||
|
||||
resolve(true);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function getArchiveItemName(skin) {
|
||||
const { filePaths } = skin;
|
||||
const filename = `${path.parse(filePaths[0]).name}`;
|
||||
const token = filename.replace(/[^0-9a-zA-Z]+/g, "_");
|
||||
return `winampskin_${token}`;
|
||||
}
|
||||
|
||||
function getArchiveTitle(skin) {
|
||||
const { filePaths } = skin;
|
||||
const filename = `${path.parse(filePaths[0]).name}`;
|
||||
return `Winamp Skin: ${filename}`;
|
||||
}
|
||||
|
||||
function archive(skin, itemName) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const { classicMd5Path, filePaths, screenshotPath } = skin;
|
||||
const filename = `${path.parse(filePaths[0]).name}`;
|
||||
childProcess.execFile(
|
||||
`python`,
|
||||
[
|
||||
"/Users/jordaneldredge/projects/ia-skins/sync.py",
|
||||
classicMd5Path,
|
||||
screenshotPath,
|
||||
`${filename}.wsz`,
|
||||
getArchiveTitle(skin),
|
||||
itemName
|
||||
],
|
||||
(err, stdout) => {
|
||||
if (err) {
|
||||
// node couldn't execute the command
|
||||
reject(err);
|
||||
}
|
||||
|
||||
resolve(stdout.trimRight());
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function skinType(skinPath) {
|
||||
if (skinPath.endsWith(".wsz")) {
|
||||
return FILE_TYPES.CLASSIC;
|
||||
} else if (skinPath.endsWith(".wal")) {
|
||||
return FILE_TYPES.MODERN;
|
||||
}
|
||||
const buffer = await fsPromises.readFile(skinPath);
|
||||
try {
|
||||
const zip = await JSZip.loadAsync(buffer);
|
||||
|
|
@ -27,12 +103,19 @@ async function skinType(skinPath) {
|
|||
// This is a collection of skins
|
||||
return FILE_TYPES.PACK;
|
||||
}
|
||||
return zip.file(/^main\.bmp$/i).length === 1
|
||||
? FILE_TYPES.CLASSIC
|
||||
: FILE_TYPES.MODERN;
|
||||
const mains = zip.file(/^(.+\/)main\.bmp$/i);
|
||||
return mains.length === 1 ? FILE_TYPES.CLASSIC : FILE_TYPES.MODERN;
|
||||
} catch (e) {
|
||||
return FILE_TYPES.INVALID;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { unique, skinType };
|
||||
module.exports = {
|
||||
unique,
|
||||
skinType,
|
||||
skinWasGenerated,
|
||||
md5File,
|
||||
archive,
|
||||
getArchiveItemName,
|
||||
getArchiveTitle
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"start": "node lib/index.js \"/Volumes/Mobile Backup/skins/skins/\" \"/Volumes/Mobile Backup/skins/\" \"/Volumes/Mobile Backup/skins/pathnames.txt\"",
|
||||
"sync-skins": "aws s3 sync \"/Volumes/Mobile Backup/skins/md5Skins/\" s3://webamp-uploaded-skins/skins/",
|
||||
"sync-screenshots": "aws s3 sync \"/Volumes/Mobile Backup/skins/md5Screenshots/\" s3://webamp-uploaded-skins/screenshots/",
|
||||
"sync-skins": "aws s3 sync --acl=public-read \"/Volumes/Mobile Backup/skins/md5Skins/\" s3://webamp-uploaded-skins/skins/",
|
||||
"sync-screenshots": "aws s3 sync --acl=public-read \"/Volumes/Mobile Backup/skins/md5Screenshots/\" s3://webamp-uploaded-skins/screenshots/",
|
||||
"sync": "npm run sync-skins && npm run sync-screenshots",
|
||||
"clean": "cd md5Skins && find . -delete"
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue