Mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp.git synced 2026-07-27 03:54:24 +00:00
Code Issues Projects Releases Wiki Activity

Improve skin upload

Browse source
This commit is contained in:
Jordan Eldredge 2020-04-19 23:29:04 -07:00
parent fae252966f
commit c4d4649501
8 changed files with 163 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

39
experiments/skin-database/addSkin.js Normal file
View file

@ -0,0 +1,39 @@
const fs = require("fs");
const md5Buffer = require("md5");
const Skins = require("./data/skins");
const S3 = require("./s3");
const Shooter = require("./shooter");
const temp = require("temp").track();
const Analyser = require("./analyser");
// TODO
// Extract the readme
// Extract the emails
// Extract the average color
// Upload to Internet Archive
// Store the Internet Archive item name
// Construct IA Webamp UR
async function addSkinFromBuffer(buffer, filePath, uploader) {
const md5 = md5Buffer(buffer);
const skin = await Skins.getSkinByMd5(md5);
if (skin != null) {
return { md5, status: "FOUND" };
}
const tempFile = temp.path({ suffix: ".wsz" });
fs.writeFileSync(tempFile, buffer);
const tempScreenshotPath = temp.path({ suffix: ".png" });
await Shooter.withShooter((shooter) =>
shooter.takeScreenshot(tempFile, tempScreenshotPath, {
minify: true,
})
);
const averageColor = await Analyser.getColor(tempScreenshotPath);
await S3.putScreenshot(md5, fs.readFileSync(tempScreenshotPath));
await S3.putSkin(md5, buffer);
await Skins.addSkin({ md5, filePath, uploader, averageColor });
return { md5, status: "ADDED", averageColor };
}
module.exports = { addSkinFromBuffer };

20
experiments/skin-database/analyser.js Normal file
View file

@ -0,0 +1,20 @@
// Functions for deriving information from skins
const exec = require("child_process").exec;
const shellescape = require("shell-escape");
function getColor(imgPath) {
return new Promise((resolve, reject) => {
const excapedImgPath = shellescape([imgPath]);
const command = `convert ${excapedImgPath} -scale 1x1\! -format '%[pixel:u]' info:-`;
exec(command, (error, stdout) => {
if (error !== null) {
reject(error);
return;
}
resolve(stdout.slice(1));
});
});
}
module.exports = { getColor };

98
experiments/skin-database/cli.js
View file

@ -10,61 +10,71 @@ const Skins = require("./data/skins");
const db = require("./db");
const Discord = require("discord.js");
const tweet = require("./tasks/tweet");
const { addSkinFromBuffer } = require("./addSkin");
async function main() {
const client = new Discord.Client();
// The Winston transport logs in the client.
await DiscordWinstonTransport.addToLogger(client, logger);
switch (argv._[0]) {
case "image-hash":
const hashes = new Map();
try {
switch (argv._[0]) {
case "image-hash":
const hashes = new Map();
fs.readFileSync(path.join(__dirname, "./hash.txt"), "utf8")
.split("\n")
.forEach(line => {
const [md5, imgHash] = line.split(" ");
hashes.set(md5, imgHash);
});
fs.readFileSync(path.join(__dirname, "./hash.txt"), "utf8")
.split("\n")
.forEach((line) => {
const [md5, imgHash] = line.split(" ");
hashes.set(md5, imgHash);
});
for (const [md5, imgHash] of hashes.entries()) {
await Skins.setImageHash(md5, imgHash);
process.stderr.write(".");
for (const [md5, imgHash] of hashes.entries()) {
await Skins.setImageHash(md5, imgHash);
process.stderr.write(".");
}
break;
case "tweet":
await tweet(client);
break;
case "fetch-metadata":
console.log("Going to download metadata from the Internet Archive");
await fetchInternetArchiveMetadata();
break;
case "ensure-md5s":
await ensureInternetArchiveItemsIndexByMd5();
break;
case "metadata": {
const hash = argv._[1];
console.log(await Skins.getInternetArchiveUrl(hash));
break;
}
break;
case "tweet":
await tweet(client);
break;
case "fetch-metadata":
console.log("Going to download metadata from the Internet Archive");
await fetchInternetArchiveMetadata();
break;
case "ensure-md5s":
await ensureInternetArchiveItemsIndexByMd5();
break;
case "metadata": {
const hash = argv._[1];
console.log(await Skins.getInternetArchiveUrl(hash));
break;
case "reconcile": {
await Skins.reconcile();
break;
}
case "skin": {
const hash = argv._[1];
logger.info({ hash });
console.log(await Skins.getSkinByMd5(hash));
break;
}
case "add": {
const filePath = argv._[1];
const buffer = fs.readFileSync(filePath);
console.log(await addSkinFromBuffer(buffer, filePath, "cli-user"));
break;
}
default:
console.log(`Unknown command ${argv._[0]}`);
}
case "reconcile": {
await Skins.reconcile();
break;
}
case "skin": {
const hash = argv._[1];
logger.info({ hash });
console.log(await Skins.getSkinByMd5(hash));
break;
}
default:
console.log(`Unknown command ${argv._[0]}`);
} finally {
logger.close();
client.destroy();
await db.close();
}
logger.close();
client.destroy();
await db.close();
}
main();

11
experiments/skin-database/data/skins.js
View file

@ -18,6 +18,7 @@ const REVIEWABLE_QUERY = {
rejected: { $ne: true },
type: "CLASSIC",
};
function getSkinRecord(skin) {
const {
md5,
@ -55,8 +56,14 @@ async function getProp(md5, prop) {
return value == null ? null : value;
}
async function addSkin({ md5, filePath, uploader }) {
skins.insert({ md5, type: "CLASSIC", filePaths: [filePath], uploader });
async function addSkin({ md5, filePath, uploader, averageColor }) {
skins.insert({
md5,
type: "CLASSIC",
filePaths: [filePath],
uploader,
averageColor,
});
}
const IA_URL = /^(https:\/\/)?archive.org\/details\/([^\/]+)\/?/;

92
experiments/skin-database/discord-bot/commands/archive.js
View file

@ -1,11 +1,6 @@
const fs = require("fs");
const fetch = require("node-fetch");
const md5Buffer = require("md5");
const Skins = require("../../data/skins");
const Utils = require("../utils");
const S3 = require("../../s3");
const Shooter = require("../../shooter");
const temp = require("temp").track();
const { addSkinFromBuffer } = require("../../addSkin");
async function handler(message) {
const { attachments } = message;
@ -20,73 +15,34 @@ async function handler(message) {
console.log("got response");
const buffer = await response.buffer();
console.log("got buffer");
const md5 = md5Buffer(buffer);
console.log("got md5", md5);
return { filename, buffer, md5 };
return { filename, buffer };
})
);
// TODO
// Extract the readme
// Extract the emails
// Extract the average color
// Upload to Internet Archive
// Store the Internet Archive item name
// Construct IA Webamp URL
const shooter = new Shooter("https://webamp.org");
try {
for (const file of files) {
const skin = await Skins.getSkinByMd5(file.md5);
if (skin != null) {
await message.channel.send(`This skin is already in our collection.`);
await Utils.postSkin({
md5: file.md5,
dest: message.channel,
});
} else {
await message.channel.send(
`Thanks! ${file.filename} is a brand new skin. 👏 Uploading to our server and taking a screenshot...`
);
const tempFile = temp.path({ suffix: ".wsz" });
fs.writeFileSync(tempFile, file.buffer);
const tempScreenshotPath = temp.path({ suffix: ".png" });
try {
await shooter.takeScreenshot(tempFile, tempScreenshotPath, {
minify: true,
});
} catch (e) {
console.error(e);
await message.channel.send(
`Something went wrong taking the screenshot of ${file.filename}. Sorry.`
);
continue;
}
await S3.putScreenshot(file.md5, fs.readFileSync(tempScreenshotPath));
await S3.putSkin(file.md5, file.buffer);
await Skins.addSkin({
md5: file.md5,
filePath: file.filename,
uploader: message.author.username,
});
await message.channel.send(
`Done! ${file.filename} has been queued for archiving. In the mean time, here's a screenshot and a link to it on Webamp.`
);
await Utils.postSkin({
md5: file.md5,
dest: message.channel,
});
}
for (const file of files) {
let result;
try {
result = await addSkinFromBuffer(
file.buffer,
file.filename,
message.author.username
);
} catch (e) {
console.error(e);
message.channel.send(
"There was an error archiving your skin. Please ping @captbaritone."
);
throw e;
}
} catch (e) {
console.error(e);
message.channel.send(
"There was an error archiving your skin. Please ping @captbaritone."
);
if (result.status === "FOUND") {
await message.channel.send(`This skin is already in our collection.`);
} else if (result.status === "ADDED") {
await message.channel.send(
`Thanks! ${file.filename} is a brand new skin. 👏 It has been queued for archiving. In the mean time, here's a screenshot and a link to it on Webamp.`
);
}
await Utils.postSkin({ md5: result.md5, dest: message.channel });
}
shooter.dispose();
}
module.exports = {

5
experiments/skin-database/package.json
View file

@ -7,16 +7,17 @@
"aws-sdk": "^2.469.0",
"discord.js": "^11.5.1",
"express": "^4.17.1",
"image-hash": "^3.5.1",
"imagemin": "^7.0.0",
"express-graphql": "^0.9.0",
"graphql": "^14.5.8",
"image-hash": "^3.5.1",
"imagemin": "^7.0.0",
"imagemin-optipng": "^7.0.0",
"md5": "^2.2.1",
"monk": "^7.0.0",
"node-fetch": "^2.6.0",
"puppeteer": "^1.17.0",
"rgb-hex": "^2.1.0",
"shell-escape": "^0.2.0",
"temp": "^0.9.0",
"winston": "^3.2.1",
"yargs": "^13.2.4"

9
experiments/skin-database/shooter.js
View file

@ -15,6 +15,15 @@ class Shooter {
this._url = url;
}
static async withShooter(cb) {
const shooter = new Shooter("https://webamp.org");
try {
await cb(shooter);
} finally {
shooter.dispose();
}
}
async init() {
this._browser = await puppeteer.launch();
this._page = await this._browser.newPage();

5
experiments/skin-database/yarn.lock
View file

@ -2301,6 +2301,11 @@ shebang-regex@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
shell-escape@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/shell-escape/-/shell-escape-0.2.0.tgz#68fd025eb0490b4f567a027f0bf22480b5f84133"
integrity sha1-aP0CXrBJC09WegJ/C/IkgLX4QTM=
signal-exit@^3.0.0:
version "3.0.2"
resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"