From b73b44bfd9cf8531b85f8a01ac6f662f9cfa16a1 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sun, 19 Apr 2020 01:39:38 -0400 Subject: [PATCH] Upload to S3 and Screenshot when you upload --- experiments/skin-database/.gitignore | 3 ++ experiments/skin-database/data/skins.js | 11 +++- .../discord-bot/commands/archive.js | 53 +++++++++++++------ experiments/skin-database/graphql.js | 5 ++ experiments/skin-database/s3.js | 38 +++++++++++++ 5 files changed, 91 insertions(+), 19 deletions(-) diff --git a/experiments/skin-database/.gitignore b/experiments/skin-database/.gitignore index edb6142f..a96f3e58 100644 --- a/experiments/skin-database/.gitignore +++ b/experiments/skin-database/.gitignore @@ -1,3 +1,6 @@ node_modules/ yarn-error.log config.js +uploads/ +combined.log +hash.txt diff --git a/experiments/skin-database/data/skins.js b/experiments/skin-database/data/skins.js index b395e8a0..c50e1d43 100644 --- a/experiments/skin-database/data/skins.js +++ b/experiments/skin-database/data/skins.js @@ -28,6 +28,7 @@ function getSkinRecord(skin) { readmeText, filePaths, imageHash, + uploader, } = skin; const fileNames = filePaths.map((p) => path.basename(p)); const skinUrl = `https://s3.amazonaws.com/webamp-uploaded-skins/skins/${md5}.wsz`; @@ -44,6 +45,7 @@ function getSkinRecord(skin) { webampUrl: `https://webamp.org?skinUrl=${skinUrl}`, readmeText, imageHash, + uploader, }; } @@ -53,6 +55,10 @@ async function getProp(md5, prop) { return value == null ? null : value; } +async function addSkin({ md5, filePath, uploader }) { + skins.insert({ md5, type: "CLASSIC", filePaths: [filePath], uploader }); +} + const IA_URL = /^(https:\/\/)?archive.org\/details\/([^\/]+)\/?/; const MD5 = /([a-fA-F0-9]{32})/; @@ -176,7 +182,7 @@ async function reject(md5) { async function getSkinToReview() { const reviewable = await skins.aggregate([ { $match: REVIEWABLE_QUERY }, - { $sample: { size: 1 } } + { $sample: { size: 1 } }, ]); const skin = reviewable[0]; const { canonicalFilename, md5 } = getSkinRecord(skin); @@ -186,7 +192,7 @@ async function getSkinToReview() { async function getSkinToTweet() { const tweetables = await skins.aggregate([ { $match: TWEETABLE_QUERY }, - { $sample: { size: 1 } } + { $sample: { size: 1 } }, ]); const skin = tweetables[0]; if (skin == null) { @@ -228,6 +234,7 @@ async function setImageHash(md5, imageHash) { } module.exports = { + addSkin, getMd5sMatchingImageHash, getInternetArchiveItem, getMd5ByAnything, diff --git a/experiments/skin-database/discord-bot/commands/archive.js b/experiments/skin-database/discord-bot/commands/archive.js index ddb5263d..d18e7a31 100644 --- a/experiments/skin-database/discord-bot/commands/archive.js +++ b/experiments/skin-database/discord-bot/commands/archive.js @@ -1,10 +1,11 @@ const fs = require("fs"); -const path = require("path"); const fetch = require("node-fetch"); const md5Buffer = require("md5"); -const config = require("../../config"); const Skins = require("../../data/skins"); const Utils = require("../utils"); +const S3 = require("../../s3"); +const Shooter = require("../../shooter"); +const temp = require("temp").track(); async function handler(message) { const { attachments } = message; @@ -25,17 +26,14 @@ async function handler(message) { }) ); - // Compute MD5 - // Upload to S3 - // Record the filepath - // Take a screenshot - // Upload screenshot to S3 + // TODO // Extract the readme // Extract the emails // Extract the average color // Upload to Internet Archive // Store the Internet Archive item name // Construct IA Webamp URL + const shooter = new Shooter("https://webamp.org"); for (const file of files) { const skin = await Skins.getSkinByMd5(file.md5); @@ -46,18 +44,39 @@ async function handler(message) { dest: message.channel, }); } else { - fs.writeFileSync( - path.join( - config.uploadDir, - // TODO: Use a sub directory using md5 to avoid collision - // file.md5, - file.filename - ), - file.buffer - ); await message.channel.send( - `Thanks! ${file.filename} is a brand new skin. 👏 It has been queued for archiving.` + `Thanks! ${file.filename} is a brand new skin. 👏 Uploading to our server and taking a screenshot...` ); + const tempFile = temp.path({ suffix: ".wsz" }); + fs.writeFileSync(tempFile, file.buffer); + const tempScreenshotPath = temp.path({ suffix: ".png" }); + + try { + await shooter.takeScreenshot(tempFile, tempScreenshotPath, { + minify: true, + }); + } catch (e) { + console.error(e); + await message.channel.send( + `Something went wrong taking the screenshot of ${file.filename}. Sorry.` + ); + continue; + } + + await S3.putScreenshot(file.md5, fs.readFileSync(tempScreenshotPath)); + await S3.putSkin(file.md5, file.buffer); + await Skins.addSkin({ + md5: file.md5, + filePath: file.filename, + uploader: message.author.username, + }); + await message.channel.send( + `Done! ${file.filename} has been queued for archiving. In the mean time, here's a screenshot and a link to it on Webamp.` + ); + await Utils.postSkin({ + md5: file.md5, + dest: message.channel, + }); } } } diff --git a/experiments/skin-database/graphql.js b/experiments/skin-database/graphql.js index 6c47a568..4487d93a 100644 --- a/experiments/skin-database/graphql.js +++ b/experiments/skin-database/graphql.js @@ -28,6 +28,7 @@ type InternetArchiveItem { readmeText: String internetArchiveItem: InternetArchiveItem imageHash: String + uploader: String } type Query { skin(md5: String!): Skin @@ -119,6 +120,10 @@ class Skin { return this._get((skin) => skin.imageHash); } + async uploader() { + return this._get(skin => skin.upload); + } + internetArchiveItem() { return new InternetArchiveItem(this._md5); } diff --git a/experiments/skin-database/s3.js b/experiments/skin-database/s3.js index 9c31da16..1be78a2e 100644 --- a/experiments/skin-database/s3.js +++ b/experiments/skin-database/s3.js @@ -30,6 +30,42 @@ function putFile(key, body) { }); } +function putSkin(md5, buffer) { + return new Promise((resolve, rejectPromise) => { + const bucketName = "webamp-uploaded-skins"; + const key = `skins/${md5}.wsz`; + s3.putObject( + { Bucket: bucketName, Key: key, Body: buffer, ACL: "public-read" }, + err => { + if (err) { + rejectPromise(err); + return; + } + console.log(`Upladed skin to ${bucketName} ${key}`); + resolve(); + } + ); + }); +} + +function putScreenshot(md5, buffer) { + return new Promise((resolve, rejectPromise) => { + const bucketName = "webamp-uploaded-skins"; + const key = `screenshots/${md5}.png`; + s3.putObject( + { Bucket: bucketName, Key: key, Body: buffer, ACL: "public-read" }, + err => { + if (err) { + rejectPromise(err); + return; + } + console.log(`Upladed screenshot to ${bucketName} ${key}`); + resolve(); + } + ); + }); +} + function getLines(body) { return body .trim() @@ -155,6 +191,8 @@ async function markAsTweeted(md5) { } module.exports = { + putScreenshot, + putSkin, getSkinToReview, approve, reject,