Unify discord-bot and skin-database

experiments/skin-database/cli.js

@@ -1,7 +1,9 @@
 #!/usr/bin/env node
 const argv = require("yargs").argv;
 const findTweetableSkin = require("./tasks/findTweetableSkins");
+const fetchInternetArchiveMetadata = require("./tasks/fetchInternetArchiveMetadata");
 const path = require("path");
+const Skins = require("./data/skins");
 
 const { spawn } = require("child_process");
 
@@ -44,9 +46,27 @@ async function main() {
           //, "--dry"
         ]
       );
+      break;
       console.log({ output });
 
       console.log("Done");
+    case "fetch-metadata":
+      console.log("Going to download metadata from the Internet Archive");
+      await fetchInternetArchiveMetadata();
+      console.log("Done");
+      break;
+    case "metadata": {
+      const hash = argv._[1];
+      console.log(await Skins.getInternetArchiveUrl(hash));
+      break;
+    }
+    case "skin": {
+      const hash = argv._[1];
+      console.log(await Skins.getSkinByMd5(hash));
+      break;
+    }
+    default:
+      console.log(`Unknown command ${argv._[0]}`);
   }
 }
 

experiments/skin-database/data/skins.js

@@ -0,0 +1,132 @@
+const db = require("../db");
+const path = require("path");
+const skins = db.get("skins");
+const iaItems = db.get("internetArchiveItems");
+const S3 = require("../s3");
+
+function getFilenames(skin) {
+  return Array.from(new Set((skin.filePaths || []).map(p => path.basename(p))));
+}
+
+async function getAll() {
+  const skinRecords = await skins.find(
+    { type: "CLASSIC" },
+    {
+      limit: 10,
+      fields: {
+        md5: 1,
+        filenames: 1,
+        averageColor: 1,
+        emails: 1,
+        filePaths: 1,
+        screenshotUrl: 1,
+        tweetUrl: 1,
+        twitterLikes: 1,
+      },
+    }
+  );
+
+  return skinRecords.map(getSkinRecord);
+}
+
+function getSkinRecord(skin) {
+  const {
+    md5,
+    averageColor,
+    emails,
+    tweetUrl,
+    twitterLikes,
+    readmeText,
+    filePaths,
+  } = skin;
+  console.log(skin);
+  const fileNames = filePaths.map(p => path.basename(p));
+  const skinUrl = `https://s3.amazonaws.com/webamp-uploaded-skins/skins/${md5}.wsz`;
+  return {
+    skinUrl,
+    screenshotUrl: `https://s3.amazonaws.com/webamp-uploaded-skins/screenshots/${md5}.png`,
+    md5,
+    averageColor,
+    fileNames,
+    canonicalFilename: fileNames != null ? fileNames[0] : null,
+    emails,
+    tweetUrl,
+    twitterLikes,
+    webampUrl: `https://webamp.org?skinUrl=${skinUrl}`,
+    readmeText,
+  };
+}
+
+async function getProp(md5, prop) {
+  const skin = await skins.findOne({ md5, type: "CLASSIC" });
+  const value = skin && skin[prop];
+  return value == null ? null : value;
+}
+
+async function getSkinByMd5(md5) {
+  const skin = await skins.findOne({ md5, type: "CLASSIC" });
+  if (skin == null) {
+    return null;
+  }
+  const internetArchiveItemName = await getInternetArchiveItemName(md5);
+  const internetArchiveUrl = await getInternetArchiveUrl(md5);
+  const tweetStatus = await getTweetStatus(md5);
+  return {
+    ...getSkinRecord(skin),
+    internetArchiveUrl,
+    internetArchiveItemName,
+    tweetStatus,
+  };
+}
+
+async function getReadme(md5) {
+  return getProp(md5, "readmeText");
+}
+
+async function getScreenshotUrl(md5) {
+  return getProp(md5, "screenshotUrl");
+}
+
+async function getSkinUrl(md5) {
+  return getProp(md5, "skinUrl");
+}
+
+async function getInternetArchiveItemName(md5) {
+  const item = await getInternetArchiveItem(md5);
+  if (item == null) {
+    return null;
+  }
+  return item.identifier;
+}
+async function getInternetArchiveItem(md5) {
+  return iaItems.findOne(
+    { "metadata.files.md5": md5 },
+    {
+      fields: {
+        metadata: 1,
+        identifier: 1,
+      },
+    }
+  );
+}
+
+async function getInternetArchiveUrl(md5) {
+  const itemName = await getInternetArchiveItemName(md5);
+  if (itemName == null) {
+    return null;
+  }
+  return `https://archive.org/details/${itemName}`;
+}
+
+async function getTweetStatus(md5) {
+  return S3.getStatus(md5);
+}
+
+module.exports = {
+  getReadme,
+  getScreenshotUrl,
+  getSkinUrl,
+  getInternetArchiveUrl,
+  getTweetStatus,
+  getSkinByMd5,
+};

experiments/skin-database/discord-bot/commands/approve.js

@@ -6,11 +6,6 @@ const TWEET_BOT_CHANNEL_ID = "445577489834704906";
 
 async function handler(message, args) {
   const [md5] = args;
-  const filename = getFilename(md5);
-  if (filename == null) {
-    await message.channel.send(`Could not find a skin matching ${md5}`);
-    return;
-  }
   const status = await getStatus(md5);
   if (status !== "UNREVIEWED") {
     await message.channel.send(`This skin has already been reviewed.`);
@@ -19,9 +14,8 @@ async function handler(message, args) {
   await approve(md5);
   const tweetBotChannel = message.client.channels.get(TWEET_BOT_CHANNEL_ID);
   await Utils.postSkin({
-    filename,
     md5,
-    title: `Approved: ${filename}`,
+    title: filename => `Approved: ${filename}`,
     dest: tweetBotChannel
   });
   await tweetBotChannel.send(

experiments/skin-database/discord-bot/commands/random.js

@@ -7,11 +7,8 @@ async function handler(message) {
   const skins = Object.values(cache).filter(skin => skin.type === "CLASSIC");
   const skin = skins[Math.floor(Math.random() * skins.length)];
   const { md5 } = skin;
-  const filename = getFilename(md5);
   await Utils.postSkin({
-    filename,
     md5,
-    title: `${filename}`,
     dest: message.channel
   });
 }

experiments/skin-database/discord-bot/commands/reject.js

@@ -6,11 +6,6 @@ const TWEET_BOT_CHANNEL_ID = "445577489834704906";
 
 async function handler(message, args) {
   const [md5] = args;
-  const filename = getFilename(md5);
-  if (filename == null) {
-    await message.channel.send(`Could not find a skin matching ${md5}`);
-    return;
-  }
   const status = await getStatus(md5);
   if (status !== "UNREVIEWED") {
     await message.channel.send(`This skin has already been reviewed.`);
@@ -19,9 +14,8 @@ async function handler(message, args) {
   await reject(md5);
   const tweetBotChannel = message.client.channels.get(TWEET_BOT_CHANNEL_ID);
   await Utils.postSkin({
-    filename,
     md5,
-    title: `Rejected: ${filename}`,
+    title: filename => `Rejected: ${filename}`,
     dest: tweetBotChannel
   });
   await tweetBotChannel.send(

experiments/skin-database/discord-bot/commands/review.js

@@ -1,26 +1,22 @@
 const { getSkinToReview } = require("../../s3");
 const Utils = require("../utils");
+const Skins = require("../../data/skins");
 
 async function reviewSkin(message) {
-  const { filename, md5 } = await getSkinToReview();
+  const { md5 } = await getSkinToReview();
   await Utils.postSkin({
-    filename,
     md5,
-    title: `Review: ${filename}`,
+    title: filename => `Review: ${filename}`,
     dest: message.channel
   });
 }
 
 async function handler(message, args) {
-  let [count] = args;
+  let count = args[0] || 1;
   if (count > 50) {
-    if (count > 1) {
-      message.channel.send(
-        `You can only review up to ${count} skins at a time.`
-      );
-      message.channel.send(`Going to show ${count} skins to review`);
-      count = 50;
-    }
+    message.channel.send(`You can only review up to ${count} skins at a time.`);
+    message.channel.send(`Going to show ${count} skins to review`);
+    count = 50;
   }
   message.channel.send(`Going to show ${count} skins to review.`);
   let i = Number(count);

experiments/skin-database/discord-bot/commands/skin.js

@@ -2,11 +2,8 @@ const { getFilename } = require("../info");
 const Utils = require("../utils");
 async function handler(message, args) {
   const [md5] = args;
-  const filename = getFilename(md5);
   await Utils.postSkin({
-    filename,
     md5,
-    title: `${filename}`,
     dest: message.channel
   });
 }

experiments/skin-database/discord-bot/info.js

@@ -6,10 +6,7 @@ let cache = null;
 function getCache() {
   if (cache == null) {
     cache = JSON.parse(
-      fs.readFileSync(
-        path.join("/Volumes/Mobile Backup/skins/cache", "info.json"),
-        "utf8"
-      )
+      fs.readFileSync(path.join(__dirname, "../info.json"), "utf8")
     );
   }
   return cache;

experiments/skin-database/discord-bot/utils.js

@@ -1,63 +1,76 @@
 const Discord = require("discord.js");
 const rgbHex = require("rgb-hex");
-const { getInfo } = require("./info");
+const Skins = require("../data/skins");
+const { getInfo, getFilename } = require("./info");
 const { approve, reject, getStatus } = require("./s3");
 
 const filter = reaction => {
   return ["👍", "👎"].some(name => reaction.emoji.name === name);
 };
 
-async function postSkin({ filename, md5, title, dest }) {
-  const screenshotUrl = `https://s3.amazonaws.com/webamp-uploaded-skins/screenshots/${md5}.png`;
-  const skinUrl = `https://s3.amazonaws.com/webamp-uploaded-skins/skins/${md5}.wsz`;
-  const webampUrl = `https://webamp.org?skinUrl=${skinUrl}`;
+async function postSkin({ md5, title, dest }) {
+  const skin = await Skins.getSkinByMd5(md5);
+  const {
+    canonicalFilename,
+    screenshotUrl,
+    skinUrl,
+    webampUrl,
+    averageColor,
+    emails,
+    tweetUrl,
+    twitterLikes,
+    tweetStatus,
+    internetArchiveUrl,
+    internetArchiveItemName,
+    readmeText
+  } = skin;
+  console.log(skin);
+  title = title ? title(canonicalFilename) : canonicalFilename;
 
   const embed = new Discord.RichEmbed()
     .setTitle(title)
     .addField("Try Online", `[webamp.org](${webampUrl})`, true)
-    .addField("Download", `[${filename}](${skinUrl})`, true)
+    .addField("Download", `[${canonicalFilename}](${skinUrl})`, true)
     .addField("Md5", md5, true)
     .setImage(screenshotUrl);
 
-  const info = getInfo(md5);
-  if (info == null) {
-    embed.addField("Warning", "Cached metadata not found", true);
+  if (readmeText) {
+    embed.setDescription(`\`\`\`${readmeText}\`\`\``);
+  }
+  if (averageColor) {
+    try {
+      const color = rgbHex(averageColor);
+      if (String(color).length === 6) {
+        embed.setColor(`#${color}`);
+      } else {
+        console.log("Did not get a safe color from ", averageColor);
+        console.log("Got ", color);
+      }
+    } catch (e) {
+      console.error("could not use color", averageColor);
+    }
+  }
+  if (emails != null && emails.length) {
+    embed.addField("Emails", emails.join(", "), true);
+  }
+  if (tweetUrl != null) {
+    let likes = "";
+    if (twitterLikes != null) {
+      likes = `(${Number(twitterLikes).toLocaleString()} likes) `;
+    }
+    embed.addField("Tweet Status", `[Tweeted](${tweetUrl}) ${likes}🐦`, true);
   } else {
-    if (info.averageColor) {
-      try {
-        const color = rgbHex(info.averageColor);
-        if (String(color).length === 6) {
-          embed.setColor(`#${color}`);
-        } else {
-          console.log("Did not get a safe color from ", info.averageColor);
-          console.log("Got ", color);
-        }
-      } catch (e) {
-        console.error("could not use color", info.averageColor);
-      }
-    }
-    if (info.emails != null && info.emails.length) {
-      const uniqueEmails = Array.from(new Set(info.emails));
-      // TODO: Fix email parsing
-      embed.addField("Emails", uniqueEmails.join(", "), true);
-    }
-    if (info.tweetUrl != null) {
-      let likes = "";
-      if (info.twitterLikes != null) {
-        likes = `(${Number(info.twitterLikes).toLocaleString()} likes) `;
-      }
-      embed.addField(
-        "Tweet Status",
-        `[Tweeted](${info.tweetUrl}) ${likes}🐦`,
-        true
-      );
-    } else {
-      const status = await getStatus(md5);
-      if (status === "UNREVIEWED") {
-        embed.setFooter("React with 👍 or 👎 to approve or deny");
-      }
-      embed.addField("Tweet Status", getPrettyTwitterStatus(status), true);
+    if (tweetStatus === "UNREVIEWED") {
+      embed.setFooter("React with 👍 or 👎 to approve or deny");
     }
+    embed.addField("Tweet Status", getPrettyTwitterStatus(tweetStatus), true);
+  }
+  if (internetArchiveUrl) {
+    embed.addField(
+      "Internet Archive",
+      `[${internetArchiveItemName || "Permalink"}](${internetArchiveUrl})`,
+      true
+    );
   }
 
   const msg = await dest.send(embed);
@@ -69,13 +82,17 @@ async function postSkin({ filename, md5, title, dest }) {
       case "👍":
         await approve(md5);
         console.log(`${user.username} approved ${md5}`);
-        await msg.channel.send(`${filename} was approved by ${user.username}`);
+        await msg.channel.send(
+          `${canonicalFilename} was approved by ${user.username}`
+        );
         msg.react("✅");
         break;
       case "👎":
         await reject(md5);
         console.log(`${user.username} rejected ${md5}`);
-        await msg.channel.send(`${filename} was rejected by ${user.username}`);
+        await msg.channel.send(
+          `${canonicalFilename} was rejected by ${user.username}`
+        );
         msg.react("❌");
         break;
     }

experiments/skin-database/index.js

@@ -2,92 +2,20 @@ const express = require("express");
 const path = require("path");
 const app = express();
 const db = require("./db");
-const skins = db.get("skins");
 const iaItems = db.get("internetArchiveItems");
 // const info = require("/Volumes/Mobile Backup/skins/cache/info.json");
-const { getStatus } = require("./s3");
-const port = 3000;
+const Skins = require("./data/skins");
+const port = 3001;
 
 // TODO: Look into 766c4fad9088037ab4839b18292be8b1
 // Has huge number of filenames in info.json
 
-function getFilenames(skin) {
-  return Array.from(new Set((skin.filePaths || []).map(p => path.basename(p))));
-}
-
-function getSkinRecord(skin) {
-  const {
-    md5,
-    averageColor,
-    emails,
-    screenshotUrl,
-    skinUrl,
-    tweetUrl,
-    twitterLikes,
-  } = skin;
-  return {
-    skinUrl,
-    screenshotUrl,
-    md5,
-    averageColor,
-    fileNames: getFilenames(skin),
-    emails,
-    tweetUrl,
-    twitterLikes,
-  };
-}
-
-async function getProp(md5, prop) {
-  const skin = await skins.findOne({ md5, type: "CLASSIC" });
-  const value = skin && skin[prop];
-  return value == null ? null : value;
-}
-
 app.set("json spaces", 2);
 
 app.get("/", async (req, res) => {
   res.send("Hello World!");
 });
 
-app.get("/skins/", async (req, res) => {
-  const skinRecords = await skins.find(
-    { type: "CLASSIC" },
-    {
-      limit: 10,
-      fields: {
-        md5: 1,
-        filenames: 1,
-        averageColor: 1,
-        emails: 1,
-        filePaths: 1,
-        screenshotUrl: 1,
-        skinUrl: 1,
-        tweetUrl: 1,
-        twitterLikes: 1,
-      },
-    }
-  );
-
-  res.json(skinRecords.map(getSkinRecord));
-});
-
-app.get("/items/", async (req, res) => {
-  const items = await iaItems.find(
-    { metadata: { $ne: null } },
-    {
-      limit: 10,
-      fields: {
-        metadata: 1,
-        identifier: 1,
-      },
-    }
-  );
-
-  const todo = await iaItems.count({ metadata: { $eq: null } });
-
-  res.json({ items, todo });
-});
-
 app.get("/items/:identifier", async (req, res) => {
   const { identifier } = req.params;
   const item = await iaItems.findOne({ identifier });
@@ -98,42 +26,19 @@ app.get("/items/:identifier", async (req, res) => {
   res.json(item);
 });
 
-app.get("/tweets/", async (req, res) => {
-  const skinRecords = await skins.find(
-    { type: "CLASSIC", tweetUrl: { $ne: null } },
-    {
-      limit: 10,
-      fields: {
-        md5: 1,
-        filenames: 1,
-        averageColor: 1,
-        emails: 1,
-        filePaths: 1,
-        screenshotUrl: 1,
-        skinUrl: 1,
-        tweetUrl: 1,
-        twitterLikes: 1,
-        // md5: 1,
-      },
-    }
-  );
-
-  res.json(skinRecords.map(getSkinRecord));
-});
-
 app.get("/skins/:md5", async (req, res) => {
   const { md5 } = req.params;
-  const skin = await skins.findOne({ md5, type: "CLASSIC" });
+  const skin = await Skins.getSkinByMd5(md5);
   if (skin == null) {
     res.status(404).json();
     return;
   }
-  res.json({ ...getSkinRecord(skin), tweetStatus: await getStatus(md5) });
+  res.json(skin);
 });
 
 app.get("/skins/:md5/readme.txt", async (req, res) => {
   const { md5 } = req.params;
-  const readmeText = await getProp(md5, "readmeText");
+  const readmeText = await Skins.getReadme(md5);
   if (readmeText == null) {
     // TODO: make this 404
     res.send("");
@@ -144,7 +49,7 @@ app.get("/skins/:md5/readme.txt", async (req, res) => {
 
 app.get("/skins/:md5/screenshot.png", async (req, res) => {
   const { md5 } = req.params;
-  const screenshotUrl = await getProp(md5, "screenshotUrl");
+  const screenshotUrl = await Skins.getScreenshotUrl(md5);
   if (screenshotUrl == null) {
     res.status(404).send();
     return;
@@ -154,7 +59,7 @@ app.get("/skins/:md5/screenshot.png", async (req, res) => {
 
 app.get("/skins/:md5/download", async (req, res) => {
   const { md5 } = req.params;
-  const skinUrl = await getProp(md5, "skinUrl");
+  const skinUrl = await Skins.getSkinUrl(md5);
   if (skinUrl == null) {
     res.status(404).send();
     return;

experiments/skin-database/package.json

@@ -5,7 +5,7 @@
   "license": "MIT",
   "dependencies": {
     "aws-sdk": "^2.469.0",
-    "discord.js": "^11.4.2",
+    "discord.js": "^11.5.1",
     "express": "^4.17.1",
     "imagemin": "^6.1.0",
     "imagemin-optipng": "^7.0.0",
@@ -18,8 +18,9 @@
     "yargs": "^13.2.4"
   },
   "scripts": {
-    "serve": "node index.js",
+    "start": "node index.js",
     "tweet": "./cli.js tweet",
+    "fetch-metadata": "./cli.js fetch-metadata",
     "bot": "node ./discord-bot/index.js"
   }
 }

experiments/skin-database/shooter.js

@@ -86,4 +86,3 @@ class Shooter {
 }
 
 module.exports = Shooter;
-s;

experiments/skin-database/tasks/fetchInternetArchiveMetadata.js

@@ -1,5 +1,5 @@
 const fetch = require("node-fetch");
-const db = require("./db");
+const db = require("../db");
 const iaItems = db.get("internetArchiveItems");
 
 async function fetchMetadata(identifier) {