From 964112e04027e751049c616216f7ef6fda28335e Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Fri, 27 Dec 2019 10:08:52 -0500 Subject: [PATCH] Image hash --- experiments/skin-database/cli.js | 18 ++++++++++ experiments/skin-database/data/skins.js | 31 +++++++++++++++-- .../discord-bot/commands/skin.js | 4 +++ .../skin-database/discord-bot/utils.js | 1 + experiments/skin-database/graphql.js | 33 ++++++++++++------- 5 files changed, 73 insertions(+), 14 deletions(-) diff --git a/experiments/skin-database/cli.js b/experiments/skin-database/cli.js index a8205f7c..395f10ac 100755 --- a/experiments/skin-database/cli.js +++ b/experiments/skin-database/cli.js @@ -1,4 +1,6 @@ #!/usr/bin/env node +const path = require("path"); +const fs = require("fs"); const argv = require("yargs").argv; const fetchInternetArchiveMetadata = require("./tasks/fetchInternetArchiveMetadata"); const ensureInternetArchiveItemsIndexByMd5 = require("./tasks/ensureInternetArchiveItemsIndexByMd5"); @@ -15,6 +17,22 @@ async function main() { await DiscordWinstonTransport.addToLogger(client, logger); switch (argv._[0]) { + case "image-hash": + const hashes = new Map(); + + fs.readFileSync(path.join(__dirname, "./hash.txt"), "utf8") + .split("\n") + .forEach(line => { + const [md5, imgHash] = line.split(" "); + hashes.set(md5, imgHash); + }); + + for (const [md5, imgHash] of hashes.entries()) { + await Skins.setImageHash(md5, imgHash); + process.stderr.write("."); + } + break; + case "tweet": await tweet(client); break; diff --git a/experiments/skin-database/data/skins.js b/experiments/skin-database/data/skins.js index a30aa890..9590d11e 100644 --- a/experiments/skin-database/data/skins.js +++ b/experiments/skin-database/data/skins.js @@ -9,14 +9,14 @@ const TWEETABLE_QUERY = { tweeted: { $ne: true }, approved: true, rejected: { $ne: true }, - type: "CLASSIC" + type: "CLASSIC", }; const REVIEWABLE_QUERY = { tweeted: { $ne: true }, approved: { $ne: true }, rejected: { $ne: true }, - type: "CLASSIC" + type: "CLASSIC", }; function getSkinRecord(skin) { const { @@ -27,6 +27,7 @@ function getSkinRecord(skin) { twitterLikes, readmeText, filePaths, + imageHash, } = skin; const fileNames = filePaths.map(p => path.basename(p)); const skinUrl = `https://s3.amazonaws.com/webamp-uploaded-skins/skins/${md5}.wsz`; @@ -42,6 +43,7 @@ function getSkinRecord(skin) { twitterLikes, webampUrl: `https://webamp.org?skinUrl=${skinUrl}`, readmeText, + imageHash, }; } @@ -58,7 +60,10 @@ async function getMd5ByAnything(anything) { const md5Match = anything.match(MD5); if (md5Match != null) { const md5 = md5Match[1]; - return md5; + const found = await skins.findOne({ md5, type: "CLASSIC" }); + if (found != null) { + return md5; + } } const itemMatchResult = anything.match(IA_URL); if (itemMatchResult != null) { @@ -72,6 +77,10 @@ async function getMd5ByAnything(anything) { if (md5 != null) { return md5; } + + const imageHashMd5 = await getMd5FromImageHash(anything); + + return imageHashMd5; } async function getSkinByMd5(md5) { @@ -117,6 +126,16 @@ async function getMd5FromInternetArchvieItemName(itemName) { return item == null ? null : item.md5; } +async function getMd5FromImageHash(imageHash) { + const item = await skins.findOne({ imageHash }, { md5: 1 }); + console.log(item); + return item == null ? null : item.md5; +} + +async function getMd5sMatchingImageHash(imageHash) { + return skins.find({ imageHash }, { md5: 1 }); +} + function getInternetArchiveUrl(itemName) { return itemName == null ? null : `https://archive.org/details/${itemName}`; } @@ -196,7 +215,12 @@ async function reconcile() { ]); } +async function setImageHash(md5, imageHash) { + await skins.findOneAndUpdate({ md5 }, { $set: { imageHash } }); +} + module.exports = { + getMd5sMatchingImageHash, getInternetArchiveItem, getMd5ByAnything, getReadme, @@ -213,4 +237,5 @@ module.exports = { getTweetableSkinCount, reconcile, getSkinToTweet, + setImageHash, }; diff --git a/experiments/skin-database/discord-bot/commands/skin.js b/experiments/skin-database/discord-bot/commands/skin.js index dd45435a..75a2d9e8 100644 --- a/experiments/skin-database/discord-bot/commands/skin.js +++ b/experiments/skin-database/discord-bot/commands/skin.js @@ -1,8 +1,12 @@ const Utils = require("../utils"); const Skins = require("../../data/skins"); + async function handler(message, args) { const [anything] = args; const md5 = await Skins.getMd5ByAnything(anything); + if (md5 == null) { + message.channel.send(`Could not find a skin matching ${anything}`); + } await Utils.postSkin({ md5, dest: message.channel diff --git a/experiments/skin-database/discord-bot/utils.js b/experiments/skin-database/discord-bot/utils.js index 0bf0e536..5be0611d 100644 --- a/experiments/skin-database/discord-bot/utils.js +++ b/experiments/skin-database/discord-bot/utils.js @@ -10,6 +10,7 @@ const filter = reaction => { async function postSkin({ md5, title, dest }) { const skin = await Skins.getSkinByMd5(md5); if (skin == null) { + console.warn("Could not find skin for md5", { md5, alert: true }); logger.warn("Could not find skin for md5", { md5, alert: true }); return; } diff --git a/experiments/skin-database/graphql.js b/experiments/skin-database/graphql.js index 9c418e80..3700a4a6 100644 --- a/experiments/skin-database/graphql.js +++ b/experiments/skin-database/graphql.js @@ -27,9 +27,11 @@ type InternetArchiveItem { tweetStatus: TweetStatus readmeText: String internetArchiveItem: InternetArchiveItem + imageHash: String } type Query { skin(md5: String!): Skin + skins(imageHash: String!): [Skin] } `); @@ -57,21 +59,19 @@ class InternetArchiveItem { } class Skin { - constructor(md5) { - this._md5 = md5; - this._skinPromise = Skins.getSkinByMd5(this._md5); + static async fromMd5(md5) { + return new Skin(await Skins.getSkinByMd5(md5)); + } + constructor(data) { + this._data = data; } md5() { - return this._md5; + return this._data.md5; } async _get(getter) { - const skin = await this._skinPromise; - if (skin == null) { - return null; - } - return getter(skin); + return getter(this._data); } async canonicalFilename() { @@ -114,6 +114,10 @@ class Skin { return this._get(skin => skin.averageColor); } + async imageHash() { + return this._get(skin => skin.imageHash); + } + internetArchiveItem() { return new InternetArchiveItem(this._md5); } @@ -121,8 +125,15 @@ class Skin { // The root provides a resolver function for each API endpoint var root = { - skin: ({ md5 }) => { - return new Skin(md5); + skin: async ({ md5, query }) => { + return Skin.fromMd5(md5); + }, + skins: async ({ imageHash }) => { + if (imageHash) { + const matches = await Skins.getMd5sMatchingImageHash(imageHash); + return matches.map(({ md5 }) => Skin.fromMd5(md5)); + } + return []; }, };