diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index cebd64f2..c9b65c7c 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node import path from "path"; import fs from "fs"; +import { db, knex } from "./db"; import { argv } from "yargs"; import fetchInternetArchiveMetadata from "./tasks/fetchInternetArchiveMetadata"; import ensureInternetArchiveItemsIndexByMd5 from "./tasks/ensureInternetArchiveItemsIndexByMd5"; @@ -90,6 +91,8 @@ async function main() { console.log(`Unknown command ${argv._[0]}`); } } finally { + knex.destroy(); + db.close(); logger.close(); client.destroy(); } diff --git a/packages/skin-database/data/skins.ts b/packages/skin-database/data/skins.ts index 6f03e4cb..e86f0595 100644 --- a/packages/skin-database/data/skins.ts +++ b/packages/skin-database/data/skins.ts @@ -13,7 +13,7 @@ const SKIN_TYPE = { INVALID: 4, }; -const skins = db.get("skins"); +const skins_DEPRECATED = db.get("skins"); const iaItems = db.get("internetArchiveItems"); const CLASSIC_QUERY = { @@ -55,7 +55,7 @@ function getWebampUrl(skin: DBSkinRecord): string { } export async function addSkin({ md5, filePath, uploader, averageColor }) { - skins.insert({ + skins_DEPRECATED.insert({ md5, type: "CLASSIC", filePaths: [filePath], @@ -83,13 +83,14 @@ export async function addSkin({ md5, filePath, uploader, averageColor }) { const IA_URL = /^(https:\/\/)?archive.org\/details\/([^\/]+)\/?/; const MD5 = /([a-fA-F0-9]{32})/; +// TODO: SQLITE export async function getMd5ByAnything( anything: string ): Promise { const md5Match = anything.match(MD5); if (md5Match != null) { const md5 = md5Match[1]; - const found = await skins.findOne({ md5, type: "CLASSIC" }); + const found = await skins_DEPRECATED.findOne({ md5, type: "CLASSIC" }); if (found != null) { return md5; } @@ -105,8 +106,9 @@ export async function getMd5ByAnything( return getMd5FromInternetArchvieItemName(anything); } +// TODO: SQLITE export async function getSkinByMd5(md5: string): Promise { - const skin: DBSkinRecord | null = await skins.findOne({ + const skin: DBSkinRecord | null = await skins_DEPRECATED.findOne({ md5, type: "CLASSIC", }); @@ -136,7 +138,7 @@ export async function getSkinByMd5(md5: string): Promise { }; } -export async function getInternetArchiveItem(md5: string): Promise { +async function getInternetArchiveItem(md5: string): Promise { return iaItems.findOne({ md5: md5 }); } @@ -145,12 +147,8 @@ async function getMd5FromInternetArchvieItemName(itemName: string) { return item == null ? null : item.md5; } -export async function getUnarchived() { - return skins.find({ itemName: null }, { md5: 1 }); -} - export async function getMissingNsfwPredictions() { - const results = await skins.find( + const results = await skins_DEPRECATED.find( { nsfwPredictions: null, type: "CLASSIC" }, { md5: 1 } ); @@ -162,22 +160,22 @@ export function getInternetArchiveUrl(itemName: string | null): string | null { } export function getTweetableSkinCount(): Promise { - return skins.count(TWEETABLE_QUERY); + return skins_DEPRECATED.count(TWEETABLE_QUERY); } export function getClassicSkinCount(): Promise { - return skins.count(CLASSIC_QUERY); + return skins_DEPRECATED.count(CLASSIC_QUERY); } // TODO: Also pass id export async function markAsTweeted(md5: string, url: string): Promise { - await skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } }); + await skins_DEPRECATED.findOneAndUpdate({ md5 }, { $set: { tweeted: true } }); await knex("tweets").insert({ skin_md5: md5, url }, []); } // TODO: Also path actor export async function markAsNSFW(md5: string): Promise { - await skins.findOneAndUpdate({ md5 }, { $set: { nsfw: true } }); + await skins_DEPRECATED.findOneAndUpdate({ md5 }, { $set: { nsfw: true } }); const indexes = [{ objectID: md5, nsfw: true }]; // TODO: Await here, but for some reason this never completes new Promise((resolve, reject) => { @@ -190,28 +188,39 @@ export async function markAsNSFW(md5: string): Promise { } export async function getStatus(md5: string): Promise { - const skin = await skins.findOne({ md5 }); - if (skin.tweeted) { + const tweeted = await knex("tweets").where({ skin_md5: md5 }).limit(1); + if (tweeted) { return "TWEETED"; } - if (skin.rejected) { + const reviewRows = await knex("skin_reviews") + .where({ skin_md5: md5 }) + .limit(1); + const reviews = new Set(reviewRows.map((row) => row.review)); + if (reviews.has("REJECTED")) { return "REJECTED"; } - if (skin.approved) { + if (reviews.has("APPROVED")) { return "APPROVED"; } + return "UNREVIEWED"; } // TODO: Also path actor export async function approve(md5: string): Promise { - await skins.findOneAndUpdate({ md5 }, { $set: { approved: true } }); + await skins_DEPRECATED.findOneAndUpdate( + { md5 }, + { $set: { approved: true } } + ); await knex("skin_reviews").insert({ skin_md5: md5, review: "APPROVED" }, []); } // TODO: Also path actor export async function reject(md5: string): Promise { - await skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } }); + await skins_DEPRECATED.findOneAndUpdate( + { md5 }, + { $set: { rejected: true } } + ); await knex("skin_reviews").insert({ skin_md5: md5, review: "REJECTED" }, []); } @@ -219,7 +228,7 @@ export async function getSkinToReview(): Promise<{ filename: string | null; md5: string; }> { - const reviewable = await skins.aggregate([ + const reviewable = await skins_DEPRECATED.aggregate([ { $match: REVIEWABLE_QUERY }, { $sample: { size: 1 } }, ]); @@ -231,7 +240,7 @@ export async function getSkinToReviewForNsfw(): Promise<{ filename: string | null; md5: string; }> { - const reviewable = await skins.find(REVIEWABLE_QUERY, { + const reviewable = await skins_DEPRECATED.find(REVIEWABLE_QUERY, { limit: 1, sort: { "nsfwPredictions.porn": -1 }, }); @@ -240,7 +249,7 @@ export async function getSkinToReviewForNsfw(): Promise<{ } export async function getSkinToTweet(): Promise { - const tweetables = await skins.aggregate([ + const tweetables = await skins_DEPRECATED.aggregate([ { $match: TWEETABLE_QUERY }, { $sample: { size: 1 } }, ]); @@ -257,15 +266,15 @@ export async function getStats(): Promise<{ tweeted: number; tweetable: number; }> { - const approved = await skins.count({ approved: true }); - const rejected = await skins.count({ rejected: true }); - const tweeted = await skins.count({ tweeted: true }); + const approved = await skins_DEPRECATED.count({ approved: true }); + const rejected = await skins_DEPRECATED.count({ rejected: true }); + const tweeted = await skins_DEPRECATED.count({ tweeted: true }); const tweetable = await getTweetableSkinCount(); return { approved, rejected, tweeted, tweetable }; } export async function getRandomClassicSkinMd5(): Promise { - const random = await skins.aggregate([ + const random = await skins_DEPRECATED.aggregate([ { $match: CLASSIC_QUERY }, { $sample: { size: 1 } }, ]); @@ -291,7 +300,10 @@ export async function setNsfwPredictions( md5: string, nsfwPredictions: NsfwPrediction ): Promise { - await skins.findOneAndUpdate({ md5 }, { $set: { nsfwPredictions } }); + await skins_DEPRECATED.findOneAndUpdate( + { md5 }, + { $set: { nsfwPredictions } } + ); await knex("nsfw_predictions").insert( { skin_md5: md5, ...nsfwPredictions }, [] @@ -303,7 +315,7 @@ export async function setTweetInfo( likes: number, tweetId: string ): Promise { - await skins.findOneAndUpdate( + await skins_DEPRECATED.findOneAndUpdate( { md5 }, { $set: { twitterLikes: likes, tweetId } } ); @@ -327,7 +339,7 @@ export async function getMuseumPage({ }): Promise< Array<{ color: string; fileName: string; md5: string; nsfw: boolean }> > { - const reviewable = await skins.find( + const reviewable = await skins_DEPRECATED.find( { type: "CLASSIC" }, { limit: first, diff --git a/packages/skin-database/discord-bot/index.js b/packages/skin-database/discord-bot/index.js index 827d4588..6987a129 100644 --- a/packages/skin-database/discord-bot/index.js +++ b/packages/skin-database/discord-bot/index.js @@ -10,12 +10,15 @@ const handlers = { help: handleHelp, }; -const commands = fs - .readdirSync(path.resolve(__dirname, "./commands")) - .filter((file) => file.endsWith(".js")) - .map((file) => { - return require(`./commands/${file}`); - }); +const commands = [ + require("./commands/archive"), + require("./commands/random"), + require("./commands/review"), + require("./commands/screenshot"), + require("./commands/skin"), + require("./commands/stats"), + require("./commands/tweet"), +]; for (const command of commands) { handlers[command.command] = command.handler; diff --git a/packages/skin-database/package.json b/packages/skin-database/package.json index 61ba64ed..7f68e365 100644 --- a/packages/skin-database/package.json +++ b/packages/skin-database/package.json @@ -23,16 +23,18 @@ "shell-escape": "^0.2.0", "sqlite3": "^4.2.0", "temp": "^0.9.0", + "ts-node": "^8.10.2", "winston": "^3.2.1", "yargs": "^13.2.4" }, "scripts": { - "start": "tsc && node dist/index.js", - "tweet": "tsc && ./dist/cli.js tweet", - "fetch-metadata": "tsc && .dist/cli.js fetch-metadata", - "bot": "tsc && node ./dist/discord-bot/index.js", - "cli": "tsc && node ./dist/cli.js", - "index": "tsc && node ./dist/tasks/searchIndex.js" + "start": "ts-node --transpile-only index.js", + "tweet": "ts-node --transpile-only ./cli.ts tweet", + "fetch-metadata": "ts-node --transpile-only ./cli.ts fetch-metadata", + "bot": "ts-node --transpile-only ./discord-bot/index.js", + "cli": "ts-node --transpile-only ./cli.ts", + "repro": "ts-node --transpile-only ./repro.ts", + "index": "ts-node --transpile-only ./tasks/searchIndex.js" }, "prettier": {}, "devDependencies": { diff --git a/packages/skin-database/tsconfig.json b/packages/skin-database/tsconfig.json index 67837324..a1c37d49 100644 --- a/packages/skin-database/tsconfig.json +++ b/packages/skin-database/tsconfig.json @@ -4,7 +4,7 @@ "esModuleInterop": true, "target": "es6", "moduleResolution": "node", - "sourceMap": true, + "sourceMap": false, "allowJs": true, "outDir": "dist", "strictNullChecks": true, diff --git a/yarn.lock b/yarn.lock index 9a6294bb..8be22b52 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1803,6 +1803,10 @@ are-we-there-yet@~1.1.2: delegates "^1.0.0" readable-stream "^2.0.6" +arg@^4.1.0: + version "4.1.3" + resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" + argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -3598,6 +3602,10 @@ diff-sequences@^24.9.0: version "24.9.0" resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-24.9.0.tgz#5715d6244e2aa65f48bba0bc972db0b0b11e95b5" +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + diffie-hellman@^5.0.0: version "5.0.3" resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875" @@ -6920,6 +6928,10 @@ make-dir@^3.0.0: dependencies: semver "^6.0.0" +make-error@^1.1.1: + version "1.3.6" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + make-iterator@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/make-iterator/-/make-iterator-1.0.1.tgz#29b33f312aa8f547c4a5e490f56afcec99133ad6" @@ -9695,7 +9707,7 @@ source-map-resolve@^0.5.0: source-map-url "^0.4.0" urix "^0.1.0" -source-map-support@^0.5.16, source-map-support@^0.5.6, source-map-support@~0.5.12: +source-map-support@^0.5.16, source-map-support@^0.5.17, source-map-support@^0.5.6, source-map-support@~0.5.12: version "0.5.19" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" dependencies: @@ -10398,6 +10410,16 @@ tryer@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8" +ts-node@^8.10.2: + version "8.10.2" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.10.2.tgz#eee03764633b1234ddd37f8db9ec10b75ec7fb8d" + dependencies: + arg "^4.1.0" + diff "^4.0.1" + make-error "^1.1.1" + source-map-support "^0.5.17" + yn "3.1.1" + tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: version "1.13.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043" @@ -11178,3 +11200,7 @@ yauzl@^2.10.0, yauzl@^2.4.2: dependencies: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" + +yn@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"