From 68200edeb5f8f2e6a4f400bad7af3b02eae2823b Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sun, 14 Jun 2020 16:32:06 -0400 Subject: [PATCH] Start writing to sqlite db --- packages/skin-database/cli.ts | 3 +- packages/skin-database/data/skins.ts | 44 +++++++++++++++++-- packages/skin-database/db.js | 13 +++++- .../ensureInternetArchiveItemsIndexByMd5.js | 2 +- .../tasks/fetchInternetArchiveMetadata.js | 2 +- .../tasks/findInternetArchiveItems.js | 2 +- packages/skin-database/tasks/searchIndex.js | 2 +- packages/skin-database/tasks/syncToSqlite.js | 11 +---- packages/skin-database/tasks/tweet.ts | 4 +- 9 files changed, 62 insertions(+), 21 deletions(-) diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index c96823c8..cebd64f2 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -10,6 +10,7 @@ import * as Skins from "./data/skins"; import Discord from "discord.js"; import { tweet } from "./tasks/tweet"; import { addSkinFromBuffer } from "./addSkin"; +import { PROJECT_ROOT } from "./config"; async function main() { const client = new Discord.Client(); @@ -67,7 +68,7 @@ async function main() { case "tweet-data": { // From running `tweet.py sort` const file = fs.readFileSync( - path.join(__dirname, "../../tweetBot/likes.txt"), + path.join(PROJECT_ROOT, "../tweetBot/likes.txt"), { encoding: "utf8" } ); diff --git a/packages/skin-database/data/skins.ts b/packages/skin-database/data/skins.ts index c9b4d48c..091fd062 100644 --- a/packages/skin-database/data/skins.ts +++ b/packages/skin-database/data/skins.ts @@ -1,4 +1,4 @@ -import db from "../db"; +import { db, knex } from "../db"; import path from "path"; import logger from "../logger"; import { searchIndex } from "../algolia"; @@ -6,6 +6,13 @@ import { DBSkinRecord, SkinRecord, DBIARecord, TweetStatus } from "../types"; import fetch from "node-fetch"; import { analyseBuffer, NsfwPrediction } from "../nsfwImage"; +const SKIN_TYPE = { + CLASSIC: 1, + MODERN: 2, + PACK: 3, + INVALID: 4, +}; + const skins = db.get("skins"); const iaItems = db.get("internetArchiveItems"); @@ -55,6 +62,22 @@ export async function addSkin({ md5, filePath, uploader, averageColor }) { uploader, averageColor, }); + await knex("skins").insert( + { + md5, + skin_type: SKIN_TYPE.CLASSIC, + average_color: averageColor, + }, + [] + ); + await knex("files").insert( + { + skin_md5: md5, + file_path: filePath, + source_attribution: uploader, + }, + [] + ); } const IA_URL = /^(https:\/\/)?archive.org\/details\/([^\/]+)\/?/; @@ -146,10 +169,13 @@ export function getClassicSkinCount(): Promise { return skins.count(CLASSIC_QUERY); } +// TODO: Also pass id export async function markAsTweeted(md5: string): Promise { await skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } }); + await knex("tweets").insert({ skin_md5: md5 }, []); } +// TODO: Also path actor export async function markAsNSFW(md5: string): Promise { await skins.findOneAndUpdate({ md5 }, { $set: { nsfw: true } }); const indexes = [{ objectID: md5, nsfw: true }]; @@ -160,6 +186,7 @@ export async function markAsNSFW(md5: string): Promise { resolve(content); }); }); + await knex("skin_reviews").insert({ skin_md5: md5, review: "NSFW" }, []); } export async function getStatus(md5: string): Promise { @@ -176,12 +203,16 @@ export async function getStatus(md5: string): Promise { return "UNREVIEWED"; } +// TODO: Also path actor export async function approve(md5: string): Promise { await skins.findOneAndUpdate({ md5 }, { $set: { approved: true } }); + await knex("skin_reviews").insert({ skin_md5: md5, review: "APPROVED" }, []); } +// TODO: Also path actor export async function reject(md5: string): Promise { await skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } }); + await knex("skin_reviews").insert({ skin_md5: md5, review: "REJECTED" }, []); } export async function getSkinToReview(): Promise<{ @@ -233,13 +264,13 @@ export async function getStats(): Promise<{ return { approved, rejected, tweeted, tweetable }; } -export async function getRandomClassicSkinMd5() { +export async function getRandomClassicSkinMd5(): Promise { const random = await skins.aggregate([ { $match: CLASSIC_QUERY }, { $sample: { size: 1 } }, ]); if (random.length === 0) { - return null; + throw new Error("Could not find any classic skins"); } return random[0].md5; } @@ -261,6 +292,10 @@ export async function setNsfwPredictions( nsfwPredictions: NsfwPrediction ): Promise { await skins.findOneAndUpdate({ md5 }, { $set: { nsfwPredictions } }); + await knex("nsfw_predictions").insert( + { skin_md5: md5, ...nsfwPredictions }, + [] + ); } export async function setTweetInfo( @@ -272,6 +307,9 @@ export async function setTweetInfo( { md5 }, { $set: { twitterLikes: likes, tweetId } } ); + await knex("tweets") + .where({ skin_md5: md5 }) + .update({ tweet_id: tweetId, likes }, []); } export async function computeAndSetNsfwPredictions(md5: string): Promise { diff --git a/packages/skin-database/db.js b/packages/skin-database/db.js index 3a4ba2d4..340925e9 100644 --- a/packages/skin-database/db.js +++ b/packages/skin-database/db.js @@ -1,3 +1,14 @@ +const path = require("path"); +const { PROJECT_ROOT } = require("./config"); const db = require("monk")("mongodb://127.0.0.1:27017/winamp"); -module.exports = db; +const knex = require("knex")({ + client: "sqlite3", + connection: { + filename: path.join(PROJECT_ROOT, "./skins.sqlite3"), + }, + useNullAsDefault: true, + debug: false, +}); + +module.exports = { db, knex }; diff --git a/packages/skin-database/tasks/ensureInternetArchiveItemsIndexByMd5.js b/packages/skin-database/tasks/ensureInternetArchiveItemsIndexByMd5.js index 8bec7957..47656255 100644 --- a/packages/skin-database/tasks/ensureInternetArchiveItemsIndexByMd5.js +++ b/packages/skin-database/tasks/ensureInternetArchiveItemsIndexByMd5.js @@ -1,4 +1,4 @@ -const db = require("../db"); +const { db } = require("../db"); const iaItems = db.get("internetArchiveItems"); module.exports = async function main() { diff --git a/packages/skin-database/tasks/fetchInternetArchiveMetadata.js b/packages/skin-database/tasks/fetchInternetArchiveMetadata.js index 4e3c7bfd..5fc61340 100644 --- a/packages/skin-database/tasks/fetchInternetArchiveMetadata.js +++ b/packages/skin-database/tasks/fetchInternetArchiveMetadata.js @@ -1,5 +1,5 @@ const fetch = require("node-fetch"); -const db = require("../db"); +const { db } = require("../db"); const iaItems = db.get("internetArchiveItems"); async function fetchMetadata(identifier) { diff --git a/packages/skin-database/tasks/findInternetArchiveItems.js b/packages/skin-database/tasks/findInternetArchiveItems.js index 74cc38c0..26315cd2 100644 --- a/packages/skin-database/tasks/findInternetArchiveItems.js +++ b/packages/skin-database/tasks/findInternetArchiveItems.js @@ -1,5 +1,5 @@ const fetch = require("node-fetch"); -const db = require("../db"); +const { db } = require("../db"); const iaItems = db.get("internetArchiveItems"); module.exports = async function main() { diff --git a/packages/skin-database/tasks/searchIndex.js b/packages/skin-database/tasks/searchIndex.js index f715f050..cb299791 100644 --- a/packages/skin-database/tasks/searchIndex.js +++ b/packages/skin-database/tasks/searchIndex.js @@ -1,6 +1,6 @@ const { searchIndex } = require("../algolia"); const path = require("path"); -const db = require("../db"); +const { db } = require("../db"); function tuncate(str, len) { const overflow = str.length - len; diff --git a/packages/skin-database/tasks/syncToSqlite.js b/packages/skin-database/tasks/syncToSqlite.js index dbf21309..27a3bccb 100644 --- a/packages/skin-database/tasks/syncToSqlite.js +++ b/packages/skin-database/tasks/syncToSqlite.js @@ -1,14 +1,5 @@ const path = require("path"); -const db = require("../db"); - -var knex = require("knex")({ - client: "sqlite3", - connection: { - filename: path.join(__dirname, "../skins.sqlite3"), - }, - useNullAsDefault: true, - debug: false, -}); +const { db, knex } = require("../db"); const skins = db.get("skins"); const iaItems = db.get("internetArchiveItems"); diff --git a/packages/skin-database/tasks/tweet.ts b/packages/skin-database/tasks/tweet.ts index 97779c20..2cd85841 100644 --- a/packages/skin-database/tasks/tweet.ts +++ b/packages/skin-database/tasks/tweet.ts @@ -5,6 +5,7 @@ import { TWEET_BOT_CHANNEL_ID } from "../config"; import { spawn } from "child_process"; import { Client } from "discord.js"; import { SkinRecord } from "../types"; +import { PROJECT_ROOT } from "../config"; function spawnPromise(command: string, args: string[]): Promise { return new Promise((resolve, reject) => { @@ -83,8 +84,7 @@ export async function tweet(discordClient: Client, anything: string | null) { throw new Error(`Could not find filename for skin with hash ${md5}`); } const output = await spawnPromise( - // This will be run from the dist directory - path.resolve(__dirname, "../../../../packages/tweetBot/tweet.py"), + path.resolve(PROJECT_ROOT, "../tweetBot/tweet.py"), [ "tweet", md5,