Mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp.git synced 2026-07-21 00:59:29 +00:00
Code Issues Projects Releases Wiki Activity

Speed up TS and start doing more sqlite

Browse source
This commit is contained in:
Jordan Eldredge 2020-06-24 23:15:52 -04:00
parent 372e55c4f4
commit 202ca53a20
6 changed files with 90 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

3
packages/skin-database/cli.ts
View file

@ -1,6 +1,7 @@
#!/usr/bin/env node
import path from "path";
import fs from "fs";
import { db, knex } from "./db";
import { argv } from "yargs";
import fetchInternetArchiveMetadata from "./tasks/fetchInternetArchiveMetadata";
import ensureInternetArchiveItemsIndexByMd5 from "./tasks/ensureInternetArchiveItemsIndexByMd5";
@ -90,6 +91,8 @@ async function main() {
console.log(`Unknown command ${argv._[0]}`);
}
} finally {
knex.destroy();
db.close();
logger.close();
client.destroy();
}

72
packages/skin-database/data/skins.ts
View file

@ -13,7 +13,7 @@ const SKIN_TYPE = {
INVALID: 4,
};
const skins = db.get("skins");
const skins_DEPRECATED = db.get("skins");
const iaItems = db.get("internetArchiveItems");
const CLASSIC_QUERY = {
@ -55,7 +55,7 @@ function getWebampUrl(skin: DBSkinRecord): string {
}
export async function addSkin({ md5, filePath, uploader, averageColor }) {
skins.insert({
skins_DEPRECATED.insert({
md5,
type: "CLASSIC",
filePaths: [filePath],
@ -83,13 +83,14 @@ export async function addSkin({ md5, filePath, uploader, averageColor }) {
const IA_URL = /^(https:\/\/)?archive.org\/details\/([^\/]+)\/?/;
const MD5 = /([a-fA-F0-9]{32})/;
// TODO: SQLITE
export async function getMd5ByAnything(
anything: string
): Promise<string | null> {
const md5Match = anything.match(MD5);
if (md5Match != null) {
const md5 = md5Match[1];
const found = await skins.findOne({ md5, type: "CLASSIC" });
const found = await skins_DEPRECATED.findOne({ md5, type: "CLASSIC" });
if (found != null) {
return md5;
}
@ -105,8 +106,9 @@ export async function getMd5ByAnything(
return getMd5FromInternetArchvieItemName(anything);
}
// TODO: SQLITE
export async function getSkinByMd5(md5: string): Promise<SkinRecord | null> {
const skin: DBSkinRecord | null = await skins.findOne({
const skin: DBSkinRecord | null = await skins_DEPRECATED.findOne({
md5,
type: "CLASSIC",
});
@ -136,7 +138,7 @@ export async function getSkinByMd5(md5: string): Promise<SkinRecord | null> {
};
}
export async function getInternetArchiveItem(md5: string): Promise<DBIARecord> {
async function getInternetArchiveItem(md5: string): Promise<DBIARecord> {
return iaItems.findOne({ md5: md5 });
}
@ -145,12 +147,8 @@ async function getMd5FromInternetArchvieItemName(itemName: string) {
return item == null ? null : item.md5;
}
export async function getUnarchived() {
return skins.find({ itemName: null }, { md5: 1 });
}
export async function getMissingNsfwPredictions() {
const results = await skins.find(
const results = await skins_DEPRECATED.find(
{ nsfwPredictions: null, type: "CLASSIC" },
{ md5: 1 }
);
@ -162,22 +160,22 @@ export function getInternetArchiveUrl(itemName: string | null): string | null {
}
export function getTweetableSkinCount(): Promise<number> {
return skins.count(TWEETABLE_QUERY);
return skins_DEPRECATED.count(TWEETABLE_QUERY);
}
export function getClassicSkinCount(): Promise<number> {
return skins.count(CLASSIC_QUERY);
return skins_DEPRECATED.count(CLASSIC_QUERY);
}
// TODO: Also pass id
export async function markAsTweeted(md5: string, url: string): Promise<void> {
await skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } });
await skins_DEPRECATED.findOneAndUpdate({ md5 }, { $set: { tweeted: true } });
await knex("tweets").insert({ skin_md5: md5, url }, []);
}
// TODO: Also path actor
export async function markAsNSFW(md5: string): Promise<void> {
await skins.findOneAndUpdate({ md5 }, { $set: { nsfw: true } });
await skins_DEPRECATED.findOneAndUpdate({ md5 }, { $set: { nsfw: true } });
const indexes = [{ objectID: md5, nsfw: true }];
// TODO: Await here, but for some reason this never completes
new Promise((resolve, reject) => {
@ -190,28 +188,39 @@ export async function markAsNSFW(md5: string): Promise<void> {
}
export async function getStatus(md5: string): Promise<TweetStatus> {
const skin = await skins.findOne({ md5 });
if (skin.tweeted) {
const tweeted = await knex("tweets").where({ skin_md5: md5 }).limit(1);
if (tweeted) {
return "TWEETED";
}
if (skin.rejected) {
const reviewRows = await knex("skin_reviews")
.where({ skin_md5: md5 })
.limit(1);
const reviews = new Set(reviewRows.map((row) => row.review));
if (reviews.has("REJECTED")) {
return "REJECTED";
}
if (skin.approved) {
if (reviews.has("APPROVED")) {
return "APPROVED";
}
return "UNREVIEWED";
}
// TODO: Also path actor
export async function approve(md5: string): Promise<void> {
await skins.findOneAndUpdate({ md5 }, { $set: { approved: true } });
await skins_DEPRECATED.findOneAndUpdate(
{ md5 },
{ $set: { approved: true } }
);
await knex("skin_reviews").insert({ skin_md5: md5, review: "APPROVED" }, []);
}
// TODO: Also path actor
export async function reject(md5: string): Promise<void> {
await skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } });
await skins_DEPRECATED.findOneAndUpdate(
{ md5 },
{ $set: { rejected: true } }
);
await knex("skin_reviews").insert({ skin_md5: md5, review: "REJECTED" }, []);
}
@ -219,7 +228,7 @@ export async function getSkinToReview(): Promise<{
filename: string | null;
md5: string;
}> {
const reviewable = await skins.aggregate([
const reviewable = await skins_DEPRECATED.aggregate([
{ $match: REVIEWABLE_QUERY },
{ $sample: { size: 1 } },
]);
@ -231,7 +240,7 @@ export async function getSkinToReviewForNsfw(): Promise<{
filename: string | null;
md5: string;
}> {
const reviewable = await skins.find(REVIEWABLE_QUERY, {
const reviewable = await skins_DEPRECATED.find(REVIEWABLE_QUERY, {
limit: 1,
sort: { "nsfwPredictions.porn": -1 },
});
@ -240,7 +249,7 @@ export async function getSkinToReviewForNsfw(): Promise<{
}
export async function getSkinToTweet(): Promise<SkinRecord | null> {
const tweetables = await skins.aggregate([
const tweetables = await skins_DEPRECATED.aggregate([
{ $match: TWEETABLE_QUERY },
{ $sample: { size: 1 } },
]);
@ -257,15 +266,15 @@ export async function getStats(): Promise<{
tweeted: number;
tweetable: number;
}> {
const approved = await skins.count({ approved: true });
const rejected = await skins.count({ rejected: true });
const tweeted = await skins.count({ tweeted: true });
const approved = await skins_DEPRECATED.count({ approved: true });
const rejected = await skins_DEPRECATED.count({ rejected: true });
const tweeted = await skins_DEPRECATED.count({ tweeted: true });
const tweetable = await getTweetableSkinCount();
return { approved, rejected, tweeted, tweetable };
}
export async function getRandomClassicSkinMd5(): Promise<string> {
const random = await skins.aggregate([
const random = await skins_DEPRECATED.aggregate([
{ $match: CLASSIC_QUERY },
{ $sample: { size: 1 } },
]);
@ -291,7 +300,10 @@ export async function setNsfwPredictions(
md5: string,
nsfwPredictions: NsfwPrediction
): Promise<void> {
await skins.findOneAndUpdate({ md5 }, { $set: { nsfwPredictions } });
await skins_DEPRECATED.findOneAndUpdate(
{ md5 },
{ $set: { nsfwPredictions } }
);
await knex("nsfw_predictions").insert(
{ skin_md5: md5, ...nsfwPredictions },
[]
@ -303,7 +315,7 @@ export async function setTweetInfo(
likes: number,
tweetId: string
): Promise<void> {
await skins.findOneAndUpdate(
await skins_DEPRECATED.findOneAndUpdate(
{ md5 },
{ $set: { twitterLikes: likes, tweetId } }
);
@ -327,7 +339,7 @@ export async function getMuseumPage({
}): Promise<
Array<{ color: string; fileName: string; md5: string; nsfw: boolean }>
> {
const reviewable = await skins.find(
const reviewable = await skins_DEPRECATED.find(
{ type: "CLASSIC" },
{
limit: first,

15
packages/skin-database/discord-bot/index.js
View file

@ -10,12 +10,15 @@ const handlers = {
help: handleHelp,
};
const commands = fs
.readdirSync(path.resolve(__dirname, "./commands"))
.filter((file) => file.endsWith(".js"))
.map((file) => {
return require(`./commands/${file}`);
});
const commands = [
require("./commands/archive"),
require("./commands/random"),
require("./commands/review"),
require("./commands/screenshot"),
require("./commands/skin"),
require("./commands/stats"),
require("./commands/tweet"),
];
for (const command of commands) {
handlers[command.command] = command.handler;

14
packages/skin-database/package.json
View file

@ -23,16 +23,18 @@
"shell-escape": "^0.2.0",
"sqlite3": "^4.2.0",
"temp": "^0.9.0",
"ts-node": "^8.10.2",
"winston": "^3.2.1",
"yargs": "^13.2.4"
},
"scripts": {
"start": "tsc && node dist/index.js",
"tweet": "tsc && ./dist/cli.js tweet",
"fetch-metadata": "tsc && .dist/cli.js fetch-metadata",
"bot": "tsc && node ./dist/discord-bot/index.js",
"cli": "tsc && node ./dist/cli.js",
"index": "tsc && node ./dist/tasks/searchIndex.js"
"start": "ts-node --transpile-only index.js",
"tweet": "ts-node --transpile-only ./cli.ts tweet",
"fetch-metadata": "ts-node --transpile-only ./cli.ts fetch-metadata",
"bot": "ts-node --transpile-only ./discord-bot/index.js",
"cli": "ts-node --transpile-only ./cli.ts",
"repro": "ts-node --transpile-only ./repro.ts",
"index": "ts-node --transpile-only ./tasks/searchIndex.js"
},
"prettier": {},
"devDependencies": {

2
packages/skin-database/tsconfig.json
View file

@ -4,7 +4,7 @@
"esModuleInterop": true,
"target": "es6",
"moduleResolution": "node",
"sourceMap": true,
"sourceMap": false,
"allowJs": true,
"outDir": "dist",
"strictNullChecks": true,

28
yarn.lock
View file

@ -1803,6 +1803,10 @@ are-we-there-yet@~1.1.2:
delegates "^1.0.0"
readable-stream "^2.0.6"
arg@^4.1.0:
version "4.1.3"
resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
argparse@^1.0.7:
version "1.0.10"
resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@ -3598,6 +3602,10 @@ diff-sequences@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-24.9.0.tgz#5715d6244e2aa65f48bba0bc972db0b0b11e95b5"
diff@^4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
diffie-hellman@^5.0.0:
version "5.0.3"
resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875"
@ -6920,6 +6928,10 @@ make-dir@^3.0.0:
dependencies:
semver "^6.0.0"
make-error@^1.1.1:
version "1.3.6"
resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
make-iterator@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/make-iterator/-/make-iterator-1.0.1.tgz#29b33f312aa8f547c4a5e490f56afcec99133ad6"
@ -9695,7 +9707,7 @@ source-map-resolve@^0.5.0:
source-map-url "^0.4.0"
urix "^0.1.0"
source-map-support@^0.5.16, source-map-support@^0.5.6, source-map-support@~0.5.12:
source-map-support@^0.5.16, source-map-support@^0.5.17, source-map-support@^0.5.6, source-map-support@~0.5.12:
version "0.5.19"
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61"
dependencies:
@ -10398,6 +10410,16 @@ tryer@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8"
ts-node@^8.10.2:
version "8.10.2"
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.10.2.tgz#eee03764633b1234ddd37f8db9ec10b75ec7fb8d"
dependencies:
arg "^4.1.0"
diff "^4.0.1"
make-error "^1.1.1"
source-map-support "^0.5.17"
yn "3.1.1"
tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3:
version "1.13.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043"
@ -11178,3 +11200,7 @@ yauzl@^2.10.0, yauzl@^2.4.2:
dependencies:
buffer-crc32 "~0.2.3"
fd-slicer "~1.1.0"
yn@3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"