From 6fb4deb19c36148075345bddbf9caccbcff8d4e1 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Wed, 30 Dec 2020 02:17:34 -0500 Subject: [PATCH] Improve debug data --- .../skin-database/data/ArchiveFileModel.ts | 42 +++++++++++++++++++ packages/skin-database/data/SkinModel.ts | 26 +++++++++++- packages/skin-database/data/UploadModel.ts | 37 ++++++++++++++++ packages/skin-database/data/skins.ts | 39 ++++++----------- .../discord-bot/commands/debug.ts | 12 +++--- packages/skin-database/types.ts | 18 ++++++++ 6 files changed, 141 insertions(+), 33 deletions(-) create mode 100644 packages/skin-database/data/ArchiveFileModel.ts create mode 100644 packages/skin-database/data/UploadModel.ts diff --git a/packages/skin-database/data/ArchiveFileModel.ts b/packages/skin-database/data/ArchiveFileModel.ts new file mode 100644 index 00000000..d0732a02 --- /dev/null +++ b/packages/skin-database/data/ArchiveFileModel.ts @@ -0,0 +1,42 @@ +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; +import { ArchiveFileRow } from "../types"; +import DataLoader from "dataloader"; +import { knex } from "../db"; + +export type ArchiveFileDebugData = { + row: ArchiveFileRow; +}; + +export default class ArchiveFileModel { + constructor(readonly ctx: UserContext, readonly row: ArchiveFileRow) {} + + static async fromMd5( + ctx: UserContext, + md5: string + ): Promise { + const rows = await getArchiveFilesLoader(ctx).load(md5); + return rows.map((row) => new ArchiveFileModel(ctx, row)); + } + + getFileName(): string { + return this.row.file_name; + } + + async debug(): Promise { + return { + row: this.row, + }; + } +} + +const getArchiveFilesLoader = ctxWeakMapMemoize< + DataLoader +>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("archive_files") + .whereIn("skin_md5", md5s) + .select(); + return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); + }) +); diff --git a/packages/skin-database/data/SkinModel.ts b/packages/skin-database/data/SkinModel.ts index dcf047d8..1d6e5c7b 100644 --- a/packages/skin-database/data/SkinModel.ts +++ b/packages/skin-database/data/SkinModel.ts @@ -1,5 +1,5 @@ import { getScreenshotUrl, getSkinUrl } from "./skins"; -import { TweetStatus, SkinRow, ReviewRow } from "../types"; +import { TweetStatus, SkinRow, ReviewRow, UploadStatus } from "../types"; import UserContext, { ctxWeakMapMemoize } from "./UserContext"; import TweetModel, { TweetDebugData } from "./TweetModel"; import IaItemModel, { IaItemDebugData } from "./IaItemModel"; @@ -7,6 +7,9 @@ import FileModel, { FileDebugData } from "./FileModel"; import { MD5_REGEX } from "../utils"; import DataLoader from "dataloader"; import { knex } from "../db"; +import UploadModel, { UploadDebugData } from "./UploadModel"; +import ArchiveFileModel, { ArchiveFileDebugData } from "./ArchiveFileModel"; +import * as Skins from "./skins"; export default class SkinModel { constructor(readonly ctx: UserContext, readonly row: SkinRow) {} @@ -72,6 +75,19 @@ export default class SkinModel { return FileModel.fromMd5(this.ctx, this.row.md5); } + getArchiveFiles(): Promise { + return ArchiveFileModel.fromMd5(this.ctx, this.row.md5); + } + + getUploadStatuses(): Promise { + return UploadModel.fromMd5(this.ctx, this.row.md5); + } + + async getUploadStatus(): Promise { + const status = await Skins.getUploadStatuses([this.row.md5]); + return (status[this.getMd5()] as UploadStatus) || null; + } + async getIsNsfw(): Promise { const reviews = await this.getReviews(); return reviews.some((review) => review.review === "NSFW"); @@ -136,16 +152,24 @@ export default class SkinModel { reviews: ReviewRow[]; tweets: TweetDebugData[]; files: FileDebugData[]; + archiveFiles: ArchiveFileDebugData[]; iaItem: IaItemDebugData | null; + uploadStatuses: UploadDebugData[]; }> { const tweets = await this.getTweets(); const files = await this.getFiles(); + const archiveFiles = await this.getArchiveFiles(); const iaItem = await this.getIaItem(); + const uploadStatuses = await this.getUploadStatuses(); return { row: this.row, reviews: await this.getReviews(), tweets: await Promise.all(tweets.map((tweet) => tweet.debug())), files: await Promise.all(files.map((file) => file.debug())), + archiveFiles: await Promise.all(archiveFiles.map((file) => file.debug())), + uploadStatuses: await Promise.all( + uploadStatuses.map((upload) => upload.debug()) + ), iaItem: iaItem == null ? null : await iaItem.debug(), }; } diff --git a/packages/skin-database/data/UploadModel.ts b/packages/skin-database/data/UploadModel.ts new file mode 100644 index 00000000..428e6ff3 --- /dev/null +++ b/packages/skin-database/data/UploadModel.ts @@ -0,0 +1,37 @@ +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; +import { UploadRow, UploadStatus } from "../types"; +import DataLoader from "dataloader"; +import { knex } from "../db"; + +export type UploadDebugData = { + row: UploadRow; +}; + +export default class UploadModel { + constructor(readonly ctx: UserContext, readonly row: UploadRow) {} + + static async fromMd5(ctx: UserContext, md5: string): Promise { + const rows = await getUploadLoader(ctx).load(md5); + return rows.map((row) => new UploadModel(ctx, row)); + } + + getStatus(): UploadStatus { + return this.row.status; + } + + async debug(): Promise { + return { + row: this.row, + }; + } +} + +const getUploadLoader = ctxWeakMapMemoize>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("skin_uploads") + .whereIn("skin_md5", md5s) + .select(); + return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); + }) +); diff --git a/packages/skin-database/data/skins.ts b/packages/skin-database/data/skins.ts index bc9a8f79..fb8e6e7e 100644 --- a/packages/skin-database/data/skins.ts +++ b/packages/skin-database/data/skins.ts @@ -5,6 +5,7 @@ import { searchIndex } from "../algolia"; import { truncate } from "../utils"; import fetch from "node-fetch"; import * as S3 from "../s3"; +import * as CloudFlare from "../CloudFlare"; import SkinModel from "./SkinModel"; import UserContext from "./UserContext"; @@ -74,33 +75,6 @@ export async function setContentHash(md5: string): Promise { return contentHash; } -export async function getSkinDebugData(md5: string): Promise { - const skin = await knex("skins").where({ md5 }).select(); - const searchIndexUpdates = await knex("algolia_field_updates") - .where({ skin_md5: md5 }) - .select(); - const reviews = await knex("skin_reviews").where({ skin_md5: md5 }).select(); - const tweets = await knex("tweets").where({ skin_md5: md5 }).select(); - const internetArchive = await knex("ia_items") - .where({ skin_md5: md5 }) - .select(); - const uploads = await knex("skin_uploads").where({ skin_md5: md5 }).select(); - const archiveFiles = await knex("archive_files") - .where({ skin_md5: md5 }) - .select(); - const files = await knex("files").where({ skin_md5: md5 }).select(); - return { - skin, - searchIndexUpdates, - reviews, - tweets, - internetArchive, - uploads, - archiveFiles, - files, - }; -} - export async function getTweetableSkinCount(): Promise { const tweetable = await knex("skins") .leftJoin("skin_reviews", "skin_reviews.skin_md5", "=", "skins.md5") @@ -154,6 +128,7 @@ export async function getUploadStatuses( ): Promise<{ [md5: string]: string }> { const skins = await knex("skin_uploads") .whereIn("skin_md5", md5s) + .orderBy("id", "desc") .select("skin_md5", "status"); const statuses: { [md5: string]: string } = {}; @@ -229,12 +204,22 @@ export async function deleteSkin(md5: string): Promise { await knex("ia_items").where({ skin_md5: md5 }).delete(); console.log(`... sqlite "archive_files"`); await knex("archive_files").where({ skin_md5: md5 }).delete(); + console.log(`... sqlite "tweets"`); + await knex("tweets").where({ skin_md5: md5 }).delete(); + console.log(`... sqlite "algolia_field_updates"`); + await knex("algolia_field_updates").where({ skin_md5: md5 }).delete(); + console.log(`... sqlite "skin_uploads"`); + await knex("skin_uploads").where({ skin_md5: md5 }).delete(); + console.log(`... sqlite "screenshot_updates"`); + await knex("screenshot_updates").where({ skin_md5: md5 }).delete(); console.log(`... removing from Algolia index`); await searchIndex.deleteObjects([md5]); console.log(`... removing skin from S3`); await S3.deleteSkin(md5); console.log(`... removing screenshot from S3`); await S3.deleteScreenshot(md5); + console.log(`... purging screenshot and skin from CloudFlare`); + await CloudFlare.purgeFiles([getScreenshotUrl(md5), getSkinUrl(md5)]); console.log(`Done deleting skin ${md5}.`); } diff --git a/packages/skin-database/discord-bot/commands/debug.ts b/packages/skin-database/discord-bot/commands/debug.ts index fc506e44..21834e1a 100644 --- a/packages/skin-database/discord-bot/commands/debug.ts +++ b/packages/skin-database/discord-bot/commands/debug.ts @@ -3,8 +3,6 @@ import { Message } from "discord.js"; import UserContext from "../../data/UserContext"; import SkinModel from "../../data/SkinModel"; -const TRIPPLE = "```"; - async function handler(message: Message, args: [string]) { const ctx = new UserContext(); const [anything] = args; @@ -18,9 +16,13 @@ async function handler(message: Message, args: [string]) { return; } const data = await skin.debug(); - await message.channel.send( - [TRIPPLE, JSON.stringify(data, null, 2), TRIPPLE].join("") - ); + const pageSize = 1975; + const jsonString = JSON.stringify(data, null, 2); + for (let i = 0; i < jsonString.length; i += pageSize) { + await message.channel.send( + "```" + jsonString.slice(i, i + pageSize) + "```" + ); + } } module.exports = { diff --git a/packages/skin-database/types.ts b/packages/skin-database/types.ts index 936087f4..19a9130b 100644 --- a/packages/skin-database/types.ts +++ b/packages/skin-database/types.ts @@ -34,7 +34,25 @@ export type FileRow = { file_path: string; }; +export type ArchiveFileRow = { + skin_md5: string; + file_name: string; + file_md5: string; +}; + export type IaItemRow = { skin_md5: string; identifier: string; }; + +export type UploadStatus = + | "UPLOAD_REPORTED" + | "URL_REQUESTED" + | "ERRORED" + | "ARCHIVED"; + +export type UploadRow = { + skin_md5: string; + id: string; + status: UploadStatus; +};