Improve debug data

packages/skin-database/data/ArchiveFileModel.ts

@@ -0,0 +1,42 @@
+import UserContext, { ctxWeakMapMemoize } from "./UserContext";
+import { ArchiveFileRow } from "../types";
+import DataLoader from "dataloader";
+import { knex } from "../db";
+
+export type ArchiveFileDebugData = {
+  row: ArchiveFileRow;
+};
+
+export default class ArchiveFileModel {
+  constructor(readonly ctx: UserContext, readonly row: ArchiveFileRow) {}
+
+  static async fromMd5(
+    ctx: UserContext,
+    md5: string
+  ): Promise<ArchiveFileModel[]> {
+    const rows = await getArchiveFilesLoader(ctx).load(md5);
+    return rows.map((row) => new ArchiveFileModel(ctx, row));
+  }
+
+  getFileName(): string {
+    return this.row.file_name;
+  }
+
+  async debug(): Promise<ArchiveFileDebugData> {
+    return {
+      row: this.row,
+    };
+  }
+}
+
+const getArchiveFilesLoader = ctxWeakMapMemoize<
+  DataLoader<string, ArchiveFileRow[]>
+>(
+  () =>
+    new DataLoader<string, ArchiveFileRow[]>(async (md5s) => {
+      const rows = await knex("archive_files")
+        .whereIn("skin_md5", md5s)
+        .select();
+      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
+    })
+);

packages/skin-database/data/SkinModel.ts

@@ -1,5 +1,5 @@
 import { getScreenshotUrl, getSkinUrl } from "./skins";
-import { TweetStatus, SkinRow, ReviewRow } from "../types";
+import { TweetStatus, SkinRow, ReviewRow, UploadStatus } from "../types";
 import UserContext, { ctxWeakMapMemoize } from "./UserContext";
 import TweetModel, { TweetDebugData } from "./TweetModel";
 import IaItemModel, { IaItemDebugData } from "./IaItemModel";
@@ -7,6 +7,9 @@ import FileModel, { FileDebugData } from "./FileModel";
 import { MD5_REGEX } from "../utils";
 import DataLoader from "dataloader";
 import { knex } from "../db";
+import UploadModel, { UploadDebugData } from "./UploadModel";
+import ArchiveFileModel, { ArchiveFileDebugData } from "./ArchiveFileModel";
+import * as Skins from "./skins";
 
 export default class SkinModel {
   constructor(readonly ctx: UserContext, readonly row: SkinRow) {}
@@ -72,6 +75,19 @@ export default class SkinModel {
     return FileModel.fromMd5(this.ctx, this.row.md5);
   }
 
+  getArchiveFiles(): Promise<ArchiveFileModel[]> {
+    return ArchiveFileModel.fromMd5(this.ctx, this.row.md5);
+  }
+
+  getUploadStatuses(): Promise<UploadModel[]> {
+    return UploadModel.fromMd5(this.ctx, this.row.md5);
+  }
+
+  async getUploadStatus(): Promise<UploadStatus | null> {
+    const status = await Skins.getUploadStatuses([this.row.md5]);
+    return (status[this.getMd5()] as UploadStatus) || null;
+  }
+
   async getIsNsfw(): Promise<boolean> {
     const reviews = await this.getReviews();
     return reviews.some((review) => review.review === "NSFW");
@@ -136,16 +152,24 @@ export default class SkinModel {
     reviews: ReviewRow[];
     tweets: TweetDebugData[];
     files: FileDebugData[];
+    archiveFiles: ArchiveFileDebugData[];
     iaItem: IaItemDebugData | null;
+    uploadStatuses: UploadDebugData[];
   }> {
     const tweets = await this.getTweets();
     const files = await this.getFiles();
+    const archiveFiles = await this.getArchiveFiles();
     const iaItem = await this.getIaItem();
+    const uploadStatuses = await this.getUploadStatuses();
     return {
       row: this.row,
       reviews: await this.getReviews(),
       tweets: await Promise.all(tweets.map((tweet) => tweet.debug())),
       files: await Promise.all(files.map((file) => file.debug())),
+      archiveFiles: await Promise.all(archiveFiles.map((file) => file.debug())),
+      uploadStatuses: await Promise.all(
+        uploadStatuses.map((upload) => upload.debug())
+      ),
       iaItem: iaItem == null ? null : await iaItem.debug(),
     };
   }

packages/skin-database/data/UploadModel.ts

@@ -0,0 +1,37 @@
+import UserContext, { ctxWeakMapMemoize } from "./UserContext";
+import { UploadRow, UploadStatus } from "../types";
+import DataLoader from "dataloader";
+import { knex } from "../db";
+
+export type UploadDebugData = {
+  row: UploadRow;
+};
+
+export default class UploadModel {
+  constructor(readonly ctx: UserContext, readonly row: UploadRow) {}
+
+  static async fromMd5(ctx: UserContext, md5: string): Promise<UploadModel[]> {
+    const rows = await getUploadLoader(ctx).load(md5);
+    return rows.map((row) => new UploadModel(ctx, row));
+  }
+
+  getStatus(): UploadStatus {
+    return this.row.status;
+  }
+
+  async debug(): Promise<UploadDebugData> {
+    return {
+      row: this.row,
+    };
+  }
+}
+
+const getUploadLoader = ctxWeakMapMemoize<DataLoader<string, UploadRow[]>>(
+  () =>
+    new DataLoader<string, UploadRow[]>(async (md5s) => {
+      const rows = await knex("skin_uploads")
+        .whereIn("skin_md5", md5s)
+        .select();
+      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
+    })
+);

packages/skin-database/data/skins.ts

@@ -5,6 +5,7 @@ import { searchIndex } from "../algolia";
 import { truncate } from "../utils";
 import fetch from "node-fetch";
 import * as S3 from "../s3";
+import * as CloudFlare from "../CloudFlare";
 import SkinModel from "./SkinModel";
 import UserContext from "./UserContext";
 
@@ -74,33 +75,6 @@ export async function setContentHash(md5: string): Promise<string | null> {
   return contentHash;
 }
 
-export async function getSkinDebugData(md5: string): Promise<any | null> {
-  const skin = await knex("skins").where({ md5 }).select();
-  const searchIndexUpdates = await knex("algolia_field_updates")
-    .where({ skin_md5: md5 })
-    .select();
-  const reviews = await knex("skin_reviews").where({ skin_md5: md5 }).select();
-  const tweets = await knex("tweets").where({ skin_md5: md5 }).select();
-  const internetArchive = await knex("ia_items")
-    .where({ skin_md5: md5 })
-    .select();
-  const uploads = await knex("skin_uploads").where({ skin_md5: md5 }).select();
-  const archiveFiles = await knex("archive_files")
-    .where({ skin_md5: md5 })
-    .select();
-  const files = await knex("files").where({ skin_md5: md5 }).select();
-  return {
-    skin,
-    searchIndexUpdates,
-    reviews,
-    tweets,
-    internetArchive,
-    uploads,
-    archiveFiles,
-    files,
-  };
-}
-
 export async function getTweetableSkinCount(): Promise<number> {
   const tweetable = await knex("skins")
     .leftJoin("skin_reviews", "skin_reviews.skin_md5", "=", "skins.md5")
@@ -154,6 +128,7 @@ export async function getUploadStatuses(
 ): Promise<{ [md5: string]: string }> {
   const skins = await knex("skin_uploads")
     .whereIn("skin_md5", md5s)
+    .orderBy("id", "desc")
     .select("skin_md5", "status");
 
   const statuses: { [md5: string]: string } = {};
@@ -229,12 +204,22 @@ export async function deleteSkin(md5: string): Promise<void> {
   await knex("ia_items").where({ skin_md5: md5 }).delete();
   console.log(`... sqlite "archive_files"`);
   await knex("archive_files").where({ skin_md5: md5 }).delete();
+  console.log(`... sqlite "tweets"`);
+  await knex("tweets").where({ skin_md5: md5 }).delete();
+  console.log(`... sqlite "algolia_field_updates"`);
+  await knex("algolia_field_updates").where({ skin_md5: md5 }).delete();
+  console.log(`... sqlite "skin_uploads"`);
+  await knex("skin_uploads").where({ skin_md5: md5 }).delete();
+  console.log(`... sqlite "screenshot_updates"`);
+  await knex("screenshot_updates").where({ skin_md5: md5 }).delete();
   console.log(`... removing from Algolia index`);
   await searchIndex.deleteObjects([md5]);
   console.log(`... removing skin from S3`);
   await S3.deleteSkin(md5);
   console.log(`... removing screenshot from S3`);
   await S3.deleteScreenshot(md5);
+  console.log(`... purging screenshot and skin from CloudFlare`);
+  await CloudFlare.purgeFiles([getScreenshotUrl(md5), getSkinUrl(md5)]);
   console.log(`Done deleting skin ${md5}.`);
 }
 

packages/skin-database/discord-bot/commands/debug.ts

@@ -3,8 +3,6 @@ import { Message } from "discord.js";
 import UserContext from "../../data/UserContext";
 import SkinModel from "../../data/SkinModel";
 
-const TRIPPLE = "```";
-
 async function handler(message: Message, args: [string]) {
   const ctx = new UserContext();
   const [anything] = args;
@@ -18,9 +16,13 @@ async function handler(message: Message, args: [string]) {
     return;
   }
   const data = await skin.debug();
-  await message.channel.send(
-    [TRIPPLE, JSON.stringify(data, null, 2), TRIPPLE].join("")
-  );
+  const pageSize = 1975;
+  const jsonString = JSON.stringify(data, null, 2);
+  for (let i = 0; i < jsonString.length; i += pageSize) {
+    await message.channel.send(
+      "```" + jsonString.slice(i, i + pageSize) + "```"
+    );
+  }
 }
 
 module.exports = {

packages/skin-database/types.ts

@@ -34,7 +34,25 @@ export type FileRow = {
   file_path: string;
 };
 
+export type ArchiveFileRow = {
+  skin_md5: string;
+  file_name: string;
+  file_md5: string;
+};
+
 export type IaItemRow = {
   skin_md5: string;
   identifier: string;
 };
+
+export type UploadStatus =
+  | "UPLOAD_REPORTED"
+  | "URL_REQUESTED"
+  | "ERRORED"
+  | "ARCHIVED";
+
+export type UploadRow = {
+  skin_md5: string;
+  id: string;
+  status: UploadStatus;
+};