Progress

packages/skin-database/api/__mocks__/processUserUploads.ts

@@ -1,3 +1,3 @@
-export async function processUserUploads() {
+export const processUserUploads = jest.fn(async () => {
   // Mock
-}
+});

packages/skin-database/api/__tests__/router.test.ts

@@ -3,6 +3,7 @@ import { knex } from "../../db";
 import request from "supertest"; // supertest is a framework that allows to easily test web apis
 import { createApp } from "../app";
 import * as S3 from "../../s3";
+import { processUserUploads } from "../processUserUploads";
 jest.mock("../../s3");
 jest.mock("../processUserUploads");
 
@@ -135,8 +136,10 @@ test("/skins/get_upload_urls", async () => {
 });
 
 test("An Upload Flow", async () => {
+  // Request an upload URL
   const md5 = "3b73bcd43c30b85d4cad3083e8ac9695";
-  const skins = { [md5]: "a_fake_new_file.wsz" };
+  const filename = "a_fake_new_file.wsz";
+  const skins = { [md5]: filename };
   const getUrlsResponse = await request(app)
     .post("/skins/get_upload_urls")
     .send({ skins });
@@ -147,11 +150,29 @@ test("An Upload Flow", async () => {
     [md5]: { id: expect.any(Number), url: "<MOCK_S3_UPLOAD_URL>" },
   });
 
+  const requestedUpload = await knex("skin_uploads").where({ id }).first();
+  expect(requestedUpload).toEqual({
+    filename,
+    id,
+    skin_md5: md5,
+    status: "URL_REQUESTED",
+  });
+
+  // Report that we've uploaded the skin to S3 (we lie)
   const uploadedResponse = await request(app)
     .post(`/skins/${md5}/uploaded`)
     .query({ id })
     .send({ skins });
   expect(uploadedResponse.body).toEqual({ done: true });
+  expect(processUserUploads).toHaveBeenCalled();
+
+  const reportedUpload = await knex("skin_uploads").where({ id }).first();
+  expect(reportedUpload).toEqual({
+    filename,
+    id,
+    skin_md5: md5,
+    status: "UPLOAD_REPORTED",
+  });
 });
 
 test("/stylegan.json", async () => {

packages/skin-database/data/FileModel.ts

@@ -1,6 +1,8 @@
 import path from "path";
-import UserContext from "./UserContext";
+import UserContext, { ctxWeakMapMemoize } from "./UserContext";
 import { FileRow } from "../types";
+import DataLoader from "dataloader";
+import { knex } from "../db";
 
 export type FileDebugData = {
   row: FileRow;
@@ -10,7 +12,7 @@ export default class FileModel {
   constructor(readonly ctx: UserContext, readonly row: FileRow) {}
 
   static async fromMd5(ctx: UserContext, md5: string): Promise<FileModel[]> {
-    const rows = await ctx.files.load(md5);
+    const rows = await getFilesLoader(ctx).load(md5);
     return rows.map((row) => new FileModel(ctx, row));
   }
 
@@ -24,3 +26,11 @@ export default class FileModel {
     };
   }
 }
+
+const getFilesLoader = ctxWeakMapMemoize<DataLoader<string, FileRow[]>>(
+  () =>
+    new DataLoader<string, FileRow[]>(async (md5s) => {
+      const rows = await knex("files").whereIn("skin_md5", md5s).select();
+      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
+    })
+);

packages/skin-database/data/IaItemModel.ts

@@ -1,6 +1,8 @@
-import UserContext from "./UserContext";
+import UserContext, { ctxWeakMapMemoize } from "./UserContext";
 import { IaItemRow } from "../types";
 import SkinModel from "./SkinModel";
+import DataLoader from "dataloader";
+import { knex } from "../db";
 
 const IA_URL = /^(https:\/\/)?archive.org\/details\/([^/]+)\/?/;
 
@@ -11,7 +13,7 @@ export default class IaItemModel {
     ctx: UserContext,
     md5: string
   ): Promise<IaItemModel | null> {
-    const row = await ctx.iaItem.load(md5);
+    const row = await getIaItemLoader(ctx).load(md5);
     return row == null ? null : new IaItemModel(ctx, row);
   }
 
@@ -19,7 +21,7 @@ export default class IaItemModel {
     ctx: UserContext,
     identifier: string
   ): Promise<IaItemModel | null> {
-    const row = await ctx.iaItemByIdentifier.load(identifier);
+    const row = await getIaItemByItentifierLoader(ctx).load(identifier);
     return row == null ? null : new IaItemModel(ctx, row);
   }
 
@@ -64,3 +66,25 @@ export default class IaItemModel {
     return identifier;
   }
 }
+
+const getIaItemLoader = ctxWeakMapMemoize<DataLoader<string, IaItemRow>>(
+  () =>
+    new DataLoader(async (md5s) => {
+      const rows = await knex("ia_items").whereIn("skin_md5", md5s).select();
+      return md5s.map((md5) => rows.find((x) => x.skin_md5 === md5));
+    })
+);
+
+const getIaItemByItentifierLoader = ctxWeakMapMemoize<
+  DataLoader<string, IaItemRow>
+>(
+  () =>
+    new DataLoader(async (identifiers) => {
+      const rows = await knex("ia_items")
+        .whereIn("identifier", identifiers)
+        .select();
+      return identifiers.map((identifier) =>
+        rows.find((x) => x.identifier === identifier)
+      );
+    })
+);

packages/skin-database/data/SkinModel.ts

@@ -1,10 +1,12 @@
 import { getScreenshotUrl, getSkinUrl } from "./skins";
 import { TweetStatus, SkinRow, ReviewRow } from "../types";
-import UserContext from "./UserContext";
+import UserContext, { ctxWeakMapMemoize } from "./UserContext";
 import TweetModel, { TweetDebugData } from "./TweetModel";
 import IaItemModel from "./IaItemModel";
 import FileModel, { FileDebugData } from "./FileModel";
 import { MD5_REGEX } from "../utils";
+import DataLoader from "dataloader";
+import { knex } from "../db";
 
 export default class SkinModel {
   constructor(readonly ctx: UserContext, readonly row: SkinRow) {}
@@ -13,7 +15,7 @@ export default class SkinModel {
     ctx: UserContext,
     md5: string
   ): Promise<SkinModel | null> {
-    const row = await ctx.skin.load(md5);
+    const row = await getSkinLoader(ctx).load(md5);
     return row == null ? null : new SkinModel(ctx, row);
   }
 
@@ -34,7 +36,7 @@ export default class SkinModel {
   }
 
   static async exists(ctx: UserContext, md5: string): Promise<boolean> {
-    const row = await ctx.skin.load(md5);
+    const row = await getSkinLoader(ctx).load(md5);
     return row != null;
   }
 
@@ -48,8 +50,7 @@ export default class SkinModel {
   }
 
   async getTweets(): Promise<TweetModel[]> {
-    const rows = await this.ctx.tweets.load(this.row.md5);
-    return rows.map((row) => new TweetModel(this.ctx, row));
+    return TweetModel.fromMd5(this.ctx, this.row.md5);
   }
 
   getIaItem(): Promise<IaItemModel | null> {
@@ -57,7 +58,7 @@ export default class SkinModel {
   }
 
   getReviews(): Promise<ReviewRow[]> {
-    return this.ctx.reviews.load(this.row.md5);
+    return getReviewsLoader(this.ctx).load(this.row.md5);
   }
 
   getFiles(): Promise<FileModel[]> {
@@ -136,3 +137,21 @@ export default class SkinModel {
     };
   }
 }
+
+const getSkinLoader = ctxWeakMapMemoize<DataLoader<string, SkinRow | null>>(
+  () =>
+    new DataLoader(async (md5s) => {
+      const rows = await knex("skins").whereIn("md5", md5s).select();
+      return md5s.map((md5) => rows.find((x) => x.md5 === md5));
+    })
+);
+
+const getReviewsLoader = ctxWeakMapMemoize<DataLoader<string, ReviewRow[]>>(
+  () =>
+    new DataLoader(async (md5s) => {
+      const rows = await knex("skin_reviews")
+        .whereIn("skin_md5", md5s)
+        .select();
+      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
+    })
+);

packages/skin-database/data/TweetModel.ts

@@ -1,5 +1,7 @@
-import UserContext from "./UserContext";
+import UserContext, { ctxWeakMapMemoize } from "./UserContext";
 import { TweetRow } from "../types";
+import DataLoader from "dataloader";
+import { knex } from "../db";
 
 export type TweetDebugData = {
   row: TweetRow;
@@ -9,7 +11,7 @@ export default class TweetModel {
   constructor(readonly ctx: UserContext, readonly row: TweetRow) {}
 
   static async fromMd5(ctx: UserContext, md5: string): Promise<TweetModel[]> {
-    const rows = await ctx.tweets.load(md5);
+    const rows = await getTweetsLoader(ctx).load(md5);
     return rows.map((row) => new TweetModel(ctx, row));
   }
 
@@ -29,3 +31,11 @@ export default class TweetModel {
     };
   }
 }
+
+const getTweetsLoader = ctxWeakMapMemoize<DataLoader<string, TweetRow[]>>(
+  () =>
+    new DataLoader(async (md5s) => {
+      const rows = await knex("tweets").whereIn("skin_md5", md5s).select();
+      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
+    })
+);

packages/skin-database/data/UserContext.ts

@@ -1,50 +1,13 @@
-import { knex } from "../db";
+// Currently only used as a WeakMap key
+export default class UserContext {}
 
-import DataLoader from "dataloader";
-import { SkinRow, TweetRow, ReviewRow, FileRow, IaItemRow } from "../types";
-
-export default class UserContext {
-  skin: DataLoader<string, SkinRow | null>;
-  tweets: DataLoader<string, TweetRow[]>;
-  reviews: DataLoader<string, ReviewRow[]>;
-  file: DataLoader<string, FileRow>;
-  files: DataLoader<string, FileRow[]>;
-  iaItem: DataLoader<string, IaItemRow>;
-  iaItemByIdentifier: DataLoader<string, IaItemRow>;
-  constructor() {
-    this.skin = new DataLoader(async (md5s) => {
-      const rows = await knex("skins").whereIn("md5", md5s).select();
-      return md5s.map((md5) => rows.find((x) => x.md5 === md5));
-    });
-    this.tweets = new DataLoader(async (md5s) => {
-      const rows = await knex("tweets").whereIn("skin_md5", md5s).select();
-      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
-    });
-    this.reviews = new DataLoader(async (md5s) => {
-      const rows = await knex("skin_reviews")
-        .whereIn("skin_md5", md5s)
-        .select();
-      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
-    });
-    this.file = new DataLoader(async (md5s) => {
-      const rows = await knex("files").whereIn("skin_md5", md5s).select();
-      return md5s.map((md5) => rows.find((x) => x.skin_md5 === md5));
-    });
-    this.files = new DataLoader<string, FileRow[]>(async (md5s) => {
-      const rows = await knex("files").whereIn("skin_md5", md5s).select();
-      return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
-    });
-    this.iaItem = new DataLoader(async (md5s) => {
-      const rows = await knex("ia_items").whereIn("skin_md5", md5s).select();
-      return md5s.map((md5) => rows.find((x) => x.skin_md5 === md5));
-    });
-    this.iaItemByIdentifier = new DataLoader(async (identifiers) => {
-      const rows = await knex("ia_items")
-        .whereIn("identifier", identifiers)
-        .select();
-      return identifiers.map((identifier) =>
-        rows.find((x) => x.identifier === identifier)
-      );
-    });
-  }
+export function ctxWeakMapMemoize<T>(factory: () => T) {
+  const cache: WeakMap<UserContext, T> = new WeakMap();
+  return function get(ctx: UserContext): T {
+    if (!cache.has(ctx)) {
+      cache.set(ctx, factory());
+    }
+    // @ts-ignore We just put the value in there
+    return cache.get(ctx);
+  };
 }