diff --git a/packages/skin-database/api/__mocks__/processUserUploads.ts b/packages/skin-database/api/__mocks__/processUserUploads.ts index 5660c3a5..2868e359 100644 --- a/packages/skin-database/api/__mocks__/processUserUploads.ts +++ b/packages/skin-database/api/__mocks__/processUserUploads.ts @@ -1,3 +1,3 @@ -export async function processUserUploads() { +export const processUserUploads = jest.fn(async () => { // Mock -} +}); diff --git a/packages/skin-database/api/__tests__/router.test.ts b/packages/skin-database/api/__tests__/router.test.ts index 574ad6f1..c64f0e8f 100644 --- a/packages/skin-database/api/__tests__/router.test.ts +++ b/packages/skin-database/api/__tests__/router.test.ts @@ -3,6 +3,7 @@ import { knex } from "../../db"; import request from "supertest"; // supertest is a framework that allows to easily test web apis import { createApp } from "../app"; import * as S3 from "../../s3"; +import { processUserUploads } from "../processUserUploads"; jest.mock("../../s3"); jest.mock("../processUserUploads"); @@ -135,8 +136,10 @@ test("/skins/get_upload_urls", async () => { }); test("An Upload Flow", async () => { + // Request an upload URL const md5 = "3b73bcd43c30b85d4cad3083e8ac9695"; - const skins = { [md5]: "a_fake_new_file.wsz" }; + const filename = "a_fake_new_file.wsz"; + const skins = { [md5]: filename }; const getUrlsResponse = await request(app) .post("/skins/get_upload_urls") .send({ skins }); @@ -147,11 +150,29 @@ test("An Upload Flow", async () => { [md5]: { id: expect.any(Number), url: "" }, }); + const requestedUpload = await knex("skin_uploads").where({ id }).first(); + expect(requestedUpload).toEqual({ + filename, + id, + skin_md5: md5, + status: "URL_REQUESTED", + }); + + // Report that we've uploaded the skin to S3 (we lie) const uploadedResponse = await request(app) .post(`/skins/${md5}/uploaded`) .query({ id }) .send({ skins }); expect(uploadedResponse.body).toEqual({ done: true }); + expect(processUserUploads).toHaveBeenCalled(); + + const reportedUpload = await knex("skin_uploads").where({ id }).first(); + expect(reportedUpload).toEqual({ + filename, + id, + skin_md5: md5, + status: "UPLOAD_REPORTED", + }); }); test("/stylegan.json", async () => { diff --git a/packages/skin-database/data/FileModel.ts b/packages/skin-database/data/FileModel.ts index 12fd737e..0b113a4b 100644 --- a/packages/skin-database/data/FileModel.ts +++ b/packages/skin-database/data/FileModel.ts @@ -1,6 +1,8 @@ import path from "path"; -import UserContext from "./UserContext"; +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; import { FileRow } from "../types"; +import DataLoader from "dataloader"; +import { knex } from "../db"; export type FileDebugData = { row: FileRow; @@ -10,7 +12,7 @@ export default class FileModel { constructor(readonly ctx: UserContext, readonly row: FileRow) {} static async fromMd5(ctx: UserContext, md5: string): Promise { - const rows = await ctx.files.load(md5); + const rows = await getFilesLoader(ctx).load(md5); return rows.map((row) => new FileModel(ctx, row)); } @@ -24,3 +26,11 @@ export default class FileModel { }; } } + +const getFilesLoader = ctxWeakMapMemoize>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("files").whereIn("skin_md5", md5s).select(); + return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); + }) +); diff --git a/packages/skin-database/data/IaItemModel.ts b/packages/skin-database/data/IaItemModel.ts index 8b875574..2f272849 100644 --- a/packages/skin-database/data/IaItemModel.ts +++ b/packages/skin-database/data/IaItemModel.ts @@ -1,6 +1,8 @@ -import UserContext from "./UserContext"; +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; import { IaItemRow } from "../types"; import SkinModel from "./SkinModel"; +import DataLoader from "dataloader"; +import { knex } from "../db"; const IA_URL = /^(https:\/\/)?archive.org\/details\/([^/]+)\/?/; @@ -11,7 +13,7 @@ export default class IaItemModel { ctx: UserContext, md5: string ): Promise { - const row = await ctx.iaItem.load(md5); + const row = await getIaItemLoader(ctx).load(md5); return row == null ? null : new IaItemModel(ctx, row); } @@ -19,7 +21,7 @@ export default class IaItemModel { ctx: UserContext, identifier: string ): Promise { - const row = await ctx.iaItemByIdentifier.load(identifier); + const row = await getIaItemByItentifierLoader(ctx).load(identifier); return row == null ? null : new IaItemModel(ctx, row); } @@ -64,3 +66,25 @@ export default class IaItemModel { return identifier; } } + +const getIaItemLoader = ctxWeakMapMemoize>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("ia_items").whereIn("skin_md5", md5s).select(); + return md5s.map((md5) => rows.find((x) => x.skin_md5 === md5)); + }) +); + +const getIaItemByItentifierLoader = ctxWeakMapMemoize< + DataLoader +>( + () => + new DataLoader(async (identifiers) => { + const rows = await knex("ia_items") + .whereIn("identifier", identifiers) + .select(); + return identifiers.map((identifier) => + rows.find((x) => x.identifier === identifier) + ); + }) +); diff --git a/packages/skin-database/data/SkinModel.ts b/packages/skin-database/data/SkinModel.ts index 45650800..a123bca8 100644 --- a/packages/skin-database/data/SkinModel.ts +++ b/packages/skin-database/data/SkinModel.ts @@ -1,10 +1,12 @@ import { getScreenshotUrl, getSkinUrl } from "./skins"; import { TweetStatus, SkinRow, ReviewRow } from "../types"; -import UserContext from "./UserContext"; +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; import TweetModel, { TweetDebugData } from "./TweetModel"; import IaItemModel from "./IaItemModel"; import FileModel, { FileDebugData } from "./FileModel"; import { MD5_REGEX } from "../utils"; +import DataLoader from "dataloader"; +import { knex } from "../db"; export default class SkinModel { constructor(readonly ctx: UserContext, readonly row: SkinRow) {} @@ -13,7 +15,7 @@ export default class SkinModel { ctx: UserContext, md5: string ): Promise { - const row = await ctx.skin.load(md5); + const row = await getSkinLoader(ctx).load(md5); return row == null ? null : new SkinModel(ctx, row); } @@ -34,7 +36,7 @@ export default class SkinModel { } static async exists(ctx: UserContext, md5: string): Promise { - const row = await ctx.skin.load(md5); + const row = await getSkinLoader(ctx).load(md5); return row != null; } @@ -48,8 +50,7 @@ export default class SkinModel { } async getTweets(): Promise { - const rows = await this.ctx.tweets.load(this.row.md5); - return rows.map((row) => new TweetModel(this.ctx, row)); + return TweetModel.fromMd5(this.ctx, this.row.md5); } getIaItem(): Promise { @@ -57,7 +58,7 @@ export default class SkinModel { } getReviews(): Promise { - return this.ctx.reviews.load(this.row.md5); + return getReviewsLoader(this.ctx).load(this.row.md5); } getFiles(): Promise { @@ -136,3 +137,21 @@ export default class SkinModel { }; } } + +const getSkinLoader = ctxWeakMapMemoize>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("skins").whereIn("md5", md5s).select(); + return md5s.map((md5) => rows.find((x) => x.md5 === md5)); + }) +); + +const getReviewsLoader = ctxWeakMapMemoize>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("skin_reviews") + .whereIn("skin_md5", md5s) + .select(); + return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); + }) +); diff --git a/packages/skin-database/data/TweetModel.ts b/packages/skin-database/data/TweetModel.ts index 8048e37f..596a23b9 100644 --- a/packages/skin-database/data/TweetModel.ts +++ b/packages/skin-database/data/TweetModel.ts @@ -1,5 +1,7 @@ -import UserContext from "./UserContext"; +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; import { TweetRow } from "../types"; +import DataLoader from "dataloader"; +import { knex } from "../db"; export type TweetDebugData = { row: TweetRow; @@ -9,7 +11,7 @@ export default class TweetModel { constructor(readonly ctx: UserContext, readonly row: TweetRow) {} static async fromMd5(ctx: UserContext, md5: string): Promise { - const rows = await ctx.tweets.load(md5); + const rows = await getTweetsLoader(ctx).load(md5); return rows.map((row) => new TweetModel(ctx, row)); } @@ -29,3 +31,11 @@ export default class TweetModel { }; } } + +const getTweetsLoader = ctxWeakMapMemoize>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("tweets").whereIn("skin_md5", md5s).select(); + return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); + }) +); diff --git a/packages/skin-database/data/UserContext.ts b/packages/skin-database/data/UserContext.ts index 172822bd..6a07b43f 100644 --- a/packages/skin-database/data/UserContext.ts +++ b/packages/skin-database/data/UserContext.ts @@ -1,50 +1,13 @@ -import { knex } from "../db"; +// Currently only used as a WeakMap key +export default class UserContext {} -import DataLoader from "dataloader"; -import { SkinRow, TweetRow, ReviewRow, FileRow, IaItemRow } from "../types"; - -export default class UserContext { - skin: DataLoader; - tweets: DataLoader; - reviews: DataLoader; - file: DataLoader; - files: DataLoader; - iaItem: DataLoader; - iaItemByIdentifier: DataLoader; - constructor() { - this.skin = new DataLoader(async (md5s) => { - const rows = await knex("skins").whereIn("md5", md5s).select(); - return md5s.map((md5) => rows.find((x) => x.md5 === md5)); - }); - this.tweets = new DataLoader(async (md5s) => { - const rows = await knex("tweets").whereIn("skin_md5", md5s).select(); - return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); - }); - this.reviews = new DataLoader(async (md5s) => { - const rows = await knex("skin_reviews") - .whereIn("skin_md5", md5s) - .select(); - return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); - }); - this.file = new DataLoader(async (md5s) => { - const rows = await knex("files").whereIn("skin_md5", md5s).select(); - return md5s.map((md5) => rows.find((x) => x.skin_md5 === md5)); - }); - this.files = new DataLoader(async (md5s) => { - const rows = await knex("files").whereIn("skin_md5", md5s).select(); - return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); - }); - this.iaItem = new DataLoader(async (md5s) => { - const rows = await knex("ia_items").whereIn("skin_md5", md5s).select(); - return md5s.map((md5) => rows.find((x) => x.skin_md5 === md5)); - }); - this.iaItemByIdentifier = new DataLoader(async (identifiers) => { - const rows = await knex("ia_items") - .whereIn("identifier", identifiers) - .select(); - return identifiers.map((identifier) => - rows.find((x) => x.identifier === identifier) - ); - }); - } +export function ctxWeakMapMemoize(factory: () => T) { + const cache: WeakMap = new WeakMap(); + return function get(ctx: UserContext): T { + if (!cache.has(ctx)) { + cache.set(ctx, factory()); + } + // @ts-ignore We just put the value in there + return cache.get(ctx); + }; }