Implement all museum API functionality in GraphQL

packages/skin-database/api/app.ts

@@ -47,6 +47,7 @@ declare global {
       notify(action: ApiAction): void;
       log(message: string): void;
       logError(message: string): void;
+      startTime: number;
       session: {
         username: string | undefined;
       };
@@ -66,6 +67,11 @@ export function createApp({ eventHandler, extraMiddleware, logger }: Options) {
     app.use(Sentry.Handlers.requestHandler());
   }
 
+  app.use(function (req, res, next) {
+    req.startTime = Date.now();
+    next();
+  });
+
   // https://expressjs.com/en/guide/behind-proxies.html
   // This is needed in order to allow `cookieSession({secure: true})` cookies to be sent.
   app.set("trust proxy", "loopback");

packages/skin-database/api/graphql/SkinsConnection.ts

@@ -2,6 +2,37 @@ import SkinModel from "../../data/SkinModel";
 import * as Skins from "../../data/skins";
 import { knex } from "../../db";
 import SkinResolver from "./resolvers/SkinResolver";
+import LRU from "lru-cache";
+
+const options = {
+  max: 100,
+  maxAge: 1000 * 60 * 60,
+};
+let skinCount: number | null = null;
+const cache = new LRU<string, Skins.MuseumPage>(options);
+
+// A supery hacky global cache for common requests.
+async function getMuseumSkinCountFromCache() {
+  if (skinCount == null) {
+    skinCount = await Skins.getClassicSkinCount();
+  }
+  return skinCount;
+}
+
+// A supery hacky global cache for common requests.
+async function getSkinMuseumPageFromCache(first: number, offset: number) {
+  const key = `${first}-${offset}`;
+  const cached = cache.get(key);
+  if (cached != null) {
+    return cached;
+  }
+  const skins = await Skins.getMuseumPage({
+    offset: Number(offset),
+    first: Number(first),
+  });
+  cache.set(key, skins);
+  return skins;
+}
 
 export default class SkinsConnection {
   _first: number;
@@ -43,6 +74,10 @@ export default class SkinsConnection {
   }
 
   async count() {
+    if (this._sort === "MUSEUM") {
+      // This is the common case, so serve it from cache.
+      return getMuseumSkinCountFromCache();
+    }
     const count = await this._getQuery().count("*", { as: "count" });
     return count[0].count;
   }
@@ -54,10 +89,7 @@ export default class SkinsConnection {
           "We don't support combining sorting and filtering at the same time."
         );
       }
-      const items = await Skins.getMuseumPage({
-        first: this._first,
-        offset: this._offset,
-      });
+      const items = await getSkinMuseumPageFromCache(this._first, this._offset);
       return Promise.all(
         items.map(async (item) => {
           const model = await SkinModel.fromMd5Assert(ctx, item.md5);

packages/skin-database/api/graphql/index.ts

@@ -12,6 +12,23 @@ const schema = buildSchema(fs.readFileSync(schemaPath, "utf8"));
 
 const router = Router();
 
+const extensions = ({
+  variables,
+  operationName,
+  context: req,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  document,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  result,
+}) => {
+  const runTime = Date.now() - req.startTime;
+  const vars = JSON.stringify(variables);
+  req.log(
+    `Handled GraphQL Query: "${operationName}" with variables ${vars} in ${runTime}ms`
+  );
+  return { runTime };
+};
+
 router.use(
   "/",
   graphqlHTTP({
@@ -29,6 +46,7 @@ router.use(
       stack: error.stack ? error.stack.split("\n") : [],
       path: error.path,
     }),
+    extensions,
   })
 );
 

packages/skin-database/api/graphql/resolvers/MutationResolver.ts

@@ -0,0 +1,104 @@
+import * as Parallel from "async-parallel";
+import SkinModel from "../../../data/SkinModel";
+import * as S3 from "../../../s3";
+import * as Skins from "../../../data/skins";
+import { processUserUploads } from "../../processUserUploads";
+
+// We don't use a resolver here, just return the value directly.
+type UploadUrl = { id: string; url: string; md5: string };
+
+class UploadMutationResolver {
+  async get_upload_urls(
+    { files }: { files: { filename: string; md5: string }[] },
+    { ctx }
+  ): Promise<UploadUrl[]> {
+    const missing: UploadUrl[] = [];
+    await Parallel.each(
+      files,
+      async ({ md5, filename }) => {
+        if (!(await SkinModel.exists(ctx, md5))) {
+          const id = await Skins.recordUserUploadRequest(md5, filename);
+          const url = S3.getSkinUploadUrl(md5, id);
+          missing.push({ id, url, md5 });
+        }
+      },
+      5
+    );
+
+    return missing;
+  }
+
+  async report_skin_uploaded({ id, md5 }, req) {
+    // TODO: Validate md5 and id;
+    await Skins.recordUserUploadComplete(md5, id);
+    // Don't await, just kick off the task.
+    processUserUploads(req.notify);
+    return true;
+  }
+}
+
+function requireAuthed(handler) {
+  return (args, req) => {
+    if (!req.ctx.authed()) {
+      throw new Error("You must be logged in to read this field.");
+    } else {
+      return handler(args, req);
+    }
+  };
+}
+
+export default class MutationResolver {
+  async upload() {
+    return new UploadMutationResolver();
+  }
+  async send_feedback({ message, email, url }, req) {
+    req.notify({
+      type: "GOT_FEEDBACK",
+      url,
+      message,
+      email,
+    });
+    return true;
+  }
+
+  reject_skin = requireAuthed(async ({ md5 }, req) => {
+    req.log(`Rejecting skin with hash "${md5}"`);
+    const skin = await SkinModel.fromMd5Assert(req.ctx, md5);
+    if (skin == null) {
+      return false;
+    }
+    await Skins.reject(req.ctx, md5);
+    req.notify({ type: "REJECTED_SKIN", md5 });
+    return true;
+  });
+
+  approve_skin = requireAuthed(async ({ md5 }, req) => {
+    req.log(`Approving skin with hash "${md5}"`);
+    const skin = await SkinModel.fromMd5(req.ctx, md5);
+    if (skin == null) {
+      return false;
+    }
+    await Skins.approve(req.ctx, md5);
+    req.notify({ type: "APPROVED_SKIN", md5 });
+    return true;
+  });
+
+  mark_skin_nsfw = requireAuthed(async ({ md5 }, req) => {
+    req.log(`Approving skin with hash "${md5}"`);
+    const skin = await SkinModel.fromMd5(req.ctx, md5);
+    if (skin == null) {
+      return false;
+    }
+    await Skins.markAsNSFW(req.ctx, md5);
+    req.notify({ type: "MARKED_SKIN_NSFW", md5 });
+    return true;
+  });
+
+  async request_nsfw_review_for_skin({ md5 }, req) {
+    req.log(`Reporting skin with hash "${md5}"`);
+    // Blow up if there is no skin with this hash
+    await SkinModel.fromMd5Assert(req.ctx, md5);
+    req.notify({ type: "REVIEW_REQUESTED", md5 });
+    return true;
+  }
+}

packages/skin-database/api/graphql/resolvers/RootResolver.ts

@@ -7,13 +7,17 @@ import UserResolver from "../resolvers/UserResolver";
 import SkinsConnection from "../SkinsConnection";
 import TweetsConnection from "../TweetsConnection";
 import InternetArchiveItemResolver from "./InternetArchiveItemResolver";
+import * as Skins from "../../../data/Skins";
+
 import algoliasearch from "algoliasearch";
+import MutationResolver from "./MutationResolver";
+import { knex } from "../../../db";
 
 // These keys are already in the web client, so they are not secret at all.
 const client = algoliasearch("HQ9I5Z6IM5", "6466695ec3f624a5fccf46ec49680e51");
 const index = client.initIndex("Skins");
 
-class RootResolver {
+class RootResolver extends MutationResolver {
   async fetch_skin_by_md5({ md5 }, { ctx }) {
     const skin = await SkinModel.fromMd5(ctx, md5);
     if (skin == null) {
@@ -61,6 +65,11 @@ class RootResolver {
     }
     return new SkinsConnection(first, offset, sort, filter);
   }
+  async skin_to_review(_args, { ctx }) {
+    const { md5 } = await Skins.getSkinToReview();
+    const model = await SkinModel.fromMd5Assert(ctx, md5);
+    return new SkinResolver(model);
+  }
   async tweets({ first, offset, sort }) {
     if (first > 1000) {
       throw new Error("Maximum limit is 1000");
@@ -70,6 +79,30 @@ class RootResolver {
   me() {
     return new UserResolver();
   }
+  async upload_statuses_by_md5({ md5s }, { ctx }) {
+    return this._upload_statuses({ keyName: "md5", keys: md5s }, ctx);
+  }
+
+  async upload_statuses({ ids }, { ctx }) {
+    return this._upload_statuses({ keyName: "id", keys: ids }, ctx);
+  }
+
+  // Shared implementation for upload_statuses and upload_statuses_by_md5
+  async _upload_statuses({ keyName, keys }, ctx) {
+    const skins = await knex("skin_uploads")
+      .whereIn(keyName, keys)
+      .orderBy("id", "desc")
+      .select("id", "skin_md5", "status");
+
+    return Promise.all(
+      skins.map(async ({ id, skin_md5, status }) => {
+        // TODO: Could we avoid fetching the skin if it's not read?
+        const skinModel = await SkinModel.fromMd5(ctx, skin_md5);
+        const skin = skinModel == null ? null : new SkinResolver(skinModel);
+        return { id, skin, status, upload_md5: skin_md5 };
+      })
+    );
+  }
 }
 
 export default RootResolver;

packages/skin-database/api/graphql/schema.graphql

@@ -268,6 +268,137 @@ enum SkinsFilterOption {
   TWEETED
 }
 
+"""
+The current status of a pending upload.
+
+**Note:** Expect more values here as we try to be more transparent about
+the status of a pending uploads.
+"""
+enum SkinUploadStatus {
+  """
+  The user has requested a URL, but the skin has not yet been processed.
+  """
+  URL_REQUESTED
+
+  """
+  The user has notified us that the skin has been uploaded, but we haven't yet
+  processed it.
+  """
+  UPLOAD_REPORTED
+
+  """
+  An error occured processing the skin. Usually this is a transient error, and
+  the skin will be retried at a later time.
+  """
+  ERRORED
+
+  """
+  The skin has been successfully added to the Museum.
+  """
+  ARCHIVED
+}
+
+"""
+Information about an attempt to upload a skin to the Museum.
+"""
+type SkinUpload {
+  id: String
+  status: SkinUploadStatus
+
+  """
+  Skin that was uploaded. **Note:** This is null if the skin has not yet been
+  fully processed. (status == ARCHIVED)
+  """
+  skin: Skin
+
+  """
+  Md5 hash given when requesting the upload URL.
+  """
+  upload_md5: String
+}
+
+"""
+A URL that the client can use to upload a skin to S3, and then notify the server
+when they're done.
+"""
+type UploadUrl {
+  id: String
+  url: String
+  md5: String
+}
+
+"""
+Input object used for a user to request an UploadUrl
+"""
+input UploadUrlRequest {
+  md5: String!
+  filename: String!
+}
+
+"""
+Mutations for the upload flow
+
+1. The user finds the md5 hash of their local files.
+2. (`get_upload_urls`) The user requests upload URLs for each of their files.
+3. The server returns upload URLs for each of their files which are not already in the collection.
+4. The user uploads each of their files to the URLs returned in step 3.
+5. (`report_skin_uploaded`) The user notifies the server that they're done uploading.
+6. (TODO) The user polls for the status of their uploads.
+"""
+type UploadMutations {
+  """
+  Get a (possibly incompelte) list of UploadUrls for each of the files. If an
+  UploadUrl is not returned for a given hash, it means the file is already in
+  the collection.
+  """
+  get_upload_urls(files: [UploadUrlRequest!]!): [UploadUrl]
+
+  """
+  Notify the server that the user is done uploading.
+  """
+  report_skin_uploaded(id: String!, md5: String!): Boolean
+}
+
+type Mutation {
+  """
+  Mutations for the upload flow
+  """
+  upload: UploadMutations
+
+  """
+  Send a message to the admin of the site. Currently this appears in Discord.
+  """
+  send_feedback(message: String!, email: String, url: String): Boolean
+
+  """
+  Reject skin for tweeting
+
+  **Note:** Requires being logged in
+  """
+  reject_skin(md5: String): Boolean
+
+  """
+  Approve skin for tweeting
+
+  **Note:** Requires being logged in
+  """
+  approve_skin(md5: String): Boolean
+
+  """
+  Mark a skin as NSFW
+
+  **Note:** Requires being logged in
+  """
+  mark_skin_nsfw(md5: String): Boolean
+
+  """
+  Request that an admin check if this skin is NSFW.
+  Unlike other review mutaiton endpoints, this one does not require being logged
+  in.
+  """
+  request_nsfw_review_for_skin(md5: String): Boolean
+}
+
 type Query {
   """
   The currently authenticated user, if any.
@@ -284,6 +415,11 @@ type Query {
   """
   fetch_tweet_by_url(url: String!): Tweet
 
+  """
+  A random skin that needs to be reviewed
+  """
+  skin_to_review: Skin
+
   """
   Get an archive.org item by its identifier. You can find this in the URL:
 
@@ -320,4 +456,17 @@ type Query {
     offset: Int = 0
     sort: TweetsSortOption
   ): TweetsConnection
+
+  """
+  Get the status of a batch of uploads by md5s
+  """
+  upload_statuses_by_md5(md5s: [String!]!): [SkinUpload]
+    @deprecated(
+      reason: "Prefer `upload_statuses` instead, were we operate on ids."
+    )
+
+  """
+  Get the status of a batch of uploads by ids
+  """
+  upload_statuses(ids: [String!]!): [SkinUpload]
 }

packages/skin-database/api/router.ts

@@ -23,6 +23,7 @@ const options = {
 let skinCount: number | null = null;
 const cache = new LRU<string, MuseumPage>(options);
 
+// Purposefully REST
 router.get(
   "/auth/",
   asyncHandler(async (req, res) => {
@@ -37,6 +38,7 @@ router.get(
   })
 );
 
+// @deprecated Use GraphQL
 router.get(
   "/authed/",
   asyncHandler(async (req, res) => {
@@ -44,6 +46,7 @@ router.get(
   })
 );
 
+// Purposefully REST
 router.get(
   "/auth/discord",
   asyncHandler(async (req, res) => {
@@ -65,6 +68,7 @@ router.get(
   })
 );
 
+// @deprecated Use GraphQL
 router.get(
   "/skins/",
   asyncHandler(async (req, res) => {
@@ -93,6 +97,7 @@ router.get(
   })
 );
 
+// @deprecated Use GraphQL
 router.post(
   "/skins/get_upload_urls",
   asyncHandler(async (req, res) => {
@@ -113,6 +118,7 @@ router.post(
   })
 );
 
+// @deprecated Use GraphQL
 router.post(
   "/feedback",
   asyncHandler(async (req, res) => {
@@ -131,6 +137,7 @@ router.post(
   })
 );
 
+// @deprecate Use GraphQL
 router.post(
   "/skins/status",
   asyncHandler(async (req, res) => {
@@ -139,6 +146,7 @@ router.post(
   })
 );
 
+// @deprecated Use GraphQL
 router.get(
   "/skins/:md5",
   asyncHandler(async (req, res) => {
@@ -157,6 +165,7 @@ router.get(
   })
 );
 
+// @deprecated Use GraphQL
 router.get(
   "/skins/:md5/metadata",
   asyncHandler(async (req, res) => {
@@ -176,6 +185,7 @@ router.get(
   })
 );
 
+// @deprecated Use GraphQL
 router.get(
   "/skins/:md5/debug",
   asyncHandler(async (req, res) => {
@@ -199,6 +209,7 @@ function requireAuthed(req, res, next) {
   }
 }
 
+// @deprecated Use GraphQL
 router.get(
   "/to_review",
   requireAuthed,
@@ -208,6 +219,7 @@ router.get(
   })
 );
 
+// @deprecated Use GraphQL
 router.post(
   "/skins/:md5/reject",
   requireAuthed,
@@ -225,6 +237,7 @@ router.post(
   })
 );
 
+// @deprecated Use GraphQL
 router.post(
   "/skins/:md5/approve",
   requireAuthed,
@@ -242,6 +255,7 @@ router.post(
   })
 );
 
+// @deprecated Use GraphQL
 // Unlike /report, this marks the skin NSFW right away without sending to
 // Discord. Because of this, it requires auth.
 router.post(
@@ -261,6 +275,7 @@ router.post(
   })
 );
 
+// @deprecated Use GraphQL
 router.post(
   "/skins/:md5/report",
   asyncHandler(async (req, res) => {
@@ -273,6 +288,7 @@ router.post(
   })
 );
 
+// @deprecated Use GraphQL
 // User reports that they uploaded a skin
 router.post(
   "/skins/:md5/uploaded",
@@ -290,6 +306,7 @@ router.post(
   })
 );
 
+// @deprecated Use GraphQL
 router.get(
   "/approved",
   asyncHandler(async (req, res) => {
@@ -298,6 +315,7 @@ router.get(
   })
 );
 
+// @deprecated Special purpose URL
 router.get(
   "/stylegan.json",
   asyncHandler(async (req, res) => {