From d7200892d968d13b63aad189cc669d442a8637d5 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sun, 20 Feb 2022 14:11:22 -0800 Subject: [PATCH] Implement all museum API functionality in GraphQL --- packages/skin-database/api/app.ts | 6 + .../api/graphql/SkinsConnection.ts | 40 ++++- packages/skin-database/api/graphql/index.ts | 18 +++ .../api/graphql/resolvers/MutationResolver.ts | 104 ++++++++++++ .../api/graphql/resolvers/RootResolver.ts | 35 +++- .../skin-database/api/graphql/schema.graphql | 149 ++++++++++++++++++ packages/skin-database/api/router.ts | 18 +++ 7 files changed, 365 insertions(+), 5 deletions(-) create mode 100644 packages/skin-database/api/graphql/resolvers/MutationResolver.ts diff --git a/packages/skin-database/api/app.ts b/packages/skin-database/api/app.ts index 323a1cbf..50ea5895 100644 --- a/packages/skin-database/api/app.ts +++ b/packages/skin-database/api/app.ts @@ -47,6 +47,7 @@ declare global { notify(action: ApiAction): void; log(message: string): void; logError(message: string): void; + startTime: number; session: { username: string | undefined; }; @@ -66,6 +67,11 @@ export function createApp({ eventHandler, extraMiddleware, logger }: Options) { app.use(Sentry.Handlers.requestHandler()); } + app.use(function (req, res, next) { + req.startTime = Date.now(); + next(); + }); + // https://expressjs.com/en/guide/behind-proxies.html // This is needed in order to allow `cookieSession({secure: true})` cookies to be sent. app.set("trust proxy", "loopback"); diff --git a/packages/skin-database/api/graphql/SkinsConnection.ts b/packages/skin-database/api/graphql/SkinsConnection.ts index c091f5d3..4364ffd7 100644 --- a/packages/skin-database/api/graphql/SkinsConnection.ts +++ b/packages/skin-database/api/graphql/SkinsConnection.ts @@ -2,6 +2,37 @@ import SkinModel from "../../data/SkinModel"; import * as Skins from "../../data/skins"; import { knex } from "../../db"; import SkinResolver from "./resolvers/SkinResolver"; +import LRU from "lru-cache"; + +const options = { + max: 100, + maxAge: 1000 * 60 * 60, +}; +let skinCount: number | null = null; +const cache = new LRU(options); + +// A supery hacky global cache for common requests. +async function getMuseumSkinCountFromCache() { + if (skinCount == null) { + skinCount = await Skins.getClassicSkinCount(); + } + return skinCount; +} + +// A supery hacky global cache for common requests. +async function getSkinMuseumPageFromCache(first: number, offset: number) { + const key = `${first}-${offset}`; + const cached = cache.get(key); + if (cached != null) { + return cached; + } + const skins = await Skins.getMuseumPage({ + offset: Number(offset), + first: Number(first), + }); + cache.set(key, skins); + return skins; +} export default class SkinsConnection { _first: number; @@ -43,6 +74,10 @@ export default class SkinsConnection { } async count() { + if (this._sort === "MUSEUM") { + // This is the common case, so serve it from cache. + return getMuseumSkinCountFromCache(); + } const count = await this._getQuery().count("*", { as: "count" }); return count[0].count; } @@ -54,10 +89,7 @@ export default class SkinsConnection { "We don't support combining sorting and filtering at the same time." ); } - const items = await Skins.getMuseumPage({ - first: this._first, - offset: this._offset, - }); + const items = await getSkinMuseumPageFromCache(this._first, this._offset); return Promise.all( items.map(async (item) => { const model = await SkinModel.fromMd5Assert(ctx, item.md5); diff --git a/packages/skin-database/api/graphql/index.ts b/packages/skin-database/api/graphql/index.ts index 25a4562e..4d011705 100644 --- a/packages/skin-database/api/graphql/index.ts +++ b/packages/skin-database/api/graphql/index.ts @@ -12,6 +12,23 @@ const schema = buildSchema(fs.readFileSync(schemaPath, "utf8")); const router = Router(); +const extensions = ({ + variables, + operationName, + context: req, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + document, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + result, +}) => { + const runTime = Date.now() - req.startTime; + const vars = JSON.stringify(variables); + req.log( + `Handled GraphQL Query: "${operationName}" with variables ${vars} in ${runTime}ms` + ); + return { runTime }; +}; + router.use( "/", graphqlHTTP({ @@ -29,6 +46,7 @@ router.use( stack: error.stack ? error.stack.split("\n") : [], path: error.path, }), + extensions, }) ); diff --git a/packages/skin-database/api/graphql/resolvers/MutationResolver.ts b/packages/skin-database/api/graphql/resolvers/MutationResolver.ts new file mode 100644 index 00000000..4aa12e57 --- /dev/null +++ b/packages/skin-database/api/graphql/resolvers/MutationResolver.ts @@ -0,0 +1,104 @@ +import * as Parallel from "async-parallel"; +import SkinModel from "../../../data/SkinModel"; +import * as S3 from "../../../s3"; +import * as Skins from "../../../data/skins"; +import { processUserUploads } from "../../processUserUploads"; + +// We don't use a resolver here, just return the value directly. +type UploadUrl = { id: string; url: string; md5: string }; + +class UploadMutationResolver { + async get_upload_urls( + { files }: { files: { filename: string; md5: string }[] }, + { ctx } + ): Promise { + const missing: UploadUrl[] = []; + await Parallel.each( + files, + async ({ md5, filename }) => { + if (!(await SkinModel.exists(ctx, md5))) { + const id = await Skins.recordUserUploadRequest(md5, filename); + const url = S3.getSkinUploadUrl(md5, id); + missing.push({ id, url, md5 }); + } + }, + 5 + ); + + return missing; + } + + async report_skin_uploaded({ id, md5 }, req) { + // TODO: Validate md5 and id; + await Skins.recordUserUploadComplete(md5, id); + // Don't await, just kick off the task. + processUserUploads(req.notify); + return true; + } +} + +function requireAuthed(handler) { + return (args, req) => { + if (!req.ctx.authed()) { + throw new Error("You must be logged in to read this field."); + } else { + return handler(args, req); + } + }; +} + +export default class MutationResolver { + async upload() { + return new UploadMutationResolver(); + } + async send_feedback({ message, email, url }, req) { + req.notify({ + type: "GOT_FEEDBACK", + url, + message, + email, + }); + return true; + } + + reject_skin = requireAuthed(async ({ md5 }, req) => { + req.log(`Rejecting skin with hash "${md5}"`); + const skin = await SkinModel.fromMd5Assert(req.ctx, md5); + if (skin == null) { + return false; + } + await Skins.reject(req.ctx, md5); + req.notify({ type: "REJECTED_SKIN", md5 }); + return true; + }); + + approve_skin = requireAuthed(async ({ md5 }, req) => { + req.log(`Approving skin with hash "${md5}"`); + const skin = await SkinModel.fromMd5(req.ctx, md5); + if (skin == null) { + return false; + } + await Skins.approve(req.ctx, md5); + req.notify({ type: "APPROVED_SKIN", md5 }); + return true; + }); + + mark_skin_nsfw = requireAuthed(async ({ md5 }, req) => { + req.log(`Approving skin with hash "${md5}"`); + const skin = await SkinModel.fromMd5(req.ctx, md5); + if (skin == null) { + return false; + } + await Skins.markAsNSFW(req.ctx, md5); + req.notify({ type: "MARKED_SKIN_NSFW", md5 }); + return true; + }); + + async request_nsfw_review_for_skin({ md5 }, req) { + req.log(`Reporting skin with hash "${md5}"`); + // Blow up if there is no skin with this hash + await SkinModel.fromMd5Assert(req.ctx, md5); + req.notify({ type: "REVIEW_REQUESTED", md5 }); + return true; + } +} diff --git a/packages/skin-database/api/graphql/resolvers/RootResolver.ts b/packages/skin-database/api/graphql/resolvers/RootResolver.ts index fb7626c5..3367f1e7 100644 --- a/packages/skin-database/api/graphql/resolvers/RootResolver.ts +++ b/packages/skin-database/api/graphql/resolvers/RootResolver.ts @@ -7,13 +7,17 @@ import UserResolver from "../resolvers/UserResolver"; import SkinsConnection from "../SkinsConnection"; import TweetsConnection from "../TweetsConnection"; import InternetArchiveItemResolver from "./InternetArchiveItemResolver"; +import * as Skins from "../../../data/Skins"; + import algoliasearch from "algoliasearch"; +import MutationResolver from "./MutationResolver"; +import { knex } from "../../../db"; // These keys are already in the web client, so they are not secret at all. const client = algoliasearch("HQ9I5Z6IM5", "6466695ec3f624a5fccf46ec49680e51"); const index = client.initIndex("Skins"); -class RootResolver { +class RootResolver extends MutationResolver { async fetch_skin_by_md5({ md5 }, { ctx }) { const skin = await SkinModel.fromMd5(ctx, md5); if (skin == null) { @@ -61,6 +65,11 @@ class RootResolver { } return new SkinsConnection(first, offset, sort, filter); } + async skin_to_review(_args, { ctx }) { + const { md5 } = await Skins.getSkinToReview(); + const model = await SkinModel.fromMd5Assert(ctx, md5); + return new SkinResolver(model); + } async tweets({ first, offset, sort }) { if (first > 1000) { throw new Error("Maximum limit is 1000"); @@ -70,6 +79,30 @@ class RootResolver { me() { return new UserResolver(); } + async upload_statuses_by_md5({ md5s }, { ctx }) { + return this._upload_statuses({ keyName: "md5", keys: md5s }, ctx); + } + + async upload_statuses({ ids }, { ctx }) { + return this._upload_statuses({ keyName: "id", keys: ids }, ctx); + } + + // Shared implementation for upload_statuses and upload_statuses_by_md5 + async _upload_statuses({ keyName, keys }, ctx) { + const skins = await knex("skin_uploads") + .whereIn(keyName, keys) + .orderBy("id", "desc") + .select("id", "skin_md5", "status"); + + return Promise.all( + skins.map(async ({ id, skin_md5, status }) => { + // TODO: Could we avoid fetching the skin if it's not read? + const skinModel = await SkinModel.fromMd5(ctx, skin_md5); + const skin = skinModel == null ? null : new SkinResolver(skinModel); + return { id, skin, status, upload_md5: skin_md5 }; + }) + ); + } } export default RootResolver; diff --git a/packages/skin-database/api/graphql/schema.graphql b/packages/skin-database/api/graphql/schema.graphql index 21c571be..6ca50d6f 100644 --- a/packages/skin-database/api/graphql/schema.graphql +++ b/packages/skin-database/api/graphql/schema.graphql @@ -268,6 +268,137 @@ enum SkinsFilterOption { TWEETED } +""" +The current status of a pending upload. + +**Note:** Expect more values here as we try to be more transparent about +the status of a pending uploads. +""" +enum SkinUploadStatus { + """ + The user has requested a URL, but the skin has not yet been processed. + """ + URL_REQUESTED + + """ + The user has notified us that the skin has been uploaded, but we haven't yet + processed it. + """ + UPLOAD_REPORTED + + """ + An error occured processing the skin. Usually this is a transient error, and + the skin will be retried at a later time. + """ + ERRORED + + """ + The skin has been successfully added to the Museum. + """ + ARCHIVED +} + +""" +Information about an attempt to upload a skin to the Museum. +""" +type SkinUpload { + id: String + status: SkinUploadStatus + + """ + Skin that was uploaded. **Note:** This is null if the skin has not yet been + fully processed. (status == ARCHIVED) + """ + skin: Skin + + """ + Md5 hash given when requesting the upload URL. + """ + upload_md5: String +} + +""" +A URL that the client can use to upload a skin to S3, and then notify the server +when they're done. +""" +type UploadUrl { + id: String + url: String + md5: String +} + +""" +Input object used for a user to request an UploadUrl +""" +input UploadUrlRequest { + md5: String! + filename: String! +} + +""" +Mutations for the upload flow + +1. The user finds the md5 hash of their local files. +2. (`get_upload_urls`) The user requests upload URLs for each of their files. +3. The server returns upload URLs for each of their files which are not already in the collection. +4. The user uploads each of their files to the URLs returned in step 3. +5. (`report_skin_uploaded`) The user notifies the server that they're done uploading. +6. (TODO) The user polls for the status of their uploads. +""" +type UploadMutations { + """ + Get a (possibly incompelte) list of UploadUrls for each of the files. If an + UploadUrl is not returned for a given hash, it means the file is already in + the collection. + """ + get_upload_urls(files: [UploadUrlRequest!]!): [UploadUrl] + + """ + Notify the server that the user is done uploading. + """ + report_skin_uploaded(id: String!, md5: String!): Boolean +} + +type Mutation { + """ + Mutations for the upload flow + """ + upload: UploadMutations + + """ + Send a message to the admin of the site. Currently this appears in Discord. + """ + send_feedback(message: String!, email: String, url: String): Boolean + + """ + Reject skin for tweeting + + **Note:** Requires being logged in + """ + reject_skin(md5: String): Boolean + + """ + Approve skin for tweeting + + **Note:** Requires being logged in + """ + approve_skin(md5: String): Boolean + + """ + Mark a skin as NSFW + + **Note:** Requires being logged in + """ + mark_skin_nsfw(md5: String): Boolean + + """ + Request that an admin check if this skin is NSFW. + Unlike other review mutaiton endpoints, this one does not require being logged + in. + """ + request_nsfw_review_for_skin(md5: String): Boolean +} + type Query { """ The currently authenticated user, if any. @@ -284,6 +415,11 @@ type Query { """ fetch_tweet_by_url(url: String!): Tweet + """ + A random skin that needs to be reviewed + """ + skin_to_review: Skin + """ Get an archive.org item by its identifier. You can find this in the URL: @@ -320,4 +456,17 @@ type Query { offset: Int = 0 sort: TweetsSortOption ): TweetsConnection + + """ + Get the status of a batch of uploads by md5s + """ + upload_statuses_by_md5(md5s: [String!]!): [SkinUpload] + @deprecated( + reason: "Prefer `upload_statuses` instead, were we operate on ids." + ) + + """ + Get the status of a batch of uploads by ids + """ + upload_statuses(ids: [String!]!): [SkinUpload] } diff --git a/packages/skin-database/api/router.ts b/packages/skin-database/api/router.ts index 8d8f35fa..9d03261c 100644 --- a/packages/skin-database/api/router.ts +++ b/packages/skin-database/api/router.ts @@ -23,6 +23,7 @@ const options = { let skinCount: number | null = null; const cache = new LRU(options); +// Purposefully REST router.get( "/auth/", asyncHandler(async (req, res) => { @@ -37,6 +38,7 @@ router.get( }) ); +// @deprecated Use GraphQL router.get( "/authed/", asyncHandler(async (req, res) => { @@ -44,6 +46,7 @@ router.get( }) ); +// Purposefully REST router.get( "/auth/discord", asyncHandler(async (req, res) => { @@ -65,6 +68,7 @@ router.get( }) ); +// @deprecated Use GraphQL router.get( "/skins/", asyncHandler(async (req, res) => { @@ -93,6 +97,7 @@ router.get( }) ); +// @deprecated Use GraphQL router.post( "/skins/get_upload_urls", asyncHandler(async (req, res) => { @@ -113,6 +118,7 @@ router.post( }) ); +// @deprecated Use GraphQL router.post( "/feedback", asyncHandler(async (req, res) => { @@ -131,6 +137,7 @@ router.post( }) ); +// @deprecate Use GraphQL router.post( "/skins/status", asyncHandler(async (req, res) => { @@ -139,6 +146,7 @@ router.post( }) ); +// @deprecated Use GraphQL router.get( "/skins/:md5", asyncHandler(async (req, res) => { @@ -157,6 +165,7 @@ router.get( }) ); +// @deprecated Use GraphQL router.get( "/skins/:md5/metadata", asyncHandler(async (req, res) => { @@ -176,6 +185,7 @@ router.get( }) ); +// @deprecated Use GraphQL router.get( "/skins/:md5/debug", asyncHandler(async (req, res) => { @@ -199,6 +209,7 @@ function requireAuthed(req, res, next) { } } +// @deprecated Use GraphQL router.get( "/to_review", requireAuthed, @@ -208,6 +219,7 @@ router.get( }) ); +// @deprecated Use GraphQL router.post( "/skins/:md5/reject", requireAuthed, @@ -225,6 +237,7 @@ router.post( }) ); +// @deprecated Use GraphQL router.post( "/skins/:md5/approve", requireAuthed, @@ -242,6 +255,7 @@ router.post( }) ); +// @deprecated Use GraphQL // Unlike /report, this marks the skin NSFW right away without sending to // Discord. Because of this, it requires auth. router.post( @@ -261,6 +275,7 @@ router.post( }) ); +// @deprecated Use GraphQL router.post( "/skins/:md5/report", asyncHandler(async (req, res) => { @@ -273,6 +288,7 @@ router.post( }) ); +// @deprecated Use GraphQL // User reports that they uploaded a skin router.post( "/skins/:md5/uploaded", @@ -290,6 +306,7 @@ router.post( }) ); +// @deprecated Use GraphQL router.get( "/approved", asyncHandler(async (req, res) => { @@ -298,6 +315,7 @@ router.get( }) ); +// @deprecated Special purpose URL router.get( "/stylegan.json", asyncHandler(async (req, res) => {