From 4165b9a4f5eb19f8d585cfe54e4a6d552a0cfd45 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sun, 20 Feb 2022 03:02:27 -0800 Subject: [PATCH] Add search, metadata and split some stuff up. --- .../skin-database/api/graphql/defaultQuery.ts | 51 +++ packages/skin-database/api/graphql/index.ts | 305 +----------------- .../resolvers/InternetArchiveItemResolver.ts | 11 + .../api/graphql/resolvers/RootResolver.ts | 75 +++++ packages/skin-database/api/graphql/schema.ts | 242 ++++++++++++++ packages/skin-database/data/IaItemModel.ts | 16 + packages/skin-database/types.ts | 1 + 7 files changed, 400 insertions(+), 301 deletions(-) create mode 100644 packages/skin-database/api/graphql/defaultQuery.ts create mode 100644 packages/skin-database/api/graphql/resolvers/RootResolver.ts create mode 100644 packages/skin-database/api/graphql/schema.ts diff --git a/packages/skin-database/api/graphql/defaultQuery.ts b/packages/skin-database/api/graphql/defaultQuery.ts new file mode 100644 index 00000000..6cf24602 --- /dev/null +++ b/packages/skin-database/api/graphql/defaultQuery.ts @@ -0,0 +1,51 @@ +const DEFAULT_QUERY = `# Winamp Skins GraphQL API +# +# https://skins.webamp.org +# +# This is a GraphQL API for the Winamp Skin Museum's database. +# It's mostly intended for exploring the data set. Please feel +# free to have a look around and see what you can find! +# +# The GraphiQL environment has many helpful features to help +# you discover what fields exist and what they mean, so be bold! +# +# If you have any questions or feedback, please get in touch: +# - Twitter: @captbaritone +# - Email: jordan@jordaneldredge.com +# - Discord: https://webamp.org/chat + + +# An example query to get you started... +query MyQuery { + # Get info about a @winampskins tweet + # (spoiler, it's Luigihann's ZeldaAmp) + fetch_tweet_by_url(url: "https://twitter.com/winampskins/status/1056605906597629953") { + skin { + + # The filename of the skin that the tweet is about + filename + download_url + screenshot_url + museum_url + + # All the tweets that shared this skin + tweets { + likes + retweets + url + } + + # Information about the files contained within the skin + archive_files { + filename + # For image files, try hovering the + # returned url ---> + url + # Date the file was created according to the zip file + date + } + } + } +}`; + +export default DEFAULT_QUERY; diff --git a/packages/skin-database/api/graphql/index.ts b/packages/skin-database/api/graphql/index.ts index 8075f8f1..0819ff20 100644 --- a/packages/skin-database/api/graphql/index.ts +++ b/packages/skin-database/api/graphql/index.ts @@ -1,309 +1,12 @@ import { Router } from "express"; import { graphqlHTTP } from "express-graphql"; -import { buildSchema } from "graphql"; -import SkinModel from "../../data/SkinModel"; -import TweetModel from "../../data/TweetModel"; -import SkinResolver from "./resolvers/SkinResolver"; -import TweetResolver from "./resolvers/TweetResolver"; -import UserResolver from "./resolvers/UserResolver"; -import SkinsConnection from "./SkinsConnection"; -import TweetsConnection from "./TweetsConnection"; +import RootResolver from "./resolvers/RootResolver"; +import schema from "./schema"; +import DEFAULT_QUERY from "./defaultQuery"; const router = Router(); -const schema = buildSchema(` -"""A classic Winamp skin""" -type Skin { - """Database ID of the skin""" - id: Int, - - """MD5 hash of the skin's file""" - md5: String, - - """URL of the skin on the Winamp Skin Museum""" - museum_url: String, - - """URL of webamp.org with the skin loaded""" - webamp_url: String, - - """URL of a screenshot of the skin""" - screenshot_url: String, - - """URL to download the skin""" - download_url: String, - - """ - Filename of skin when uploaded to the Museum. Note: In some cases a skin - has been uploaded under multiple names. Here we just pick one. - """ - filename: String, - - """Text of the readme file extracted from the skin""" - readme_text: String, - - """Has the skin been flagged as "not safe for wrok"?""" - nsfw: Boolean, - - """String representation (rgb usually) of the skin's average color""" - average_color: String, - - """Has the skin been tweeted?""" - tweeted: Boolean - - """List of @winampskins tweets that mentioned the skin.""" - tweets: [Tweet] - - """List of files contained within the skin's .wsz archive""" - archive_files: [ArchiveFile] - - """The skin's "item" at archive.org""" - internet_archive_item: InternetArchiveItem - - """ - Times that the skin has been reviewed either on the Museum's Tinder-style - reivew page, or via the Discord bot. - """ - reviews: [Review] -} - - -"""The judgement made about a skin by a moderator""" -enum Rating { - APPROVED - REJECTED - NSFW -} - -""" -A review of a skin. Done either on the Museum's Tinder-style -reivew page, or via the Discord bot. -""" -type Review { - """The skin that was reviewed""" - skin: Skin - - """ - The user who made the review (if known). **Note:** In the early days we didn't - track this, so many will be null. - """ - reviewer: String - - """The rating that the user gave the skin""" - rating: Rating -} - - -"""A file found within a Winamp Skin's .wsz archive""" -type ArchiveFile { - """Filename of the file within the archive""" - filename: String, - - """ - A URL to download the file. **Note:** This is powered by a little - serverless Cloudflare function which tries to exctact the file on the fly. - It may not work for all files. - """ - url: String - - """ - The date on the file inside the archive. Given in simplified extended ISO - format (ISO 8601). - """ - date: String - -} - -"""A tweet made by @winampskins mentioning a Winamp skin""" -type Tweet { - """ - URL of the tweet. **Note:** Early on in the bot's life we just recorded - _which_ skins were tweeted, not any info about the actual tweet. This means we - don't always know the URL of the tweet. - """ - url: String - - """Number of likes the tweet has received. Updated nightly. (Note: Recent likes on older tweets may not be reflected here)""" - likes: Int - - """Number of retweets the tweet has received. Updated nightly. (Note: Recent retweets on older tweets may not be reflected here)""" - retweets: Int - skin: Skin -} - -type InternetArchiveItem { - """The Internet Archive's unique identifier for this item""" - identifier: String - - """The URL where this item can be viewed on the Internet Archive""" - url: String - - """The skin that this item contains""" - skin: Skin -} - -"""A collection of tweets made by the @winampskins bot""" -type TweetsConnection { - """The total number of tweets""" - count: Int - - """The list of tweets""" - nodes: [Tweet] -} - -enum TweetsSortOption { - LIKES - RETWEETS -} - -"""A collection of classic Winamp skins""" -type SkinsConnection { - """The total number of skins matching the filter""" - count: Int - - """The list of skins""" - nodes: [Skin] -} - -type User { - username: String -} - -enum SkinsSortOption { - """ - the Museum's (https://skins.webamp.org) special sorting rules. - - Roughly speaking, it's: - - 1. The four classic default skins - 2. Tweeted skins first (sorted by the number of likes/retweets) - 3. Approved, but not tweeted yet, skins - 4. Unreviwed skins - 5. Rejected skins - 6. NSFW skins - """ - MUSEUM -} - -enum SkinsFilterOption { - """All the skins that have been approved for tweeting""" - APPROVED -} - -type Query { - """The currently authenticated user, if any.""" - me: User - - """Get a skin by its MD5 hash""" - fetch_skin_by_md5(md5: String!): Skin - - """Get a tweet by its URL""" - fetch_tweet_by_url(url: String!): Tweet - - """ - All skins in the database - - **Note:** We don't currently support combining sorting and filtering. - """ - skins( - first: Int, - offset: Int, - sort: SkinsSortOption, - filter: SkinsFilterOption - ): SkinsConnection - - """ - Tweets tweeted by @winampskins - """ - tweets( - first: Int, - offset: Int, - sort: TweetsSortOption - ): TweetsConnection -}`); - -const root = { - async fetch_skin_by_md5({ md5 }, { ctx }) { - const skin = await SkinModel.fromMd5(ctx, md5); - if (skin == null) { - return null; - } - return new SkinResolver(skin); - }, - async fetch_tweet_by_url({ url }, { ctx }) { - const tweet = await TweetModel.fromAnything(ctx, url); - if (tweet == null) { - return null; - } - return new TweetResolver(tweet); - }, - async skins({ first, offset, sort, filter }) { - if (first > 1000) { - throw new Error("Maximum limit is 1000"); - } - return new SkinsConnection(first, offset, sort, filter); - }, - async tweets({ first, offset, sort }) { - if (first > 1000) { - throw new Error("Maximum limit is 1000"); - } - return new TweetsConnection(first, offset, sort); - }, - me() { - return new UserResolver(); - }, -}; - -const DEFAULT_QUERY = `# Winamp Skins GraphQL API -# -# https://skins.webamp.org -# -# This is a GraphQL API for the Winamp Skin Museum's database. -# It's mostly intended for exploring the data set. Please feel -# free to have a look around and see what you can find! -# -# The GraphiQL environment has many helpful features to help -# you discover what fields exist and what they mean, so be bold! -# -# If you have any questions or feedback, please get in touch: -# - Twitter: @captbaritone -# - Email: jordan@jordaneldredge.com -# - Discord: https://webamp.org/chat - - -# An example query to get you started... -query MyQuery { - # Get info about a @winampskins tweet - # (spoiler, it's Luigihann's ZeldaAmp) - fetch_tweet_by_url(url: "https://twitter.com/winampskins/status/1056605906597629953") { - skin { - - # The filename of the skin that the tweet is about - filename - download_url - screenshot_url - museum_url - - # All the tweets that shared this skin - tweets { - likes - retweets - url - } - - # Information about the files contained within the skin - archive_files { - filename - # For image files, try hovering the - # returned url ---> - url - # Date the file was created according to the zip file - date - } - } - } -}`; - router.use( "/", graphqlHTTP({ @@ -311,7 +14,7 @@ router.use( throw new Error("We probably need to implement typeResolver"); }, schema: schema, - rootValue: root, + rootValue: new RootResolver(), graphiql: { defaultQuery: DEFAULT_QUERY, }, diff --git a/packages/skin-database/api/graphql/resolvers/InternetArchiveItemResolver.ts b/packages/skin-database/api/graphql/resolvers/InternetArchiveItemResolver.ts index 77e23c0b..0c475199 100644 --- a/packages/skin-database/api/graphql/resolvers/InternetArchiveItemResolver.ts +++ b/packages/skin-database/api/graphql/resolvers/InternetArchiveItemResolver.ts @@ -12,6 +12,17 @@ export default class InternetArchiveItemResolver { url() { return this._model.getUrl(); } + metadata_url() { + return this._model.getMetadataUrl(); + } + + last_metadata_scrape_date_UNSTABLE() { + return this._model.getMetadataTimestamp(); + } + + raw_metadata_json() { + return this._model.getMetadataJSON(); + } async skin() { const skin = await this._model.getSkin(); if (skin == null) { diff --git a/packages/skin-database/api/graphql/resolvers/RootResolver.ts b/packages/skin-database/api/graphql/resolvers/RootResolver.ts new file mode 100644 index 00000000..fb7626c5 --- /dev/null +++ b/packages/skin-database/api/graphql/resolvers/RootResolver.ts @@ -0,0 +1,75 @@ +import IaItemModel from "../../../data/IaItemModel"; +import SkinModel from "../../../data/SkinModel"; +import TweetModel from "../../../data/TweetModel"; +import SkinResolver from "../resolvers/SkinResolver"; +import TweetResolver from "../resolvers/TweetResolver"; +import UserResolver from "../resolvers/UserResolver"; +import SkinsConnection from "../SkinsConnection"; +import TweetsConnection from "../TweetsConnection"; +import InternetArchiveItemResolver from "./InternetArchiveItemResolver"; +import algoliasearch from "algoliasearch"; + +// These keys are already in the web client, so they are not secret at all. +const client = algoliasearch("HQ9I5Z6IM5", "6466695ec3f624a5fccf46ec49680e51"); +const index = client.initIndex("Skins"); + +class RootResolver { + async fetch_skin_by_md5({ md5 }, { ctx }) { + const skin = await SkinModel.fromMd5(ctx, md5); + if (skin == null) { + return null; + } + return new SkinResolver(skin); + } + async fetch_tweet_by_url({ url }, { ctx }) { + const tweet = await TweetModel.fromAnything(ctx, url); + if (tweet == null) { + return null; + } + return new TweetResolver(tweet); + } + async fetch_internet_archive_item_by_identifier({ identifier }, { ctx }) { + const iaItem = await IaItemModel.fromIdentifier(ctx, identifier); + if (iaItem == null) { + return null; + } + return new InternetArchiveItemResolver(iaItem); + } + + async search_skins({ query, first, offset }, { ctx }) { + if (first > 1000) { + throw new Error("Can only query 1000 records via search."); + } + + const results = await index.search(query, { + attributesToRetrieve: ["md5"], + length: first, + offset, + }); + + return Promise.all( + results.hits.map(async (hit) => { + const model = await SkinModel.fromMd5Assert(ctx, hit.md5); + return new SkinResolver(model); + }) + ); + } + + async skins({ first, offset, sort, filter }) { + if (first > 1000) { + throw new Error("Maximum limit is 1000"); + } + return new SkinsConnection(first, offset, sort, filter); + } + async tweets({ first, offset, sort }) { + if (first > 1000) { + throw new Error("Maximum limit is 1000"); + } + return new TweetsConnection(first, offset, sort); + } + me() { + return new UserResolver(); + } +} + +export default RootResolver; diff --git a/packages/skin-database/api/graphql/schema.ts b/packages/skin-database/api/graphql/schema.ts new file mode 100644 index 00000000..fec32d6d --- /dev/null +++ b/packages/skin-database/api/graphql/schema.ts @@ -0,0 +1,242 @@ +import { buildSchema } from "graphql"; + +const schema = buildSchema(` +"""A classic Winamp skin""" +type Skin { + """Database ID of the skin""" + id: Int, + + """MD5 hash of the skin's file""" + md5: String, + + """URL of the skin on the Winamp Skin Museum""" + museum_url: String, + + """URL of webamp.org with the skin loaded""" + webamp_url: String, + + """URL of a screenshot of the skin""" + screenshot_url: String, + + """URL to download the skin""" + download_url: String, + + """ + Filename of skin when uploaded to the Museum. Note: In some cases a skin + has been uploaded under multiple names. Here we just pick one. + """ + filename: String, + + """Text of the readme file extracted from the skin""" + readme_text: String, + + """Has the skin been flagged as "not safe for wrok"?""" + nsfw: Boolean, + + """String representation (rgb usually) of the skin's average color""" + average_color: String, + + """Has the skin been tweeted?""" + tweeted: Boolean + + """List of @winampskins tweets that mentioned the skin.""" + tweets: [Tweet] + + """List of files contained within the skin's .wsz archive""" + archive_files: [ArchiveFile] + + """The skin's "item" at archive.org""" + internet_archive_item: InternetArchiveItem + + """ + Times that the skin has been reviewed either on the Museum's Tinder-style + reivew page, or via the Discord bot. + """ + reviews: [Review] +} + + +"""The judgement made about a skin by a moderator""" +enum Rating { + APPROVED + REJECTED + NSFW +} + +""" +A review of a skin. Done either on the Museum's Tinder-style +reivew page, or via the Discord bot. +""" +type Review { + """The skin that was reviewed""" + skin: Skin + + """ + The user who made the review (if known). **Note:** In the early days we didn't + track this, so many will be null. + """ + reviewer: String + + """The rating that the user gave the skin""" + rating: Rating +} + + +"""A file found within a Winamp Skin's .wsz archive""" +type ArchiveFile { + """Filename of the file within the archive""" + filename: String, + + """ + A URL to download the file. **Note:** This is powered by a little + serverless Cloudflare function which tries to exctact the file on the fly. + It may not work for all files. + """ + url: String + + """ + The date on the file inside the archive. Given in simplified extended ISO + format (ISO 8601). + """ + date: String + +} + +"""A tweet made by @winampskins mentioning a Winamp skin""" +type Tweet { + """ + URL of the tweet. **Note:** Early on in the bot's life we just recorded + _which_ skins were tweeted, not any info about the actual tweet. This means we + don't always know the URL of the tweet. + """ + url: String + + """Number of likes the tweet has received. Updated nightly. (Note: Recent likes on older tweets may not be reflected here)""" + likes: Int + + """Number of retweets the tweet has received. Updated nightly. (Note: Recent retweets on older tweets may not be reflected here)""" + retweets: Int + skin: Skin +} + +type InternetArchiveItem { + """The Internet Archive's unique identifier for this item""" + identifier: String + + """The URL where this item can be viewed on the Internet Archive""" + url: String + + """URL to get the Internet Archive's metadata for this item in JSON form.""" + metadata_url: String + + """ + Our cached version of the metadata avaliable at \`metadata_url\` (above) + """ + raw_metadata_json: String + + """ + The date and time that we last scraped this item's metadata. + **Note:** This field is temporary and will be removed in the future. + The date format is just what we get from the database, and it's ambiguous. + """ + last_metadata_scrape_date_UNSTABLE: String + + """The skin that this item contains""" + skin: Skin + +} + +"""A collection of tweets made by the @winampskins bot""" +type TweetsConnection { + """The total number of tweets""" + count: Int + + """The list of tweets""" + nodes: [Tweet] +} + +enum TweetsSortOption { + LIKES + RETWEETS +} + +"""A collection of classic Winamp skins""" +type SkinsConnection { + """The total number of skins matching the filter""" + count: Int + + """The list of skins""" + nodes: [Skin] +} + +type User { + username: String +} + +enum SkinsSortOption { + """ + the Museum's (https://skins.webamp.org) special sorting rules. + + Roughly speaking, it's: + + 1. The four classic default skins + 2. Tweeted skins first (sorted by the number of likes/retweets) + 3. Approved, but not tweeted yet, skins + 4. Unreviwed skins + 5. Rejected skins + 6. NSFW skins + """ + MUSEUM +} + +enum SkinsFilterOption { + """All the skins that have been approved for tweeting""" + APPROVED +} + +type Query { + """The currently authenticated user, if any.""" + me: User + + """Get a skin by its MD5 hash""" + fetch_skin_by_md5(md5: String!): Skin + + """Get a tweet by its URL""" + fetch_tweet_by_url(url: String!): Tweet + + """ + Get an archive.org item by its identifier. You can find this in the URL: + + https://archive.org/details// + """ + fetch_internet_archive_item_by_identifier(identifier: String!): InternetArchiveItem + + """ + All skins in the database + + **Note:** We don't currently support combining sorting and filtering. + """ + skins( + first: Int = 10, + offset: Int = 0, + sort: SkinsSortOption, + filter: SkinsFilterOption + ): SkinsConnection + + """ + Search the database using the Algolia search index used by the Museum. + + Useful for locating a particular skin. + """ + search_skins(query: String!, first: Int = 10, offset: Int = 0): [Skin] + + """ + Tweets tweeted by @winampskins + """ + tweets( + first: Int = 10, + offset: Int = 0, + sort: TweetsSortOption + ): TweetsConnection +}`); +export default schema; diff --git a/packages/skin-database/data/IaItemModel.ts b/packages/skin-database/data/IaItemModel.ts index 0565da42..41b277f4 100644 --- a/packages/skin-database/data/IaItemModel.ts +++ b/packages/skin-database/data/IaItemModel.ts @@ -61,6 +61,14 @@ export default class IaItemModel { return `https://archive.org/details/${this.getIdentifier()}`; } + getMetadataUrl(): string { + return `https://archive.org/metadata/${this.getIdentifier()}`; + } + + getMetadataTimestamp(): string | null { + return this.row.metadata_timestamp; + } + getIdentifier(): string { const { identifier } = this.row; if (identifier == null) { @@ -71,6 +79,14 @@ export default class IaItemModel { return identifier; } + getMetadataJSON(): string | null { + return this.row.metadata; + } + + getMetadataScrapeDate(): string | null { + return this.row.metadata; + } + getAllFiles(): any[] { if (!this.row.metadata) { return []; diff --git a/packages/skin-database/types.ts b/packages/skin-database/types.ts index f63479fa..0df9e055 100644 --- a/packages/skin-database/types.ts +++ b/packages/skin-database/types.ts @@ -52,6 +52,7 @@ export type IaItemRow = { skin_md5: string; identifier: string; metadata: string; // JSON from the internet archive + metadata_timestamp: string; }; export type UploadStatus =