Mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp.git synced 2026-01-23 02:15:01 +00:00
Code Issues Projects Releases Wiki Activity

Refactor museum query

Browse source
This commit is contained in:
Jordan Eldredge 2022-06-19 16:20:57 -07:00
parent 5a3b21a460
commit c939022b92
4 changed files with 127 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

33
packages/skin-database/cli.ts
View file

@ -72,7 +72,7 @@ program
.command("share")
.description(
"Share a skin on Twitter and Instagram. If no md5 is " +
"given, random approved skins are shared."
"given, random approved skins are shared."
)
.argument("[md5]", "md5 of the skin to share")
.option("-t, --twitter", "Share on Twitter")
@ -101,12 +101,16 @@ program
.option(
"--delete",
"Delete a skin from the database, including its S3 files " +
"CloudFlare cache and seach index entries."
"CloudFlare cache and seach index entries."
)
.option(
"--hide",
"Hide a skin from the museum main page. Useful for removing aparent dupes."
)
.option(
"--delete-local",
"Delete a skin from the database only, NOT including its S3 files " +
"CloudFlare cache and seach index entries."
"CloudFlare cache and seach index entries."
)
.option("--index", "Update the seach index for a skin.")
.option(
@ -118,12 +122,15 @@ program
.action(
async (
md5,
{ delete: del, deleteLocal, index, refresh, reject, metadata }
{ delete: del, deleteLocal, index, refresh, reject, metadata, hide }
) => {
const ctx = new UserContext("CLI");
if (del) {
await Skins.deleteSkin(md5);
}
if (hide) {
await Skins.hideSkin(md5);
}
if (deleteLocal) {
await Skins.deleteLocalSkin(md5);
}
@ -182,18 +189,18 @@ program
.option(
"--fetch-metadata <count>",
"Fetch missing metadata for <count> items from the Internet " +
"Archive. Currently it only fetches missing metadata. In the " +
"future it could refresh stale metadata."
"Archive. Currently it only fetches missing metadata. In the " +
"future it could refresh stale metadata."
)
.option(
"--fetch-items",
"Seach the Internet Archive for items that we don't know about" +
"and add them to our database."
"and add them to our database."
)
.option(
"--update-metadata <count>",
"Find <count> items in our database that have incorrect or incomplete " +
"metadata, and update the Internet Archive"
"metadata, and update the Internet Archive"
)
.option(
"--upload-new",
@ -227,7 +234,7 @@ program
.command("stats")
.description(
"Report information about skins in the database. " +
"Identical to `!stats` in Discord."
"Identical to `!stats` in Discord."
)
.action(async () => {
console.table([await Skins.getStats()]);
@ -273,17 +280,17 @@ program
.option(
"--likes",
"Scrape @winampskins tweets for like and retweet counts, " +
"and update the database."
"and update the database."
)
.option(
"--milestones",
"Check the most recent @winampskins tweets to see if they have " +
"passed a milestone. If so, notify the Discord channel."
"passed a milestone. If so, notify the Discord channel."
)
.option(
"--followers",
"Check if @winampskins has passed a follower count milestone. " +
"If so, notify the Discord channel."
"If so, notify the Discord channel."
)
.action(async ({ likes, milestones, followers }) => {
if (likes) {
@ -311,7 +318,7 @@ program
.option(
"--upload-ia-screenshot <md5>",
"Upload a screenshot of a skin to the skin's Internet Archive itme. " +
"[[Warning!]] This might result in multiple screenshots on the item."
"[[Warning!]] This might result in multiple screenshots on the item."
)
.option(
"--upload-missing-screenshots",

105
packages/skin-database/data/skins.ts
View file

@ -215,6 +215,10 @@ export async function updateSearchIndex(
return updateSearchIndexs(ctx, [md5]);
}
export async function hideSkin(md5: string): Promise<void> {
await knex("museum_sort_overrides").insert({ skin_md5: md5, score: -1 });
}
// Note: This might leave behind some files in file_info.
export async function deleteSkin(md5: string): Promise<void> {
await deleteLocalSkin(md5);
@ -587,51 +591,62 @@ export async function getMuseumPage({
}): Promise<MuseumPage> {
const skins = await knex.raw(
`
SELECT skins.md5,
skin_reviews.review = 'NSFW' AS nsfw,
skin_reviews.review = 'APPROVED' AS approved,
skin_reviews.review = 'REJECTED' AS rejected,
(IFNULL(tweets.likes, 0) + (IFNULL(tweets.retweets,0) * 1.5)) AS tweet_score,
files.file_path,
CASE skins.md5
WHEN "5e4f10275dcb1fb211d4a8b4f1bda236" THEN 0 -- Base
WHEN "cd251187a5e6ff54ce938d26f1f2de02" THEN 1 -- Winamp3 Classified
WHEN "b0fb83cc20af3abe264291bb17fb2a13" THEN 2 -- Winamp5 Classified
WHEN "d6010aa35bed659bc1311820daa4b341" THEN 3 -- Bento Classified
ELSE 1000
END
priority
FROM skins
LEFT JOIN (
SELECT
skin_md5,
MAX(likes) as likes,
MAX(retweets) as retweets
FROM tweets
GROUP BY skin_md5
) as tweets ON tweets.skin_md5 = skins.md5
LEFT JOIN skin_reviews ON skin_reviews.skin_md5 = skins.md5
LEFT JOIN files ON files.skin_md5 = skins.md5
LEFT JOIN refreshes ON refreshes.skin_md5 = skins.md5
WHERE skin_type = 1 AND refreshes.error IS NULL
--
-- WHERE
-- skin_type = 1
-- AND refreshes.error IS NULL
-- AND skins.md5 != "d7541f8c5be768cf23b9aeee1d6e70c7" -- Duplicate Garfield
-- AND skins.md5 != "25a932542e307416ca86da4e16be1b32" -- Duplicate Vault-tec
-- AND skins.md5 != "89643da06361e4bcc269fe811f07c4a3" -- Another duplicate Vault-tec
-- AND skins.md5 != "db1f2e128f6dd6c702b7a448751fbe84" -- Duplicate Fallout
-- AND skins.md5 != "be2de111c4710af306fea0813440f275" -- Duplicate Microchip
-- AND skins.md5 != "66cf0af3593d79fc8a5080dd17f8b07d" -- Another duplicate Microchip
GROUP BY skins.md5
ORDER BY
priority ASC,
tweet_score DESC,
nsfw ASC,
approved DESC,
rejected ASC
LIMIT ? offset ?`,
-- A tweet score for each skin based on its tweets.
WITH skin_tweets as (
SELECT
skin_md5,
MAX(likes) as likes,
MAX(retweets) as retweets,
(IFNULL(likes, 0) + (IFNULL(retweets, 0) * 1.5)) AS tweet_score
FROM
tweets
GROUP BY
skin_md5
)
SELECT
skins.md5,
files.file_path,
skin_reviews.review = 'NSFW' AS nsfw
FROM
skins
LEFT JOIN museum_sort_overrides ON museum_sort_overrides.skin_md5 = skins.md5
LEFT JOIN skin_tweets ON skin_tweets.skin_md5 = skins.md5
LEFT JOIN skin_reviews ON skin_reviews.skin_md5 = skins.md5
LEFT JOIN files ON files.skin_md5 = skins.md5
LEFT JOIN refreshes ON refreshes.skin_md5 = skins.md5
WHERE
-- Only show classic skins
skin_type = 1
-- Hide skins that are dupes or we otherwise want to hide
AND (museum_sort_overrides.score IS NULL OR museum_sort_overrides.score > 0)
-- Hides skins that might not have a valid screenshot
AND refreshes.error IS NULL
GROUP BY
skins.md5
ORDER BY
-- The secret sauce of the Winamp Skin Museum.
-- We try to rank skins based on how interesting they are to a modern
-- audience by leveraging data accumulated by the @winampskins Twitter bot.
-- 1. Manaully currated skins (the default skin and classic ports of the default modern skins)
-- 2. All tweeted skins ranked by (likes + retweets * 1.5)
-- 3. All approved skins that have not yet been tweeted
-- 4. All unreviewed skins
-- 5. All rejected skins
-- 6. All NSFW skins
-- Show manually currated skins (default skins) first
museum_sort_overrides.score DESC,
-- Sort skins by their popularity on Twitter
tweet_score DESC,
-- Push NSFW skins to the bottom
skin_reviews.review = 'NSFW' ASC,
-- Skins that have been approved are better than others
skin_reviews.review = 'APPROVED' DESC,
-- Skins that have been rejected are worse than those that have not been reviewed
skin_reviews.review = 'REJECTED' ASC
LIMIT ? offset ?`,
[first, offset]
);

9
packages/skin-database/docs/database.md
View file

@ -145,3 +145,12 @@ Metadata about migrations that have been run on the database. Used for making da
## knex_migrations_lock
Used to ensure migrations are applied correctly.
## museum_sort_overrides
Used for making editorial decisions about how individual skins show up in the main scroll of the Winamp Skin Museum. Used for boosting the default skins and hiding aparent duplicates. In reality there are many many near or actual dupes, but we manually cull duplicates that appear in the first few pages.
- `id` A unique ID for this override (local to this database)
- `skin_md5` The md5 hash of the skin that is being overridden
- `score` A score for how highly rated this skin should be. Negative numbers mean the skin should be hidden.
- `comment` Explains why the skin was ranked this way

38
packages/skin-database/migrations/20220619155617_museum_overrides.ts Normal file
View file

@ -0,0 +1,38 @@
import * as Knex from "knex";
export async function up(knex: Knex): Promise<void> {
await knex.raw(`
CREATE TABLE museum_sort_overrides (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
skin_md5 TEXT NOT NULL UNIQUE,
comment TEXT,
score INTEGER NOT NULL,
FOREIGN KEY (skin_md5) REFERENCES skins (md5) ON DELETE CASCADE ON UPDATE NO ACTION
);`);
await knex.raw(`INSERT INTO
museum_sort_overrides (skin_md5, comment, score)
VALUES
("5e4f10275dcb1fb211d4a8b4f1bda236", "Base", 5),
("cd251187a5e6ff54ce938d26f1f2de02", "Winamp3 Classified", 4),
("b0fb83cc20af3abe264291bb17fb2a13", "Winamp5 Classified", 3),
("d6010aa35bed659bc1311820daa4b341", "Bento Classified", 2),
("d7541f8c5be768cf23b9aeee1d6e70c7", "Duplicate Garfield", -1),
("25a932542e307416ca86da4e16be1b32", "Duplicate Vault-tec", -1),
("89643da06361e4bcc269fe811f07c4a3", "Duplicate Vault-tec", -1),
("fb1ca386260ee4d4e44b7a3a2e029729", "Duplicate Vault-tec", -1),
("db1f2e128f6dd6c702b7a448751fbe84", "Duplicate Fallout", -1),
("be2de111c4710af306fea0813440f275", "Duplicate Microchip", -1),
("66cf0af3593d79fc8a5080dd17f8b07d", "Duplicate Microchip", -1),
("4269b10d8d27bd201f8608c59295680c", "Duplicate Doom", -1),
("44c8f2bf4889f7ea5565e82f332f4a20", "Duplicate Mtn Dew", -1);
`)
}
export async function down(knex: Knex): Promise<void> {
await knex.raw(`DROP TABLE museum_sort_overrides;`);
}