diff --git a/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts b/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts index da0dd158..ae50ec97 100644 --- a/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts +++ b/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts @@ -95,7 +95,7 @@ export default class ClassicSkinResolver /** * URL of a screenshot of the skin * @gqlField */ - screenshot_url(): string { + async screenshot_url(): Promise { return this._model.getScreenshotUrl(); } /** diff --git a/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts b/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts index aac6ef70..bf885d28 100644 --- a/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts +++ b/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts @@ -90,10 +90,8 @@ export interface ISkin { */ webamp_url(): string | null; - /** - * @gqlField - * @deprecated Needed for migration */ - screenshot_url(): string | null; + /** @gqlField */ + screenshot_url(): Promise; /** * @gqlField diff --git a/packages/skin-database/api/graphql/resolvers/ModernSkinResolver.ts b/packages/skin-database/api/graphql/resolvers/ModernSkinResolver.ts index 657f8366..cfd40967 100644 --- a/packages/skin-database/api/graphql/resolvers/ModernSkinResolver.ts +++ b/packages/skin-database/api/graphql/resolvers/ModernSkinResolver.ts @@ -7,6 +7,7 @@ import ReviewResolver from "./ReviewResolver"; import InternetArchiveItemResolver from "./InternetArchiveItemResolver"; import ArchiveFileResolver from "./ArchiveFileResolver"; import TweetResolver from "./TweetResolver"; +import { XMLParser } from "fast-xml-parser"; /** * A "modern" Winamp skin. These skins use the `.wal` file extension and are free-form. @@ -109,11 +110,45 @@ export default class ModernSkinResolver return null; } - /** - * @gqlField - * @deprecated Needed for migration */ - screenshot_url(): string | null { - return null; + /** @gqlField */ + async screenshot_url(): Promise { + const archiveFiles = await this._model.getArchiveFiles(); + const skinXml = archiveFiles.find((f) => + f.getFileName().match(/skin\.xml$/i) + ); + + if (skinXml == null) { + return null; + } + const xmlContent = await skinXml.getTextContent(); + if (xmlContent == null) { + return null; + } + + const parser = new XMLParser(); + const parsedXml = parser.parse(xmlContent); + const screenshotPath = + parsedXml?.WinampAbstractionLayer?.skininfo?.screenshot; + if (screenshotPath == null) { + return null; + } + + const screenshotFile = archiveFiles.find((f) => { + const fileName = f.getFileName().toLowerCase(); + const normalizedScreenshotPath = screenshotPath.toLowerCase(); + return ( + fileName === normalizedScreenshotPath || + fileName === `/${normalizedScreenshotPath}` || + fileName === `\\${normalizedScreenshotPath}` + ); + }); + + console.log({ screenshotFile }); + if (screenshotFile == null) { + return null; + } + + return screenshotFile.getUrl(); } /** * @gqlField diff --git a/packages/skin-database/api/graphql/schema.graphql b/packages/skin-database/api/graphql/schema.graphql index 4a611d12..a4e0b660 100644 --- a/packages/skin-database/api/graphql/schema.graphql +++ b/packages/skin-database/api/graphql/schema.graphql @@ -192,7 +192,7 @@ type ModernSkin implements Node & Skin { reivew page, or via the Discord bot. """ reviews: [Review] - screenshot_url: String @deprecated(reason: "Needed for migration") + screenshot_url: String """Has the skin been tweeted?""" tweeted: Boolean """List of @winampskins tweets that mentioned the skin.""" @@ -358,7 +358,7 @@ interface Skin { reivew page, or via the Discord bot. """ reviews: [Review] - screenshot_url: String @deprecated(reason: "Needed for migration") + screenshot_url: String """Has the skin been tweeted?""" tweeted: Boolean """List of @winampskins tweets that mentioned the skin.""" diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index 39e6e5e3..f4a0ae3e 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -74,7 +74,7 @@ program .command("share") .description( "Share a skin on Twitter and Instagram. If no md5 is " + - "given, random approved skins are shared." + "given, random approved skins are shared." ) .argument("[md5]", "md5 of the skin to share") .option("-t, --twitter", "Share on Twitter") @@ -107,7 +107,7 @@ program .option( "--delete", "Delete a skin from the database, including its S3 files " + - "CloudFlare cache and seach index entries." + "CloudFlare cache and seach index entries." ) .option( "--hide", @@ -116,7 +116,7 @@ program .option( "--delete-local", "Delete a skin from the database only, NOT including its S3 files " + - "CloudFlare cache and seach index entries." + "CloudFlare cache and seach index entries." ) .option("--index", "Update the seach index for a skin.") .option( @@ -195,18 +195,18 @@ program .option( "--fetch-metadata ", "Fetch missing metadata for items from the Internet " + - "Archive. Currently it only fetches missing metadata. In the " + - "future it could refresh stale metadata." + "Archive. Currently it only fetches missing metadata. In the " + + "future it could refresh stale metadata." ) .option( "--fetch-items", "Seach the Internet Archive for items that we don't know about" + - "and add them to our database." + "and add them to our database." ) .option( "--update-metadata ", "Find items in our database that have incorrect or incomplete " + - "metadata, and update the Internet Archive" + "metadata, and update the Internet Archive" ) .option( "--upload-new", @@ -240,7 +240,7 @@ program .command("stats") .description( "Report information about skins in the database. " + - "Identical to `!stats` in Discord." + "Identical to `!stats` in Discord." ) .action(async () => { console.table([await Skins.getStats()]); @@ -286,17 +286,17 @@ program .option( "--likes", "Scrape @winampskins tweets for like and retweet counts, " + - "and update the database." + "and update the database." ) .option( "--milestones", "Check the most recent @winampskins tweets to see if they have " + - "passed a milestone. If so, notify the Discord channel." + "passed a milestone. If so, notify the Discord channel." ) .option( "--followers", "Check if @winampskins has passed a follower count milestone. " + - "If so, notify the Discord channel." + "If so, notify the Discord channel." ) .action(async ({ likes, milestones, followers }) => { if (likes) { @@ -325,7 +325,7 @@ program .option( "--upload-ia-screenshot ", "Upload a screenshot of a skin to the skin's Internet Archive itme. " + - "[[Warning!]] This might result in multiple screenshots on the item." + "[[Warning!]] This might result in multiple screenshots on the item." ) .option( "--upload-missing-screenshots", @@ -377,7 +377,7 @@ program [500] ); const md5s = rows.map((row) => row.md5); - console.log(md5s.length) + console.log(md5s.length); console.log(await Skins.updateSearchIndexs(ctx, md5s)); } if (refreshContentHash) { @@ -401,17 +401,22 @@ program const skinRows = await knex("skins") .leftJoin("archive_files", "skins.md5", "archive_files.skin_md5") .leftJoin("file_info", "file_info.file_md5", "archive_files.file_md5") - .where("skin_type", 1) + .where("skin_type", "in", [1, 2]) .where((builder) => { return builder.where("file_info.file_md5", null); }) - .limit(90000) + .limit(1000) .groupBy("skins.md5") .select(); console.log(`Found ${skinRows.length} skins to update`); const skins = skinRows.map((row) => new SkinModel(ctx, row)); for (const skin of skins) { - await setHashesForSkin(skin); + console.log("Working on", skin.getMd5(), await skin.getFileName()); + try { + await setHashesForSkin(skin); + } catch (e) { + console.error(e); + } // await Skins.setContentHash(skin.getMd5()); process.stdout.write("."); } @@ -429,15 +434,15 @@ program }); async function main() { - process.on('unhandledRejection', (reason, promise) => { - console.error("Unhanded rejection") - console.error(reason, promise) + process.on("unhandledRejection", (reason, promise) => { + console.error("Unhanded rejection"); + console.error(reason, promise); }); try { await program.parseAsync(process.argv); } finally { knex.destroy(); - console.log("CLOSING THE LOGGER") + console.log("CLOSING THE LOGGER"); logger.close(); } } diff --git a/packages/skin-database/data/SkinModel.ts b/packages/skin-database/data/SkinModel.ts index a1680610..41f1c1f1 100644 --- a/packages/skin-database/data/SkinModel.ts +++ b/packages/skin-database/data/SkinModel.ts @@ -350,15 +350,14 @@ export default class SkinModel { return 0; } const text = await region.getTextContent(); - if(text == null) { + if (text == null) { return 0; } try { - - return getTransparentAreaSize(text); - } catch(e) { - console.error(`Failed: ${this.getMd5()}`) - return 0 + return getTransparentAreaSize(text); + } catch (e) { + console.error(`Failed: ${this.getMd5()}`); + return 0; } } diff --git a/packages/skin-database/package.json b/packages/skin-database/package.json index 00998912..4c49ea22 100644 --- a/packages/skin-database/package.json +++ b/packages/skin-database/package.json @@ -27,6 +27,7 @@ "express-fileupload": "^1.1.7-alpha.3", "express-graphql": "^0.12.0", "express-sitemap-xml": "^2.0.0", + "fast-xml-parser": "^4.2.2", "graphql": "14.7.0", "grats": "^0.0.0-main-8de1c8ac", "imagemin": "^7.0.0", @@ -49,7 +50,7 @@ "scripts": { "lint": "eslint .", "start": "ts-node --transpile-only api/server.ts", - "dev": "NODE_ENV=development ts-node --transpile-only api/server.ts", + "dev": "NODE_ENV=production ts-node --transpile-only api/server.ts", "tweet": "ts-node --transpile-only ./cli.ts tweet", "fetch-metadata": "ts-node --transpile-only ./cli.ts fetch-metadata", "bot": "ts-node --transpile-only ./discord-bot/index.js", diff --git a/packages/skin-database/skinHash.ts b/packages/skin-database/skinHash.ts index 9dc9968e..ccb66deb 100644 --- a/packages/skin-database/skinHash.ts +++ b/packages/skin-database/skinHash.ts @@ -13,7 +13,10 @@ type FileData = { }; function mightBeText(filename: string): boolean { - return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm|ai|log|uue|)$/i.test( + if (/skin\.xml/i.test(filename)) { + return true; + } + return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm|ai|log|uue|xml)$/i.test( filename ); } @@ -21,7 +24,8 @@ function mightBeText(filename: string): boolean { async function getFileData(file: JSZip.JSZipObject): Promise { try { let textContent: string | null = null; - if (mightBeText(file.name)) { + const extractText = mightBeText(file.name); + if (extractText) { textContent = await file.async("text"); } const blob = await file.async("nodebuffer"); @@ -56,19 +60,24 @@ export async function setHashesForSkin(skin: SkinModel): Promise { const hashes = await getSkinFileData(skin); const rows = hashes .filter(notEmpty) - .map(({ fileName, md5, date, isDirectory }) => { - return { - skin_md5: skin.getMd5(), - file_name: fileName, - file_md5: md5, // TODO: Consider using an id into file_info table - file_date: date, - // text_content: textContent, - // uncompressed_size: uncompressedSize, - is_directory: isDirectory, - }; - }); + .map( + ({ fileName, md5, date, isDirectory, textContent, uncompressedSize }) => { + return { + skin_md5: skin.getMd5(), + file_name: fileName, + file_md5: md5, // TODO: Consider using an id into file_info table + file_date: date, + text_content: textContent, + uncompressed_size: uncompressedSize, + is_directory: isDirectory, + }; + } + ); await knex("archive_files").where("skin_md5", skin.getMd5()).delete(); - await knex("archive_files").insert(rows); + // Avoid bulk insert because size can be large + for (const row of rows) { + await knex("archive_files").insert(row); + } const fileInfoRows = hashes .filter(notEmpty) diff --git a/packages/skin-database/tasks/syncToArchive.ts b/packages/skin-database/tasks/syncToArchive.ts index 058eee19..384f892b 100644 --- a/packages/skin-database/tasks/syncToArchive.ts +++ b/packages/skin-database/tasks/syncToArchive.ts @@ -169,7 +169,7 @@ export async function identifierExists(identifier: string): Promise { } async function getNewIdentifier(filename: string): Promise { - // The internet archvie has a max identifier length of 80 chars. + // The internet archive has a max identifier length of 80 chars. const identifierBase = `winampskins_${sanitize( path.parse(filename).name )}`.slice(0, 76); @@ -222,6 +222,14 @@ export async function syncToArchive(handler: DiscordEventHandler) { await Parallel.map( unarchived, async ({ md5 }) => { + if ( + md5 === "513fdd06bf39391e52f3ac5b233dd147" || + md5 === "91477bec2b599bc5085f87f0fca3a4d5" + ) { + // The internet archive claims this one is corrupt for some reason. + console.warn(`Skipping this skin. It's known to not upload correctly.`); + return null; + } const skin = await SkinModel.fromMd5Assert(ctx, md5); try { console.log(`Attempting to upload ${md5}`); @@ -231,18 +239,14 @@ export async function syncToArchive(handler: DiscordEventHandler) { } catch (e) { console.log("Archive failed..."); errorCount++; - // The internet archive claims this one is corrupt for some reason. - if (md5 === "513fdd06bf39391e52f3ac5b233dd147") { - console.warn(`This skin is known to not upload correctly.`); - } if (/error checking archive/.test(e.message)) { - console.log(`Corrupt archvie: ${skin.getMd5()}`); + console.log(`Corrupt archive: ${skin.getMd5()}`); } else if ( /archive files are not allowed to contain encrypted content/.test( e.message ) ) { - console.log(`Corrupt archvie (encrypted): ${skin.getMd5()}`); + console.log(`Corrupt archive (encrypted): ${skin.getMd5()}`); } else if (/case alias may already exist/.test(e.message)) { console.log( `Invalid name (case alias): ${skin.getMd5()} with ${e.message}` diff --git a/yarn.lock b/yarn.lock index f80bb401..6a9ca466 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10098,9 +10098,9 @@ comma-separated-tokens@^1.0.0: integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== commander@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.0.tgz#71797971162cd3cf65f0b9d24eb28f8d303acdf1" - integrity sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA== + version "10.0.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" + integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== commander@^2.19.0, commander@^2.20.0, commander@^2.20.3, commander@^2.3.0: version "2.20.3" @@ -13510,6 +13510,13 @@ fast-stringify@^1.1.0: resolved "https://registry.npmjs.org/fast-stringify/-/fast-stringify-1.1.2.tgz" integrity sha512-SfslXjiH8km0WnRiuPfpUKwlZjW5I878qsOm+2x8x3TgqmElOOLh1rgJFb+PolNdNRK3r8urEefqx0wt7vx1dA== +fast-xml-parser@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.2.tgz#cb7310d1e9cf42d22c687b0fae41f3c926629368" + integrity sha512-DLzIPtQqmvmdq3VUKR7T6omPK/VCRNqgFlGtbESfyhcH2R4I8EzK1/K6E8PkRCK2EabWrUHK32NjYRbEFnnz0Q== + dependencies: + strnum "^1.0.5" + fastest-levenshtein@^1.0.7, fastest-levenshtein@^1.0.9: version "1.0.16" resolved "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz" @@ -14771,22 +14778,15 @@ graphql-ws@^5.4.1: resolved "https://registry.npmjs.org/graphql-ws/-/graphql-ws-5.11.1.tgz" integrity sha512-AlOO/Gt0fXuSHXe/Weo6o3rIQVnH5MW7ophzeYzL+vYNlkf0NbWRJ6IIFgtSLcv9JpTlQdxSpB3t0SnM47/BHA== -graphql@14.7.0: - version "14.7.0" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.7.0.tgz#7fa79a80a69be4a31c27dda824dc04dac2035a72" - integrity sha512-l0xWZpoPKpppFzMfvVyFmp9vLN7w/ZZJPefUicMCepfJeQ8sMcztloGYY9DfjVPo6tIUDzU5Hw3MUbIjj9AVVA== - dependencies: - iterall "^1.2.2" +graphql@14.7.0, graphql@15.3.0, graphql@^16.6.0: + version "15.3.0" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.3.0.tgz#3ad2b0caab0d110e3be4a5a9b2aa281e362b5278" + integrity sha512-GTCJtzJmkFLWRfFJuoo9RWWa/FfamUHgiFosxi/X1Ani4AVWbeyBenZTNX6dM+7WSbbFfTo/25eh0LLkwHMw2w== -graphql@^16.6.0: - version "16.6.0" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.6.0.tgz#c2dcffa4649db149f6282af726c8c83f1c7c5fdb" - integrity sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw== - -grats@^0.0.0-main-44ac2e23: - version "0.0.0-main-44ac2e23" - resolved "https://registry.yarnpkg.com/grats/-/grats-0.0.0-main-44ac2e23.tgz#b788e44c3d3eaac28588741a7c34ff4c69832183" - integrity sha512-IFVtiWuOtcjw24+TsvuZvYZci4e8jPvs7WiPrEQtsvFkZ4HEJqn+PUsMPq+R5D7Ox4NQSSTZP/r7iFUwsi19TQ== +grats@^0.0.0-main-8de1c8ac: + version "0.0.0-main-8de1c8ac" + resolved "https://registry.yarnpkg.com/grats/-/grats-0.0.0-main-8de1c8ac.tgz#ddd0eb60c290951e76307028143b24a3f8608ea4" + integrity sha512-lP/CpAU/6qE/Cn7+MUPbQT100kzuxh5rJ2fAiDbtAzxvG3MFhtXySy8Cz6sFt0eBjHv77a2G3J3Ua6+xoM6GnA== dependencies: "@graphql-tools/utils" "^9.2.1" commander "^10.0.0" @@ -16506,11 +16506,6 @@ isurl@^1.0.0-alpha5: has-to-string-tag-x "^1.2.0" is-object "^1.0.1" -iterall@^1.2.2: - version "1.3.0" - resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea" - integrity sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg== - jake@^10.8.5: version "10.8.5" resolved "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz" @@ -25027,6 +25022,11 @@ strip-outer@^1.0.0: dependencies: escape-string-regexp "^1.0.2" +strnum@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db" + integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA== + strtok3@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/strtok3/-/strtok3-2.3.0.tgz"