From 3713b262e17464bb6f3aa79fb8ee95b79d2e7462 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Fri, 25 Feb 2022 23:09:30 -0800 Subject: [PATCH] Add file_info and rm redme_text column --- packages/skin-database/addSkin.ts | 28 +++++--- packages/skin-database/analyser.ts | 54 -------------- packages/skin-database/analyzer.test.ts | 29 -------- packages/skin-database/cli.ts | 4 +- .../skin-database/data/ArchiveFileModel.ts | 22 ++++-- packages/skin-database/data/FileInfoModel.ts | 48 +++++++++++++ packages/skin-database/data/SkinModel.ts | 18 ++++- packages/skin-database/data/skins.ts | 70 ++++++++++--------- .../20220225101216_add_file_contents_table.ts | 16 +++++ packages/skin-database/skinHash.ts | 49 +++++++++---- packages/skin-database/tasks/refresh.ts | 6 +- packages/skin-database/types.ts | 8 ++- 12 files changed, 201 insertions(+), 151 deletions(-) delete mode 100644 packages/skin-database/analyser.ts delete mode 100644 packages/skin-database/analyzer.test.ts create mode 100644 packages/skin-database/data/FileInfoModel.ts create mode 100644 packages/skin-database/migrations/20220225101216_add_file_contents_table.ts diff --git a/packages/skin-database/addSkin.ts b/packages/skin-database/addSkin.ts index c46f6c06..d506bf82 100644 --- a/packages/skin-database/addSkin.ts +++ b/packages/skin-database/addSkin.ts @@ -4,11 +4,11 @@ import md5Buffer from "md5"; import * as S3 from "./s3"; import Shooter from "./shooter"; import _temp from "temp"; -import * as Analyser from "./analyser"; import { SkinType } from "./types"; import SkinModel from "./data/SkinModel"; import UserContext from "./data/UserContext"; import JSZip from "jszip"; +import { setHashesForSkin } from "./skinHash"; // TODO Move this into the function so that we clean up on each run? const temp = _temp.track(); @@ -36,11 +36,11 @@ export async function addSkinFromBuffer( // Note: This will thrown on invalid skins. const zip = await JSZip.loadAsync(buffer); - const skinType = await Analyser.getSkinType(zip); + const skinType = await getSkinType(zip); switch (skinType) { case "CLASSIC": - return addClassicSkinFromBuffer(buffer, md5, filePath, uploader); + return addClassicSkinFromBuffer(ctx, buffer, md5, filePath, uploader); case "MODERN": return addModernSkinFromBuffer(buffer, md5, filePath, uploader); } @@ -61,13 +61,13 @@ async function addModernSkinFromBuffer( filePath, uploader, modern: true, - readmeText: null, }); return { md5, status: "ADDED", skinType: "MODERN" }; } async function addClassicSkinFromBuffer( + ctx: UserContext, buffer: Buffer, md5: string, filePath: string, @@ -95,17 +95,29 @@ async function addClassicSkinFromBuffer( throw e; } await S3.putSkin(md5, buffer, "wsz"); - const zip = await JSZip.loadAsync(buffer); - const readmeText = await Analyser.getReadme(zip); + await Skins.addSkin({ md5, filePath, uploader, modern: false, - readmeText, }); - await Skins.updateSearchIndex(md5); + const skin = await SkinModel.fromMd5Assert(ctx, md5); + + await setHashesForSkin(skin); + + await Skins.updateSearchIndex(ctx, md5); return { md5, status: "ADDED", skinType: "CLASSIC" }; } + +export async function getSkinType(zip: JSZip): Promise { + if (zip.file(/main\.bmp$/i).length > 0) { + return "CLASSIC"; + } + if (zip.file(/skin\.xml$/i).length > 0) { + return "MODERN"; + } + throw new Error("Not a skin"); +} diff --git a/packages/skin-database/analyser.ts b/packages/skin-database/analyser.ts deleted file mode 100644 index 5024a6ff..00000000 --- a/packages/skin-database/analyser.ts +++ /dev/null @@ -1,54 +0,0 @@ -// Functions for deriving information from skins - -import { knex } from "./db"; -import JSZip from "jszip"; -import { SkinType } from "./types"; -import * as Skins from "./data/skins"; -import SkinModel from "./data/SkinModel"; - -export async function setReadmeForSkin(skin: SkinModel): Promise { - let text: string | null; - try { - text = await getReadme(await skin.getZip()); - } catch (e) { - console.error(e.message); - return; - } - - if (skin.getReadme() !== text) { - await knex("skins") - .where({ md5: skin.getMd5() }) - .update({ readme_text: text }); - await Skins.updateSearchIndex(skin.getMd5()); - } -} - -const IS_README = /(file_id\.diz)|(\.txt)$/i; -// Skinning Updates.txt ? -const IS_NOT_README = - /(genex\.txt)|(genexinfo\.txt)|(gen_gslyrics\.txt)|(region\.txt)|(pledit\.txt)|(viscolor\.txt)|(winampmb\.txt)|("gen_ex help\.txt)|(mbinner\.txt)$/i; - -export async function getReadme(zip: JSZip): Promise { - const readmeFiles = zip.filter((filePath) => { - return IS_README.test(filePath) && !IS_NOT_README.test(filePath); - }); - - if (readmeFiles.length === 0) { - return null; - } - - // TODO, could we try to get more than one? - const readme = readmeFiles[0]; - - return readme.async("text"); -} - -export async function getSkinType(zip: JSZip): Promise { - if (zip.file(/main\.bmp$/i).length > 0) { - return "CLASSIC"; - } - if (zip.file(/skin\.xml$/i).length > 0) { - return "MODERN"; - } - throw new Error("Not a skin"); -} diff --git a/packages/skin-database/analyzer.test.ts b/packages/skin-database/analyzer.test.ts deleted file mode 100644 index 3dbfd655..00000000 --- a/packages/skin-database/analyzer.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import fsPromises from "fs"; -import path from "path"; -import * as Analyzer from "./analyser"; -import JSZip from "jszip"; - -test("getReadme", async () => { - const zip = await getSkinZip("Sonic_Attitude.wsz"); - const readme = await Analyzer.getReadme(zip); - if (readme == null) { - throw new Error("Expected to find readme."); - } - expect(readme.length).toBe(387); - expect(readme.split("\n")[0].trim()).toMatchInlineSnapshot( - `"SONIC ATTITUDE - By LuigiHann (luigihann@aol.com)"` - ); -}); - -test("getSkinType", async () => { - const zip = await getSkinZip("Sonic_Attitude.wsz"); - const skinType = await Analyzer.getSkinType(zip); - expect(skinType).toBe("CLASSIC"); -}); - -function getSkinZip(filename: string): Promise { - const buffer = fsPromises.readFileSync( - path.join(__dirname, "../webamp/demo/skins/", filename) - ); - return JSZip.loadAsync(buffer); -} diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index 96513ce1..7a306e3b 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -324,9 +324,7 @@ program .leftJoin("archive_files", "skins.md5", "archive_files.skin_md5") .where("skin_type", 1) .where((builder) => { - return builder - .where("archive_files.skin_md5", null) - .orWhere("archive_files.uncompressed_size", null); + return builder.where("archive_files.skin_md5", null); }) .limit(80000) .groupBy("skins.md5") diff --git a/packages/skin-database/data/ArchiveFileModel.ts b/packages/skin-database/data/ArchiveFileModel.ts index 07411a98..c891d9a5 100644 --- a/packages/skin-database/data/ArchiveFileModel.ts +++ b/packages/skin-database/data/ArchiveFileModel.ts @@ -3,6 +3,7 @@ import { ArchiveFileRow } from "../types"; import DataLoader from "dataloader"; import { knex } from "../db"; import SkinModel from "./SkinModel"; +import FileInfoModel from "./FileInfoModel"; export type ArchiveFileDebugData = { row: ArchiveFileRow; @@ -50,12 +51,20 @@ export default class ArchiveFileModel { } // Null if directory - getFileSize(): number | null { - return this.row.uncompressed_size; + async getFileSize(): Promise { + const info = await this._getFileInfo(); + if (info == null) { + return null; + } + return info.getFileSize(); } - getTextContent(): string | null { - return this.row.text_content; + async getTextContent(): Promise { + const info = await this._getFileInfo(); + if (info == null) { + return null; + } + return info.getTextContent(); } getIsDirectory(): boolean { @@ -71,6 +80,11 @@ export default class ArchiveFileModel { return SkinModel.fromMd5Assert(this.ctx, this.getMd5()); } + // Let's try to keep this as an implementation detail + async _getFileInfo(): Promise { + return FileInfoModel.fromFileMd5(this.ctx, this.getFileMd5()); + } + async debug(): Promise { return { row: this.row, diff --git a/packages/skin-database/data/FileInfoModel.ts b/packages/skin-database/data/FileInfoModel.ts new file mode 100644 index 00000000..aff25ec0 --- /dev/null +++ b/packages/skin-database/data/FileInfoModel.ts @@ -0,0 +1,48 @@ +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; +import { FileInfoRow } from "../types"; +import DataLoader from "dataloader"; +import { knex } from "../db"; + +export type FileInfoDebugData = { + row: FileInfoRow; +}; + +export default class FileInfoModel { + constructor(readonly ctx: UserContext, readonly row: FileInfoRow) {} + + static async fromFileMd5( + ctx: UserContext, + md5: string + ): Promise { + const row = await getFileInfoByFileMd5Loader(ctx).load(md5); + return row == null ? null : new FileInfoModel(ctx, row); + } + + getFileMd5(): string { + return this.row.file_md5; + } + + getFileSize(): number | null { + return this.row.size_in_bytes; + } + + getTextContent(): string | null { + return this.row.text_content; + } + + async debug(): Promise { + return { + row: this.row, + }; + } +} + +const getFileInfoByFileMd5Loader = ctxWeakMapMemoize< + DataLoader +>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("file_info").whereIn("file_md5", md5s).select(); + return md5s.map((md5) => rows.find((x) => x.file_md5 === md5)); + }) +); diff --git a/packages/skin-database/data/SkinModel.ts b/packages/skin-database/data/SkinModel.ts index dae82f23..356bf3bf 100644 --- a/packages/skin-database/data/SkinModel.ts +++ b/packages/skin-database/data/SkinModel.ts @@ -21,6 +21,11 @@ import JSZip from "jszip"; import fs from "fs/promises"; import path from "path"; +export const IS_README = /(file_id\.diz)|(\.txt)$/i; +// Skinning Updates.txt ? +export const IS_NOT_README = + /(genex\.txt)|(genexinfo\.txt)|(gen_gslyrics\.txt)|(region\.txt)|(pledit\.txt)|(viscolor\.txt)|(winampmb\.txt)|("gen_ex help\.txt)|(mbinner\.txt)$/i; + export default class SkinModel { constructor(readonly ctx: UserContext, readonly row: SkinRow) {} @@ -174,8 +179,17 @@ export default class SkinModel { return emails ? emails.split(" ") : []; } - getReadme(): string | null { - return this.row.readme_text || null; + async getReadme(): Promise { + const files = await this.getArchiveFiles(); + const readme = files.find((file) => { + const filename = file.getFileName(); + return IS_README.test(filename) && !IS_NOT_README.test(filename); + }); + + if (readme == null) { + return null; + } + return readme.getTextContent(); } getMuseumUrl(): string { diff --git a/packages/skin-database/data/skins.ts b/packages/skin-database/data/skins.ts index 400b28d0..14e63ba4 100644 --- a/packages/skin-database/data/skins.ts +++ b/packages/skin-database/data/skins.ts @@ -8,6 +8,7 @@ import * as S3 from "../s3"; import * as CloudFlare from "../CloudFlare"; import SkinModel from "./SkinModel"; import UserContext from "./UserContext"; +import TweetModel from "./TweetModel"; export const SKIN_TYPE = { CLASSIC: 1, @@ -29,19 +30,16 @@ export async function addSkin({ filePath, uploader, modern, - readmeText, }: { md5: string; filePath: string; uploader: string; modern: boolean; - readmeText: string | null; }) { await knex("skins").insert( { md5, skin_type: modern ? SKIN_TYPE.MODERN : SKIN_TYPE.CLASSIC, - readme_text: readmeText, }, [] ); @@ -158,36 +156,40 @@ type SearchIndex = { twitterLikes: number; }; -async function getSearchIndexes(md5s: string[]): Promise { - const skins = await knex("skins") - .leftJoin("tweets", "tweets.skin_md5", "=", "skins.md5") - .leftJoin("skin_reviews", "skin_reviews.skin_md5", "=", "skins.md5") - .leftJoin("files", "files.skin_md5", "=", "skins.md5") - .where("skin_type", SKIN_TYPE.CLASSIC) - .whereIn("md5", md5s) - .groupBy("skins.md5") - .select( - "skins.md5", - "skins.readme_text", - "tweets.likes", - "skin_reviews.review", - "files.file_path" - ); +async function getSearchIndexes( + ctx: UserContext, + md5s: string[] +): Promise { + const skins = await Promise.all( + md5s.map((md5) => { + return SkinModel.fromMd5Assert(ctx, md5); + }) + ); - return skins.map((skin) => { - return { - objectID: skin.md5, - md5: skin.md5, - nsfw: skin.review === "NSFW", - readmeText: skin.readme_text ? truncate(skin.readme_text, 4800) : null, - fileName: path.basename(skin.file_path), - twitterLikes: Number(skin.likes || 0), - }; - }); + return Promise.all( + skins.map(async (skin) => { + const readmeText = await skin.getReadme(); + const tweets = await skin.getTweets(); + const likes = tweets.reduce((acc: number, tweet: TweetModel) => { + return Math.max(acc, tweet.getLikes()); + }, 0); + return { + objectID: skin.getMd5(), + md5: skin.getMd5(), + nsfw: await skin.getIsNsfw(), + readmeText: readmeText ? truncate(readmeText, 4800) : null, + fileName: await skin.getFileName(), + twitterLikes: likes, + }; + }) + ); } -export async function updateSearchIndexs(md5s: string[]): Promise { - const skinIndexes = await getSearchIndexes(md5s); +export async function updateSearchIndexs( + ctx: UserContext, + md5s: string[] +): Promise { + const skinIndexes = await getSearchIndexes(ctx, md5s); const results = await searchIndex.partialUpdateObjects(skinIndexes, { createIfNotExists: true, @@ -199,10 +201,14 @@ export async function updateSearchIndexs(md5s: string[]): Promise { return results; } -export async function updateSearchIndex(md5: string): Promise { - return updateSearchIndexs([md5]); +export async function updateSearchIndex( + ctx: UserContext, + md5: string +): Promise { + return updateSearchIndexs(ctx, [md5]); } +// Note: This might leave behind some files in file_info. export async function deleteSkin(md5: string): Promise { console.log(`Deleting skin ${md5}...`); console.log(`... sqlite "skins"`); diff --git a/packages/skin-database/migrations/20220225101216_add_file_contents_table.ts b/packages/skin-database/migrations/20220225101216_add_file_contents_table.ts new file mode 100644 index 00000000..711cfcb9 --- /dev/null +++ b/packages/skin-database/migrations/20220225101216_add_file_contents_table.ts @@ -0,0 +1,16 @@ +import * as Knex from "knex"; + +export async function up(knex: Knex): Promise { + await knex.raw( + `CREATE TABLE "file_info" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + "file_md5" TEXT NOT NULL UNIQUE, + text_content TEXT, + size_in_bytes INTEGER + );` + ); +} + +export async function down(knex: Knex): Promise { + await knex.raw(`DROP TABLE "file_info"`); +} diff --git a/packages/skin-database/skinHash.ts b/packages/skin-database/skinHash.ts index 5ca52e1c..aeac2373 100644 --- a/packages/skin-database/skinHash.ts +++ b/packages/skin-database/skinHash.ts @@ -13,7 +13,7 @@ type FileData = { }; function mightBeText(filename: string): boolean { - return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm)$/i.test( + return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm|ai|log|uue|)$/i.test( filename ); } @@ -56,19 +56,40 @@ export async function setHashesForSkin(skin: SkinModel): Promise { const hashes = await getSkinFileData(skin); const rows = hashes .filter(notEmpty) - .map( - ({ fileName, md5, date, uncompressedSize, textContent, isDirectory }) => { - return { - skin_md5: skin.getMd5(), - file_name: fileName, - file_md5: md5, - file_date: date, - text_content: textContent, - uncompressed_size: uncompressedSize, - is_directory: isDirectory, - }; - } - ); + .map(({ fileName, md5, date, isDirectory }) => { + return { + skin_md5: skin.getMd5(), + file_name: fileName, + file_md5: md5, // TODO: Consider using an id into file_info table + file_date: date, + // text_content: textContent, + // uncompressed_size: uncompressedSize, + is_directory: isDirectory, + }; + }); await knex("archive_files").where("skin_md5", skin.getMd5()).delete(); await knex("archive_files").insert(rows); + + const fileInfoRows = hashes + .filter(notEmpty) + .filter(({ isDirectory }) => !isDirectory) + .map(({ md5, uncompressedSize, textContent }) => { + return { + file_md5: md5, + text_content: textContent, + size_in_bytes: uncompressedSize, + }; + }); + + for (const row of fileInfoRows) { + const countRows = await knex("file_info") + .where({ file_md5: row.file_md5 }) + .count({ count: "*" }); + if (!countRows[0].count) { + await knex("file_info").insert(row); + console.log("Inserted file info:", row); + } else { + console.log("Already have file info for", row.file_md5); + } + } } diff --git a/packages/skin-database/tasks/refresh.ts b/packages/skin-database/tasks/refresh.ts index 1d2be825..14e2e077 100644 --- a/packages/skin-database/tasks/refresh.ts +++ b/packages/skin-database/tasks/refresh.ts @@ -2,11 +2,11 @@ import UserContext from "../data/UserContext"; import SkinModel from "../data/SkinModel"; import { knex } from "../db"; import { setHashesForSkin } from "../skinHash"; -import * as Analyser from "../analyser"; import Shooter from "../shooter"; import { screenshot } from "./screenshotSkin"; import * as Skins from "../data/skins"; import { SkinType } from "../types"; +import { getSkinType } from "../addSkin"; // TODO Move this into the function so that we clean up on each run? @@ -76,11 +76,9 @@ export async function _refresh( await setHashesForSkin(skin); await Skins.setContentHash(skin.getMd5()); - await Analyser.setReadmeForSkin(skin); - let skinType: SkinType; try { - skinType = await Analyser.getSkinType(await skin.getZip()); + skinType = await getSkinType(await skin.getZip()); } catch (e) { throw new Error("Not a skin (no main.bmp/skin.xml)"); } diff --git a/packages/skin-database/types.ts b/packages/skin-database/types.ts index 5f745f66..e1ab4c5b 100644 --- a/packages/skin-database/types.ts +++ b/packages/skin-database/types.ts @@ -12,7 +12,7 @@ export type SkinRow = { md5: string; skin_type: number; emails: string; - readme_text: string; + // readme_text: string; average_color: string; }; @@ -51,6 +51,12 @@ export type ArchiveFileRow = { is_directory: number; // SQLite uses integers for booleans }; +export type FileInfoRow = { + file_md5: string; + text_content: string | null; + size_in_bytes: number; +}; + export type IaItemRow = { skin_md5: string; identifier: string;