Add file_info and rm redme_text column

packages/skin-database/addSkin.ts

@@ -4,11 +4,11 @@ import md5Buffer from "md5";
 import * as S3 from "./s3";
 import Shooter from "./shooter";
 import _temp from "temp";
-import * as Analyser from "./analyser";
 import { SkinType } from "./types";
 import SkinModel from "./data/SkinModel";
 import UserContext from "./data/UserContext";
 import JSZip from "jszip";
+import { setHashesForSkin } from "./skinHash";
 
 // TODO Move this into the function so that we clean up on each run?
 const temp = _temp.track();
@@ -36,11 +36,11 @@ export async function addSkinFromBuffer(
 
   // Note: This will thrown on invalid skins.
   const zip = await JSZip.loadAsync(buffer);
-  const skinType = await Analyser.getSkinType(zip);
+  const skinType = await getSkinType(zip);
 
   switch (skinType) {
     case "CLASSIC":
-      return addClassicSkinFromBuffer(buffer, md5, filePath, uploader);
+      return addClassicSkinFromBuffer(ctx, buffer, md5, filePath, uploader);
     case "MODERN":
       return addModernSkinFromBuffer(buffer, md5, filePath, uploader);
   }
@@ -61,13 +61,13 @@ async function addModernSkinFromBuffer(
     filePath,
     uploader,
     modern: true,
-    readmeText: null,
   });
 
   return { md5, status: "ADDED", skinType: "MODERN" };
 }
 
 async function addClassicSkinFromBuffer(
+  ctx: UserContext,
   buffer: Buffer,
   md5: string,
   filePath: string,
@@ -95,17 +95,29 @@ async function addClassicSkinFromBuffer(
     throw e;
   }
   await S3.putSkin(md5, buffer, "wsz");
-  const zip = await JSZip.loadAsync(buffer);
-  const readmeText = await Analyser.getReadme(zip);
+
   await Skins.addSkin({
     md5,
     filePath,
     uploader,
     modern: false,
-    readmeText,
   });
 
-  await Skins.updateSearchIndex(md5);
+  const skin = await SkinModel.fromMd5Assert(ctx, md5);
+
+  await setHashesForSkin(skin);
+
+  await Skins.updateSearchIndex(ctx, md5);
 
   return { md5, status: "ADDED", skinType: "CLASSIC" };
 }
+
+export async function getSkinType(zip: JSZip): Promise<SkinType> {
+  if (zip.file(/main\.bmp$/i).length > 0) {
+    return "CLASSIC";
+  }
+  if (zip.file(/skin\.xml$/i).length > 0) {
+    return "MODERN";
+  }
+  throw new Error("Not a skin");
+}

packages/skin-database/analyser.ts

@@ -1,54 +0,0 @@
-// Functions for deriving information from skins
-
-import { knex } from "./db";
-import JSZip from "jszip";
-import { SkinType } from "./types";
-import * as Skins from "./data/skins";
-import SkinModel from "./data/SkinModel";
-
-export async function setReadmeForSkin(skin: SkinModel): Promise<void> {
-  let text: string | null;
-  try {
-    text = await getReadme(await skin.getZip());
-  } catch (e) {
-    console.error(e.message);
-    return;
-  }
-
-  if (skin.getReadme() !== text) {
-    await knex("skins")
-      .where({ md5: skin.getMd5() })
-      .update({ readme_text: text });
-    await Skins.updateSearchIndex(skin.getMd5());
-  }
-}
-
-const IS_README = /(file_id\.diz)|(\.txt)$/i;
-// Skinning Updates.txt ?
-const IS_NOT_README =
-  /(genex\.txt)|(genexinfo\.txt)|(gen_gslyrics\.txt)|(region\.txt)|(pledit\.txt)|(viscolor\.txt)|(winampmb\.txt)|("gen_ex help\.txt)|(mbinner\.txt)$/i;
-
-export async function getReadme(zip: JSZip): Promise<string | null> {
-  const readmeFiles = zip.filter((filePath) => {
-    return IS_README.test(filePath) && !IS_NOT_README.test(filePath);
-  });
-
-  if (readmeFiles.length === 0) {
-    return null;
-  }
-
-  // TODO, could we try to get more than one?
-  const readme = readmeFiles[0];
-
-  return readme.async("text");
-}
-
-export async function getSkinType(zip: JSZip): Promise<SkinType> {
-  if (zip.file(/main\.bmp$/i).length > 0) {
-    return "CLASSIC";
-  }
-  if (zip.file(/skin\.xml$/i).length > 0) {
-    return "MODERN";
-  }
-  throw new Error("Not a skin");
-}

packages/skin-database/analyzer.test.ts

@@ -1,29 +0,0 @@
-import fsPromises from "fs";
-import path from "path";
-import * as Analyzer from "./analyser";
-import JSZip from "jszip";
-
-test("getReadme", async () => {
-  const zip = await getSkinZip("Sonic_Attitude.wsz");
-  const readme = await Analyzer.getReadme(zip);
-  if (readme == null) {
-    throw new Error("Expected to find readme.");
-  }
-  expect(readme.length).toBe(387);
-  expect(readme.split("\n")[0].trim()).toMatchInlineSnapshot(
-    `"SONIC ATTITUDE - By LuigiHann (luigihann@aol.com)"`
-  );
-});
-
-test("getSkinType", async () => {
-  const zip = await getSkinZip("Sonic_Attitude.wsz");
-  const skinType = await Analyzer.getSkinType(zip);
-  expect(skinType).toBe("CLASSIC");
-});
-
-function getSkinZip(filename: string): Promise<JSZip> {
-  const buffer = fsPromises.readFileSync(
-    path.join(__dirname, "../webamp/demo/skins/", filename)
-  );
-  return JSZip.loadAsync(buffer);
-}

packages/skin-database/cli.ts

@@ -324,9 +324,7 @@ program
           .leftJoin("archive_files", "skins.md5", "archive_files.skin_md5")
           .where("skin_type", 1)
           .where((builder) => {
-            return builder
-              .where("archive_files.skin_md5", null)
-              .orWhere("archive_files.uncompressed_size", null);
+            return builder.where("archive_files.skin_md5", null);
           })
           .limit(80000)
           .groupBy("skins.md5")

packages/skin-database/data/ArchiveFileModel.ts

@@ -3,6 +3,7 @@ import { ArchiveFileRow } from "../types";
 import DataLoader from "dataloader";
 import { knex } from "../db";
 import SkinModel from "./SkinModel";
+import FileInfoModel from "./FileInfoModel";
 
 export type ArchiveFileDebugData = {
   row: ArchiveFileRow;
@@ -50,12 +51,20 @@ export default class ArchiveFileModel {
   }
 
   // Null if directory
-  getFileSize(): number | null {
-    return this.row.uncompressed_size;
+  async getFileSize(): Promise<number | null> {
+    const info = await this._getFileInfo();
+    if (info == null) {
+      return null;
+    }
+    return info.getFileSize();
   }
 
-  getTextContent(): string | null {
-    return this.row.text_content;
+  async getTextContent(): Promise<string | null> {
+    const info = await this._getFileInfo();
+    if (info == null) {
+      return null;
+    }
+    return info.getTextContent();
   }
 
   getIsDirectory(): boolean {
@@ -71,6 +80,11 @@ export default class ArchiveFileModel {
     return SkinModel.fromMd5Assert(this.ctx, this.getMd5());
   }
 
+  // Let's try to keep this as an implementation detail
+  async _getFileInfo(): Promise<FileInfoModel | null> {
+    return FileInfoModel.fromFileMd5(this.ctx, this.getFileMd5());
+  }
+
   async debug(): Promise<ArchiveFileDebugData> {
     return {
       row: this.row,

packages/skin-database/data/FileInfoModel.ts

@@ -0,0 +1,48 @@
+import UserContext, { ctxWeakMapMemoize } from "./UserContext";
+import { FileInfoRow } from "../types";
+import DataLoader from "dataloader";
+import { knex } from "../db";
+
+export type FileInfoDebugData = {
+  row: FileInfoRow;
+};
+
+export default class FileInfoModel {
+  constructor(readonly ctx: UserContext, readonly row: FileInfoRow) {}
+
+  static async fromFileMd5(
+    ctx: UserContext,
+    md5: string
+  ): Promise<FileInfoModel | null> {
+    const row = await getFileInfoByFileMd5Loader(ctx).load(md5);
+    return row == null ? null : new FileInfoModel(ctx, row);
+  }
+
+  getFileMd5(): string {
+    return this.row.file_md5;
+  }
+
+  getFileSize(): number | null {
+    return this.row.size_in_bytes;
+  }
+
+  getTextContent(): string | null {
+    return this.row.text_content;
+  }
+
+  async debug(): Promise<FileInfoDebugData> {
+    return {
+      row: this.row,
+    };
+  }
+}
+
+const getFileInfoByFileMd5Loader = ctxWeakMapMemoize<
+  DataLoader<string, FileInfoRow>
+>(
+  () =>
+    new DataLoader<string, FileInfoRow>(async (md5s) => {
+      const rows = await knex("file_info").whereIn("file_md5", md5s).select();
+      return md5s.map((md5) => rows.find((x) => x.file_md5 === md5));
+    })
+);

packages/skin-database/data/SkinModel.ts

@@ -21,6 +21,11 @@ import JSZip from "jszip";
 import fs from "fs/promises";
 import path from "path";
 
+export const IS_README = /(file_id\.diz)|(\.txt)$/i;
+// Skinning Updates.txt ?
+export const IS_NOT_README =
+  /(genex\.txt)|(genexinfo\.txt)|(gen_gslyrics\.txt)|(region\.txt)|(pledit\.txt)|(viscolor\.txt)|(winampmb\.txt)|("gen_ex help\.txt)|(mbinner\.txt)$/i;
+
 export default class SkinModel {
   constructor(readonly ctx: UserContext, readonly row: SkinRow) {}
 
@@ -174,8 +179,17 @@ export default class SkinModel {
     return emails ? emails.split(" ") : [];
   }
 
-  getReadme(): string | null {
-    return this.row.readme_text || null;
+  async getReadme(): Promise<string | null> {
+    const files = await this.getArchiveFiles();
+    const readme = files.find((file) => {
+      const filename = file.getFileName();
+      return IS_README.test(filename) && !IS_NOT_README.test(filename);
+    });
+
+    if (readme == null) {
+      return null;
+    }
+    return readme.getTextContent();
   }
 
   getMuseumUrl(): string {

packages/skin-database/data/skins.ts

@@ -8,6 +8,7 @@ import * as S3 from "../s3";
 import * as CloudFlare from "../CloudFlare";
 import SkinModel from "./SkinModel";
 import UserContext from "./UserContext";
+import TweetModel from "./TweetModel";
 
 export const SKIN_TYPE = {
   CLASSIC: 1,
@@ -29,19 +30,16 @@ export async function addSkin({
   filePath,
   uploader,
   modern,
-  readmeText,
 }: {
   md5: string;
   filePath: string;
   uploader: string;
   modern: boolean;
-  readmeText: string | null;
 }) {
   await knex("skins").insert(
     {
       md5,
       skin_type: modern ? SKIN_TYPE.MODERN : SKIN_TYPE.CLASSIC,
-      readme_text: readmeText,
     },
     []
   );
@@ -158,36 +156,40 @@ type SearchIndex = {
   twitterLikes: number;
 };
 
-async function getSearchIndexes(md5s: string[]): Promise<SearchIndex[]> {
-  const skins = await knex("skins")
-    .leftJoin("tweets", "tweets.skin_md5", "=", "skins.md5")
-    .leftJoin("skin_reviews", "skin_reviews.skin_md5", "=", "skins.md5")
-    .leftJoin("files", "files.skin_md5", "=", "skins.md5")
-    .where("skin_type", SKIN_TYPE.CLASSIC)
-    .whereIn("md5", md5s)
-    .groupBy("skins.md5")
-    .select(
-      "skins.md5",
-      "skins.readme_text",
-      "tweets.likes",
-      "skin_reviews.review",
-      "files.file_path"
-    );
+async function getSearchIndexes(
+  ctx: UserContext,
+  md5s: string[]
+): Promise<SearchIndex[]> {
+  const skins = await Promise.all(
+    md5s.map((md5) => {
+      return SkinModel.fromMd5Assert(ctx, md5);
+    })
+  );
 
-  return skins.map((skin) => {
-    return {
-      objectID: skin.md5,
-      md5: skin.md5,
-      nsfw: skin.review === "NSFW",
-      readmeText: skin.readme_text ? truncate(skin.readme_text, 4800) : null,
-      fileName: path.basename(skin.file_path),
-      twitterLikes: Number(skin.likes || 0),
-    };
-  });
+  return Promise.all(
+    skins.map(async (skin) => {
+      const readmeText = await skin.getReadme();
+      const tweets = await skin.getTweets();
+      const likes = tweets.reduce((acc: number, tweet: TweetModel) => {
+        return Math.max(acc, tweet.getLikes());
+      }, 0);
+      return {
+        objectID: skin.getMd5(),
+        md5: skin.getMd5(),
+        nsfw: await skin.getIsNsfw(),
+        readmeText: readmeText ? truncate(readmeText, 4800) : null,
+        fileName: await skin.getFileName(),
+        twitterLikes: likes,
+      };
+    })
+  );
 }
 
-export async function updateSearchIndexs(md5s: string[]): Promise<any> {
-  const skinIndexes = await getSearchIndexes(md5s);
+export async function updateSearchIndexs(
+  ctx: UserContext,
+  md5s: string[]
+): Promise<any> {
+  const skinIndexes = await getSearchIndexes(ctx, md5s);
 
   const results = await searchIndex.partialUpdateObjects(skinIndexes, {
     createIfNotExists: true,
@@ -199,10 +201,14 @@ export async function updateSearchIndexs(md5s: string[]): Promise<any> {
   return results;
 }
 
-export async function updateSearchIndex(md5: string): Promise<any | null> {
-  return updateSearchIndexs([md5]);
+export async function updateSearchIndex(
+  ctx: UserContext,
+  md5: string
+): Promise<any | null> {
+  return updateSearchIndexs(ctx, [md5]);
 }
 
+// Note: This might leave behind some files in file_info.
 export async function deleteSkin(md5: string): Promise<void> {
   console.log(`Deleting skin ${md5}...`);
   console.log(`... sqlite "skins"`);

packages/skin-database/migrations/20220225101216_add_file_contents_table.ts

@@ -0,0 +1,16 @@
+import * as Knex from "knex";
+
+export async function up(knex: Knex): Promise<any> {
+  await knex.raw(
+    `CREATE TABLE "file_info" (
+        "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+        "file_md5" TEXT NOT NULL UNIQUE,
+        text_content TEXT,
+        size_in_bytes INTEGER
+    );`
+  );
+}
+
+export async function down(knex: Knex): Promise<any> {
+  await knex.raw(`DROP TABLE "file_info"`);
+}

packages/skin-database/skinHash.ts

@@ -13,7 +13,7 @@ type FileData = {
 };
 
 function mightBeText(filename: string): boolean {
-  return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm)$/i.test(
+  return !/\.(bmp|pdf|png|zip|wsz|cur|ani|jpg|jpeg|db|gif|avs|psd|psp|ico|ttf|milk|bak|maki|m|doc|tmp|_bm|ai|log|uue|)$/i.test(
     filename
   );
 }
@@ -56,19 +56,40 @@ export async function setHashesForSkin(skin: SkinModel): Promise<void> {
   const hashes = await getSkinFileData(skin);
   const rows = hashes
     .filter(notEmpty)
-    .map(
-      ({ fileName, md5, date, uncompressedSize, textContent, isDirectory }) => {
-        return {
-          skin_md5: skin.getMd5(),
-          file_name: fileName,
-          file_md5: md5,
-          file_date: date,
-          text_content: textContent,
-          uncompressed_size: uncompressedSize,
-          is_directory: isDirectory,
-        };
-      }
-    );
+    .map(({ fileName, md5, date, isDirectory }) => {
+      return {
+        skin_md5: skin.getMd5(),
+        file_name: fileName,
+        file_md5: md5, // TODO: Consider using an id into file_info table
+        file_date: date,
+        // text_content: textContent,
+        // uncompressed_size: uncompressedSize,
+        is_directory: isDirectory,
+      };
+    });
   await knex("archive_files").where("skin_md5", skin.getMd5()).delete();
   await knex("archive_files").insert(rows);
+
+  const fileInfoRows = hashes
+    .filter(notEmpty)
+    .filter(({ isDirectory }) => !isDirectory)
+    .map(({ md5, uncompressedSize, textContent }) => {
+      return {
+        file_md5: md5,
+        text_content: textContent,
+        size_in_bytes: uncompressedSize,
+      };
+    });
+
+  for (const row of fileInfoRows) {
+    const countRows = await knex("file_info")
+      .where({ file_md5: row.file_md5 })
+      .count({ count: "*" });
+    if (!countRows[0].count) {
+      await knex("file_info").insert(row);
+      console.log("Inserted file info:", row);
+    } else {
+      console.log("Already have file info for", row.file_md5);
+    }
+  }
 }

packages/skin-database/tasks/refresh.ts

@@ -2,11 +2,11 @@ import UserContext from "../data/UserContext";
 import SkinModel from "../data/SkinModel";
 import { knex } from "../db";
 import { setHashesForSkin } from "../skinHash";
-import * as Analyser from "../analyser";
 import Shooter from "../shooter";
 import { screenshot } from "./screenshotSkin";
 import * as Skins from "../data/skins";
 import { SkinType } from "../types";
+import { getSkinType } from "../addSkin";
 
 // TODO Move this into the function so that we clean up on each run?
 
@@ -76,11 +76,9 @@ export async function _refresh(
   await setHashesForSkin(skin);
   await Skins.setContentHash(skin.getMd5());
 
-  await Analyser.setReadmeForSkin(skin);
-
   let skinType: SkinType;
   try {
-    skinType = await Analyser.getSkinType(await skin.getZip());
+    skinType = await getSkinType(await skin.getZip());
   } catch (e) {
     throw new Error("Not a skin (no main.bmp/skin.xml)");
   }

packages/skin-database/types.ts

@@ -12,7 +12,7 @@ export type SkinRow = {
   md5: string;
   skin_type: number;
   emails: string;
-  readme_text: string;
+  // readme_text: string;
   average_color: string;
 };
 
@@ -51,6 +51,12 @@ export type ArchiveFileRow = {
   is_directory: number; // SQLite uses integers for booleans
 };
 
+export type FileInfoRow = {
+  file_md5: string;
+  text_content: string | null;
+  size_in_bytes: number;
+};
+
 export type IaItemRow = {
   skin_md5: string;
   identifier: string;