Refactors and work toward fixing consistency with archive.org

packages/skin-database/api/router.ts

@@ -266,10 +266,8 @@ router.post(
   asyncHandler(async (req, res) => {
     const { md5 } = req.params;
     req.log(`Reporting skin with hash "${md5}"`);
-    const skin = await SkinModel.fromMd5(req.ctx, md5);
-    if (skin == null) {
-      throw new Error(`Cold not locate as skin with md5 ${md5}`);
-    }
+    // Blow up if there is no skin with this hash
+    await SkinModel.fromMd5Assert(req.ctx, md5);
     req.notify({ type: "REVIEW_REQUESTED", md5 });
     res.send("The skin has been reported and will be reviewed shortly.");
   })

packages/skin-database/cli.ts

@@ -11,8 +11,16 @@ import { addSkinFromBuffer } from "./addSkin";
 import { scrapeLikeData } from "./tasks/scrapeLikes";
 import { followerCount, popularTweets } from "./tasks/tweetMilestones";
 import UserContext from "./data/UserContext";
-import { integrityCheck } from "./tasks/integrityCheck";
-import { ensureWebampLinks, syncWithArchive } from "./tasks/syncWithArchive";
+import {
+  integrityCheck,
+  checkInternetArchiveMetadata,
+} from "./tasks/integrityCheck";
+import {
+  ensureWebampLinks,
+  findItemsMissingImages,
+  syncWithArchive,
+  uploadScreenshotIfSafe,
+} from "./tasks/syncToArchive";
 import { fillMissingMetadata, syncFromArchive } from "./tasks/syncFromArchive";
 import { refreshSkins } from "./tasks/refresh";
 import {
@@ -110,10 +118,7 @@ program
       console.log(await Skins.updateSearchIndex(md5));
     }
     if (refresh) {
-      const skin = await SkinModel.fromMd5(ctx, md5);
-      if (skin == null) {
-        throw new Error(`Could not find skin ${md5}`);
-      }
+      const skin = await SkinModel.fromMd5Assert(ctx, md5);
       await refreshSkins([skin]);
     }
     if (reject) {
@@ -172,7 +177,7 @@ program
       "and add them."
   )
   .option(
-    "--upload",
+    "--upload-new",
     "Find newly uploaded skins, and publish them to the Internet Archive."
   )
   .action(async ({ metadata, webampLinks, fetchItems, upload }) => {
@@ -223,8 +228,16 @@ program
 program
   .command("integrity-check")
   .description("Perfrom a non-exhaustive list of database consistency checks")
-  .action(async () => {
-    await integrityCheck();
+  .option(
+    "--ia",
+    "Check the Internet Archive for items that are missing files."
+  )
+  .action(async ({ ia }) => {
+    if (ia) {
+      await checkInternetArchiveMetadata();
+    } else {
+      await integrityCheck();
+    }
   });
 
 /**
@@ -265,6 +278,41 @@ program
     }
   });
 
+/**
+ * Commands thare are still in development
+ */
+
+program
+  .command("dev")
+  .description("Grab bag of commands that don't have a place to live yet")
+  .option(
+    "--upload-ia-screenshot <md5>",
+    "Upload a screenshot of a skin to the skin's Internet Archive itme. " +
+      "[[Warning!]] This might result in multiple screenshots on the item."
+  )
+  .option(
+    "--upload-missing-screenshots",
+    "Find all IA items that are missing screenshots, and upload the missing ones."
+  )
+  .action(async ({ uploadIaScreenshot, uploadMissingScreenshots }) => {
+    if (uploadIaScreenshot) {
+      const md5 = uploadIaScreenshot;
+      if (!(await uploadScreenshotIfSafe(md5))) {
+        console.log("Did not upload screenshot");
+      }
+    }
+    if (uploadMissingScreenshots) {
+      const md5s = await findItemsMissingImages();
+      for (const md5 of md5s) {
+        if (await uploadScreenshotIfSafe(md5)) {
+          console.log("Upladed screenshot for ", md5);
+        } else {
+          console.log("Did not upload screenshot for ", md5);
+        }
+      }
+    }
+  });
+
 async function main() {
   try {
     await program.parseAsync(process.argv);

packages/skin-database/data/IaItemModel.ts

@@ -3,6 +3,7 @@ import { IaItemRow } from "../types";
 import SkinModel from "./SkinModel";
 import DataLoader from "dataloader";
 import { knex } from "../db";
+import { exec } from "../utils";
 
 const IA_URL = /^(https:\/\/)?archive.org\/details\/([^/]+)\/?/;
 
@@ -70,6 +71,88 @@ export default class IaItemModel {
     return identifier;
   }
 
+  getAllFiles(): any[] {
+    if (this.row.metadata == null) {
+      return [];
+    }
+    return JSON.parse(this.row.metadata).files;
+  }
+
+  getUploadedFiles(): any {
+    return this.getAllFiles().filter(isNotGeneratedFile);
+  }
+
+  // There should be exactly one, but in error cases there can be more or none.
+  getSkinFiles(): any[] {
+    return this.getUploadedFiles().filter((file) => file.name.endsWith(".wsz"));
+  }
+
+  async getTasks(): Promise<any[]> {
+    const result = await exec(`ia tasks ${this.getIdentifier()}`, {
+      encoding: "utf8",
+    });
+    return result.stdout
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line));
+  }
+
+  async hasRunningTasks(): Promise<boolean> {
+    const tasks = await this.getTasks();
+    const hasTasks = tasks.some((task) => {
+      // I'm not really sure what the schema is here. From my limited observations:
+      //  - All completed tasks have a cateory of "history"
+      //  - All completed tasks have a finished timestamp
+      //  - All running tasks have a status of "running"
+      return (
+        task.status === "running" ||
+        task.category !== "history" ||
+        task.finished == null
+      );
+    });
+
+    // Just to be sure, let's invalidate the metadata we have.
+    if (hasTasks) {
+      await this.invalidateMetadata();
+    }
+    return hasTasks;
+  }
+
+  // Fetch new metadata assuming there a no running tasks.
+  async updateMetadataUnsafe(): Promise<void> {
+    // TODO: Move some of this into a IA service.
+    const identifier = this.getIdentifier();
+    const r = await fetch(`https://archive.org/metadata/${identifier}`);
+    if (!r.ok) {
+      console.error(await r.json());
+      throw new Error(`Could not fetch metadata for ${identifier}`);
+    }
+    const response = await r.json();
+    await knex("ia_items")
+      .where("identifier", identifier)
+      .update({ metadata: JSON.stringify(response, null, 2) })
+      .update("metadata_timestamp", knex.fn.now());
+  }
+
+  // Check if there are any running tasks and, if not, update the metadata.
+  async updateMetadata(): Promise<boolean> {
+    if (await this.hasRunningTasks()) {
+      return false;
+    }
+    await this.updateMetadataUnsafe();
+
+    return true;
+  }
+
+  // Clear our local cache of metadata. Call this any time you update an IA item
+  // in such a way that it might change the metadata.
+  async invalidateMetadata(): Promise<void> {
+    this.row.metadata = "";
+    await knex("ia_items")
+      .where("identifier", this.getIdentifier())
+      .update({ metadata: "", metadata_timestamp: null });
+  }
+
   async debug(): Promise<IaItemDebugData> {
     return {
       row: this.row,
@@ -77,6 +160,21 @@ export default class IaItemModel {
   }
 }
 
+function isNotGeneratedFile(file) {
+  switch (file.source) {
+    case "derivative":
+    case "metadata":
+      return false;
+  }
+  switch (file.format) {
+    case "Metadata":
+    case "Item Tile":
+    case "JPEG Thumb":
+      return false;
+  }
+  return true;
+}
+
 const getIaItemLoader = ctxWeakMapMemoize<DataLoader<string, IaItemRow>>(
   () =>
     new DataLoader(async (md5s) => {

packages/skin-database/data/InstagramPostModel.ts

@@ -45,11 +45,7 @@ export default class InstagramPostModel {
   }
 
   async getSkin(): Promise<SkinModel> {
-    const skin = await SkinModel.fromMd5(this.ctx, this.getMd5());
-    if (skin == null) {
-      throw new Error(`Could not find skin for md5 "${this.getMd5()}"`);
-    }
-    return skin;
+    return SkinModel.fromMd5Assert(this.ctx, this.getMd5());
   }
 
   async debug(): Promise<InstagramPostDebugData> {
@@ -64,7 +60,9 @@ const getInstagramPostsLoader = ctxWeakMapMemoize<
 >(
   () =>
     new DataLoader(async (md5s) => {
-      const rows = await knex("instagram_posts").whereIn("skin_md5", md5s).select();
+      const rows = await knex("instagram_posts")
+        .whereIn("skin_md5", md5s)
+        .select();
       return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5));
     })
 );
@@ -74,7 +72,9 @@ const getInstagramPostsByPostIdLoader = ctxWeakMapMemoize<
 >(
   () =>
     new DataLoader(async (ids) => {
-      const rows = await knex("instagram_posts").whereIn("post_id", ids).select();
+      const rows = await knex("instagram_posts")
+        .whereIn("post_id", ids)
+        .select();
       return ids.map((id) => rows.find((x) => x.tweet_id === id));
     })
 );

packages/skin-database/data/SkinModel.ts

@@ -10,7 +10,7 @@ import UserContext, { ctxWeakMapMemoize } from "./UserContext";
 import TweetModel, { TweetDebugData } from "./TweetModel";
 import IaItemModel, { IaItemDebugData } from "./IaItemModel";
 import FileModel, { FileDebugData } from "./FileModel";
-import { MD5_REGEX } from "../utils";
+import { MD5_REGEX, withUrlAsTempFile } from "../utils";
 import DataLoader from "dataloader";
 import { knex } from "../db";
 import UploadModel, { UploadDebugData } from "./UploadModel";
@@ -30,6 +30,13 @@ export default class SkinModel {
     return row == null ? null : new SkinModel(ctx, row);
   }
 
+  static async fromMd5Assert(
+    ctx: UserContext,
+    md5: string
+  ): Promise<SkinModel> {
+    return SkinModel.fromMd5Assert(ctx, md5);
+  }
+
   static async fromAnything(
     ctx: UserContext,
     anything: string
@@ -136,7 +143,16 @@ export default class SkinModel {
     if (files.length === 0) {
       throw new Error(`Could not find file for skin with md5 ${this.getMd5()}`);
     }
-    return files[0].getFileName();
+    const filename = files[0].getFileName();
+    if (!filename.match(/\.(zip)|(wsz)$/i)) {
+      throw new Error("Expected filename to end with zip or wsz.");
+    }
+    return filename;
+  }
+
+  async getScreenshotFileName(): Promise<string> {
+    const skinFilename = await this.getFileName();
+    return skinFilename.replace(/\.(wsz|zip)$/, ".png");
   }
 
   getMd5(): string {
@@ -169,22 +185,41 @@ export default class SkinModel {
     return this.row.average_color;
   }
 
-  getBuffer = mem(
-    async (): Promise<Buffer> => {
-      const response = await fetch(this.getSkinUrl());
-      if (!response.ok) {
-        throw new Error(`Could not fetch skin at "${this.getSkinUrl()}"`);
-      }
-      return response.buffer();
+  getBuffer = mem(async (): Promise<Buffer> => {
+    const response = await fetch(this.getSkinUrl());
+    if (!response.ok) {
+      throw new Error(`Could not fetch skin at "${this.getSkinUrl()}"`);
     }
-  );
+    return response.buffer();
+  });
 
-  getZip = mem(
-    async (): Promise<JSZip> => {
-      const buffer = await this.getBuffer();
-      return JSZip.loadAsync(buffer);
+  getScreenshotBuffer = mem(async (): Promise<Buffer> => {
+    const response = await fetch(this.getScreenshotUrl());
+    if (!response.ok) {
+      throw new Error(
+        `Could not fetch skin screenshot at "${this.getScreenshotUrl()}"`
+      );
     }
-  );
+    return response.buffer();
+  });
+
+  async withTempFile(cb: (file: string) => Promise<void>): Promise<void> {
+    const filename = await this.getFileName();
+
+    return withUrlAsTempFile(this.getSkinUrl(), filename, cb);
+  }
+
+  async withScreenshotTempFile(
+    cb: (file: string) => Promise<void>
+  ): Promise<void> {
+    const screenshotFilename = await this.getScreenshotFileName();
+    return withUrlAsTempFile(this.getScreenshotUrl(), screenshotFilename, cb);
+  }
+
+  getZip = mem(async (): Promise<JSZip> => {
+    const buffer = await this.getBuffer();
+    return JSZip.loadAsync(buffer);
+  });
 
   async debug(): Promise<{
     row: SkinRow;

packages/skin-database/data/TweetModel.ts

@@ -61,11 +61,7 @@ export default class TweetModel {
   }
 
   async getSkin(): Promise<SkinModel> {
-    const skin = await SkinModel.fromMd5(this.ctx, this.getMd5());
-    if (skin == null) {
-      throw new Error(`Could not find skin for md5 "${this.getMd5()}"`);
-    }
-    return skin;
+    return SkinModel.fromMd5Assert(this.ctx, this.getMd5());
   }
 
   async debug(): Promise<TweetDebugData> {

packages/skin-database/tasks/insta.ts

@@ -93,10 +93,7 @@ const UPSCALE = 4;
 
 async function post(md5: string): Promise<string> {
   const ctx = new UserContext();
-  const skin = await SkinModel.fromMd5(ctx, md5);
-  if (skin == null) {
-    throw new Error("Could not find skin");
-  }
+  const skin = await SkinModel.fromMd5Assert(ctx, md5);
   const screenshot = await Skins.getScreenshotBuffer(md5);
   const { width, height } = await sharp(screenshot).metadata();
 

packages/skin-database/tasks/integrityCheck.ts

@@ -2,6 +2,66 @@ import fs from "fs";
 import { knex } from "../db";
 import { TWEET_SNOWFLAKE_REGEX, MD5_REGEX } from "../utils";
 
+function isNotGeneratedFile(file) {
+  switch (file.source) {
+    case "derivative":
+    case "metadata":
+      return false;
+  }
+  switch (file.format) {
+    case "Metadata":
+    case "Item Tile":
+    case "JPEG Thumb":
+      return false;
+  }
+  return true;
+}
+
+export async function checkInternetArchiveMetadata(): Promise<void> {
+  const results = await knex.raw(
+    'SELECT skin_md5, identifier, metadata FROM ia_items WHERE metadata != "";'
+  );
+
+  const tooMany: string[] = [];
+  const tooFew: string[] = [];
+  const missingSkin: string[] = [];
+
+  for (const item of results) {
+    const { identifier, metadata, skin_md5 } = item;
+    try {
+      const allFiles = JSON.parse(metadata).files;
+      const files = allFiles.filter(isNotGeneratedFile);
+      if (files.length > 2) {
+        tooMany.push(skin_md5);
+        continue;
+        console.warn("Too many files", { files, identifier, skin_md5 });
+      }
+      const skinFile = files.find((file) => file.md5 === skin_md5);
+      if (skinFile == null) {
+        missingSkin.push(skin_md5);
+        continue;
+        console.warn("No skin file", { identifier, skin_md5 });
+      }
+      if (files.length < 2) {
+        console.log({skin_md5, identifier, length: files.length});
+        tooFew.push(skin_md5);
+        continue;
+        console.warn("Too few files", { identifier, skin_md5 });
+      }
+    } catch (e) {
+      console.log(metadata);
+    }
+  }
+
+
+  console.table({
+    total: results.length,
+    tooMany: tooMany.length,
+    tooFew: tooFew.length,
+    missingSkin: missingSkin.length,
+  });
+}
+
 export async function integrityCheck(): Promise<void> {
   await noDuplicateTweetIds();
   await noDuplicateTweetMd5s();

packages/skin-database/tasks/syncFromArchive.ts

@@ -1,20 +1,9 @@
 import { knex } from "../db";
 import fetch from "node-fetch";
 import UserContext from "../data/UserContext";
-import SkinModel from "../data/SkinModel";
-import child_process from "child_process";
 import * as Parallel from "async-parallel";
-import { chunk } from "../utils";
-import util from "util";
-const exec = util.promisify(child_process.exec);
-
-function flatten<T>(matrix: T[][]): T[] {
-  const flat: T[] = [];
-  matrix.forEach((arr) => {
-    flat.push(...arr);
-  });
-  return flat;
-}
+import { chunk, flatten } from "../utils";
+import IaItemModel from "../data/IaItemModel";
 
 async function _filterOutKnownIdentifiers(
   identifiers: string[]
@@ -69,47 +58,8 @@ async function allItems(): Promise<string[]> {
   return items.map((item: { identifier: string }) => item.identifier);
 }
 
-async function ensureIaRecord(
-  ctx: UserContext,
-  identifier: string
-): Promise<void> {
-  const response = await updateMetadata(identifier);
-  const files = response.files;
-  const skins = files.filter((file) => file.name.endsWith(".wsz"));
-  if (skins.length === 0) {
-    // TODO TODO TODO TODO
-    // TODO TODO TODO TODO
-    //
-    // What if the skin ends in .zip?
-    //
-    // TODO TODO TODO TODO
-    // TODO TODO TODO TODO
-    console.log(`No skins found in ${identifier}. Deleting... (YOLO)`);
-    const command = `ia delete ${identifier} --all`;
-    // await exec(command, { encoding: "utf8" });
-    console.log(`Deleted ${identifier}`);
-    return;
-  }
-  if (skins.length !== 1) {
-    console.error(
-      `Expected to find one skin file for "${identifier}", found ${skins.length}`
-    );
-    return;
-  }
-  const md5 = skins[0].md5;
-  const skin = await SkinModel.fromMd5(ctx, md5);
-  if (skin == null) {
-    console.error(
-      `We don't have a record for the skin found in "${identifier}"`
-    );
-    return;
-  }
-
-  await knex("ia_items").insert({ skin_md5: md5, identifier });
-  console.log(`Inserted "${identifier}".`);
-}
-
 export async function fillMissingMetadata(count: number) {
+  const ctx = new UserContext();
   const skins = await knex("skins")
     .leftJoin("ia_items", "skins.md5", "ia_items.skin_md5")
     .where("ia_items.metadata", null)
@@ -117,27 +67,19 @@ export async function fillMissingMetadata(count: number) {
     .limit(count)
     .select("skins.md5", "ia_items.identifier");
   for (const { md5, identifier } of skins) {
-    await updateMetadata(identifier);
+    const iaItem = await IaItemModel.fromIdentifier(ctx, identifier);
+    if (iaItem == null) {
+      console.error(`Could not find IA item for ${identifier}`);
+      break;
+    }
+    await iaItem.updateMetadata();
     console.log(`Updated metadata for ${identifier} ${md5}`);
   }
   console.log("Done updating metadata.");
 }
 
-async function updateMetadata(identifier: string): Promise<any> {
-  const r = await fetch(`https://archive.org/metadata/${identifier}`);
-  if (!r.ok) {
-    console.error(await r.json())
-    throw new Error(`Could not fetch metadata for ${identifier}`);
-  }
-  const response = await r.json();
-  await knex("ia_items")
-    .where("identifier", identifier)
-    .update({ metadata: JSON.stringify(response, null, 2) })
-    .update("metadata_timestamp", knex.fn.now());
-  return response;
-}
-
 export async function syncFromArchive() {
+  throw new Error("This needs to be rewritten");
   const ctx = new UserContext();
   // Ensure we know about all items in the `winampskins` collection.
   console.log("Going to ensure we know about all archive items");
@@ -146,7 +88,8 @@ export async function syncFromArchive() {
   await Parallel.each(
     unknownItems,
     async (identifier) => {
-      await ensureIaRecord(ctx, identifier);
+      console.log(identifier, ctx);
+      // await ensureIaRecord(ctx, identifier);
     },
     CONCURRENT
   );

packages/skin-database/tasks/syncToArchive.ts

@@ -3,14 +3,70 @@ import path from "path";
 import fetch from "node-fetch";
 import _temp from "temp";
 import fs from "fs";
-import child_process from "child_process";
 import UserContext from "../data/UserContext";
 import SkinModel from "../data/SkinModel";
-import util from "util";
 import * as Parallel from "async-parallel";
 import IaItemModel from "../data/IaItemModel";
 import DiscordEventHandler from "../api/DiscordEventHandler";
-const exec = util.promisify(child_process.exec);
+import { exec } from "../utils";
+
+export async function findItemsMissingImages(): Promise<string[]> {
+  const ctx = new UserContext();
+  const results = await knex.raw("SELECT * FROM ia_items;");
+
+  const md5s: string[] = [];
+
+  for (const row of results) {
+    const iaItem = new IaItemModel(ctx, row);
+    if (iaItem == null) {
+      throw new Error("Expected to find IA item");
+    }
+    if (
+      iaItem.getUploadedFiles().length >= 2 ||
+      iaItem.getSkinFiles().length !== 1
+    ) {
+      continue;
+    }
+
+    md5s.push(iaItem.getMd5());
+  }
+
+  return md5s;
+}
+
+// Uploads the screenshot to IA if it's safe to do so. In general this should
+// not be needed, since we usually upload both files at once.
+export async function uploadScreenshotIfSafe(md5: string): Promise<boolean> {
+  const ctx = new UserContext();
+  const skin = await SkinModel.fromMd5Assert(ctx, md5);
+  const iaItem = await skin.getIaItem();
+  if (iaItem == null) {
+    throw new Error("Expected ia item to exist");
+  }
+  if (!iaItem.row.metadata) {
+    return false;
+  }
+  if (await iaItem.hasRunningTasks()) {
+    return false;
+  }
+  const skinFiles = iaItem.getSkinFiles();
+  if (skinFiles.length != 1) {
+    return false;
+  }
+  const uploadedFiles = iaItem.getUploadedFiles();
+  if (uploadedFiles.length !== skinFiles.length) {
+    return false;
+  }
+
+  await skin.withScreenshotTempFile(async (screenshotFile) => {
+    const command = `ia upload ${iaItem.getIdentifier()} "${screenshotFile}"`;
+    await exec(command, { encoding: "utf8" });
+  });
+  await iaItem.invalidateMetadata();
+  return true;
+}
+
+/** LEGACY BELOW HERE */
 
 const CONCURRENT = 1;
 
@@ -74,24 +130,8 @@ async function getNewIdentifier(filename: string): Promise<string> {
 
 export async function archive(skin: SkinModel): Promise<string> {
   const filename = await skin.getFileName();
-  if (filename == null) {
-    throw new Error(
-      `Couldn't archive skin. Filename not found. ${skin.getMd5()}`
-    );
-  }
 
-  if (
-    !(
-      filename.toLowerCase().endsWith(".wsz") ||
-      filename.toLowerCase().endsWith(".zip")
-    )
-  ) {
-    throw new Error(
-      `Unexpected file extension for ${skin.getMd5()}: ${filename}`
-    );
-  }
-
-  const screenshotFilename = filename.replace(/\.(wsz|zip)$/, ".png");
+  const screenshotFilename = await skin.getScreenshotFileName();
   const title = `Winamp Skin: ${filename}`;
 
   const [skinFile, screenshotFile] = await Promise.all([
@@ -126,15 +166,11 @@ export async function syncWithArchive(handler: DiscordEventHandler) {
   await Parallel.map(
     unarchived,
     async ({ md5 }) => {
-        // The internet archive claims this one is corrupt for some reason.
-        if (md5 === "513fdd06bf39391e52f3ac5b233dd147") {
-
-            return;
-        }
-      const skin = await SkinModel.fromMd5(ctx, md5);
-      if (skin == null) {
-        throw new Error(`Expected to get skin for ${md5}`);
+      // The internet archive claims this one is corrupt for some reason.
+      if (md5 === "513fdd06bf39391e52f3ac5b233dd147") {
+        return;
       }
+      const skin = await SkinModel.fromMd5Assert(ctx, md5);
       try {
         console.log(`Attempting to upload ${md5}`);
         const identifier = await archive(skin);
@@ -152,7 +188,9 @@ export async function syncWithArchive(handler: DiscordEventHandler) {
         ) {
           console.log(`Corrupt archvie (encrypted): ${skin.getMd5()}`);
         } else if (/case alias may already exist/.test(e.message)) {
-          console.log(`Invalid name (case alias): ${skin.getMd5()} with ${e.message}`);
+          console.log(
+            `Invalid name (case alias): ${skin.getMd5()} with ${e.message}`
+          );
         } else {
           console.error(e);
         }

packages/skin-database/types.ts

@@ -50,6 +50,7 @@ export type ArchiveFileRow = {
 export type IaItemRow = {
   skin_md5: string;
   identifier: string;
+  metadata: string; // JSON from the internet archive
 };
 
 export type UploadStatus =

packages/skin-database/utils.ts

@@ -1,3 +1,20 @@
+import fs from "fs";
+import _temp from "temp";
+import fetch from "node-fetch";
+import util from "util";
+import child_process from "child_process";
+import path from "path";
+
+export const exec = util.promisify(child_process.exec);
+
+const temp = _temp.track();
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
 export function truncate(str: string, len: number): string {
   const overflow = str.length - len;
   if (overflow < 0) {
@@ -19,6 +36,14 @@ export function chunk<T>(items: T[], chunkSize: number): T[][] {
   return chunks;
 }
 
+export function flatten<T>(matrix: T[][]): T[] {
+  const flat: T[] = [];
+  matrix.forEach((arr) => {
+    flat.push(...arr);
+  });
+  return flat;
+}
+
 export const MD5_REGEX = /([a-fA-F0-9]{32})/;
 export const TWEET_SNOWFLAKE_REGEX = /([0-9]{19})/;
 
@@ -27,3 +52,21 @@ export function throwAfter(message, ms) {
     setTimeout(() => reject(new Error(message)), ms);
   });
 }
+
+export async function withUrlAsTempFile(
+  url: string,
+  filename: string,
+  cb: (file: string) => Promise<void>
+): Promise<void> {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download ${url}.`);
+  }
+  const result = await response.buffer();
+  const tempDir = temp.mkdirSync();
+  const tempFile = path.join(tempDir, filename);
+  fs.writeFileSync(tempFile, result);
+  await cb(tempFile);
+  fs.unlinkSync(tempFile);
+  fs.rmdirSync(tempDir);
+}