diff --git a/packages/skin-database/api/router.ts b/packages/skin-database/api/router.ts index 818754f5..8d8f35fa 100644 --- a/packages/skin-database/api/router.ts +++ b/packages/skin-database/api/router.ts @@ -266,10 +266,8 @@ router.post( asyncHandler(async (req, res) => { const { md5 } = req.params; req.log(`Reporting skin with hash "${md5}"`); - const skin = await SkinModel.fromMd5(req.ctx, md5); - if (skin == null) { - throw new Error(`Cold not locate as skin with md5 ${md5}`); - } + // Blow up if there is no skin with this hash + await SkinModel.fromMd5Assert(req.ctx, md5); req.notify({ type: "REVIEW_REQUESTED", md5 }); res.send("The skin has been reported and will be reviewed shortly."); }) diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index ddee79de..6a09918a 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -11,8 +11,16 @@ import { addSkinFromBuffer } from "./addSkin"; import { scrapeLikeData } from "./tasks/scrapeLikes"; import { followerCount, popularTweets } from "./tasks/tweetMilestones"; import UserContext from "./data/UserContext"; -import { integrityCheck } from "./tasks/integrityCheck"; -import { ensureWebampLinks, syncWithArchive } from "./tasks/syncWithArchive"; +import { + integrityCheck, + checkInternetArchiveMetadata, +} from "./tasks/integrityCheck"; +import { + ensureWebampLinks, + findItemsMissingImages, + syncWithArchive, + uploadScreenshotIfSafe, +} from "./tasks/syncToArchive"; import { fillMissingMetadata, syncFromArchive } from "./tasks/syncFromArchive"; import { refreshSkins } from "./tasks/refresh"; import { @@ -110,10 +118,7 @@ program console.log(await Skins.updateSearchIndex(md5)); } if (refresh) { - const skin = await SkinModel.fromMd5(ctx, md5); - if (skin == null) { - throw new Error(`Could not find skin ${md5}`); - } + const skin = await SkinModel.fromMd5Assert(ctx, md5); await refreshSkins([skin]); } if (reject) { @@ -172,7 +177,7 @@ program "and add them." ) .option( - "--upload", + "--upload-new", "Find newly uploaded skins, and publish them to the Internet Archive." ) .action(async ({ metadata, webampLinks, fetchItems, upload }) => { @@ -223,8 +228,16 @@ program program .command("integrity-check") .description("Perfrom a non-exhaustive list of database consistency checks") - .action(async () => { - await integrityCheck(); + .option( + "--ia", + "Check the Internet Archive for items that are missing files." + ) + .action(async ({ ia }) => { + if (ia) { + await checkInternetArchiveMetadata(); + } else { + await integrityCheck(); + } }); /** @@ -265,6 +278,41 @@ program } }); +/** + * Commands thare are still in development + */ + +program + .command("dev") + .description("Grab bag of commands that don't have a place to live yet") + .option( + "--upload-ia-screenshot ", + "Upload a screenshot of a skin to the skin's Internet Archive itme. " + + "[[Warning!]] This might result in multiple screenshots on the item." + ) + .option( + "--upload-missing-screenshots", + "Find all IA items that are missing screenshots, and upload the missing ones." + ) + .action(async ({ uploadIaScreenshot, uploadMissingScreenshots }) => { + if (uploadIaScreenshot) { + const md5 = uploadIaScreenshot; + if (!(await uploadScreenshotIfSafe(md5))) { + console.log("Did not upload screenshot"); + } + } + if (uploadMissingScreenshots) { + const md5s = await findItemsMissingImages(); + for (const md5 of md5s) { + if (await uploadScreenshotIfSafe(md5)) { + console.log("Upladed screenshot for ", md5); + } else { + console.log("Did not upload screenshot for ", md5); + } + } + } + }); + async function main() { try { await program.parseAsync(process.argv); diff --git a/packages/skin-database/data/IaItemModel.ts b/packages/skin-database/data/IaItemModel.ts index 8d27064f..39848df7 100644 --- a/packages/skin-database/data/IaItemModel.ts +++ b/packages/skin-database/data/IaItemModel.ts @@ -3,6 +3,7 @@ import { IaItemRow } from "../types"; import SkinModel from "./SkinModel"; import DataLoader from "dataloader"; import { knex } from "../db"; +import { exec } from "../utils"; const IA_URL = /^(https:\/\/)?archive.org\/details\/([^/]+)\/?/; @@ -70,6 +71,88 @@ export default class IaItemModel { return identifier; } + getAllFiles(): any[] { + if (this.row.metadata == null) { + return []; + } + return JSON.parse(this.row.metadata).files; + } + + getUploadedFiles(): any { + return this.getAllFiles().filter(isNotGeneratedFile); + } + + // There should be exactly one, but in error cases there can be more or none. + getSkinFiles(): any[] { + return this.getUploadedFiles().filter((file) => file.name.endsWith(".wsz")); + } + + async getTasks(): Promise { + const result = await exec(`ia tasks ${this.getIdentifier()}`, { + encoding: "utf8", + }); + return result.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + } + + async hasRunningTasks(): Promise { + const tasks = await this.getTasks(); + const hasTasks = tasks.some((task) => { + // I'm not really sure what the schema is here. From my limited observations: + // - All completed tasks have a cateory of "history" + // - All completed tasks have a finished timestamp + // - All running tasks have a status of "running" + return ( + task.status === "running" || + task.category !== "history" || + task.finished == null + ); + }); + + // Just to be sure, let's invalidate the metadata we have. + if (hasTasks) { + await this.invalidateMetadata(); + } + return hasTasks; + } + + // Fetch new metadata assuming there a no running tasks. + async updateMetadataUnsafe(): Promise { + // TODO: Move some of this into a IA service. + const identifier = this.getIdentifier(); + const r = await fetch(`https://archive.org/metadata/${identifier}`); + if (!r.ok) { + console.error(await r.json()); + throw new Error(`Could not fetch metadata for ${identifier}`); + } + const response = await r.json(); + await knex("ia_items") + .where("identifier", identifier) + .update({ metadata: JSON.stringify(response, null, 2) }) + .update("metadata_timestamp", knex.fn.now()); + } + + // Check if there are any running tasks and, if not, update the metadata. + async updateMetadata(): Promise { + if (await this.hasRunningTasks()) { + return false; + } + await this.updateMetadataUnsafe(); + + return true; + } + + // Clear our local cache of metadata. Call this any time you update an IA item + // in such a way that it might change the metadata. + async invalidateMetadata(): Promise { + this.row.metadata = ""; + await knex("ia_items") + .where("identifier", this.getIdentifier()) + .update({ metadata: "", metadata_timestamp: null }); + } + async debug(): Promise { return { row: this.row, @@ -77,6 +160,21 @@ export default class IaItemModel { } } +function isNotGeneratedFile(file) { + switch (file.source) { + case "derivative": + case "metadata": + return false; + } + switch (file.format) { + case "Metadata": + case "Item Tile": + case "JPEG Thumb": + return false; + } + return true; +} + const getIaItemLoader = ctxWeakMapMemoize>( () => new DataLoader(async (md5s) => { diff --git a/packages/skin-database/data/InstagramPostModel.ts b/packages/skin-database/data/InstagramPostModel.ts index 95d194bc..1c5fc531 100644 --- a/packages/skin-database/data/InstagramPostModel.ts +++ b/packages/skin-database/data/InstagramPostModel.ts @@ -45,11 +45,7 @@ export default class InstagramPostModel { } async getSkin(): Promise { - const skin = await SkinModel.fromMd5(this.ctx, this.getMd5()); - if (skin == null) { - throw new Error(`Could not find skin for md5 "${this.getMd5()}"`); - } - return skin; + return SkinModel.fromMd5Assert(this.ctx, this.getMd5()); } async debug(): Promise { @@ -64,7 +60,9 @@ const getInstagramPostsLoader = ctxWeakMapMemoize< >( () => new DataLoader(async (md5s) => { - const rows = await knex("instagram_posts").whereIn("skin_md5", md5s).select(); + const rows = await knex("instagram_posts") + .whereIn("skin_md5", md5s) + .select(); return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); }) ); @@ -74,7 +72,9 @@ const getInstagramPostsByPostIdLoader = ctxWeakMapMemoize< >( () => new DataLoader(async (ids) => { - const rows = await knex("instagram_posts").whereIn("post_id", ids).select(); + const rows = await knex("instagram_posts") + .whereIn("post_id", ids) + .select(); return ids.map((id) => rows.find((x) => x.tweet_id === id)); }) ); diff --git a/packages/skin-database/data/SkinModel.ts b/packages/skin-database/data/SkinModel.ts index 74afbeff..8f0d71fb 100644 --- a/packages/skin-database/data/SkinModel.ts +++ b/packages/skin-database/data/SkinModel.ts @@ -10,7 +10,7 @@ import UserContext, { ctxWeakMapMemoize } from "./UserContext"; import TweetModel, { TweetDebugData } from "./TweetModel"; import IaItemModel, { IaItemDebugData } from "./IaItemModel"; import FileModel, { FileDebugData } from "./FileModel"; -import { MD5_REGEX } from "../utils"; +import { MD5_REGEX, withUrlAsTempFile } from "../utils"; import DataLoader from "dataloader"; import { knex } from "../db"; import UploadModel, { UploadDebugData } from "./UploadModel"; @@ -30,6 +30,13 @@ export default class SkinModel { return row == null ? null : new SkinModel(ctx, row); } + static async fromMd5Assert( + ctx: UserContext, + md5: string + ): Promise { + return SkinModel.fromMd5Assert(ctx, md5); + } + static async fromAnything( ctx: UserContext, anything: string @@ -136,7 +143,16 @@ export default class SkinModel { if (files.length === 0) { throw new Error(`Could not find file for skin with md5 ${this.getMd5()}`); } - return files[0].getFileName(); + const filename = files[0].getFileName(); + if (!filename.match(/\.(zip)|(wsz)$/i)) { + throw new Error("Expected filename to end with zip or wsz."); + } + return filename; + } + + async getScreenshotFileName(): Promise { + const skinFilename = await this.getFileName(); + return skinFilename.replace(/\.(wsz|zip)$/, ".png"); } getMd5(): string { @@ -169,22 +185,41 @@ export default class SkinModel { return this.row.average_color; } - getBuffer = mem( - async (): Promise => { - const response = await fetch(this.getSkinUrl()); - if (!response.ok) { - throw new Error(`Could not fetch skin at "${this.getSkinUrl()}"`); - } - return response.buffer(); + getBuffer = mem(async (): Promise => { + const response = await fetch(this.getSkinUrl()); + if (!response.ok) { + throw new Error(`Could not fetch skin at "${this.getSkinUrl()}"`); } - ); + return response.buffer(); + }); - getZip = mem( - async (): Promise => { - const buffer = await this.getBuffer(); - return JSZip.loadAsync(buffer); + getScreenshotBuffer = mem(async (): Promise => { + const response = await fetch(this.getScreenshotUrl()); + if (!response.ok) { + throw new Error( + `Could not fetch skin screenshot at "${this.getScreenshotUrl()}"` + ); } - ); + return response.buffer(); + }); + + async withTempFile(cb: (file: string) => Promise): Promise { + const filename = await this.getFileName(); + + return withUrlAsTempFile(this.getSkinUrl(), filename, cb); + } + + async withScreenshotTempFile( + cb: (file: string) => Promise + ): Promise { + const screenshotFilename = await this.getScreenshotFileName(); + return withUrlAsTempFile(this.getScreenshotUrl(), screenshotFilename, cb); + } + + getZip = mem(async (): Promise => { + const buffer = await this.getBuffer(); + return JSZip.loadAsync(buffer); + }); async debug(): Promise<{ row: SkinRow; diff --git a/packages/skin-database/data/TweetModel.ts b/packages/skin-database/data/TweetModel.ts index f7c33822..7aebc6a9 100644 --- a/packages/skin-database/data/TweetModel.ts +++ b/packages/skin-database/data/TweetModel.ts @@ -61,11 +61,7 @@ export default class TweetModel { } async getSkin(): Promise { - const skin = await SkinModel.fromMd5(this.ctx, this.getMd5()); - if (skin == null) { - throw new Error(`Could not find skin for md5 "${this.getMd5()}"`); - } - return skin; + return SkinModel.fromMd5Assert(this.ctx, this.getMd5()); } async debug(): Promise { diff --git a/packages/skin-database/tasks/insta.ts b/packages/skin-database/tasks/insta.ts index d3575668..88e03032 100644 --- a/packages/skin-database/tasks/insta.ts +++ b/packages/skin-database/tasks/insta.ts @@ -93,10 +93,7 @@ const UPSCALE = 4; async function post(md5: string): Promise { const ctx = new UserContext(); - const skin = await SkinModel.fromMd5(ctx, md5); - if (skin == null) { - throw new Error("Could not find skin"); - } + const skin = await SkinModel.fromMd5Assert(ctx, md5); const screenshot = await Skins.getScreenshotBuffer(md5); const { width, height } = await sharp(screenshot).metadata(); diff --git a/packages/skin-database/tasks/integrityCheck.ts b/packages/skin-database/tasks/integrityCheck.ts index 799553d6..c279ec3c 100644 --- a/packages/skin-database/tasks/integrityCheck.ts +++ b/packages/skin-database/tasks/integrityCheck.ts @@ -2,6 +2,66 @@ import fs from "fs"; import { knex } from "../db"; import { TWEET_SNOWFLAKE_REGEX, MD5_REGEX } from "../utils"; +function isNotGeneratedFile(file) { + switch (file.source) { + case "derivative": + case "metadata": + return false; + } + switch (file.format) { + case "Metadata": + case "Item Tile": + case "JPEG Thumb": + return false; + } + return true; +} + +export async function checkInternetArchiveMetadata(): Promise { + const results = await knex.raw( + 'SELECT skin_md5, identifier, metadata FROM ia_items WHERE metadata != "";' + ); + + const tooMany: string[] = []; + const tooFew: string[] = []; + const missingSkin: string[] = []; + + for (const item of results) { + const { identifier, metadata, skin_md5 } = item; + try { + const allFiles = JSON.parse(metadata).files; + const files = allFiles.filter(isNotGeneratedFile); + if (files.length > 2) { + tooMany.push(skin_md5); + continue; + console.warn("Too many files", { files, identifier, skin_md5 }); + } + const skinFile = files.find((file) => file.md5 === skin_md5); + if (skinFile == null) { + missingSkin.push(skin_md5); + continue; + console.warn("No skin file", { identifier, skin_md5 }); + } + if (files.length < 2) { + console.log({skin_md5, identifier, length: files.length}); + tooFew.push(skin_md5); + continue; + console.warn("Too few files", { identifier, skin_md5 }); + } + } catch (e) { + console.log(metadata); + } + } + + + console.table({ + total: results.length, + tooMany: tooMany.length, + tooFew: tooFew.length, + missingSkin: missingSkin.length, + }); +} + export async function integrityCheck(): Promise { await noDuplicateTweetIds(); await noDuplicateTweetMd5s(); diff --git a/packages/skin-database/tasks/syncFromArchive.ts b/packages/skin-database/tasks/syncFromArchive.ts index 91a5adc5..016078e9 100644 --- a/packages/skin-database/tasks/syncFromArchive.ts +++ b/packages/skin-database/tasks/syncFromArchive.ts @@ -1,20 +1,9 @@ import { knex } from "../db"; import fetch from "node-fetch"; import UserContext from "../data/UserContext"; -import SkinModel from "../data/SkinModel"; -import child_process from "child_process"; import * as Parallel from "async-parallel"; -import { chunk } from "../utils"; -import util from "util"; -const exec = util.promisify(child_process.exec); - -function flatten(matrix: T[][]): T[] { - const flat: T[] = []; - matrix.forEach((arr) => { - flat.push(...arr); - }); - return flat; -} +import { chunk, flatten } from "../utils"; +import IaItemModel from "../data/IaItemModel"; async function _filterOutKnownIdentifiers( identifiers: string[] @@ -69,47 +58,8 @@ async function allItems(): Promise { return items.map((item: { identifier: string }) => item.identifier); } -async function ensureIaRecord( - ctx: UserContext, - identifier: string -): Promise { - const response = await updateMetadata(identifier); - const files = response.files; - const skins = files.filter((file) => file.name.endsWith(".wsz")); - if (skins.length === 0) { - // TODO TODO TODO TODO - // TODO TODO TODO TODO - // - // What if the skin ends in .zip? - // - // TODO TODO TODO TODO - // TODO TODO TODO TODO - console.log(`No skins found in ${identifier}. Deleting... (YOLO)`); - const command = `ia delete ${identifier} --all`; - // await exec(command, { encoding: "utf8" }); - console.log(`Deleted ${identifier}`); - return; - } - if (skins.length !== 1) { - console.error( - `Expected to find one skin file for "${identifier}", found ${skins.length}` - ); - return; - } - const md5 = skins[0].md5; - const skin = await SkinModel.fromMd5(ctx, md5); - if (skin == null) { - console.error( - `We don't have a record for the skin found in "${identifier}"` - ); - return; - } - - await knex("ia_items").insert({ skin_md5: md5, identifier }); - console.log(`Inserted "${identifier}".`); -} - export async function fillMissingMetadata(count: number) { + const ctx = new UserContext(); const skins = await knex("skins") .leftJoin("ia_items", "skins.md5", "ia_items.skin_md5") .where("ia_items.metadata", null) @@ -117,27 +67,19 @@ export async function fillMissingMetadata(count: number) { .limit(count) .select("skins.md5", "ia_items.identifier"); for (const { md5, identifier } of skins) { - await updateMetadata(identifier); + const iaItem = await IaItemModel.fromIdentifier(ctx, identifier); + if (iaItem == null) { + console.error(`Could not find IA item for ${identifier}`); + break; + } + await iaItem.updateMetadata(); console.log(`Updated metadata for ${identifier} ${md5}`); } console.log("Done updating metadata."); } -async function updateMetadata(identifier: string): Promise { - const r = await fetch(`https://archive.org/metadata/${identifier}`); - if (!r.ok) { - console.error(await r.json()) - throw new Error(`Could not fetch metadata for ${identifier}`); - } - const response = await r.json(); - await knex("ia_items") - .where("identifier", identifier) - .update({ metadata: JSON.stringify(response, null, 2) }) - .update("metadata_timestamp", knex.fn.now()); - return response; -} - export async function syncFromArchive() { + throw new Error("This needs to be rewritten"); const ctx = new UserContext(); // Ensure we know about all items in the `winampskins` collection. console.log("Going to ensure we know about all archive items"); @@ -146,7 +88,8 @@ export async function syncFromArchive() { await Parallel.each( unknownItems, async (identifier) => { - await ensureIaRecord(ctx, identifier); + console.log(identifier, ctx); + // await ensureIaRecord(ctx, identifier); }, CONCURRENT ); diff --git a/packages/skin-database/tasks/syncWithArchive.ts b/packages/skin-database/tasks/syncToArchive.ts similarity index 76% rename from packages/skin-database/tasks/syncWithArchive.ts rename to packages/skin-database/tasks/syncToArchive.ts index 330428fb..3b82075b 100644 --- a/packages/skin-database/tasks/syncWithArchive.ts +++ b/packages/skin-database/tasks/syncToArchive.ts @@ -3,14 +3,70 @@ import path from "path"; import fetch from "node-fetch"; import _temp from "temp"; import fs from "fs"; -import child_process from "child_process"; import UserContext from "../data/UserContext"; import SkinModel from "../data/SkinModel"; -import util from "util"; import * as Parallel from "async-parallel"; import IaItemModel from "../data/IaItemModel"; import DiscordEventHandler from "../api/DiscordEventHandler"; -const exec = util.promisify(child_process.exec); +import { exec } from "../utils"; + +export async function findItemsMissingImages(): Promise { + const ctx = new UserContext(); + const results = await knex.raw("SELECT * FROM ia_items;"); + + const md5s: string[] = []; + + for (const row of results) { + const iaItem = new IaItemModel(ctx, row); + if (iaItem == null) { + throw new Error("Expected to find IA item"); + } + if ( + iaItem.getUploadedFiles().length >= 2 || + iaItem.getSkinFiles().length !== 1 + ) { + continue; + } + + md5s.push(iaItem.getMd5()); + } + + return md5s; +} + +// Uploads the screenshot to IA if it's safe to do so. In general this should +// not be needed, since we usually upload both files at once. +export async function uploadScreenshotIfSafe(md5: string): Promise { + const ctx = new UserContext(); + const skin = await SkinModel.fromMd5Assert(ctx, md5); + const iaItem = await skin.getIaItem(); + if (iaItem == null) { + throw new Error("Expected ia item to exist"); + } + if (!iaItem.row.metadata) { + return false; + } + if (await iaItem.hasRunningTasks()) { + return false; + } + const skinFiles = iaItem.getSkinFiles(); + if (skinFiles.length != 1) { + return false; + } + const uploadedFiles = iaItem.getUploadedFiles(); + if (uploadedFiles.length !== skinFiles.length) { + return false; + } + + await skin.withScreenshotTempFile(async (screenshotFile) => { + const command = `ia upload ${iaItem.getIdentifier()} "${screenshotFile}"`; + await exec(command, { encoding: "utf8" }); + }); + await iaItem.invalidateMetadata(); + return true; +} + +/** LEGACY BELOW HERE */ const CONCURRENT = 1; @@ -74,24 +130,8 @@ async function getNewIdentifier(filename: string): Promise { export async function archive(skin: SkinModel): Promise { const filename = await skin.getFileName(); - if (filename == null) { - throw new Error( - `Couldn't archive skin. Filename not found. ${skin.getMd5()}` - ); - } - if ( - !( - filename.toLowerCase().endsWith(".wsz") || - filename.toLowerCase().endsWith(".zip") - ) - ) { - throw new Error( - `Unexpected file extension for ${skin.getMd5()}: ${filename}` - ); - } - - const screenshotFilename = filename.replace(/\.(wsz|zip)$/, ".png"); + const screenshotFilename = await skin.getScreenshotFileName(); const title = `Winamp Skin: ${filename}`; const [skinFile, screenshotFile] = await Promise.all([ @@ -126,15 +166,11 @@ export async function syncWithArchive(handler: DiscordEventHandler) { await Parallel.map( unarchived, async ({ md5 }) => { - // The internet archive claims this one is corrupt for some reason. - if (md5 === "513fdd06bf39391e52f3ac5b233dd147") { - - return; - } - const skin = await SkinModel.fromMd5(ctx, md5); - if (skin == null) { - throw new Error(`Expected to get skin for ${md5}`); + // The internet archive claims this one is corrupt for some reason. + if (md5 === "513fdd06bf39391e52f3ac5b233dd147") { + return; } + const skin = await SkinModel.fromMd5Assert(ctx, md5); try { console.log(`Attempting to upload ${md5}`); const identifier = await archive(skin); @@ -152,7 +188,9 @@ export async function syncWithArchive(handler: DiscordEventHandler) { ) { console.log(`Corrupt archvie (encrypted): ${skin.getMd5()}`); } else if (/case alias may already exist/.test(e.message)) { - console.log(`Invalid name (case alias): ${skin.getMd5()} with ${e.message}`); + console.log( + `Invalid name (case alias): ${skin.getMd5()} with ${e.message}` + ); } else { console.error(e); } diff --git a/packages/skin-database/types.ts b/packages/skin-database/types.ts index 15cc57e2..2d6b472a 100644 --- a/packages/skin-database/types.ts +++ b/packages/skin-database/types.ts @@ -50,6 +50,7 @@ export type ArchiveFileRow = { export type IaItemRow = { skin_md5: string; identifier: string; + metadata: string; // JSON from the internet archive }; export type UploadStatus = diff --git a/packages/skin-database/utils.ts b/packages/skin-database/utils.ts index d40c4b90..7c9134a7 100644 --- a/packages/skin-database/utils.ts +++ b/packages/skin-database/utils.ts @@ -1,3 +1,20 @@ +import fs from "fs"; +import _temp from "temp"; +import fetch from "node-fetch"; +import util from "util"; +import child_process from "child_process"; +import path from "path"; + +export const exec = util.promisify(child_process.exec); + +const temp = _temp.track(); + +export function sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + export function truncate(str: string, len: number): string { const overflow = str.length - len; if (overflow < 0) { @@ -19,6 +36,14 @@ export function chunk(items: T[], chunkSize: number): T[][] { return chunks; } +export function flatten(matrix: T[][]): T[] { + const flat: T[] = []; + matrix.forEach((arr) => { + flat.push(...arr); + }); + return flat; +} + export const MD5_REGEX = /([a-fA-F0-9]{32})/; export const TWEET_SNOWFLAKE_REGEX = /([0-9]{19})/; @@ -27,3 +52,21 @@ export function throwAfter(message, ms) { setTimeout(() => reject(new Error(message)), ms); }); } + +export async function withUrlAsTempFile( + url: string, + filename: string, + cb: (file: string) => Promise +): Promise { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to download ${url}.`); + } + const result = await response.buffer(); + const tempDir = temp.mkdirSync(); + const tempFile = path.join(tempDir, filename); + fs.writeFileSync(tempFile, result); + await cb(tempFile); + fs.unlinkSync(tempFile); + fs.rmdirSync(tempDir); +}