Get syncing to the archive working again

packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts

@@ -24,11 +24,7 @@ export default class ClassicSkinResolver implements NodeResolver, ISkin {
     return toId(this.__typename, this.md5());
   }
   async filename(normalize_extension?: boolean): Promise<string> {
-    const filename = await this._model.getFileName();
+    return await this._model.getFileName(normalize_extension);
-    if (normalize_extension) {
-      return path.parse(filename).name + ".wsz";
-    }
-    return filename;
   }
 
   museum_url(): string {

packages/skin-database/data/SkinModel.ts

@@ -158,7 +158,7 @@ export default class SkinModel {
     return "UNREVIEWED";
   }
 
-  async getFileName(): Promise<string> {
+  async getFileName(normalizeExtension?: boolean): Promise<string> {
     const files = await this.getFiles();
     if (files.length === 0) {
       throw new Error(`Could not find file for skin with md5 ${this.getMd5()}`);
@@ -167,6 +167,9 @@ export default class SkinModel {
     if (!filename.match(/\.(zip)|(wsz)|(wal)$/i)) {
       throw new Error("Expected filename to end with zip, wsz or wal.");
     }
+    if (normalizeExtension) {
+      return path.parse(filename).name + ".wsz";
+    }
     return filename;
   }
 

packages/skin-database/services/internetArchive.ts

@@ -1,5 +1,16 @@
 import fetch from "node-fetch";
-import { exec } from "../utils";
+import { execFile } from "../utils";
+import path from "path";
+
+// Path to the ia command in the virtual environment
+const IA_COMMAND = path.join(__dirname, "../.venv/bin/ia");
+
+// Environment variables for the virtual environment
+const getVenvEnv = () => ({
+  ...process.env,
+  PATH: `${path.join(__dirname, "../.venv/bin")}:${process.env.PATH}`,
+  VIRTUAL_ENV: path.join(__dirname, "../.venv"),
+});
 
 export async function fetchMetadata(identifier: string): Promise<any> {
   const r = await fetch(`https://archive.org/metadata/${identifier}`);
@@ -11,9 +22,8 @@ export async function fetchMetadata(identifier: string): Promise<any> {
 }
 
 export async function fetchTasks(identifier: string): Promise<any> {
-  const command = `ia tasks ${identifier}`;
+  const result = await execFile(IA_COMMAND, ['tasks', identifier], {
-  const result = await exec(command, {
+    env: getVenvEnv(),
-    encoding: "utf8",
   });
   return result.stdout
     .trim()
@@ -25,12 +35,47 @@ export async function uploadFile(
   identifier: string,
   filepath: string
 ): Promise<any> {
-  const command = `ia upload ${identifier} "${filepath}"`;
+  await execFile(IA_COMMAND, ['upload', identifier, filepath], {
-  await exec(command, { encoding: "utf8" });
+    env: getVenvEnv(),
+  });
+}
+
+export async function uploadFiles(
+  identifier: string,
+  filepaths: string[],
+  metadata?: { [key: string]: string }
+): Promise<any> {
+  const args = ['upload', identifier, ...filepaths];
+  
+  if (metadata) {
+    Object.entries(metadata).forEach(([key, value]) => {
+      args.push(`--metadata=${key}:${value}`);
+    });
+  }
+  
+  await execFile(IA_COMMAND, args, { env: getVenvEnv() });
+}
+
+export async function uploadFiles(
+  identifier: string,
+  filepaths: string[],
+  metadata?: { [key: string]: string }
+): Promise<any> {
+  const args = ['upload', identifier, ...filepaths];
+  
+  if (metadata) {
+    Object.entries(metadata).forEach(([key, value]) => {
+      args.push(`--metadata=${key}:${value}`);
+    });
+  }
+  
+  await execFile(IA_COMMAND, args, { env: getVenvEnv() });
 }
 
 export async function identifierExists(identifier: string): Promise<boolean> {
-  const result = await exec(`ia metadata ${identifier}`);
+  const result = await execFile(IA_COMMAND, ['metadata', identifier], {
+    env: getVenvEnv(),
+  });
   const data = JSON.parse(result.stdout);
   return Object.keys(data).length > 0;
 }
@@ -40,7 +85,6 @@ export async function setMetadata(
   data: { [key: string]: string }
 ) {
   const pairs = Object.entries(data).map(([key, value]) => `${key}:${value}`);
-  const args = pairs.map((pair) => `--modify="${pair}"`);
+  const args = ['metadata', identifier, ...pairs.map((pair) => `--modify=${pair}`)];
-  const command = `ia metadata ${identifier} ${args.join(" ")}`;
+  await execFile(IA_COMMAND, args, { env: getVenvEnv() });
-  await exec(command);
 }

packages/skin-database/tasks/syncToArchive.ts

@@ -8,7 +8,7 @@ import SkinModel from "../data/SkinModel";
 import * as Parallel from "async-parallel";
 import IaItemModel from "../data/IaItemModel";
 import DiscordEventHandler from "../api/DiscordEventHandler";
-import { exec } from "../utils";
+import { exec, execFile } from "../utils";
 import * as IAService from "../services/internetArchive";
 
 export async function findItemsMissingImages(): Promise<string[]> {
@@ -192,7 +192,7 @@ async function getNewIdentifier(filename: string): Promise<string> {
 }
 
 export async function archive(skin: SkinModel): Promise<string> {
-  const filename = await skin.getFileName();
+  const filename = await skin.getFileName(true);
 
   const screenshotFilename = await skin.getScreenshotFileName();
   const title = `Winamp Skin: ${filename}`;
@@ -207,8 +207,36 @@ export async function archive(skin: SkinModel): Promise<string> {
 
   console.log(`Going to try to upload with identifier "${identifier}"...`);
 
-  const command = `ia upload ${identifier} "${skinFile}" "${screenshotFile}" --metadata="collection:winampskins" --metadata="skintype:wsz" --metadata="mediatype:software" --metadata="title:${title}"`;
+  // Path to the ia command in the virtual environment
-  await exec(command, { encoding: "utf8" });
+  const IA_COMMAND = path.join(__dirname, "../.venv/bin/ia");
+
+  // Environment variables for the virtual environment
+  const venvEnv = {
+    ...process.env,
+    PATH: `${path.join(__dirname, "../.venv/bin")}:${process.env.PATH}`,
+    VIRTUAL_ENV: path.join(__dirname, "../.venv"),
+  };
+
+  const metadata = {
+    collection: "winampskins",
+    skintype: "wsz",
+    mediatype: "software",
+    title: title,
+  };
+
+  // Build arguments array for ia upload command
+  const args = [
+    "upload",
+    identifier,
+    skinFile,
+    screenshotFile,
+    `--metadata=collection:${metadata.collection}`,
+    `--metadata=skintype:${metadata.skintype}`,
+    `--metadata=mediatype:${metadata.mediatype}`,
+    `--metadata=title:${metadata.title}`,
+  ];
+
+  await execFile(IA_COMMAND, args, { env: venvEnv });
   await knex("ia_items").insert({ skin_md5: skin.getMd5(), identifier });
   return identifier;
 }

packages/skin-database/utils.ts

@@ -6,6 +6,7 @@ import child_process from "child_process";
 import path from "path";
 
 export const exec = util.promisify(child_process.exec);
+export const execFile = util.promisify(child_process.execFile);
 
 const temp = _temp.track();