webamp/experiments/skin-database/tasks/fetchInternetArchiveMetadata.js

const fetch = require("node-fetch");
const db = require("../db");
const iaItems = db.get("internetArchiveItems");

async function fetchMetadata(identifier) {
  const resp = await fetch(`http://archive.org/metadata/${identifier}`);
  const metadata = await resp.json();
  const metadataFetchDate = Date.now();
  await iaItems.findOneAndUpdate(
    { identifier },
    { $set: { metadata, metadataFetchDate } }
  );
}

async function fetchAllMetadata(limit) {
  const items = await iaItems.find(
    { metadata: { $eq: null } },
    {
      limit,
      fields: {
        identifier: 1,
      },
    }
  );
  console.log(`Found ${items.length} missing metadata`);
  await Promise.all(
    items.map(item => {
      return fetchMetadata(item.identifier);
    })
  );
  return items.length;
}
// TODO: Refetch collections from:
// https://archive.org/advancedsearch.php?q=collection%3Awinampskins&fl%5B%5D=identifier&rows=100000&page=1&output=json
module.exports = async function main() {
  return new Promise((resolve, reject) => {
    let delay = 60000;
    let timeout = null;
    async function go() {
      console.log("Gonna fetch some more");
      try {
        const count = await fetchAllMetadata(500);
        if (count < 1) {
          if (timeout != null) {
            // I don't think this can ever happen
            clearTimeout(timeout);
          }
          console.log("Done.");
          resolve();
          return;
        }
      } catch (e) {
        console.error(e);
        delay += 60000;
      }
      timeout = setTimeout(go, delay);
    }

    go();
  });
};