diff --git a/packages/skin-database/migrations/20220218192912_ia_metadata.ts b/packages/skin-database/migrations/20220218192912_ia_metadata.ts index 35024eba..020e8911 100644 --- a/packages/skin-database/migrations/20220218192912_ia_metadata.ts +++ b/packages/skin-database/migrations/20220218192912_ia_metadata.ts @@ -1,9 +1,9 @@ -import * as Knex from "knex"; +import * as knex from "knex"; -export async function up(knex: Knex): Promise { +export async function up(knex: knex): Promise { await knex.raw(`ALTER TABLE ia_items ADD COLUMN metadata;`); } -export async function down(knex: Knex): Promise { +export async function down(knex: knex): Promise { throw new Error("I never implemented a down migration for adding metadata."); } diff --git a/packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts b/packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts index d92aad36..45f6aaf4 100644 --- a/packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts +++ b/packages/skin-database/migrations/20220218195859_ai_metadata_timestamp.ts @@ -1,10 +1,10 @@ -import * as Knex from "knex"; +import * as knex from "knex"; -export async function up(knex: Knex): Promise { +export async function up(knex: knex): Promise { await knex.raw(`ALTER TABLE ia_items ADD COLUMN metadata_timestamp;`); } -export async function down(knex: Knex): Promise { +export async function down(knex: knex): Promise { throw new Error( "I never implemented a down migration for adding metadata_timestamp." ); diff --git a/packages/skin-database/tasks/integrityCheck.ts b/packages/skin-database/tasks/integrityCheck.ts index e5183e2d..98039626 100644 --- a/packages/skin-database/tasks/integrityCheck.ts +++ b/packages/skin-database/tasks/integrityCheck.ts @@ -1,6 +1,5 @@ -import fs from "fs"; import { knex } from "../db"; -import { TWEET_SNOWFLAKE_REGEX, MD5_REGEX } from "../utils"; +import { TWEET_SNOWFLAKE_REGEX } from "../utils"; function isNotGeneratedFile(file) { switch (file.source) { @@ -80,30 +79,6 @@ export async function integrityCheck(): Promise { console.log("Done."); } -async function findHashesForTweets(): Promise { - const todo = JSON.parse(fs.readFileSync("todo.json", "utf8")); - let other = 0; - - for (const tweet of todo) { - const match = tweet.text.match(MD5_REGEX); - if (match) { - const md5 = match[0]; - if (md5 == null) { - throw new Error("No md5?"); - } - await knex("tweets").insert( - { skin_md5: md5, tweet_id: tweet.id_str }, - [] - ); - continue; - } - // console.log({ urls, text: skin.text, id: skin.id_str }); - - other++; - } - console.log({ other }); -} - async function skinsWithoutFiles(): Promise { console.log("Checking for skins without files..."); const result = await knex("skins") @@ -143,20 +118,6 @@ async function reviewsWithoutSkins(): Promise { } } -async function noDuplicateIaItems(): Promise { - console.log("Checking for duplicate Internet Archive items..."); - const result = await knex.raw( - "SELECT identifier, COUNT(*) c FROM ia_items GROUP BY identifier HAVING c > 1;" - ); - if (result.length > 0) { - console.warn( - `Found ${result.length} Internet Archive items with duplicate identifiers.` - ); - } else { - console.log("None found."); - } -} - async function noDuplicateTweetIds(): Promise { console.log("Checking for duplicate Tweet ids..."); const result = await knex.raw( @@ -181,20 +142,6 @@ async function noDuplicateTweetMd5s(): Promise { } } -async function tweetsHaveIds(): Promise { - console.log("Checking for Tweets with URLs but no ids..."); - const results = await knex.raw( - "SELECT * FROM tweets WHERE tweet_id IS NULL;" - ); - if (results.length > 0) { - for (const row of results) { - // console.log(row); - } - } else { - console.log("None found."); - } -} - async function tweetsHaveIdIfTheyHaveURl(): Promise { console.log("Checking for Tweets with URLs but no ids..."); const results = await knex.raw(