diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index 3b72d77c..60a26aae 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -7,6 +7,7 @@ import DiscordWinstonTransport from "./DiscordWinstonTransport"; import * as Skins from "./data/skins"; import Discord from "discord.js"; import { tweet } from "./tasks/tweet"; +import { insta, } from "./tasks/insta"; import md5Buffer from "md5"; import { addSkinFromBuffer } from "./addSkin"; import { searchIndex } from "./algolia"; @@ -127,6 +128,12 @@ async function main() { await tweet(client, hash); break; } + case "insta": { + console.log("insta"); + const hash = argv._[1]; + await insta(client, hash); + break; + } case "stats": { console.log(await Skins.getStats()); break; diff --git a/packages/skin-database/data/InstagramPostModel.ts b/packages/skin-database/data/InstagramPostModel.ts new file mode 100644 index 00000000..95d194bc --- /dev/null +++ b/packages/skin-database/data/InstagramPostModel.ts @@ -0,0 +1,80 @@ +import UserContext, { ctxWeakMapMemoize } from "./UserContext"; +import { InstagramPostRow } from "../types"; +import DataLoader from "dataloader"; +import { knex } from "../db"; +import SkinModel from "./SkinModel"; + +export type InstagramPostDebugData = { + row: InstagramPostRow; +}; + +export default class InstagramPostModel { + constructor(readonly ctx: UserContext, readonly row: InstagramPostRow) {} + + static async fromMd5( + ctx: UserContext, + md5: string + ): Promise { + const rows = await getInstagramPostsLoader(ctx).load(md5); + return rows.map((row) => new InstagramPostModel(ctx, row)); + } + + static async fromInstagramPostId( + ctx: UserContext, + instagramPostId: string + ): Promise { + const row = await getInstagramPostsByPostIdLoader(ctx).load( + instagramPostId + ); + if (row == null) { + return null; + } + return new InstagramPostModel(ctx, row); + } + + getMd5(): string { + return this.row.skin_md5; + } + + getInstagramPostId(): string { + return this.row.post_id; + } + + getUrl(): string { + return this.row.url; + } + + async getSkin(): Promise { + const skin = await SkinModel.fromMd5(this.ctx, this.getMd5()); + if (skin == null) { + throw new Error(`Could not find skin for md5 "${this.getMd5()}"`); + } + return skin; + } + + async debug(): Promise { + return { + row: this.row, + }; + } +} + +const getInstagramPostsLoader = ctxWeakMapMemoize< + DataLoader +>( + () => + new DataLoader(async (md5s) => { + const rows = await knex("instagram_posts").whereIn("skin_md5", md5s).select(); + return md5s.map((md5) => rows.filter((x) => x.skin_md5 === md5)); + }) +); + +const getInstagramPostsByPostIdLoader = ctxWeakMapMemoize< + DataLoader +>( + () => + new DataLoader(async (ids) => { + const rows = await knex("instagram_posts").whereIn("post_id", ids).select(); + return ids.map((id) => rows.find((x) => x.tweet_id === id)); + }) +); diff --git a/packages/skin-database/data/SkinModel.ts b/packages/skin-database/data/SkinModel.ts index a26bda0e..74afbeff 100644 --- a/packages/skin-database/data/SkinModel.ts +++ b/packages/skin-database/data/SkinModel.ts @@ -165,6 +165,10 @@ export default class SkinModel { return getSkinUrl(this.row.md5); } + getAverageColor(): string { + return this.row.average_color; + } + getBuffer = mem( async (): Promise => { const response = await fetch(this.getSkinUrl()); diff --git a/packages/skin-database/data/skins.ts b/packages/skin-database/data/skins.ts index 560513cc..bbeeead3 100644 --- a/packages/skin-database/data/skins.ts +++ b/packages/skin-database/data/skins.ts @@ -111,6 +111,14 @@ export async function markAsTweeted( await knex("tweets").insert({ skin_md5: md5, tweet_id: tweetId }, []); } +export async function markAsPostedToInstagram( + md5: string, + postId: string, + url: string +): Promise { + await knex("instagram_posts").insert({ skin_md5: md5, post_id: postId, url }, []); +} + // TODO: Also path actor export async function markAsNSFW(ctx: UserContext, md5: string): Promise { const index = { objectID: md5, nsfw: true }; @@ -524,6 +532,16 @@ FROM skins LEFT JOIN files ON files.skin_md5 = skins.md5 LEFT JOIN refreshes ON refreshes.skin_md5 = skins.md5 WHERE skin_type = 1 AND refreshes.error IS NULL +-- +-- WHERE +-- skin_type = 1 +-- AND refreshes.error IS NULL +-- AND skins.md5 != "d7541f8c5be768cf23b9aeee1d6e70c7" -- Duplicate Garfield +-- AND skins.md5 != "25a932542e307416ca86da4e16be1b32" -- Duplicate Vault-tec +-- AND skins.md5 != "89643da06361e4bcc269fe811f07c4a3" -- Another duplicate Vault-tec +-- AND skins.md5 != "db1f2e128f6dd6c702b7a448751fbe84" -- Duplicate Fallout +-- AND skins.md5 != "be2de111c4710af306fea0813440f275" -- Duplicate Microchip +-- AND skins.md5 != "66cf0af3593d79fc8a5080dd17f8b07d" -- Another duplicate Microchip GROUP BY skins.md5 ORDER BY priority ASC, diff --git a/packages/skin-database/migrations/20220202203919_instagram_posts.ts b/packages/skin-database/migrations/20220202203919_instagram_posts.ts new file mode 100644 index 00000000..d5ee46f7 --- /dev/null +++ b/packages/skin-database/migrations/20220202203919_instagram_posts.ts @@ -0,0 +1,16 @@ +import * as Knex from "knex"; + +export async function up(knex: Knex): Promise { + await knex.raw( + `CREATE TABLE "instagram_posts" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + "skin_md5" TEXT NOT NULL, + post_id text TEXT NOT_NULL UNIQUE, + url text TEXT NOT_NULL UNIQUE + );` + ); +} + +export async function down(knex: Knex): Promise { + await knex.raw(`DROP TABLE "instagram_posts"`); +} diff --git a/packages/skin-database/s3.js b/packages/skin-database/s3.js index 0ee8c3bb..d60cd2df 100644 --- a/packages/skin-database/s3.js +++ b/packages/skin-database/s3.js @@ -127,7 +127,52 @@ function putScreenshot(md5, buffer) { }); } +function putTemp(fileName, buffer) { + return new Promise((resolve, rejectPromise) => { + const bucketName = "cdn.webampskins.org"; + const key = `temp/${fileName}`; + s3.putObject( + { + Bucket: bucketName, + Key: key, + Body: buffer, + ACL: "public-read", + }, + (err) => { + console.log("Hello...") + if (err) { + rejectPromise(err); + return; + } + resolve(); + } + ); + }); +} + +function deleteTemp(fileName) { + return new Promise((resolve, rejectPromise) => { + const bucketName = "cdn.webampskins.org"; + const key = `screenshots/${fileName}`; + s3.deleteObject( + { + Bucket: bucketName, + Key: key, + }, + (err) => { + if (err) { + rejectPromise(err); + return; + } + resolve(); + } + ); + }); +} + module.exports = { + putTemp, + deleteTemp, putScreenshot, putSkin, deleteSkin, diff --git a/packages/skin-database/tasks/insta.ts b/packages/skin-database/tasks/insta.ts new file mode 100644 index 00000000..857076dd --- /dev/null +++ b/packages/skin-database/tasks/insta.ts @@ -0,0 +1,145 @@ +import fs from "fs"; +import * as Skins from "../data/skins"; +import * as S3 from "../s3"; +import { INSTAGRAM_ACCESS_TOKEN, INSTAGRAM_ACCOUNT_ID, TWEET_BOT_CHANNEL_ID } from "../config"; +import { Client } from "discord.js"; +import sharp from "sharp"; +import SkinModel from "../data/SkinModel"; +import UserContext from "../data/UserContext"; +import fetch from "node-fetch"; + +export async function insta(discordClient: Client, md5: string): Promise { + const url = await post(md5); + + console.log("Going to post to discord"); + const tweetBotChannel = await discordClient.channels.fetch( + TWEET_BOT_CHANNEL_ID + ); + // @ts-ignore + await tweetBotChannel.send(url); + console.log("Posted to discord"); +} + + +async function createContent( + imageUrl: string, + caption: string +): Promise { + const url = new URL( + `https://graph.facebook.com/v12.0/${INSTAGRAM_ACCOUNT_ID}/media` + ); + url.searchParams.set("access_token", INSTAGRAM_ACCESS_TOKEN); + url.searchParams.set("image_url", imageUrl); + url.searchParams.set("caption", caption); + const response = await fetch(url, { method: "POST" }); + if (!response.ok) { + throw new Error(`API error: ${await response.text()}`); + } + + console.log("crate content status", response.status); + const result: { id: string } = await response.json(); + console.log(result); + + return result.id; +} + +async function publish(id: string): Promise { + const url = new URL( + `https://graph.facebook.com/v12.0/${INSTAGRAM_ACCOUNT_ID}/media_publish` + ); + url.searchParams.set("access_token", INSTAGRAM_ACCESS_TOKEN); + url.searchParams.set("creation_id", id); + //url.searchParams.set("fields", "permalink"); + + const response = await fetch(url, { method: "POST" }); + console.log("publish status", response.status); + if (!response.ok) { + throw new Error(`API error: ${await response.text()}`); + } + const result: { id: string; permalink: string } = await response.json(); + console.log("publish result", result); + return result.id; +} + +async function getPermalink(id: string): Promise { + const url = new URL(`https://graph.facebook.com/v12.0/${id}`); + url.searchParams.set("access_token", INSTAGRAM_ACCESS_TOKEN); + url.searchParams.set("fields", "permalink"); + + const response = await fetch(url); + console.log("permalink status", response.status); + if (!response.ok) { + throw new Error(`API error: ${await response.text()}`); + } + const result: { id: string; permalink: string } = await response.json(); + console.log("permalink result", result); + return result.permalink; +} + +const PADDING = 100; +const UPSCALE = 4; + +async function post(md5: string): Promise { + const ctx = new UserContext(); + const skin = await SkinModel.fromMd5(ctx, md5); + if (skin == null) { + throw new Error("Could not find skin"); + } + const screenshot = await Skins.getScreenshotBuffer(md5); + const { width, height } = await sharp(screenshot).metadata(); + + const targetWidth = width * UPSCALE; + const targetHeight = height * UPSCALE; + + const dimension = Math.max(targetWidth, targetHeight) + PADDING; + + const paddingX = (dimension - targetWidth) / 2; + const paddingY = (dimension - targetHeight) / 2; + + const background = { r: 0, g: 0, b: 0, alpha: 255 }; + + const image = await sharp(screenshot) + .resize(targetWidth, targetHeight, { + kernel: sharp.kernel.nearest, + }) + .extend({ + top: paddingY, + bottom: paddingY, + left: paddingX, + right: paddingX, + background, + }) + .toBuffer(); + + const name = await skin.getFileName(); + + const text = `${name} +. +. +. +. +. +#winampskins #winamp #nostalgia #napster #skins #windows #nullsoft #digitalart #design #software #ui #uidesign`; + console.log("Uplaoding to S3"); + const uploadFileName = `${md5}.png`; + await S3.putTemp(uploadFileName, image); + console.log("Done uploading to S3"); + const imageUrl = `https://cdn.webampskins.org/temp/${uploadFileName}`; + + console.log("Creating content..."); + const contentId = await createContent(imageUrl, text); + + console.log("Deleting temp file"); + await S3.putTemp(uploadFileName, image); + console.log("Publishing content..."); + const id = await publish(contentId); + + console.log("Getting permalink") + const permalink = await getPermalink(id); + console.log("Permalink", permalink); + + console.log("Marking posted in DB") + await Skins.markAsPostedToInstagram(md5, id, permalink); + + return permalink; +} diff --git a/packages/skin-database/tasks/scrapeLikes.ts b/packages/skin-database/tasks/scrapeLikes.ts index 3a6541de..9a9d4ef2 100644 --- a/packages/skin-database/tasks/scrapeLikes.ts +++ b/packages/skin-database/tasks/scrapeLikes.ts @@ -30,7 +30,7 @@ async function getTweets(twitterClient): Promise { let tweets: TweetPayload[] = []; let callCount = 0; - while (callCount < MAX_CALL_COUNT) { + while (callCount < (MAX_CALL_COUNT * 6)) { callCount++; const response = await twitterClient.get("statuses/user_timeline", { max_id, @@ -38,6 +38,7 @@ async function getTweets(twitterClient): Promise { exclude_replies: true, include_rts: false, }); + console.log(JSON.stringify(response.data, null, 2)); tweets = tweets.concat(response.data); const newMaxId = tweets.map((tweet) => tweet.id_str).sort()[0]; if (newMaxId === max_id) { @@ -47,10 +48,15 @@ async function getTweets(twitterClient): Promise { max_id = newMaxId; if (response.data.length <= 1) { + console.warn("Page was short") return tweets; } + if(callCount === MAX_CALL_COUNT) { + console.warn("Hit MAX (but gonna keep going)") + } } + console.warn("Hit MAX (but gonna keep going)") return tweets; } @@ -67,8 +73,9 @@ function getMd5(tweet: TweetPayload): string | null { export async function scrapeLikeData() { const twitterClient = getTwitterClient(); + const rawTweets = await getTweets(twitterClient); - const tweets = (await getTweets(twitterClient)).filter((tweet) => { + const tweets = rawTweets.filter((tweet) => { // Ignore tweets that don't have links, or are known manual tweets. return !( tweet.entities.urls.length === 0 || knownManualTweets.has(tweet.id_str) diff --git a/packages/skin-database/types.ts b/packages/skin-database/types.ts index afd124bd..15cc57e2 100644 --- a/packages/skin-database/types.ts +++ b/packages/skin-database/types.ts @@ -23,6 +23,12 @@ export type TweetRow = { retweets: number; }; +export type InstagramPostRow = { + skin_md5: string; + url: string; + post_id: string; +}; + export type ReviewRow = { skin_md5: string; review: "APPROVED" | "REJECTED" | "NSFW";