From 2a8219299f4a510728b7d5bcc98cdf91b0bf7801 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sun, 15 Dec 2024 22:26:24 -0800 Subject: [PATCH] Stub out description generation --- packages/skin-database/cli.ts | 10 +++- packages/skin-database/package.json | 1 + packages/skin-database/services/openAi.ts | 44 +++++++++++++++ yarn.lock | 68 ++++++++++++++++------- 4 files changed, 102 insertions(+), 21 deletions(-) create mode 100644 packages/skin-database/services/openAi.ts diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index f4a0ae3e..0fccbdd4 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -33,6 +33,7 @@ import { program } from "commander"; import * as config from "./config"; import { setHashesForSkin } from "./skinHash"; import * as S3 from "./s3"; +import { generateDescription } from "./services/openAi"; async function withHandler( cb: (handler: DiscordEventHandler) => Promise @@ -125,12 +126,19 @@ program ) .option("--reject", 'Give a skin a "rejected" review.') .option("--metadata", "Push metadata to the archive.") + .option("--ai", "Use AI to generate a text description of the skin.") .action( async ( md5, - { delete: del, deleteLocal, index, refresh, reject, metadata, hide } + { delete: del, deleteLocal, index, refresh, reject, metadata, hide, ai } ) => { const ctx = new UserContext("CLI"); + if (ai) { + const skin = await SkinModel.fromMd5Assert(ctx, md5); + const description = await generateDescription(skin); + console.log("Generated description for", skin.getFileName()); + console.log(description); + } if (del) { await Skins.deleteSkin(md5); } diff --git a/packages/skin-database/package.json b/packages/skin-database/package.json index 5e85088c..bcda4323 100644 --- a/packages/skin-database/package.json +++ b/packages/skin-database/package.json @@ -31,6 +31,7 @@ "mastodon-api": "^1.3.0", "md5": "^2.2.1", "node-fetch": "^2.6.7", + "openai": "^4.68.0", "polygon-clipping": "^0.15.3", "puppeteer": "^13.3.2", "sharp": "^0.31.3", diff --git a/packages/skin-database/services/openAi.ts b/packages/skin-database/services/openAi.ts new file mode 100644 index 00000000..e745efe3 --- /dev/null +++ b/packages/skin-database/services/openAi.ts @@ -0,0 +1,44 @@ +import fs from "node:fs"; +import SkinModel from "../data/SkinModel.js"; +import OpenAI from "openai"; + +const client = new OpenAI({ + apiKey: process.env["OPENAI_API_KEY"], // This is the default and can be omitted +}); + +const prompt = ` +You are a digital preservationist being presented with a screenshot of a Winamp +skin. Write a short description of the skin which can be used to index the skin +for search retrieval. + +Do not describe any features which are common to all Winamp skins, such as the +set of UI elements visible. + +The description should be concise and descriptive, and should not include any +personal opinions or judgement. The description should be written in English and +should be no longer than 200 characters.`; + +const prompt2 = `"Identify thematic and stylistic details in this screenshot of a media player. +Give a comma separated list of tags, and ignore the text."`; + +export async function generateDescription(skin: SkinModel): Promise { + const response = await client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [ + { + role: "user", + content: [ + { type: "text", text: prompt2 }, + { + type: "image_url", + image_url: { + url: skin.getScreenshotUrl(), + }, + }, + ], + }, + ], + }); + console.log(response.choices[0].message.content); + return "Done"; +} diff --git a/yarn.lock b/yarn.lock index aa8e34f1..31c8df6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6210,7 +6210,7 @@ resolved "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz" integrity sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g== -"@types/node-fetch@^2.1.6", "@types/node-fetch@^2.5.7": +"@types/node-fetch@^2.1.6", "@types/node-fetch@^2.5.7", "@types/node-fetch@^2.6.4": version "2.6.11" resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz" integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== @@ -6242,6 +6242,13 @@ resolved "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz" integrity sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ== +"@types/node@^18.11.18": + version "18.19.56" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.56.tgz#e46d3e6e5417d3b89e89ee6207d3c2cba7ad1ef2" + integrity sha512-4EMJlWwwGnVPflJAtM14p9eVSa6BOv5b92mCsh5zcM1UagNtEtrbbtaE6WE1tw2TabavatnwqXjlIpcAEuJJNg== + dependencies: + undici-types "~5.26.4" + "@types/normalize-package-data@^2.4.0": version "2.4.4" resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz" @@ -7428,7 +7435,7 @@ agentkeepalive@^2.2.0: resolved "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz" integrity sha512-TnB6ziK363p7lR8QpeLC8aMr8EGYBKZTpgzQLfqTs3bR0Oo5VbKdwKf8h0dSzsYrB7lSCgfJnMZKqShvlq5Oyg== -agentkeepalive@^4.1.3: +agentkeepalive@^4.1.3, agentkeepalive@^4.2.1: version "4.5.0" resolved "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz" integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== @@ -14230,6 +14237,11 @@ fork-ts-checker-webpack-plugin@^6.5.0: semver "^7.3.2" tapable "^1.0.0" +form-data-encoder@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" + integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A== + form-data@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz" @@ -14262,6 +14274,14 @@ format@^0.2.0: resolved "https://registry.npmjs.org/format/-/format-0.2.2.tgz" integrity sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww== +formdata-node@^4.3.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2" + integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ== + dependencies: + node-domexception "1.0.0" + web-streams-polyfill "4.0.0-beta.3" + formidable@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz" @@ -15114,16 +15134,11 @@ graphql-ws@5.12.1: resolved "https://registry.npmjs.org/graphql-ws/-/graphql-ws-5.12.1.tgz" integrity sha512-umt4f5NnMK46ChM2coO36PTFhHouBrK9stWWBczERguwYrGnPNxJ9dimU6IyOBfOkC6Izhkg4H8+F51W/8CYDg== -graphql@^16.8.1: +graphql@16.8.1, graphql@^16.8.1, graphql@^16.9.0: version "16.8.1" - resolved "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.8.1.tgz#1930a965bef1170603702acdb68aedd3f3cf6f07" integrity sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw== -graphql@^16.9.0: - version "16.9.0" - resolved "https://registry.npmjs.org/graphql/-/graphql-16.9.0.tgz" - integrity sha512-GGTKBX4SD7Wdb8mqeDLni2oaRGYQWjWHGKPQ24ZMnUtKfcsVoiv4uX8+LJr1K6U5VW2Lu1BwJnj7uiori0YtRw== - grats@^0.0.29: version "0.0.29" resolved "https://registry.yarnpkg.com/grats/-/grats-0.0.29.tgz#9efe40da41a90c35c75f7ee4617290623a4318a9" @@ -18641,7 +18656,7 @@ jss@10.10.0, jss@^10.0.0: object.assign "^4.1.4" object.values "^1.1.6" -jszip@*, jszip@^3.1.5, jszip@^3.7.0: +jszip@*, jszip@^3.1.5, jszip@^3.10.1, jszip@^3.7.0: version "3.10.1" resolved "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz" integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== @@ -18651,16 +18666,6 @@ jszip@*, jszip@^3.1.5, jszip@^3.7.0: readable-stream "~2.3.6" setimmediate "^1.0.5" -jszip@^3.10.1: - version "3.10.1" - resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" - integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== - dependencies: - lie "~3.3.0" - pako "~1.0.2" - readable-stream "~2.3.6" - setimmediate "^1.0.5" - junk@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/junk/-/junk-3.1.0.tgz" @@ -21065,6 +21070,11 @@ node-dir@^0.1.10: dependencies: minimatch "^3.0.2" +node-domexception@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + node-fetch@2.6.7: version "2.6.7" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" @@ -21661,6 +21671,19 @@ open@^8.0.9, open@^8.2.1, open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" +openai@^4.68.0: + version "4.68.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.68.0.tgz#6390d35fbcb2b8dc15f93b1a8c87a65ebeac8636" + integrity sha512-cVH0WMKd4cColyorwqo+Gn08lN8LQ8uKLMfWXFfvnedrLq3lCH6lRd0Rd0XJRunyfgNve/L9E7uZLAii39NBkw== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + opener@~1.4.0: version "1.4.3" resolved "https://registry.npmjs.org/opener/-/opener-1.4.3.tgz" @@ -28665,6 +28688,11 @@ web-namespaces@^1.0.0: resolved "https://registry.npmjs.org/web-namespaces/-/web-namespaces-1.1.4.tgz" integrity sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw== +web-streams-polyfill@4.0.0-beta.3: + version "4.0.0-beta.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" + integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== + web-streams-polyfill@^3.2.1: version "3.3.3" resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz"