diff --git a/packages/skin-database/api/graphql/resolvers/BulkDownloadConnection.ts b/packages/skin-database/api/graphql/resolvers/BulkDownloadConnection.ts new file mode 100644 index 00000000..c8e2535a --- /dev/null +++ b/packages/skin-database/api/graphql/resolvers/BulkDownloadConnection.ts @@ -0,0 +1,93 @@ +import { Int } from "grats"; +import { knex } from "../../../db"; +import SkinModel from "../../../data/SkinModel"; +import ClassicSkinResolver from "./ClassicSkinResolver"; +import ModernSkinResolver from "./ModernSkinResolver"; +import UserContext from "../../../data/UserContext"; +import { ISkin } from "./CommonSkinResolver"; +import { SkinRow } from "../../../types"; + +/** + * Connection for bulk download skin metadata + * @gqlType + */ +export class BulkDownloadConnection { + _offset: number; + _first: number; + + constructor(first: number, offset: number) { + this._first = first; + this._offset = offset; + } + + /** + * Total number of skins available for download + * @gqlField + */ + async totalCount(): Promise { + // Get count of both classic and modern skins + const [classicResult, modernResult] = await Promise.all([ + knex("skins").where({ skin_type: 1 }).count("* as count"), + knex("skins").where({ skin_type: 2 }).count("* as count"), + ]); + + const classicCount = Number(classicResult[0].count); + const modernCount = Number(modernResult[0].count); + + return classicCount + modernCount; + } + + /** + * Estimated total size in bytes (approximation for progress indication) + * @gqlField + */ + async estimatedSizeBytes(): Promise { + const totalCount = await this.totalCount(); + // Rough estimate: average skin is ~56KB + return (totalCount * 56 * 1024).toString(); + } + + /** + * List of skin metadata for bulk download + * @gqlField + */ + async nodes(ctx: UserContext): Promise> { + // Get skins ordered by skin_type (classic first, then modern) and id for consistency + const skins = await knex("skins") + .select(["id", "md5", "skin_type", "emails"]) + .orderBy([{ column: "skins.id", order: "asc" }]) + .where({ skin_type: 1 }) + .orWhere({ skin_type: 2 }) + .limit(this._first) + .offset(this._offset); + + return skins.map((skinRow) => { + const skinModel = new SkinModel(ctx, skinRow); + + if (skinRow.skin_type === 1) { + return new ClassicSkinResolver(skinModel); + } else if (skinRow.skin_type === 2) { + return new ModernSkinResolver(skinModel); + } else { + throw new Error("Expected classic or modern skin"); + } + }); + } +} + +/** + * Get metadata for bulk downloading all skins in the museum + * @gqlQueryField + */ +export function bulkDownload({ + first = 1000, + offset = 0, +}: { + first?: Int; + offset?: Int; +}): BulkDownloadConnection { + if (first > 10000) { + throw new Error("Maximum limit is 10000 for bulk download"); + } + return new BulkDownloadConnection(first, offset); +} diff --git a/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts b/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts index 8cadeb05..da9433c7 100644 --- a/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts +++ b/packages/skin-database/api/graphql/resolvers/ClassicSkinResolver.ts @@ -41,7 +41,7 @@ export default class ClassicSkinResolver implements NodeResolver, ISkin { nsfw(): Promise { return this._model.getIsNsfw(); } - average_color(): string { + average_color(): string | null { return this._model.getAverageColor(); } /** diff --git a/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts b/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts index c9d343a2..1ecd3800 100644 --- a/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts +++ b/packages/skin-database/api/graphql/resolvers/CommonSkinResolver.ts @@ -69,19 +69,34 @@ export function id(skin: ISkin): ID { * has been uploaded under multiple names. Here we just pick one. * @gqlField */ -export function filename( +export async function filename( skin: ISkin, { normalize_extension = false, + include_museum_id = false, }: { /** * If true, the the correct file extension (.wsz or .wal) will be . * Otherwise, the original user-uploaded file extension will be used. */ normalize_extension?: boolean; + /** + * If true, the museum ID will be appended to the filename to ensure filenames are globally unique. + */ + include_museum_id?: boolean; } ): Promise { - return skin.filename(normalize_extension); + const baseFilename = await skin.filename(normalize_extension); + + if (!include_museum_id) { + return baseFilename; + } + + const museumId = skin._model.getId(); + const segments = baseFilename.split("."); + const fileExtension = segments.pop(); + + return `${segments.join(".")}_[S${museumId}].${fileExtension}`; } /** diff --git a/packages/skin-database/api/graphql/schema.graphql b/packages/skin-database/api/graphql/schema.graphql index a7a02820..5379b060 100644 --- a/packages/skin-database/api/graphql/schema.graphql +++ b/packages/skin-database/api/graphql/schema.graphql @@ -93,6 +93,10 @@ interface Skin { has been uploaded under multiple names. Here we just pick one. """ filename( + """ + If true, the museum ID will be appended to the filename to ensure filenames are globally unique. + """ + include_museum_id: Boolean! = false """ If true, the the correct file extension (.wsz or .wal) will be . Otherwise, the original user-uploaded file extension will be used. @@ -165,6 +169,16 @@ type ArchiveFile { url: String } +"""Connection for bulk download skin metadata""" +type BulkDownloadConnection { + """Estimated total size in bytes (approximation for progress indication)""" + estimatedSizeBytes: String @semanticNonNull + """List of skin metadata for bulk download""" + nodes: [Skin!] @semanticNonNull + """Total number of skins available for download""" + totalCount: Int @semanticNonNull +} + """A classic Winamp skin""" type ClassicSkin implements Node & Skin { """List of files contained within the skin's .wsz archive""" @@ -178,6 +192,10 @@ type ClassicSkin implements Node & Skin { has been uploaded under multiple names. Here we just pick one. """ filename( + """ + If true, the museum ID will be appended to the filename to ensure filenames are globally unique. + """ + include_museum_id: Boolean! = false """ If true, the the correct file extension (.wsz or .wal) will be . Otherwise, the original user-uploaded file extension will be used. @@ -308,6 +326,10 @@ type ModernSkin implements Node & Skin { has been uploaded under multiple names. Here we just pick one. """ filename( + """ + If true, the museum ID will be appended to the filename to ensure filenames are globally unique. + """ + include_museum_id: Boolean! = false """ If true, the the correct file extension (.wsz or .wal) will be . Otherwise, the original user-uploaded file extension will be used. @@ -385,6 +407,8 @@ type Mutation { } type Query { + """Get metadata for bulk downloading all skins in the museum""" + bulkDownload(first: Int! = 1000, offset: Int! = 0): BulkDownloadConnection @semanticNonNull """ Fetch archive file by it's MD5 hash diff --git a/packages/skin-database/api/graphql/schema.ts b/packages/skin-database/api/graphql/schema.ts index 4d9c0855..c6f19209 100644 --- a/packages/skin-database/api/graphql/schema.ts +++ b/packages/skin-database/api/graphql/schema.ts @@ -3,8 +3,9 @@ * Do not manually edit. Regenerate by running `npx grats`. */ -import { GraphQLSchema, GraphQLDirective, DirectiveLocation, GraphQLList, GraphQLInt, specifiedDirectives, GraphQLObjectType, GraphQLString, GraphQLBoolean, GraphQLInterfaceType, GraphQLNonNull, GraphQLID, GraphQLEnumType, defaultFieldResolver, GraphQLInputObjectType } from "graphql"; +import { GraphQLSchema, GraphQLDirective, DirectiveLocation, GraphQLList, GraphQLInt, specifiedDirectives, GraphQLObjectType, GraphQLString, defaultFieldResolver, GraphQLNonNull, GraphQLInterfaceType, GraphQLBoolean, GraphQLID, GraphQLEnumType, GraphQLInputObjectType } from "graphql"; import { getUserContext } from "./index"; +import { bulkDownload as queryBulkDownloadResolver } from "./resolvers/BulkDownloadConnection"; import { fetch_archive_file_by_md5 as queryFetch_archive_file_by_md5Resolver } from "./../../data/ArchiveFileModel"; import { fetch_internet_archive_item_by_identifier as queryFetch_internet_archive_item_by_identifierResolver } from "./../../data/IaItemModel"; import { fetch_skin_by_md5 as queryFetch_skin_by_md5Resolver, search_classic_skins as querySearch_classic_skinsResolver, search_skins as querySearch_skinsResolver, skin_to_review as querySkin_to_reviewResolver } from "./resolvers/SkinResolver"; @@ -27,6 +28,75 @@ async function assertNonNull(value: T | Promise): Promise { return awaited; } export function getSchema(): GraphQLSchema { + const ArchiveFileType: GraphQLObjectType = new GraphQLObjectType({ + name: "ArchiveFile", + description: "A file found within a Winamp Skin's .wsz archive", + fields() { + return { + date: { + description: "The date on the file inside the archive. Given in simplified extended ISO\nformat (ISO 8601).", + name: "date", + type: GraphQLString, + resolve(source) { + return assertNonNull(source.getIsoDate()); + } + }, + file_md5: { + description: "The md5 hash of the file within the archive", + name: "file_md5", + type: GraphQLString, + resolve(source) { + return assertNonNull(source.getFileMd5()); + } + }, + filename: { + description: "Filename of the file within the archive", + name: "filename", + type: GraphQLString, + resolve(source) { + return assertNonNull(source.getFileName()); + } + }, + is_directory: { + description: "Is the file a directory?", + name: "is_directory", + type: GraphQLBoolean, + resolve(source) { + return assertNonNull(source.getIsDirectory()); + } + }, + size: { + description: "The uncompressed size of the file in bytes.\n\n**Note:** Will be `null` for directories", + name: "size", + type: GraphQLInt, + resolve(source) { + return source.getFileSize(); + } + }, + skin: { + description: "The skin in which this file was found", + name: "skin", + type: SkinType + }, + text_content: { + description: "The content of the file, if it's a text file", + name: "text_content", + type: GraphQLString, + resolve(source) { + return source.getTextContent(); + } + }, + url: { + description: "A URL to download the file. **Note:** This is powered by a little\nserverless Cloudflare function which tries to exctact the file on the fly.\nIt may not work for all files.", + name: "url", + type: GraphQLString, + resolve(source) { + return source.getUrl(); + } + } + }; + } + }); const InternetArchiveItemType: GraphQLObjectType = new GraphQLObjectType({ name: "InternetArchiveItem", fields() { @@ -188,6 +258,11 @@ export function getSchema(): GraphQLSchema { name: "filename", type: GraphQLString, args: { + include_museum_id: { + description: "If true, the museum ID will be appended to the filename to ensure filenames are globally unique.", + type: new GraphQLNonNull(GraphQLBoolean), + defaultValue: false + }, normalize_extension: { description: "If true, the the correct file extension (.wsz or .wal) will be .\nOtherwise, the original user-uploaded file extension will be used.", type: new GraphQLNonNull(GraphQLBoolean), @@ -253,70 +328,33 @@ export function getSchema(): GraphQLSchema { }; } }); - const ArchiveFileType: GraphQLObjectType = new GraphQLObjectType({ - name: "ArchiveFile", - description: "A file found within a Winamp Skin's .wsz archive", + const BulkDownloadConnectionType: GraphQLObjectType = new GraphQLObjectType({ + name: "BulkDownloadConnection", + description: "Connection for bulk download skin metadata", fields() { return { - date: { - description: "The date on the file inside the archive. Given in simplified extended ISO\nformat (ISO 8601).", - name: "date", + estimatedSizeBytes: { + description: "Estimated total size in bytes (approximation for progress indication)", + name: "estimatedSizeBytes", type: GraphQLString, - resolve(source) { - return assertNonNull(source.getIsoDate()); + resolve(source, args, context, info) { + return assertNonNull(defaultFieldResolver(source, args, context, info)); } }, - file_md5: { - description: "The md5 hash of the file within the archive", - name: "file_md5", - type: GraphQLString, - resolve(source) { - return assertNonNull(source.getFileMd5()); + nodes: { + description: "List of skin metadata for bulk download", + name: "nodes", + type: new GraphQLList(new GraphQLNonNull(SkinType)), + resolve(source, _args, context) { + return assertNonNull(source.nodes(getUserContext(context))); } }, - filename: { - description: "Filename of the file within the archive", - name: "filename", - type: GraphQLString, - resolve(source) { - return assertNonNull(source.getFileName()); - } - }, - is_directory: { - description: "Is the file a directory?", - name: "is_directory", - type: GraphQLBoolean, - resolve(source) { - return assertNonNull(source.getIsDirectory()); - } - }, - size: { - description: "The uncompressed size of the file in bytes.\n\n**Note:** Will be `null` for directories", - name: "size", + totalCount: { + description: "Total number of skins available for download", + name: "totalCount", type: GraphQLInt, - resolve(source) { - return source.getFileSize(); - } - }, - skin: { - description: "The skin in which this file was found", - name: "skin", - type: SkinType - }, - text_content: { - description: "The content of the file, if it's a text file", - name: "text_content", - type: GraphQLString, - resolve(source) { - return source.getTextContent(); - } - }, - url: { - description: "A URL to download the file. **Note:** This is powered by a little\nserverless Cloudflare function which tries to exctact the file on the fly.\nIt may not work for all files.", - name: "url", - type: GraphQLString, - resolve(source) { - return source.getUrl(); + resolve(source, args, context, info) { + return assertNonNull(defaultFieldResolver(source, args, context, info)); } } }; @@ -382,6 +420,11 @@ export function getSchema(): GraphQLSchema { name: "filename", type: GraphQLString, args: { + include_museum_id: { + description: "If true, the museum ID will be appended to the filename to ensure filenames are globally unique.", + type: new GraphQLNonNull(GraphQLBoolean), + defaultValue: false + }, normalize_extension: { description: "If true, the the correct file extension (.wsz or .wal) will be .\nOtherwise, the original user-uploaded file extension will be used.", type: new GraphQLNonNull(GraphQLBoolean), @@ -544,6 +587,11 @@ export function getSchema(): GraphQLSchema { name: "filename", type: GraphQLString, args: { + include_museum_id: { + description: "If true, the museum ID will be appended to the filename to ensure filenames are globally unique.", + type: new GraphQLNonNull(GraphQLBoolean), + defaultValue: false + }, normalize_extension: { description: "If true, the the correct file extension (.wsz or .wal) will be .\nOtherwise, the original user-uploaded file extension will be used.", type: new GraphQLNonNull(GraphQLBoolean), @@ -901,6 +949,24 @@ export function getSchema(): GraphQLSchema { name: "Query", fields() { return { + bulkDownload: { + description: "Get metadata for bulk downloading all skins in the museum", + name: "bulkDownload", + type: BulkDownloadConnectionType, + args: { + first: { + type: new GraphQLNonNull(GraphQLInt), + defaultValue: 1000 + }, + offset: { + type: new GraphQLNonNull(GraphQLInt), + defaultValue: 0 + } + }, + resolve(_source, args) { + return assertNonNull(queryBulkDownloadResolver(args)); + } + }, fetch_archive_file_by_md5: { description: "Fetch archive file by it's MD5 hash\n\nGet information about a file found within a skin's wsz/wal/zip archive.", name: "fetch_archive_file_by_md5", @@ -1307,6 +1373,6 @@ export function getSchema(): GraphQLSchema { })], query: QueryType, mutation: MutationType, - types: [RatingType, SkinUploadStatusType, SkinsFilterOptionType, SkinsSortOptionType, TweetsSortOptionType, NodeType, SkinType, UploadUrlRequestType, ArchiveFileType, ClassicSkinType, DatabaseStatisticsType, InternetArchiveItemType, ModernSkinType, ModernSkinsConnectionType, MutationType, QueryType, ReviewType, SkinUploadType, SkinsConnectionType, TweetType, TweetsConnectionType, UploadMutationsType, UploadUrlType, UserType] + types: [RatingType, SkinUploadStatusType, SkinsFilterOptionType, SkinsSortOptionType, TweetsSortOptionType, NodeType, SkinType, UploadUrlRequestType, ArchiveFileType, BulkDownloadConnectionType, ClassicSkinType, DatabaseStatisticsType, InternetArchiveItemType, ModernSkinType, ModernSkinsConnectionType, MutationType, QueryType, ReviewType, SkinUploadType, SkinsConnectionType, TweetType, TweetsConnectionType, UploadMutationsType, UploadUrlType, UserType] }); } diff --git a/packages/skin-database/app/bulk-download/page.tsx b/packages/skin-database/app/bulk-download/page.tsx new file mode 100644 index 00000000..5c02e52b --- /dev/null +++ b/packages/skin-database/app/bulk-download/page.tsx @@ -0,0 +1,483 @@ +"use client"; + +import { useState, useCallback, useRef, useEffect } from "react"; +import { fetchGraphql, gql } from "../../legacy-client/src/utils"; + +interface BulkDownloadSkin { + md5: string; + filename: string; + download_url: string; + __typename: string; +} + +interface DownloadProgress { + totalSkins: number; + completedSkins: number; + failedSkins: number; + estimatedSizeBytes: string; + activeDownloads: Array<{ + filename: string; + md5: string; + status: "downloading" | "failed"; + error?: string; + }>; +} + +interface DirectoryHandle { + name: string; + getDirectoryHandle: ( + name: string, + options?: { create?: boolean } + ) => Promise; + getFileHandle: ( + name: string, + options?: { create?: boolean } + ) => Promise; +} + +declare global { + interface Window { + showDirectoryPicker?: () => Promise; + } +} + +const BULK_DOWNLOAD_QUERY = gql` + query BulkDownload($offset: Int!, $first: Int!) { + bulkDownload(offset: $offset, first: $first) { + totalCount + estimatedSizeBytes + nodes { + __typename + md5 + filename(normalize_extension: true, include_museum_id: true) + download_url + } + } + } +`; + +const MAX_CONCURRENT_DOWNLOADS = 6; +const CHUNK_SIZE = 1000; + +export default function BulkDownloadPage() { + const [directoryHandle, setDirectoryHandle] = + useState(null); + const [progress, setProgress] = useState({ + totalSkins: 0, + completedSkins: 0, + failedSkins: 0, + estimatedSizeBytes: "0", + activeDownloads: [], + }); + const [isDownloading, setIsDownloading] = useState(false); + const [error, setError] = useState(null); + const [isSupported] = useState( + typeof window !== "undefined" && "showDirectoryPicker" in window + ); + const abortController = useRef(null); + + const downloadSkin = useCallback( + async ( + skin: BulkDownloadSkin, + directoryHandle: DirectoryHandle, + signal: AbortSignal + ): Promise => { + const { filename, download_url, md5 } = skin; + + // Get the target directory and file path + const targetDirectory = await getDirectoryForSkin( + filename, + directoryHandle + ); + // Check if file already exists + try { + await targetDirectory.getFileHandle(filename); + // File exists, skip download + console.log(`Skipping ${filename} - already exists`); + setProgress((prev) => ({ + ...prev, + completedSkins: prev.completedSkins + 1, + })); + return; + } catch (error) { + // File doesn't exist, continue with download + } + + // Add to active downloads + setProgress((prev) => ({ + ...prev, + activeDownloads: [ + ...prev.activeDownloads, + { + filename, + md5, + status: "downloading" as const, + }, + ], + })); + + try { + const response = await fetch(download_url, { signal }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + // We don't need individual progress tracking anymore + // const contentLength = parseInt(response.headers.get("content-length") || "0", 10); + const reader = response.body?.getReader(); + + if (!reader) { + throw new Error("No response body"); + } + + // Use the targetDirectory and finalFilename we calculated earlier + const fileHandle = await targetDirectory.getFileHandle(filename, { + create: true, + }); + const writable = await fileHandle.createWritable(); + + // Track total bytes for this file (not needed for individual progress) + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + await writable.write(value); + } + + await writable.close(); + + // Mark as completed and immediately remove from active downloads + setProgress((prev) => ({ + ...prev, + completedSkins: prev.completedSkins + 1, + activeDownloads: prev.activeDownloads.filter((d) => d.md5 !== md5), + })); + } catch (writeError) { + await writable.abort("Failed to write file"); + throw writeError; + } + } catch (error: any) { + if (error.name === "AbortError") { + console.log(`Download aborted: ${filename}`); + throw error; // Re-throw abort errors + } + + // Mark as failed and schedule removal + setProgress((prev) => ({ + ...prev, + failedSkins: prev.failedSkins + 1, + activeDownloads: prev.activeDownloads.map((d) => + d.md5 === md5 + ? { + ...d, + status: "failed" as const, + error: error.message, + } + : d + ), + })); + + // Remove failed download after 3 seconds + setTimeout(() => { + setProgress((prev) => ({ + ...prev, + activeDownloads: prev.activeDownloads.filter((d) => d.md5 !== md5), + })); + }, 3000); + + console.error(`Failed to download ${filename}:`, error); + } + }, + [getDirectoryForSkin] + ); + + // Load initial metadata when component mounts + useEffect(() => { + async function loadInitialData() { + try { + const { totalCount, estimatedSizeBytes } = await fetchSkins(0, 1); + setProgress((prev) => ({ + ...prev, + totalSkins: totalCount, + estimatedSizeBytes, + })); + } catch (error: any) { + console.error("Failed to load initial data:", error); + setError("Failed to load skin count information"); + } + } + + loadInitialData(); + }, [fetchSkins]); + + const selectDirectoryAndStart = useCallback(async () => { + // First, select directory if not already selected + if (!directoryHandle) { + if (!window.showDirectoryPicker) { + setError( + "File System Access API is not supported in this browser. Please use Chrome or Edge." + ); + return; + } + + try { + const handle = await window.showDirectoryPicker(); + setDirectoryHandle(handle); + setError(null); + + // Now start the download with the new directory + await startDownloadWithDirectory(handle as FileSystemDirectoryHandle); + } catch (err: any) { + if (err.name !== "AbortError") { + setError(`Failed to select directory: ${err.message}`); + } + } + } else { + // Directory already selected, just start download + await startDownloadWithDirectory( + directoryHandle as FileSystemDirectoryHandle + ); + } + }, [directoryHandle]); + + const startDownloadWithDirectory = useCallback( + async (handle: FileSystemDirectoryHandle) => { + setIsDownloading(true); + setError(null); + // setStartTime(Date.now()); + abortController.current = new AbortController(); + + try { + // Get initial metadata + const { totalCount, estimatedSizeBytes } = await fetchSkins(0, 1); + + setProgress({ + totalSkins: totalCount, + completedSkins: 0, + failedSkins: 0, + estimatedSizeBytes, + activeDownloads: [], + }); + + let offset = 0; + const activePromises = new Set>(); + + while (offset < totalCount && !abortController.current.signal.aborted) { + console.log(`Fetching batch: offset=${offset}, chunk=${CHUNK_SIZE}`); + + try { + const { skins } = await fetchSkins(offset, CHUNK_SIZE); + console.log(`Retrieved ${skins.length} skins in this batch`); + + if (skins.length === 0) { + console.log("No more skins to fetch, breaking"); + break; + } + + for (const skin of skins) { + if (abortController.current.signal.aborted) break; + + await waitForAvailableSlot( + activePromises, + abortController.current.signal + ); + + if (abortController.current.signal.aborted) break; + + const downloadPromise = downloadSkin( + skin, + handle, + abortController.current.signal + ).finally(() => { + activePromises.delete(downloadPromise); + }); + + activePromises.add(downloadPromise); + } + + offset += skins.length; + console.log(`Completed batch, new offset: ${offset}/${totalCount}`); + } catch (error: any) { + console.error(`Failed to fetch batch at offset ${offset}:`, error); + setError(`Failed to fetch skins: ${error.message}`); + break; + } + } + + // Wait for all remaining downloads to complete + await Promise.allSettled(activePromises); + console.log("All downloads completed!"); + } catch (error: any) { + if (error.name !== "AbortError") { + setError(`Download failed: ${error.message}`); + } + } finally { + setIsDownloading(false); + } + }, + [fetchSkins, downloadSkin] + ); + + const stopDownload = useCallback(() => { + if (abortController.current) { + abortController.current.abort("User Canceled"); + } + setIsDownloading(false); + // setStartTime(null); + }, []); + + const progressPercent = + progress.totalSkins > 0 + ? ((progress.completedSkins + progress.failedSkins) / + progress.totalSkins) * + 100 + : 0; + + if (!isSupported) { + return

Your browser does not support filesystem access.

; + } + + const gb = Math.round( + parseInt(progress.estimatedSizeBytes || "0", 10) / (1024 * 1024 * 1024) + ); + + return ( +
+
+
+
+

Bulk Download All Skins

+

+ Download the entire Winamp Skin Museum collection. +

    +
  • + Will download {progress.totalSkins.toLocaleString()} files (~ + {gb} + GB) into the selected directory +
    • +
  • + Files will be organized into directories (aa-zz, 0-9) based on + filename prefix +
    • +
  • + Supports resuming from previously interrupted bulk download +
    • +
+

+
+
+ + {error && ( +
+
{error}
+
+ )} + + {/* Download Controls */} +
+ {isDownloading ? ( + + ) : ( + + )} +
+ + {/* Progress Section */} + {(isDownloading || progress.completedSkins > 0) && ( +
+
+ + Downloaded{" "} + {( + progress.completedSkins + progress.failedSkins + ).toLocaleString()}{" "} + of {progress.totalSkins.toLocaleString()} skins + + {Math.round(progressPercent)}% complete +
+ +
+
+
+
+ )} +
+
+ ); +} + +async function getDirectoryForSkin( + filename: string, + rootHandle: DirectoryHandle +) { + // Create directory based on first two characters of filename (case insensitive) + const firstChar = filename.charAt(0).toLowerCase(); + const secondChar = + filename.length > 1 ? filename.charAt(1).toLowerCase() : ""; + + let dirName: string; + if (/[a-z]/.test(firstChar)) { + // For letters, use two-character prefix if second char is alphanumeric + if (/[a-z0-9]/.test(secondChar)) { + dirName = firstChar + secondChar; + } else { + // Fallback to single letter + 'x' for special characters + dirName = firstChar + "x"; + } + } else { + // For numbers/symbols, use "0-9" + dirName = "0-9"; + } + + try { + return await rootHandle.getDirectoryHandle(dirName, { create: true }); + } catch (err) { + console.warn(`Failed to create directory ${dirName}, using root:`, err); + return rootHandle; + } +} + +async function fetchSkins( + offset: number, + first: number +): Promise<{ + skins: BulkDownloadSkin[]; + totalCount: number; + estimatedSizeBytes: string; +}> { + const { bulkDownload } = await fetchGraphql(BULK_DOWNLOAD_QUERY, { + offset, + first, + }); + return { + skins: bulkDownload.nodes, + totalCount: bulkDownload.totalCount, + estimatedSizeBytes: bulkDownload.estimatedSizeBytes, + }; +} +// Helper function to wait for an available download slot +async function waitForAvailableSlot( + activePromises: Set>, + signal: AbortSignal +) { + while (activePromises.size >= MAX_CONCURRENT_DOWNLOADS && !signal.aborted) { + await Promise.race(activePromises); + } +} diff --git a/packages/skin-database/data/SkinModel.ts b/packages/skin-database/data/SkinModel.ts index d7543c39..450c2a02 100644 --- a/packages/skin-database/data/SkinModel.ts +++ b/packages/skin-database/data/SkinModel.ts @@ -232,8 +232,8 @@ export default class SkinModel { } } - getAverageColor(): string { - return this.row.average_color; + getAverageColor(): string | null { + return this.row.average_color ?? null; } getBuffer = mem(async (): Promise => { diff --git a/packages/skin-database/types.ts b/packages/skin-database/types.ts index d8ad0ebb..19e476f3 100644 --- a/packages/skin-database/types.ts +++ b/packages/skin-database/types.ts @@ -19,7 +19,7 @@ export type SkinRow = { skin_type: number; emails: string; // readme_text: string; - average_color: string; + average_color?: string; }; export type TweetRow = {