diff --git a/.claude/worktrees/upbeat-tharp b/.claude/worktrees/upbeat-tharp new file mode 160000 index 000000000..e603096c5 --- /dev/null +++ b/.claude/worktrees/upbeat-tharp @@ -0,0 +1 @@ +Subproject commit e603096c56d853b429ccb06a585ffb6d0aa7539e diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 22ea1f915..91272bce3 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -47,6 +47,10 @@ yarn build - `yarn typecheck` - Run TypeScript type checking - `yarn check` - Run Biome linting and formatting +### Running minio tests + +Tests for `@uppy/aws-s3` need a minio server running inside Docker, and in order to run those tests you must set the environment variable `VITE_MINIO_CONFIG="test_user,test_password,http://localhost:9002/test-bucket,us-east-1"` when running the tests. + ### Headless components When adding a new component to `@uppy/components`, you have to run `yarn migrate:components` from root diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a67c13be7..e4c8bd147 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: COMPANION_DOMAIN: localhost:3020 COMPANION_PROTOCOL: http COMPANION_REDIS_URL: redis://localhost:6379 - BUCKET_ENV_MINIO: "minio,test_user,test_password,http://localhost:9002/test-bucket,us-east-1" + VITE_MINIO_CONFIG: "test_user,test_password,http://localhost:9002/test-bucket,us-east-1" types: name: Types diff --git a/examples/aws-companion/main.js b/examples/aws-companion/main.js index 7c1dd7f43..55eb8e450 100644 --- a/examples/aws-companion/main.js +++ b/examples/aws-companion/main.js @@ -23,5 +23,5 @@ uppy.use(Dashboard, { plugins: ['GoogleDrive', 'Webcam'], }) uppy.use(AwsS3, { - endpoint: 'http://localhost:3020', + companionEndpoint: 'http://localhost:3020', }) diff --git a/packages/@uppy/aws-s3/src/S3Uploader.ts b/packages/@uppy/aws-s3/src/S3Uploader.ts new file mode 100644 index 000000000..fc6a80dc8 --- /dev/null +++ b/packages/@uppy/aws-s3/src/S3Uploader.ts @@ -0,0 +1,421 @@ +import { EventManager, type Uppy } from '@uppy/core' +import type { Body, LocalUppyFile, Meta } from '@uppy/utils' +import type S3Client from './s3-client/S3Client.js' + +/** Persisted S3 multipart state for Golden Retriever resume support */ +interface S3MultipartState { + uploadId: string + key: string +} + +declare module '@uppy/utils' { + // biome-ignore lint/correctness/noUnusedVariables: must match existing interface signature + export interface LocalUppyFile { + s3Multipart?: S3MultipartState + } + // biome-ignore lint/correctness/noUnusedVariables: must match existing interface signature + export interface RemoteUppyFile { + s3Multipart?: S3MultipartState + } +} + +// ============================================================================ +// Constants +// ============================================================================ + +const MB = 1024 * 1024 + +/** Minimum chunk size required by S3 (5MB) */ +const MIN_CHUNK_SIZE = 5 * MB + +/** Maximum number of parts allowed by S3 */ +const MAX_PARTS = 10000 + +// ============================================================================ +// S3Uploader Types +// ============================================================================ + +interface S3UploaderOptions { + uppy: Uppy + s3Client: S3Client + file: LocalUppyFile + metadata: Record + key: string + shouldUseMultipart?: boolean + getChunkSize?: (file: { size: number }) => number + onProgress?: (bytesUploaded: number, bytesTotal: number) => void + onPartComplete?: (part: { PartNumber: number; ETag: string }) => void + onSuccess?: (result: UploadResult) => void + onError?: (err: Error) => void + onAbort?: () => void + log?: Uppy['log'] +} + +export interface UploadResult { + location: string + key: string + /** Only returned for multipart uploads */ + uploadId?: string +} + +interface Chunk { + index: number + start: number + end: number + size: number +} + +interface ChunkState { + uploaded: number + etag?: string +} + +export default class S3Uploader { + readonly #data: NonNullable['data']> + #key: string | undefined + readonly #options: S3UploaderOptions + readonly #eventManager: EventManager + + #chunks: Chunk[] = [] + #chunkState: ChunkState[] = [] + #shouldUseMultipart: boolean = false + #uploadId?: string + #uploadHasStarted: boolean = false + #abortController: AbortController | undefined + + constructor(options: S3UploaderOptions) { + if (options.file.data == null) { + throw new Error(`File data is missing for file ${options.file.id}`) + } + this.#options = options + this.#data = options.file.data + this.#eventManager = new EventManager(options.uppy) + + // Detect resume state from file (persisted by Golden Retriever across page refreshes). + // Must run before #initChunks so it can force multipart mode for resumed uploads. + const resumeState = options.file.s3Multipart + if (resumeState) { + this.#key = resumeState.key + this.#uploadId = resumeState.uploadId + this.#uploadHasStarted = true + } + + const fileSize = options.file.data.size + + // Determine if we should use multipart + // If we're resuming a multipart upload, force multipart. Otherwise use + // the boolean option (true/false) and ensure the file is larger than + // S3's minimum chunk size when enabling multipart. + this.#shouldUseMultipart = + Boolean(resumeState) || + (this.#options.shouldUseMultipart === true && fileSize > MIN_CHUNK_SIZE) + + // Create chunks based on upload strategy + if (this.#shouldUseMultipart) { + // Calculate chunk size: at least MIN_CHUNK_SIZE, but may be larger for huge files + let chunkSize = this.#getChunkSize(fileSize) + chunkSize = Math.max(chunkSize, MIN_CHUNK_SIZE) + + // Ensure we don't exceed MAX_PARTS (S3 limit: 10,000 parts) + if (Math.ceil(fileSize / chunkSize) > MAX_PARTS) { + chunkSize = Math.ceil(fileSize / MAX_PARTS) + } + + // Create chunk definitions + for ( + let offset = 0, index = 0; + offset < fileSize; + offset += chunkSize, index++ + ) { + const end = Math.min(offset + chunkSize, fileSize) + this.#chunks.push({ index, start: offset, end, size: end - offset }) + } + } else { + // Simple upload: single chunk for the entire file + this.#chunks = [{ index: 0, start: 0, end: fileSize, size: fileSize }] + } + + this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 })) + + // Setup events: + const fileId = this.#options.file.id + + this.#eventManager.onFileRemove(fileId, () => { + this.abort() + this.#options.onAbort?.() + }) + + this.#eventManager.onCancelAll(fileId, () => { + this.abort() + this.#options.onAbort?.() + }) + + this.#eventManager.onFilePause(fileId, (isPaused) => { + if (isPaused) { + this.pause() + } else { + this.start() + } + }) + + this.#eventManager.onPauseAll(fileId, () => { + this.pause() + }) + + this.#eventManager.onResumeAll(fileId, () => { + this.start() + }) + + this.#eventManager.onRetry(fileId, () => { + this.start() + }) + + this.#eventManager.onRetryAll(fileId, () => { + this.start() + }) + } + + #getChunkSize(fileSize: number): number { + if (this.#options.getChunkSize) { + return this.#options.getChunkSize({ size: fileSize }) + } + return Math.ceil(fileSize / MAX_PARTS) + } + + async start(): Promise { + // Abort any pending operations (if not already aborted) + this.#abortController?.abort() + // Always create a fresh AbortController (also for resume) + this.#abortController = new AbortController() + + try { + if (this.#uploadHasStarted) { + await this.#resumeUpload() + } else { + await this.#createUpload() + } + } catch (err) { + this.#onError(err as Error) + } + } + + pause(): void { + this.#abortController?.abort() + } + + /** + * + * @param opts - `abortInS3`: Whether to also abort the multipart upload in S3. Default: true. Set to false to keep the multipart upload in S3 active, allowing for manual cleanup later and preventing accidental data loss if the user later tries to resume the upload. + */ + abort(opts?: { abortInS3?: boolean }): void { + this.#abortController?.abort() + // Clean up event listeners + this.#eventManager.remove() + if (opts?.abortInS3 !== false && this.#uploadId) { + if (!this.#key) { + throw new Error('Missing S3 object key for aborting upload') + } + this.#options.s3Client + .abortMultipartUpload(this.#key, this.#uploadId) + .catch((abortErr) => { + this.#options.log?.(abortErr, 'warning') + }) + } + } + + async #createUpload(): Promise { + this.#uploadHasStarted = true + if (this.#shouldUseMultipart) { + await this.#uploadMultipart() + } else { + await this.#uploadNonMultipart() + } + } + + async #resumeUpload(): Promise { + if (!this.#uploadId) { + await this.#createUpload() + return + } + if (!this.#key) { + throw new Error('Missing S3 object key for resuming upload') + } + const existingParts = await this.#options.s3Client.listParts( + this.#uploadId, + this.#key, + ) + // Sync local state with S3 - mark already-uploaded parts + for (const part of existingParts) { + const chunkIndex = part.partNumber - 1 + if (chunkIndex >= 0 && chunkIndex < this.#chunkState.length) { + this.#chunkState[chunkIndex].uploaded = this.#chunks[chunkIndex].size + this.#chunkState[chunkIndex].etag = part.etag + } + } + // Emit progress update to reflect already-uploaded parts + this.#onProgress() + await this.#uploadRemainingParts() + } + + async #uploadNonMultipart(): Promise { + const signal = this.#abortController?.signal + signal?.throwIfAborted() + + const { location, key } = await this.#options.s3Client.putObject( + this.#options.key, + this.#data, + this.#options.file.type || 'application/octet-stream', + this.#options.metadata, + (bytesUploaded: number) => { + this.#chunkState[0].uploaded = bytesUploaded + this.#onProgress() + }, + signal, + ) + + this.#onSuccess({ + location, + key, + }) + } + + async #uploadMultipart(): Promise { + const signal = this.#abortController?.signal + signal?.throwIfAborted() + + const { uploadId, key } = + await this.#options.s3Client.createMultipartUpload( + this.#options.key, + this.#options.file.type || 'application/octet-stream', + this.#options.metadata, + ) + if (key == null) { + throw new Error( + 'S3 client did not return object key for multipart upload', + ) + } + + this.#key = key + + // Persist resume state so Golden Retriever can restore it after page refresh + this.#options.uppy.setFileState(this.#options.file.id, { + s3Multipart: { uploadId, key }, + }) + + this.#uploadId = uploadId + + await this.#uploadRemainingParts() + } + + async #uploadRemainingParts(): Promise { + const signal = this.#abortController?.signal + + for (let i = 0; i < this.#chunks.length; i++) { + signal?.throwIfAborted() + if (this.#chunkState[i].etag) continue // already uploaded + + const chunk = this.#chunks[i] + const partNumber = i + 1 + const chunkData = this.#data.slice(chunk.start, chunk.end) + const chunkIndex = i // Capture for closure (cannot use for-loop variable i directly in a closure) + + if (this.#key == null) { + throw new Error('Missing S3 object key for uploading part') + } + const { etag } = await this.#options.s3Client.uploadPart( + this.#key, + this.#uploadId!, + chunkData, + partNumber, + (bytesUploaded: number) => { + this.#chunkState[chunkIndex].uploaded = bytesUploaded + this.#onProgress() + }, + signal, + ) + + // after part finished uploading, update chunk state + this.#chunkState[i].uploaded = chunk.size + this.#chunkState[i].etag = etag + this.#onProgress() + + if (this.#options.onPartComplete) { + this.#options.onPartComplete({ + PartNumber: partNumber, + ETag: etag, + }) + } + } + + signal?.throwIfAborted() + + const parts = this.#chunkState.flatMap((state, i) => + state.etag ? [{ partNumber: i + 1, etag: state.etag }] : [], + ) + + if (this.#key == null) { + throw new Error('Missing S3 object key for completing multipart upload') + } + + const { location, key } = + await this.#options.s3Client.completeMultipartUpload( + this.#key, + this.#uploadId!, + parts, + ) + + this.#onSuccess({ + location, + key, + uploadId: this.#uploadId, + }) + } + + #onProgress(): void { + if (!this.#options.onProgress) return + const bytesUploaded = this.#chunkState.reduce( + (sum, state) => sum + state.uploaded, + 0, + ) + this.#options.onProgress(bytesUploaded, this.#data.size) + } + + #onSuccess(result: UploadResult): void { + // If the upload was aborted (file removed mid-upload), the network request + // may still complete successfully. Don't emit success in this case since + // the file no longer exists in Uppy's state. + this.#eventManager.remove() + if (this.#abortController?.signal.aborted) { + return + } + // Clear persisted resume state — upload completed successfully. + this.#options.uppy.setFileState(this.#options.file.id, { + s3Multipart: undefined, + }) + this.#options.onSuccess?.(result) + } + + #onError(err: Error): void { + // ignore abort signals from intentional cancellation + if (err.name === 'AbortError') return + // If we intentionally aborted, don't report any subsequent errors + // (e.g., S3 returning 404 NoSuchUpload after we aborted the upload) + if (this.#abortController?.signal.aborted) return + + // Clean up event listeners so this uploader doesn't become a "zombie" + // that reacts to future retry/pause/resume events after the error. + // Without this, each failed retry leaves an orphaned uploader that + // still listens for retry-all, causing duplicate uploads on the next + // successful retry. + this.#eventManager.remove() + + // NOTE: We intentionally do NOT abort the multipart upload in S3 here. + // This allows the user to retry and resume from where they left off. + // The multipart upload is only aborted when the user cancels via the + // `abort()` method. By default `abort()` will also abort the multipart + // upload in S3 (abortInS3 = true). Pass { abortInS3: false } to keep the + // multipart upload in S3 so it can be cleaned up manually or resumed later. + + this.#options.onError?.(err) + } +} diff --git a/packages/@uppy/aws-s3/src/index.ts b/packages/@uppy/aws-s3/src/index.ts index becbedeb8..7f0dbd511 100644 --- a/packages/@uppy/aws-s3/src/index.ts +++ b/packages/@uppy/aws-s3/src/index.ts @@ -1,18 +1,28 @@ +import type { RequestClient } from '@uppy/companion-client' import { BasePlugin, type DefinePluginOpts, - EventManager, type PluginOpts, type Uppy, } from '@uppy/core' -import type { Body, Meta, UppyFile } from '@uppy/utils' +import type { + Body, + LocalUppyFile, + Meta, + RemoteUppyFile, + UppyFile, +} from '@uppy/utils' import { filterFilesToEmitUploadStarted, filterFilesToUpload, + getAllowedMetaFields, TaskQueue, } from '@uppy/utils' import packageJson from '../package.json' with { type: 'json' } -import S3mini from './s3-client/S3.js' +import S3Uploader, { type UploadResult } from './S3Uploader.js' +import S3Companion from './s3-client/CompanionS3.js' +import type S3Client from './s3-client/S3Client.js' +import S3mini from './s3-client/S3mini.js' import type * as IT from './s3-client/types.js' // ============================================================================ @@ -37,26 +47,7 @@ declare module '@uppy/core' { } } -export interface AwsS3Options - extends PluginOpts { - bucket: string - region?: string - endpoint?: string - - /** - * Custom function to sign requests. - * Called with request details, should return signed headers. - * Alternative to using Companion endpoint. - */ - signRequest?: IT.signRequestFn - - /** - * Function to retrieve temporary credentials for client-side signing. - * When provided, S3mini handles signing internally using SigV4. - * Alternative to signRequest or endpoint. - */ - getCredentials?: IT.getCredentialsFn - +export type AwsS3Options = PluginOpts & { /** * Whether to use multipart uploads. * - `true`: Always use multipart @@ -83,7 +74,34 @@ export interface AwsS3Options * Default: `{randomId}-{filename}` */ generateObjectKey?: (file: UppyFile) => string -} +} & ( + | { + /** S3 upload endpoint */ + s3Endpoint: string + + /** AWS region, required for signing */ + region?: string + + /** + * Function to retrieve temporary credentials for client-side signing. + * When provided, S3mini handles signing internally using SigV4. + * Alternative to signRequest or endpoint. + */ + getCredentials: IT.GetCredentialsFn + } + | { + /** + * Custom function to sign requests. + * Called with request details, should return signed headers. + * Alternative to using Companion endpoint. + */ + signRequest: IT.SignRequestFn + } + | { + /** Companion URL if you want to use Companion for signing */ + companionEndpoint: string + } + ) // ============================================================================ // Constants @@ -91,12 +109,6 @@ export interface AwsS3Options const MB = 1024 * 1024 -/** Minimum chunk size required by S3 (5MB) */ -const MIN_CHUNK_SIZE = 5 * MB - -/** Maximum number of parts allowed by S3 */ -const MAX_PARTS = 10000 - const defaultOptions = { shouldUseMultipart: (file: UppyFile) => (file.size || 0) > 100 * MB, allowedMetaFields: true, @@ -108,381 +120,6 @@ const defaultOptions = { // S3Uploader Types // ============================================================================ -interface S3UploaderOptions { - uppy: Uppy - s3Client: S3mini - file: UppyFile - key: string - shouldUseMultipart?: boolean - getChunkSize?: (file: { size: number }) => number - onProgress?: (bytesUploaded: number, bytesTotal: number) => void - onPartComplete?: (part: { PartNumber: number; ETag: string }) => void - onSuccess?: (result: UploadResult) => void - onError?: (err: Error) => void - onAbort?: () => void - log?: (message: string | Error, type?: 'error' | 'warning') => void -} - -interface UploadResult { - location: string - key: string - bucket?: string - uploadId?: string -} - -interface Chunk { - index: number - start: number - end: number - size: number -} - -interface ChunkState { - uploaded: number - etag?: string -} - -/** Reason for pausing (not a real error) */ -const pausingUploadReason = Symbol('pausing upload, not an actual error') - -/** Type guard to check if an error is from pausing (not a real failure) */ -const isPauseError = (err: unknown): boolean => - err instanceof Error && - (err as { cause?: unknown }).cause === pausingUploadReason - -// ============================================================================ -// S3Uploader Class -// ============================================================================ - -class S3Uploader { - readonly #s3Client: S3mini - readonly #file: UppyFile - readonly #data: Blob - readonly #key: string - readonly #options: S3UploaderOptions - readonly #eventManager: EventManager - - #chunks: Chunk[] = [] - #chunkState: ChunkState[] = [] - #shouldUseMultipart: boolean = false - #uploadId?: string - #uploadHasStarted: boolean = false - #abortController: AbortController = new AbortController() - - constructor(data: Blob, options: S3UploaderOptions) { - this.#s3Client = options.s3Client - this.#file = options.file - this.#data = data - this.#key = options.key - this.#options = options - this.#eventManager = new EventManager(options.uppy) - this.#initChunks() - this.#setupEvents() - } - - #setupEvents(): void { - const fileId = this.#file.id - - this.#eventManager.onFileRemove(fileId, () => { - this.abort() - this.#options.onAbort?.() - }) - - this.#eventManager.onCancelAll(fileId, () => { - this.abort() - this.#options.onAbort?.() - }) - - this.#eventManager.onFilePause(fileId, (isPaused) => { - if (isPaused) { - this.pause() - } else { - this.start() - } - }) - - this.#eventManager.onPauseAll(fileId, () => { - this.pause() - }) - - this.#eventManager.onResumeAll(fileId, () => { - this.start() - }) - - this.#eventManager.onRetry(fileId, () => { - this.start() - }) - - this.#eventManager.onRetryAll(fileId, () => { - this.start() - }) - } - - #initChunks(): void { - const fileSize = this.#data.size - - // Step 1: Determine if we should use multipart - // - If explicitly set to boolean, use that - // - Otherwise, use multipart for files larger than MIN_CHUNK_SIZE (5MB) - if (typeof this.#options.shouldUseMultipart === 'boolean') { - this.#shouldUseMultipart = this.#options.shouldUseMultipart - } else { - this.#shouldUseMultipart = fileSize > MIN_CHUNK_SIZE - } - - // Step 2: Force simple upload if file is too small for multipart - // (S3 requires minimum 5MB parts, except for the last part) - if (fileSize <= MIN_CHUNK_SIZE) { - this.#shouldUseMultipart = false - } - - // Step 3: Create chunks based on upload strategy - if (this.#shouldUseMultipart) { - // Calculate chunk size: at least MIN_CHUNK_SIZE, but may be larger for huge files - let chunkSize = this.#getChunkSize(fileSize) - chunkSize = Math.max(chunkSize, MIN_CHUNK_SIZE) - - // Ensure we don't exceed MAX_PARTS (S3 limit: 10,000 parts) - if (Math.ceil(fileSize / chunkSize) > MAX_PARTS) { - chunkSize = Math.ceil(fileSize / MAX_PARTS) - } - - // Create chunk definitions - for ( - let offset = 0, index = 0; - offset < fileSize; - offset += chunkSize, index++ - ) { - const end = Math.min(offset + chunkSize, fileSize) - this.#chunks.push({ index, start: offset, end, size: end - offset }) - } - } else { - // Simple upload: single chunk for the entire file - this.#chunks = [{ index: 0, start: 0, end: fileSize, size: fileSize }] - } - - this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 })) - } - - #getChunkSize(fileSize: number): number { - if (this.#options.getChunkSize) { - return this.#options.getChunkSize({ size: fileSize }) - } - return Math.max(MIN_CHUNK_SIZE, Math.ceil(fileSize / MAX_PARTS)) - } - - start(): void { - if (this.#uploadHasStarted) { - // Abort any pending operations (if not already aborted) - if (!this.#abortController.signal.aborted) { - this.#abortController.abort(pausingUploadReason) - } - // Always create a fresh AbortController for resume - this.#abortController = new AbortController() - this.#resumeUpload() - } else { - this.#createUpload() - } - } - - pause(): void { - this.#abortController.abort(pausingUploadReason) - this.#abortController = new AbortController() - } - - abort(opts?: { abortOnS3?: boolean }): void { - this.#abortController.abort() - // Clean up event listeners - this.#eventManager.remove() - if (opts?.abortOnS3 !== false && this.#uploadId) { - this.#s3Client - .abortMultipartUpload(this.#key, this.#uploadId) - .catch((abortErr) => { - this.#options.log?.(abortErr, 'warning') - }) - } - } - - async #createUpload(): Promise { - this.#uploadHasStarted = true - try { - if (this.#shouldUseMultipart) { - await this.#multipartUpload() - } else { - await this.#simpleUpload() - } - } catch (err) { - this.#onError(err as Error) - } - } - - async #resumeUpload(): Promise { - if (!this.#uploadId) { - await this.#createUpload() - return - } - try { - const existingParts = await this.#s3Client.listParts( - this.#uploadId, - this.#key, - ) - // Sync local state with S3 - mark already-uploaded parts - for (const part of existingParts) { - const chunkIndex = part.partNumber - 1 - if (chunkIndex >= 0 && chunkIndex < this.#chunkState.length) { - this.#chunkState[chunkIndex].uploaded = this.#chunks[chunkIndex].size - this.#chunkState[chunkIndex].etag = part.etag - } - } - // Emit progress update to reflect already-uploaded parts - this.#onProgress() - await this.#uploadRemainingParts() - } catch (err) { - this.#onError(err as Error) - } - } - - async #simpleUpload(): Promise { - const signal = this.#abortController.signal - if (signal.aborted) { - throw new Error('Upload aborted', { cause: signal.reason }) - } - - await this.#s3Client.putObject( - this.#key, - this.#data, - this.#file.type || 'application/octet-stream', - (bytesUploaded: number) => { - this.#chunkState[0].uploaded = bytesUploaded - this.#onProgress() - }, - signal, - ) - - this.#onSuccess({ - location: `${this.#s3Client.endpoint}/${this.#key}`, - key: this.#key, - }) - } - - async #multipartUpload(): Promise { - const signal = this.#abortController.signal - if (signal.aborted) { - throw new Error('Upload aborted', { cause: signal.reason }) - } - - this.#uploadId = await this.#s3Client.getMultipartUploadId( - this.#key, - this.#file.type || 'application/octet-stream', - ) - await this.#uploadRemainingParts() - } - - async #uploadRemainingParts(): Promise { - const signal = this.#abortController.signal - // Collect already-uploaded parts (from resume) - const parts = this.#chunkState - .map((state, i) => - state.etag ? { partNumber: i + 1, etag: state.etag } : null, - ) - .filter((p): p is NonNullable => p !== null) - - for (let i = 0; i < this.#chunks.length; i++) { - if (signal.aborted) { - throw new Error('Upload aborted', { cause: signal.reason }) - } - if (this.#chunkState[i].etag) continue - - const chunk = this.#chunks[i] - const partNumber = i + 1 - const chunkData = this.#data.slice(chunk.start, chunk.end) - const chunkIndex = i // Capture for closure - - const part = await this.#s3Client.uploadPart( - this.#key, - this.#uploadId!, - chunkData, - partNumber, - (bytesUploaded: number) => { - this.#chunkState[chunkIndex].uploaded = bytesUploaded - this.#onProgress() - }, - signal, - ) - - this.#chunkState[i].uploaded = chunk.size - this.#chunkState[i].etag = part.etag - parts.push({ partNumber: part.partNumber, etag: part.etag }) - this.#onProgress() - - if (this.#options.onPartComplete) { - this.#options.onPartComplete({ - PartNumber: part.partNumber, - ETag: part.etag, - }) - } - } - - if (signal.aborted) { - throw new Error('Upload aborted', { cause: signal.reason }) - } - - const result = await this.#s3Client.completeMultipartUpload( - this.#key, - this.#uploadId!, - parts, - ) - - this.#onSuccess({ - location: result.location, - key: result.key, - bucket: result.bucket, - uploadId: this.#uploadId, - }) - } - - #onProgress(): void { - if (!this.#options.onProgress) return - const bytesUploaded = this.#chunkState.reduce( - (sum, state) => sum + state.uploaded, - 0, - ) - this.#options.onProgress(bytesUploaded, this.#data.size) - } - - #onSuccess(result: UploadResult): void { - // If the upload was aborted (file removed mid-upload), the network request - // may still complete successfully. Don't emit success in this case since - // the file no longer exists in Uppy's state. - if (this.#abortController.signal.aborted) { - this.#eventManager.remove() - return - } - this.#eventManager.remove() - this.#options.onSuccess?.(result) - } - - #onError(err: Error): void { - if (isPauseError(err)) return - // Also ignore abort signals from intentional cancellation - if (err.name === 'AbortError') return - // If we intentionally aborted, don't report any subsequent errors - // (e.g., S3 returning 404 NoSuchUpload after we aborted the upload) - if (this.#abortController.signal.aborted) return - - // NOTE: We intentionally do NOT abort the multipart upload on S3 here. - // This allows the user to retry and resume from where they left off. - // The multipart upload is only aborted when the user explicitly cancels - // (via the abort() method with abortOnS3: true). - - this.#options.onError?.(err) - } -} - -// ============================================================================ -// Plugin Class -// ============================================================================ - export default class AwsS3 extends BasePlugin< DefinePluginOpts, keyof typeof defaultOptions>, M, @@ -490,7 +127,7 @@ export default class AwsS3 extends BasePlugin< > { static VERSION = packageJson.version - #s3Client!: S3mini + #s3Client!: S3Client #queue!: TaskQueue #uploaders: Record | null> = {} @@ -522,10 +159,6 @@ export default class AwsS3 extends BasePlugin< } } - // -------------------------------------------------------------------------- - // Resumable Uploads Capability - // -------------------------------------------------------------------------- - #setResumableUploadsCapability = (value: boolean): void => { const { capabilities } = this.uppy.getState() this.uppy.setState({ @@ -546,68 +179,42 @@ export default class AwsS3 extends BasePlugin< // -------------------------------------------------------------------------- #initS3Client(): void { - const { endpoint, signRequest, getCredentials, bucket, region } = this.opts + if ('companionEndpoint' in this.opts) { + if (typeof this.opts.companionEndpoint !== 'string') { + throw new TypeError('companionEndpoint must be a string') + } + this.#s3Client = new S3Companion({ + companionEndpoint: this.opts.companionEndpoint, + }) + } else if ('getCredentials' in this.opts) { + if (typeof this.opts.s3Endpoint !== 'string') { + throw new TypeError('s3Endpoint must be a string') + } + if (typeof this.opts.getCredentials !== 'function') { + throw new TypeError('getCredentials must be a function') + } + if (this.opts.region != null && typeof this.opts.region !== 'string') { + throw new TypeError('region must be a string') + } - if (typeof bucket !== 'string' || bucket.trim() === '') { - throw new Error( - 'AwsS3: `bucket` option is required and must be a non-empty string', - ) - } - - if (typeof region !== 'string' || region.trim() === '') { - throw new Error( - 'AwsS3: `region` option is required and must be a non-empty string', - ) - } - - const bucketName = bucket.trim() - - if (!signRequest && !getCredentials && !endpoint) { - throw new Error( - 'AwsS3: `endpoint`, `signRequest`, or `getCredentials` is required', - ) - } - - const s3Endpoint = `https://${bucketName}.s3.${region}.amazonaws.com` - - if (getCredentials) { // Mode: Temporary credentials (client-side signing) this.#s3Client = new S3mini({ - endpoint: s3Endpoint, - getCredentials, - region, + endpoint: this.opts.s3Endpoint, + getCredentials: this.opts.getCredentials, + region: this.opts.region, }) - } else if (signRequest) { + } else if ('signRequest' in this.opts) { + if (typeof this.opts.signRequest !== 'function') { + throw new TypeError('signRequest must be a function') + } // Mode: Custom signing function this.#s3Client = new S3mini({ - endpoint: s3Endpoint, - signRequest, - region, + signRequest: this.opts.signRequest, }) } else { - // Mode: Companion signing (endpoint is guaranteed to be set here) - this.#s3Client = new S3mini({ - endpoint: s3Endpoint, - signRequest: this.#createCompanionSigner(endpoint!), - region, - }) - } - } - - /** - * Creates a signing function that calls Companion's /s3/sign endpoint. - */ - #createCompanionSigner(companionUrl: string): IT.signRequestFn { - return async (request) => { - const response = await fetch(`${companionUrl}/s3/sign`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(request), - }) - if (!response.ok) { - throw new Error(`Failed to sign request: ${response.statusText}`) - } - return response.json() + throw new TypeError( + 'One of options `companionEndpoint`, `signRequest`, or `getCredentials` is required', + ) } } @@ -626,10 +233,9 @@ export default class AwsS3 extends BasePlugin< const promises = filesToUpload.map((file) => { if (file.isRemote) { - // Remote files not yet supported in this minimal implementation - return Promise.reject( - new Error('Remote file uploads not yet supported'), - ) + // Remote uploads are queued internally by RequestClient.uploadRemoteFile() + // via getQueue(), so no outer queue wrapping is needed here. + return this.#uploadRemoteFile(file) } return this.#queue.add(async () => { // File may have been removed while waiting in the queue. @@ -640,31 +246,32 @@ export default class AwsS3 extends BasePlugin< if (!currentFile) { return } - return this.#uploadLocalFile(currentFile) + return this.#uploadLocalFile(currentFile as LocalUppyFile) // assume it's still a local file since remote files aren't queued }) }) await Promise.allSettled(promises) + // After the upload batch is done, restore resumable uploads capability. + // It may have been set to false if there were remote files in this batch. + this.#setResumableUploadsCapability(true) } // -------------------------------------------------------------------------- // Local File Upload // -------------------------------------------------------------------------- - async #uploadLocalFile(file: UppyFile): Promise { - return new Promise((resolve, reject) => { - const data = file.data as Blob - const key = this.#generateKey(file) - const shouldMultipart = this.#shouldUseMultipart(file) - - try { - // Create uploader (events are wired internally) - const uploader = new S3Uploader(data, { + async #uploadLocalFile(file: LocalUppyFile): Promise { + try { + return await new Promise((resolve, reject) => { + // Create uploader (events are wired internally). + // S3Uploader detects resume state from file.s3Multipart internally. + const uploader = new S3Uploader({ uppy: this.uppy, s3Client: this.#s3Client, file, - key, - shouldUseMultipart: shouldMultipart, + metadata: this.#getAllowedMeta(file), + key: this.#generateKey(file), + shouldUseMultipart: this.#shouldUseMultipart(file), getChunkSize: this.opts.getChunkSize, log: (...args) => this.uppy.log(...args), @@ -686,22 +293,18 @@ export default class AwsS3 extends BasePlugin< body: { location: result.location, key: result.key, - bucket: result.bucket, - } as unknown as B, + } satisfies AwsBody as unknown as B, uploadURL: result.location, }) - this.#cleanup(file.id) resolve() }, onError: (err) => { this.uppy.emit('upload-error', file, err) - this.#cleanup(file.id) reject(err) }, onAbort: () => { - this.#cleanup(file.id) resolve() // Normal completion, not an error }, }) @@ -711,12 +314,11 @@ export default class AwsS3 extends BasePlugin< // Start the upload uploader.start() - } catch (err) { - // Cleanup on synchronous failure during setup - this.#cleanup(file.id) - reject(err) - } - }) + }) + } finally { + // Clean up uploader instance after upload completes or fails + delete this.#uploaders[file.id] + } } #shouldUseMultipart(file: UppyFile): boolean { @@ -728,21 +330,59 @@ export default class AwsS3 extends BasePlugin< return shouldUseMultipart } // Default: multipart for files > 100MB - return (file.size || 0) > 100 * MB + return (file.size ?? 0) > 100 * MB } #generateKey(file: UppyFile): string { - if (this.opts.generateObjectKey) { - return this.opts.generateObjectKey(file) - } - // Default: {randomId}-{filename} - const randomId = crypto.randomUUID() - return `${randomId}-${file.name}` + return this.opts.generateObjectKey?.(file) ?? file.name } - #cleanup(fileId: string): void { - if (this.#uploaders[fileId]) { - delete this.#uploaders[fileId] + // -------------------------------------------------------------------------- + // Remote File Upload + // -------------------------------------------------------------------------- + + #getAllowedMeta(file: UppyFile) { + const allowedMetaFields = getAllowedMetaFields( + this.opts.allowedMetaFields, + file.meta, + ) + return Object.fromEntries( + allowedMetaFields.map((key) => [key, file.meta[key]]), + ) + } + + /** + * Builds the request body sent to Companion's provider get endpoint. + * Tells Companion to use its server-side S3 upload path. + */ + #getCompanionClientArgs(file: RemoteUppyFile): Record { + return { + ...file.remote.body, + protocol: 's3-multipart', + size: file.data.size, + metadata: this.#getAllowedMeta(file), + } + } + + async #uploadRemoteFile(file: RemoteUppyFile): Promise { + this.#setResumableUploadsCapability(false) + + const controller = new AbortController() + + const removedHandler = (removedFile: UppyFile) => { + if (removedFile.id === file.id) controller.abort() + } + this.uppy.on('file-removed', removedHandler) + + try { + await this.uppy + .getRequestClientForFile>(file) + .uploadRemoteFile(file, this.#getCompanionClientArgs(file), { + signal: controller.signal, + getQueue: () => this.#queue, + }) + } finally { + this.uppy.off('file-removed', removedHandler) } } } @@ -753,5 +393,4 @@ export type { AwsS3Options as AwsS3MultipartOptions } export interface AwsBody extends Body { location: string key: string - bucket?: string } diff --git a/packages/@uppy/aws-s3/src/s3-client/CompanionS3.ts b/packages/@uppy/aws-s3/src/s3-client/CompanionS3.ts new file mode 100644 index 000000000..749f42f24 --- /dev/null +++ b/packages/@uppy/aws-s3/src/s3-client/CompanionS3.ts @@ -0,0 +1,212 @@ +import * as C from './consts.js' +import S3Client from './S3Client.js' +import type * as IT from './types.js' +import * as U from './utils.js' + +/** + * S3Companion is an S3Client that interacts with a Companion server to perform S3 operations. + */ +class S3Companion extends S3Client { + readonly companionEndpoint: string + + constructor({ + companionEndpoint, + requestAbortTimeout, + }: { companionEndpoint: string; requestAbortTimeout?: number | undefined }) { + super({ requestAbortTimeout }) + this.companionEndpoint = companionEndpoint + } + + private async _fetch(path: string, opts?: RequestInit) { + const response = await fetch(`${this.companionEndpoint}/s3${path}`, opts) + if (!response.ok) { + throw new Error(`Companion request failed: ${response.statusText}`) + } + return response + } + + private async _request({ + url, + method, + data, + onProgress, + signal, + contentType, + }: { + url: string + method: IT.HttpMethod + data?: XMLHttpRequestBodyInit + onProgress?: IT.OnProgressFn + signal?: AbortSignal + contentType?: string + }): Promise { + // Wait for online before starting + await this.waitForOnline(signal) + + return this.xhr({ url, method, data, onProgress, signal, contentType }) + } + + public override async putObject( + keyIn: string, + data: Blob | File, + fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE, + metadata: Record, + onProgress?: IT.OnProgressFn, + signal?: AbortSignal, + ) { + const response = await this._fetch( + `/params?${new URLSearchParams({ filename: keyIn, type: fileType, ...Object.fromEntries(Object.entries(metadata).map(([k, v]) => [`metadata[${k}]`, String(v)])) })}`, + ) + const { url, fields }: { url: string; fields: Record } = + await response.json() + + const formData = new FormData() + Object.entries(fields).forEach(([key, value]) => formData.set(key, value)) + formData.set('file', data) + + const xhr = await this._request({ + url, + method: 'POST', + data: formData, + onProgress, + signal, + }) + + return { + location: `${url}${fields.key}`, // `url` is returned by the signer as the bucket URL without any path, but trailing slash, so we need to add the key (path) to get the full object URL + etag: U.sanitizeETag(xhr.getResponseHeader('etag')), + key: fields.key, + } + } + + public override async createMultipartUpload( + keyIn: string, + fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE, + metadata: Record, + ) { + if (typeof fileType !== 'string') { + throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`) + } + + const method = 'POST' + const response = await this._fetch('/multipart', { + method, + body: JSON.stringify({ filename: keyIn, metadata, type: fileType }), + headers: { 'content-type': 'application/json' }, + }) + const { + key, + uploadId, + }: { key?: string; uploadId?: string; bucket?: string } = + await response.json() + if (uploadId == null) throw new Error('No uploadId returned') + if (key == null) throw new Error('No key returned') + + return { uploadId, key } + } + + public override async uploadPart( + key: string, + uploadId: string, + data: XMLHttpRequestBodyInit, + partNumber: number, + onProgress?: IT.OnProgressFn, + signal?: AbortSignal, + ) { + const response = await this._fetch( + `/multipart/${encodeURIComponent(uploadId)}/${encodeURIComponent(partNumber)}?${new URLSearchParams({ key })}`, + { + method: 'GET', + }, + ) + + const { url }: { url: string } = await response.json() + + const xhr = await this._request({ + url, + method: 'PUT', + data, + onProgress, + signal, + }) + + const etag = U.sanitizeETag(xhr.getResponseHeader('etag')) + if (etag == null) { + throw new Error( + `${C.ERROR_PREFIX}Missing ETag in uploadPart response headers`, + ) + } + + return { etag } + } + + public override async listParts( + uploadId: string, + key: string, + ): Promise { + if (!uploadId) { + throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED) + } + + const response = await this._fetch( + `/multipart/${encodeURIComponent(uploadId)}?${new URLSearchParams({ key })}`, + { + method: 'GET', + }, + ) + + const parts: { PartNumber: string; ETag: string }[] = await response.json() + + return parts.map((p) => ({ + partNumber: parseInt(String(p.PartNumber), 10), + etag: String(p.ETag), + })) + } + + public override async completeMultipartUpload( + key: string, + uploadId: string, + parts: Array, + ) { + const response = await this._fetch( + `/multipart/${encodeURIComponent(uploadId)}/complete?${new URLSearchParams({ key })}`, + { + method: 'POST', + body: JSON.stringify({ + parts: parts.map((part) => ({ + PartNumber: part.partNumber, + ETag: part.etag, + })), + }), + headers: { 'content-type': 'application/json' }, + }, + ) + + const { + location, + bucket, + key: resultKey, + }: { + location: string + bucket?: string + key: string + } = await response.json() + + return { + location, + bucket, + key: resultKey, + } + } + + public override async abortMultipartUpload(key: string, uploadId: string) { + await this._fetch( + `/multipart/${encodeURIComponent(uploadId)}?${new URLSearchParams({ key })}`, + { + method: 'DELETE', + }, + ) + } +} + +export default S3Companion diff --git a/packages/@uppy/aws-s3/src/s3-client/S3.ts b/packages/@uppy/aws-s3/src/s3-client/S3.ts deleted file mode 100644 index 3436095b2..000000000 --- a/packages/@uppy/aws-s3/src/s3-client/S3.ts +++ /dev/null @@ -1,866 +0,0 @@ -/** - * Taken from https://github.com/good-lly/s3mini.git, by Jølly Good, under MIT license. - * Modified to make it work with Uppy. - */ - -import { fetcher, NetworkError } from '@uppy/utils' -import * as C from './consts.js' -import { createSigV4Signer } from './signer.js' -import type * as IT from './types.js' -import * as U from './utils.js' - -/** - * S3 client for browser-compatible interaction with S3-compatible storage. - * Supports simple uploads, multipart uploads, and object deletion. - * - * @example - * // Option 1: With signRequest callback - * const s3 = new S3mini({ - * endpoint: 'https://s3.amazonaws.com/my-bucket', - * signRequest: async ({ method, url, headers }) => { - * return await fetchSignedHeaders(method, url, headers); - * }, - * }); - * - * // Option 2: With getCredentials callback (client-side signing) - * const s3 = new S3mini({ - * endpoint: 'https://s3.amazonaws.com/my-bucket', - * getCredentials: async () => { - * const resp = await fetch('/api/s3/credentials'); - * return resp.json(); // { credentials, bucket, region } - * }, - * }); - * - * await s3.putObject('file.txt', 'Hello, World!'); - */ -class S3mini { - readonly endpoint: URL - readonly region: string - readonly requestSizeInBytes: number - readonly requestAbortTimeout?: number - - private readonly getCredentials?: IT.getCredentialsFn - private cachedCredentials?: IT.CredentialsResponse - private cachedCredentialsPromise?: Promise - private signRequest!: IT.signRequestFn - - constructor({ - endpoint, - signRequest, - getCredentials, - region = 'auto', - requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES, - requestAbortTimeout = undefined, - }: IT.S3Config) { - this._validateConstructorParams(endpoint, signRequest, getCredentials) - this.endpoint = new URL(this._ensureValidUrl(endpoint)) - this.region = region - this.requestSizeInBytes = requestSizeInBytes - this.requestAbortTimeout = requestAbortTimeout - - if (signRequest) { - this.signRequest = signRequest - } else if (getCredentials) { - this.getCredentials = getCredentials - this.signRequest = this._createCredentialBasedSigner() - } - } - - /** Creates a presigner that fetches/caches credentials and generates pre-signed URLs. */ - private _createCredentialBasedSigner(): IT.signRequestFn { - return async ( - request: IT.presignableRequest, - ): Promise => { - const creds = await this._getCachedCredentials() - const presigner = createSigV4Signer({ - accessKeyId: creds.credentials.accessKeyId, - secretAccessKey: creds.credentials.secretAccessKey, - sessionToken: creds.credentials.sessionToken, - region: creds.region || this.region, - endpoint: this.endpoint.toString(), - }) - return presigner(request) - } - } - - /** Gets cached credentials or fetches new ones. */ - private async _getCachedCredentials(): Promise { - // Return Cached Credentials if available - if (this.cachedCredentials != null) { - return this.cachedCredentials - } - - // Cache the promise so concurrent calls wait for the same fetch - if (this.cachedCredentialsPromise == null) { - this.cachedCredentialsPromise = this.getCredentials!({}) - .then((creds) => { - this.cachedCredentials = creds - return creds - }) - .finally(() => { - // Clear promise cache after resolution to allow future retries - this.cachedCredentialsPromise = undefined - }) - } - - return this.cachedCredentialsPromise - } - - private _validateConstructorParams( - endpoint: string, - signRequest?: IT.signRequestFn, - getCredentials?: IT.getCredentialsFn, - ): void { - if (typeof endpoint !== 'string' || endpoint.trim().length === 0) { - throw new TypeError(C.ERROR_ENDPOINT_REQUIRED) - } - - if (!signRequest && !getCredentials) { - throw new TypeError( - 'Either signRequest or getCredentials must be provided', - ) - } - - if (signRequest && typeof signRequest !== 'function') { - throw new TypeError('signRequest must be a function') - } - - if (getCredentials && typeof getCredentials !== 'function') { - throw new TypeError('getCredentials must be a function') - } - } - - private _ensureValidUrl(raw: string): string { - const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}` - try { - new URL(candidate) - - // Find the last non-slash character - let endIndex = candidate.length - while (endIndex > 0 && candidate[endIndex - 1] === '/') { - endIndex-- - } - return endIndex === candidate.length - ? candidate - : candidate.substring(0, endIndex) - } catch { - const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"` - throw new TypeError(msg) - } - } - - private _checkKey(key: string): void { - if (typeof key !== 'string' || key.trim().length === 0) { - throw new TypeError(C.ERROR_KEY_REQUIRED) - } - } - - private _checkOpts(opts: object): void { - if (typeof opts !== 'object') { - throw new TypeError(`${C.ERROR_PREFIX}opts must be an object`) - } - } - - private _filterIfHeaders(opts: Record): { - filteredOpts: Record - conditionalHeaders: Record - } { - const filteredOpts: Record = {} - const conditionalHeaders: Record = {} - - for (const [key, value] of Object.entries(opts)) { - if ( - C.IFHEADERS.has( - key.toLowerCase() as typeof C.IFHEADERS extends Set - ? T - : never, - ) - ) { - conditionalHeaders[key] = value - } else { - filteredOpts[key] = value as string - } - } - - return { filteredOpts, conditionalHeaders } - } - - private _validateData(data: unknown): BodyInit { - if ( - typeof data === 'string' || - data instanceof ArrayBuffer || - ArrayBuffer.isView(data) || - data instanceof Blob - ) { - return data as BodyInit - } - throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED) - } - - private _validateUploadPartParams( - key: string, - uploadId: string, - partNumber: number, - ): void { - this._checkKey(key) - if (typeof uploadId !== 'string' || uploadId.trim().length === 0) { - throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED) - } - if (!Number.isInteger(partNumber) || partNumber <= 0) { - throw new TypeError( - `${C.ERROR_PREFIX}partNumber must be a positive integer`, - ) - } - } - - private async _presignedRequest( - method: IT.HttpMethod, // 'GET' | 'HEAD' | 'PUT' | 'POST' | 'DELETE' - key: string, // '' allowed for bucket‑level ops - { - uploadId, - partNumber, - body = '', - contentType, - tolerated = [], - }: { - uploadId?: string - partNumber?: number - body?: BodyInit - contentType?: string - tolerated?: number[] - } = {}, - ): Promise { - // Get pre-signed URL from callback - const { url } = await this.signRequest({ - method, - key, - uploadId, - partNumber, - contentType, - }) - - // Build request headers - const requestHeaders: Record = contentType - ? { 'Content-Type': contentType } - : {} - - try { - return await this._sendRequest( - url, - method, - requestHeaders, - body, - tolerated, - ) - } catch (err) { - // If expired token error and using getCredentials, clear cache and retry once - if ( - this.getCredentials && - err instanceof U.S3ServiceError && - err.code && - ['ExpiredToken', 'InvalidAccessKeyId'].includes(err.code) - ) { - // Clear cache - this.clearCachedCredentials() - - // Retry with fresh credentials - const fresh = await this.signRequest({ - method, - key, - uploadId, - partNumber, - }) - return this._sendRequest( - fresh.url, - method, - contentType ? { 'Content-Type': contentType } : {}, - body, - tolerated, - ) - } - throw err - } - } - - /** - * Uploads an object to S3 using XHR for progress tracking. - * @param key - Object key - * @param data - Data to upload (Blob, ArrayBuffer, Uint8Array, or string) - * @param fileType - Content type - * @param onProgress - Optional progress callback - * @param signal - Optional abort signal - */ - public async putObject( - key: string, - data: IT.BinaryData | string, - fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE, - onProgress?: IT.OnProgressFn, - signal?: AbortSignal, - ): Promise { - this._checkKey(key) - - const attemptUpload = async (): Promise => { - const { url } = await this.signRequest({ method: 'PUT', key }) - return this._xhrUpload(url, data, onProgress, signal, fileType) - } - - try { - return await attemptUpload() - } catch (err) { - if (this._isExpiredTokenError(err)) { - this.clearCachedCredentials() - return attemptUpload() - } - throw err - } - } - - /** Initiates a multipart upload and returns the upload ID. */ - public async getMultipartUploadId( - key: string, - fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE, - ): Promise { - this._checkKey(key) - if (typeof fileType !== 'string') { - throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`) - } - const res = await this._presignedRequest('POST', key, { - contentType: fileType, - }) - const parsed = U.parseXml(await res.text()) as Record - - if (parsed && typeof parsed === 'object') { - // Check for both cases of InitiateMultipartUploadResult - const uploadResult = - (parsed.initiateMultipartUploadResult as Record) || - (parsed.InitiateMultipartUploadResult as Record) - - if (uploadResult && typeof uploadResult === 'object') { - // Check for both cases of uploadId - const uploadId = uploadResult.uploadId || uploadResult.UploadId - - if (uploadId && typeof uploadId === 'string') { - return uploadId - } - } - } - - throw new Error( - `${C.ERROR_PREFIX}Failed to create multipart upload: ${JSON.stringify( - parsed, - )}`, - ) - } - - public async uploadPart( - key: string, - uploadId: string, - data: IT.BinaryData | string, - partNumber: number, - onProgress?: IT.OnProgressFn, - signal?: AbortSignal, - ): Promise { - this._validateUploadPartParams(key, uploadId, partNumber) - - const attemptUpload = async (): Promise => { - const { url } = await this.signRequest({ - method: 'PUT', - key, - uploadId, - partNumber, - }) - const result = await this._xhrUpload(url, data, onProgress, signal) - return { - partNumber, - etag: result.headers.get('etag') - ? U.sanitizeETag(result.headers.get('etag')!) - : '', - } - } - - try { - return await attemptUpload() - } catch (err) { - if (this._isExpiredTokenError(err)) { - this.clearCachedCredentials() - return attemptUpload() - } - throw err - } - } - - /** - * Helper to check if we're currently offline in a browser context. - */ - private _isOffline(): boolean { - return typeof navigator !== 'undefined' && navigator.onLine === false - } - - /** Checks if error is an expired/invalid token that can be retried with fresh credentials */ - private _isExpiredTokenError(err: unknown): boolean { - return ( - this.getCredentials != null && - err instanceof U.S3ServiceError && - err.code != null && - ['ExpiredToken', 'InvalidAccessKeyId'].includes(err.code) - ) - } - - /** - * Core XHR upload implementation using @uppy/utils/fetcher. - * - * Features: - * - Automatic retry with exponential backoff (3 attempts) - * - Offline detection with automatic resume on reconnect - * - Stall detection via ProgressTimeout - */ - private async _xhrUpload( - url: string, - data: IT.BinaryData | string, - onProgress?: IT.OnProgressFn, - signal?: AbortSignal, - contentType?: string, - ): Promise { - // Wait for online before starting - if (this._isOffline()) { - await this._waitForOnline(signal) - } - - try { - const xhr = await fetcher(url, { - method: 'PUT', - // XHR natively supports ArrayBuffer, Uint8Array, Blob, and string - body: data as XMLHttpRequestBodyInit, - headers: contentType ? { 'Content-Type': contentType } : {}, - signal, - timeout: this.requestAbortTimeout || 30_000, - retries: 3, - /** - * Retry logic: - * - Retries: 5xx server errors, 429 rate limiting - * - Skips: 4xx client errors (except 429), offline (handled separately) - */ - shouldRetry: (xhr) => { - // If offline, don't retry via fetcher - our handler will resume - if (this._isOffline()) return false - // Don't retry client errors (except 429 rate limit) - if (xhr.status >= 400 && xhr.status < 500 && xhr.status !== 429) { - return false - } - return true - }, - onUploadProgress: (event) => { - if (event.lengthComputable && onProgress) { - onProgress(event.loaded, event.total) - } - }, - onTimeout: (timeout) => { - // Log stall detection - upload will continue but may be slow - console.warn( - `[S3mini] Upload stalled - no progress for ${Math.ceil(timeout / 1000)}s`, - ) - }, - }) - - // Return Response-like object for test compatibility - return { - status: xhr.status, - ok: xhr.status >= 200 && xhr.status < 300, - headers: { - get: (name: string) => xhr.getResponseHeader(name), - }, - } - } catch (err: unknown) { - return this._handleUploadError( - err, - url, - data, - onProgress, - signal, - contentType, - ) - } - } - - /** - * Handles errors from _xhrUpload, including offline recovery and error mapping. - */ - private async _handleUploadError( - err: unknown, - url: string, - data: IT.BinaryData | string, - onProgress?: IT.OnProgressFn, - signal?: AbortSignal, - contentType?: string, - ): Promise { - // Offline during request - wait and retry - if (this._isOffline()) { - await this._waitForOnline(signal) - // Check if aborted while waiting for online - if (signal?.aborted) { - throw new DOMException('Upload aborted', 'AbortError') - } - return this._xhrUpload(url, data, onProgress, signal, contentType) - } - - // Abort errors pass through - if (err instanceof DOMException && err.name === 'AbortError') { - throw err - } - - // Network errors (from fetcher's NetworkError) - if ( - err instanceof NetworkError || - (err instanceof Error && 'isNetworkError' in err && err.isNetworkError) - ) { - const xhr = 'request' in err ? (err as NetworkError).request : null - if (!xhr?.status) { - throw new U.S3NetworkError( - 'Network error during upload', - 'NETWORK', - err, - ) - } - // HTTP errors (non-2xx responses from NetworkError) - const headerCode = xhr.getResponseHeader('x-amz-error-code') - const parsedBody = this._parseErrorXml( - (name: string) => xhr.getResponseHeader(name), - xhr.responseText, - ) - const serviceCode = headerCode ?? parsedBody.svcCode - throw new U.S3ServiceError( - `S3 returned ${xhr.status}${serviceCode ? ` – ${serviceCode}` : ''}`, - xhr.status, - serviceCode, - xhr.responseText, - ) - } - - // Handle errors with attached XHR (from onAfterResponse throws) - if ( - err instanceof Error && - 'request' in err && - err.request instanceof XMLHttpRequest - ) { - const xhr = err.request - if (!xhr.status) { - throw new U.S3NetworkError( - 'Network error during upload', - 'NETWORK', - err, - ) - } - const headerCode = xhr.getResponseHeader('x-amz-error-code') - const parsedBody = this._parseErrorXml( - (name: string) => xhr.getResponseHeader(name), - xhr.responseText, - ) - const serviceCode = headerCode ?? parsedBody.svcCode - throw new U.S3ServiceError( - `S3 returned ${xhr.status}${serviceCode ? ` – ${serviceCode}` : ''}`, - xhr.status, - serviceCode, - xhr.responseText, - ) - } - - throw err - } - - /** Lists uploaded parts for a multipart upload. */ - public async listParts( - uploadId: string, - key: string, - ): Promise { - this._checkKey(key) - if (!uploadId) { - throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED) - } - const res = await this._presignedRequest('GET', key, { - uploadId, - }) - - const parsed = U.parseXml(await res.text()) as Record - const result = (parsed.listPartsResult || - parsed.ListPartsResult || - parsed) as Record - - if (result && typeof result === 'object') { - const parts = result.Part || result.part || [] - const partsArray = Array.isArray(parts) ? parts : [parts] - - return partsArray - .filter( - (p): p is Record => - p != null && - typeof p === 'object' && - 'PartNumber' in p && - 'ETag' in p, - ) - .map((p) => ({ - partNumber: parseInt(String(p.PartNumber), 10), - etag: U.sanitizeETag(String(p.ETag)), - })) - } - return [] - } - - /** Completes a multipart upload by combining all uploaded parts. */ - public async completeMultipartUpload( - key: string, - uploadId: string, - parts: Array, - ): Promise { - const xmlBody = this._buildCompleteMultipartUploadXml(parts) - - const res = await this._presignedRequest('POST', key, { - uploadId, - body: xmlBody, - contentType: C.XML_CONTENT_TYPE, - }) - - const parsed = U.parseXml(await res.text()) as Record - if (parsed && typeof parsed === 'object') { - // Check for both cases (camelCase from our parser, PascalCase from S3) - const result = - parsed.completeMultipartUploadResult || - parsed.CompleteMultipartUploadResult || - parsed - - if (result && typeof result === 'object') { - const r = result as Record - - // S3 returns PascalCase (Location, Bucket, Key, ETag). - // Normalize to lowercase for our type interface. - const resultLocation = (r.Location || r.location) as string | undefined - const resultBucket = (r.Bucket || r.bucket) as string | undefined - const resultKey = (r.Key || r.key) as string | undefined - const rawEtag = (r.ETag || r.eTag || r.etag) as string | undefined - - if (!resultLocation || !resultKey) { - throw new Error( - `${C.ERROR_PREFIX}CompleteMultipartUpload response missing Location or Key: ${JSON.stringify(r)}`, - ) - } - - const etag = rawEtag ? U.sanitizeETag(rawEtag) : '' - - return { - location: resultLocation, - bucket: resultBucket ?? '', - key: resultKey, - etag, - } satisfies IT.CompleteMultipartUploadResult - } - } - - throw new Error( - `${C.ERROR_PREFIX}Failed to complete multipart upload: ${JSON.stringify( - parsed, - )}`, - ) - } - - /** Aborts a multipart upload and removes all uploaded parts. */ - public async abortMultipartUpload( - key: string, - uploadId: string, - ): Promise { - this._checkKey(key) - if (!uploadId) { - throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED) - } - - const res = await this._presignedRequest('DELETE', key, { - uploadId, - }) - const parsed = U.parseXml(await res.text()) as Record - if ( - parsed && - 'error' in parsed && - typeof parsed.error === 'object' && - parsed.error !== null && - 'message' in parsed.error - ) { - throw new Error( - `${C.ERROR_PREFIX}Failed to abort multipart upload: ${String( - parsed.error.message, - )}`, - ) - } - return { status: 'Aborted', key, uploadId, response: parsed } - } - - private _buildCompleteMultipartUploadXml( - parts: Array, - ): string { - let xml = '' - for (const part of parts) { - xml += `${part.partNumber}${part.etag}` - } - xml += '' - return xml - } - - /** Deletes an object from the bucket. Returns true on success. */ - public async deleteObject(key: string): Promise { - const res = await this._presignedRequest('DELETE', key, { - tolerated: [200, 204], - }) - return res.status === 200 || res.status === 204 - } - - /** - * Clears cached credentials. - * Call this method when you need to force a credential refresh on the next request. - */ - public clearCachedCredentials(): void { - this.cachedCredentials = undefined - this.cachedCredentialsPromise = undefined - } - - /** - * Waits for the browser to come back online. - * Returns a promise that resolves when the 'online' event fires, - * or rejects if the abort signal is triggered. - */ - private _waitForOnline(signal?: AbortSignal): Promise { - return new Promise((resolve, reject) => { - // Already online or not in browser - if ( - typeof navigator === 'undefined' || - navigator.onLine === true || - navigator.onLine === undefined - ) { - resolve() - return - } - - // Already aborted - if (signal?.aborted) { - reject(new DOMException('Upload aborted', 'AbortError')) - return - } - - const cleanup = () => { - window.removeEventListener('online', onOnline) - signal?.removeEventListener('abort', onAbort) - } - - const onOnline = () => { - cleanup() - resolve() - } - - const onAbort = () => { - cleanup() - reject(new DOMException('Upload aborted', 'AbortError')) - } - - window.addEventListener('online', onOnline) - signal?.addEventListener('abort', onAbort) - }) - } - - private async _sendRequest( - url: string, - method: IT.HttpMethod, - headers: Record, - body?: BodyInit, - toleratedStatusCodes: number[] = [], - ): Promise { - // Wait for online if currently offline - if (this._isOffline()) { - await this._waitForOnline() - } - - try { - const res = await fetch(url, { - method, - headers, - body: ['GET', 'HEAD'].includes(method) ? undefined : body, - signal: this.requestAbortTimeout - ? AbortSignal.timeout(this.requestAbortTimeout) - : undefined, - }) - if (res.ok || toleratedStatusCodes.includes(res.status)) { - return res - } - await this._handleErrorResponse(res) - return res - } catch (err: unknown) { - // Check if we're offline - if so, wait and retry - if (this._isOffline()) { - await this._waitForOnline() - // Retry the request - return this._sendRequest( - url, - method, - headers, - body, - toleratedStatusCodes, - ) - } - - const code = U.extractErrCode(err) - if ( - code && - ['ENOTFOUND', 'EAI_AGAIN', 'ETIMEDOUT', 'ECONNREFUSED'].includes(code) - ) { - throw new U.S3NetworkError(`S3 network error: ${code}`, code, err) - } - throw err - } - } - - private _parseErrorXml( - getHeader: (name: string) => string | null, - body: string, - ): { svcCode?: string; errorMessage?: string } { - if (getHeader('content-type') !== 'application/xml') { - return {} - } - const parsedBody = U.parseXml(body) - if ( - !parsedBody || - typeof parsedBody !== 'object' || - !('Error' in parsedBody) || - !parsedBody.Error || - typeof parsedBody.Error !== 'object' - ) { - return {} - } - const error = parsedBody.Error - return { - svcCode: - 'Code' in error && typeof error.Code === 'string' - ? error.Code - : undefined, - errorMessage: - 'Message' in error && typeof error.Message === 'string' - ? error.Message - : undefined, - } - } - - private async _handleErrorResponse(res: Response): Promise { - const errorBody = await res.text() - const parsedErrorBody = this._parseErrorXml( - (n) => res.headers.get(n), - errorBody, - ) - const svcCode = - res.headers.get('x-amz-error-code') ?? - parsedErrorBody.svcCode ?? - 'Unknown' - throw new U.S3ServiceError( - `S3 returned ${res.status} – ${svcCode}`, - res.status, - svcCode, - errorBody, - ) - } -} - -export { S3mini } -export default S3mini diff --git a/packages/@uppy/aws-s3/src/s3-client/S3Client.ts b/packages/@uppy/aws-s3/src/s3-client/S3Client.ts new file mode 100644 index 000000000..80c1923a2 --- /dev/null +++ b/packages/@uppy/aws-s3/src/s3-client/S3Client.ts @@ -0,0 +1,197 @@ +import { fetcher } from '@uppy/utils' +import * as C from './consts.js' +import type * as IT from './types.js' + +class S3Client { + readonly requestAbortTimeout?: number + + constructor({ + requestAbortTimeout, + ...rest + }: { requestAbortTimeout?: number | undefined }) { + this.requestAbortTimeout = requestAbortTimeout + } + + /** + * Helper to check if we're currently offline in a browser context. + */ + protected isOffline(): boolean { + return typeof navigator !== 'undefined' && navigator.onLine === false + } + + /** + * Waits for the browser to come back online. + * Returns a promise that resolves when the 'online' event fires, + * or rejects if the abort signal is triggered. + */ + protected waitForOnline(signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (!this.isOffline()) { + resolve() + return + } + // Already online or not in browser + if ( + typeof navigator === 'undefined' || + navigator.onLine === true || + navigator.onLine === undefined + ) { + resolve() + return + } + + // Already aborted + if (signal?.aborted) { + reject(new DOMException('Upload aborted', 'AbortError')) + return + } + + const cleanup = () => { + window.removeEventListener('online', onOnline) + signal?.removeEventListener('abort', onAbort) + } + + const onOnline = () => { + cleanup() + resolve() + } + + const onAbort = () => { + cleanup() + reject(new DOMException('Upload aborted', 'AbortError')) + } + + window.addEventListener('online', onOnline) + signal?.addEventListener('abort', onAbort) + }) + } + + protected async xhr({ + url, + method, + data, + onProgress, + signal, + contentType, + }: { + url: string + method: IT.HttpMethod + data?: XMLHttpRequestBodyInit + onProgress?: IT.OnProgressFn + signal?: AbortSignal + contentType?: string + }) { + // Check if aborted while waiting for online + if (signal?.aborted) { + throw new DOMException('Request aborted', 'AbortError') + } + + return fetcher(url, { + method, + // XHR natively supports ArrayBuffer, Uint8Array, Blob, and string + body: ['GET', 'HEAD'].includes(method) ? undefined : data, + headers: contentType ? { 'Content-Type': contentType } : {}, + signal, + timeout: this.requestAbortTimeout, + retries: 3, + /** + * Retry logic: + * - Retries: 5xx server errors, 429 rate limiting + * - Skips: 4xx client errors (except 429), offline (handled separately) + */ + shouldRetry: (xhr) => { + // If offline, don't retry via fetcher - our handler will resume + if (this.isOffline()) return false + // Don't retry client errors (except 429 rate limit) + if (xhr.status >= 400 && xhr.status < 500 && xhr.status !== 429) { + return false + } + return true + }, + onUploadProgress: (event) => { + if (event.lengthComputable && onProgress) { + onProgress(event.loaded, event.total) + } + }, + onTimeout: (timeout) => { + // Log stall detection - upload will continue but may be slow + console.warn( + `[S3mini] Upload stalled - no progress for ${Math.ceil(timeout / 1000)}s`, + ) + }, + }) + } + + public async putObject( + key: string, + data: XMLHttpRequestBodyInit, + fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE, + metadata: Record, + onProgress?: IT.OnProgressFn, + signal?: AbortSignal, + ): Promise<{ + location: string + key: string + etag: string | undefined + }> { + throw new Error('Not implemented') + } + + public async createMultipartUpload( + key: string, + fileType?: string, + // @ts-expect-error unused + metadata: Record, + ): Promise<{ + uploadId: string + key: string + }> { + throw new Error('Not implemented') + } + + public async uploadPart( + key: string, + uploadId: string, + data: XMLHttpRequestBodyInit, + partNumber: number, + onProgress?: IT.OnProgressFn, + signal?: AbortSignal, + ): Promise<{ + etag: string + }> { + throw new Error('Not implemented') + } + + public async listParts( + uploadId: string, + key: string, + ): Promise { + throw new Error('Not implemented') + } + + public async completeMultipartUpload( + key: string, + uploadId: string, + parts: IT.UploadPart[], + ): Promise<{ + location: string + bucket: string | undefined + key: string + etag?: string | undefined + }> { + throw new Error('Not implemented') + } + + public async abortMultipartUpload( + key: string, + uploadId: string, + ): Promise { + throw new Error('Not implemented') + } + + public async deleteObject(key: string): Promise { + throw new Error('Not implemented') + } +} + +export default S3Client diff --git a/packages/@uppy/aws-s3/src/s3-client/S3mini.ts b/packages/@uppy/aws-s3/src/s3-client/S3mini.ts new file mode 100644 index 000000000..060be670b --- /dev/null +++ b/packages/@uppy/aws-s3/src/s3-client/S3mini.ts @@ -0,0 +1,567 @@ +/** + * Taken from https://github.com/good-lly/s3mini.git, by Jølly Good, under MIT license. + * Modified to make it work with Uppy. + */ + +import * as C from './consts.js' +import S3Client from './S3Client.js' +import { createSigV4Signer } from './signer.js' +import type * as IT from './types.js' +import * as U from './utils.js' + +/** + * S3 client for browser-compatible interaction with S3-compatible storage. + * Supports simple uploads, multipart uploads, and object deletion. + * + * @example + * // Option 1: With signRequest callback (no endpoint needed) + * const s3 = new S3mini({ + * signRequest: async ({ method, key, uploadId, partNumber }) => { + * const resp = await fetch('/api/s3/sign', { + * method: 'POST', + * body: JSON.stringify({ method, key, uploadId, partNumber }), + * }); + * return resp.json(); // { url } + * }, + * }); + * + * // Option 2: With getCredentials callback (endpoint required for client-side signing) + * const s3 = new S3mini({ + * endpoint: 'https://s3.us-east-1.amazonaws.com/my-bucket', + * getCredentials: async () => { + * const resp = await fetch('/api/s3/credentials'); + * return resp.json(); // { credentials, region } + * }, + * }); + * + * await s3.putObject('file.txt', 'Hello, World!'); + */ +class S3mini extends S3Client { + readonly endpoint?: URL + readonly region: string + readonly requestSizeInBytes: number + + private readonly getCredentials?: IT.GetCredentialsFn + private cachedCredentials?: IT.CredentialsResponse + private cachedCredentialsPromise?: Promise + private signRequest!: IT.SignRequestFn + + constructor({ + region = 'auto', + requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES, + requestAbortTimeout, + ...rest + }: IT.S3Config) { + super({ requestAbortTimeout }) + if ('signRequest' in rest) { + const { signRequest } = rest + if (!signRequest) { + throw new TypeError( + 'Either signRequest or getCredentials must be provided', + ) + } + + if (signRequest && typeof signRequest !== 'function') { + throw new TypeError('signRequest must be a function') + } + + this.signRequest = signRequest + } else if ('getCredentials' in rest) { + const { getCredentials, endpoint } = rest + if (typeof endpoint !== 'string' || endpoint.trim().length === 0) { + throw new TypeError(C.ERROR_ENDPOINT_REQUIRED) + } + if (getCredentials && typeof getCredentials !== 'function') { + throw new TypeError('getCredentials must be a function') + } + this.endpoint = new URL(this._ensureValidUrl(endpoint)) + + this.getCredentials = getCredentials + this.signRequest = this._createCredentialBasedSigner() + } else { + throw new TypeError( + 'Either signRequest or getCredentials must be provided', + ) + } + + this.region = region + this.requestSizeInBytes = requestSizeInBytes + } + + /** Creates a presigner that fetches/caches credentials and generates pre-signed URLs. */ + private _createCredentialBasedSigner(): IT.SignRequestFn { + return async ( + request: IT.PresignableRequest, + ): Promise => { + const creds = await this._getCachedCredentials() + if (this.endpoint == null) { + throw new Error('Endpoint is required for credential-based signing') + } + const presigner = createSigV4Signer({ + accessKeyId: creds.credentials.accessKeyId, + secretAccessKey: creds.credentials.secretAccessKey, + sessionToken: creds.credentials.sessionToken, + region: creds.region || this.region, + endpoint: this.endpoint.toString(), + }) + return presigner(request) + } + } + + /** Gets cached credentials or fetches new ones. */ + private async _getCachedCredentials(): Promise { + // Return Cached Credentials if available + if (this.cachedCredentials != null) { + return this.cachedCredentials + } + + // Cache the promise so concurrent calls wait for the same fetch + if (this.cachedCredentialsPromise == null) { + this.cachedCredentialsPromise = (async () => { + try { + const creds = await this.getCredentials!({}) + this.cachedCredentials = creds + return creds + } finally { + // Clear promise cache after resolution to allow future retries + this.cachedCredentialsPromise = undefined + } + })() + } + + return this.cachedCredentialsPromise + } + + private _ensureValidUrl(raw: string): string { + const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}` + try { + new URL(candidate) + + // Find the last non-slash character + let endIndex = candidate.length + while (endIndex > 0 && candidate[endIndex - 1] === '/') { + endIndex-- + } + return endIndex === candidate.length + ? candidate + : candidate.substring(0, endIndex) + } catch { + const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"` + throw new TypeError(msg) + } + } + + private _checkKey(key: string): void { + if (typeof key !== 'string' || key.trim().length === 0) { + throw new TypeError(C.ERROR_KEY_REQUIRED) + } + } + + private _validateUploadPartParams( + key: string, + uploadId: string, + partNumber: number, + ): void { + this._checkKey(key) + if (typeof uploadId !== 'string' || uploadId.trim().length === 0) { + throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED) + } + if (!Number.isInteger(partNumber) || partNumber <= 0) { + throw new TypeError( + `${C.ERROR_PREFIX}partNumber must be a positive integer`, + ) + } + } + + /** + * Uploads an object to S3 using XHR for progress tracking. + * @param key - Object key + * @param data - Data to upload (Blob, ArrayBuffer, Uint8Array, or string) + * @param fileType - Content type + * @param onProgress - Optional progress callback + * @param signal - Optional abort signal + */ + public override async putObject( + key: string, + data: XMLHttpRequestBodyInit, + fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE, + metadata?: Record, + onProgress?: IT.OnProgressFn, + signal?: AbortSignal, + ) { + this._checkKey(key) + + const { xhr, url } = await this.request({ + request: { method: 'PUT', key }, + data, + onProgress, + signal, + contentType: fileType, + }) + + return { + location: U.removeQueryString(url), + etag: U.sanitizeETag(xhr.getResponseHeader('etag')), + key, + } + } + + /** Initiates a multipart upload and returns the upload ID. */ + public override async createMultipartUpload( + key: string, + fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE, + metadata?: Record, // todo support metadata here too? + ) { + this._checkKey(key) + if (typeof fileType !== 'string') { + throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`) + } + + const { xhr } = await this.request({ + request: { method: 'POST', key }, + contentType: fileType, + }) + + const parsed = U.parseXml(xhr.responseText) as Record + + if (parsed && typeof parsed === 'object') { + // Check for both cases of InitiateMultipartUploadResult + const uploadResult = + (parsed.initiateMultipartUploadResult as Record) || + (parsed.InitiateMultipartUploadResult as Record) + + if (uploadResult && typeof uploadResult === 'object') { + // Check for both cases of uploadId + const uploadId = uploadResult.uploadId || uploadResult.UploadId + + if (uploadId && typeof uploadId === 'string') { + return { uploadId, key } + } + } + } + + throw new Error( + `${C.ERROR_PREFIX}Failed to create multipart upload: ${JSON.stringify( + parsed, + )}`, + ) + } + + public override async uploadPart( + key: string, + uploadId: string, + data: XMLHttpRequestBodyInit, + partNumber: number, + onProgress?: IT.OnProgressFn, + signal?: AbortSignal, + ) { + this._validateUploadPartParams(key, uploadId, partNumber) + + const { xhr } = await this.request({ + request: { + method: 'PUT', + key, + uploadId, + partNumber, + }, + data, + onProgress, + signal, + }) + + const etag = U.sanitizeETag(xhr.getResponseHeader('etag')) + if (etag == null) { + throw new Error( + `${C.ERROR_PREFIX}Missing ETag in uploadPart response headers`, + ) + } + + return { etag } + } + + /** + * Core XHR upload implementation using @uppy/utils/fetcher. + * + * Features: + * - Automatic retry with exponential backoff (3 attempts) + * - Offline detection with automatic resume on reconnect + * - Stall detection via ProgressTimeout + */ + private async request({ + request, + data, + onProgress, + signal, + contentType, + shouldRetryCredentials = true, + }: { + request: IT.PresignableRequest + data?: XMLHttpRequestBodyInit + onProgress?: IT.OnProgressFn + signal?: AbortSignal + contentType?: string + shouldRetryCredentials?: boolean + }): Promise<{ xhr: XMLHttpRequest; url: string }> { + // Wait for online before starting + await this.waitForOnline(signal) + + // Check if aborted while waiting for online + if (signal?.aborted) { + throw new DOMException('Request aborted', 'AbortError') + } + + try { + const { url } = await this.signRequest(request) + + const xhr = await this.xhr({ + url, + method: request.method, + data, + onProgress, + signal, + contentType, + }) + + return { xhr, url } + } catch (err: unknown) { + // NetworkError or errors with attached XHR (from onAfterResponse throws) + if ( + err instanceof Error && + 'request' in err && + err.request instanceof XMLHttpRequest + ) { + const xhr = err.request as XMLHttpRequest + if (xhr.status === 0) { + throw new U.S3NetworkError( + 'Network error during S3 request', + 'NETWORK', + err, + ) + } + // HTTP errors (non-2xx responses from NetworkError) + const parsedBody = this._parseErrorXml( + (name: string) => xhr.getResponseHeader(name), + xhr.responseText, + ) + const serviceCode = + xhr.getResponseHeader('x-amz-error-code') ?? parsedBody.svcCode + + // If expired token error and using getCredentials, clear cache and retry once + if ( + shouldRetryCredentials && + this.getCredentials != null && + serviceCode != null && + ['ExpiredToken', 'InvalidAccessKeyId'].includes(serviceCode) + ) { + this.clearCachedCredentials() + + // Retry with fresh credentials + return this.request({ + request, + data, + onProgress, + signal, + contentType, + shouldRetryCredentials: false, // prevent infinite recursion + }) + } + + throw new U.S3ServiceError( + `S3 returned ${xhr.status}${serviceCode ? ` – ${serviceCode}` : ''}`, + xhr.status, + serviceCode, + xhr.responseText, + ) + } + + throw err + } + } + + /** Lists uploaded parts for a multipart upload. */ + public override async listParts( + uploadId: string, + key: string, + ): Promise { + this._checkKey(key) + if (!uploadId) { + throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED) + } + const { xhr } = await this.request({ + request: { method: 'GET', key, uploadId }, + }) + + const parsed = U.parseXml(xhr.responseText) as Record + const result = (parsed.listPartsResult || + parsed.ListPartsResult || + parsed) as Record + + if (result && typeof result === 'object') { + const parts = result.Part || result.part || [] + const partsArray = Array.isArray(parts) ? parts : [parts] + + return partsArray + .filter( + (p): p is Record => + p != null && + typeof p === 'object' && + 'PartNumber' in p && + 'ETag' in p, + ) + .map((p) => ({ + partNumber: parseInt(String(p.PartNumber), 10), + etag: U.sanitizeXmlETag(String(p.ETag)), + })) + } + return [] + } + + /** Completes a multipart upload by combining all uploaded parts. */ + public override async completeMultipartUpload( + key: string, + uploadId: string, + parts: Array, + ) { + const xmlBody = this._buildCompleteMultipartUploadXml(parts) + + const { xhr } = await this.request({ + request: { method: 'POST', key, uploadId }, + contentType: C.XML_CONTENT_TYPE, + data: xmlBody, + }) + + const parsed = U.parseXml(xhr.responseText) + if (parsed && typeof parsed === 'object') { + // Check for both cases (camelCase from our parser, PascalCase from S3) + const result = + parsed.completeMultipartUploadResult || + parsed.CompleteMultipartUploadResult || + parsed + + if (result && typeof result === 'object') { + const r = result as Record + + // S3 returns PascalCase (Location, Bucket, Key, ETag). + // Normalize to lowercase for our type interface. + const resultLocation = (r.Location || r.location) as string | undefined + const resultBucket = (r.Bucket || r.bucket) as string | undefined + const resultKey = (r.Key || r.key) as string | undefined + const rawEtag = (r.ETag || r.eTag || r.etag) as string | undefined + + if (!resultLocation || !resultKey) { + throw new Error( + `${C.ERROR_PREFIX}CompleteMultipartUpload response missing Location or Key: ${JSON.stringify(r)}`, + ) + } + + const etag = rawEtag ? U.sanitizeXmlETag(rawEtag) : undefined + + return { + location: resultLocation, + bucket: resultBucket, + key: resultKey, + etag, + } + } + } + + throw new Error( + `${C.ERROR_PREFIX}Failed to complete multipart upload: ${JSON.stringify( + parsed, + )}`, + ) + } + + /** Aborts a multipart upload and removes all uploaded parts. */ + public override async abortMultipartUpload(key: string, uploadId: string) { + this._checkKey(key) + if (!uploadId) { + throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED) + } + + const { xhr } = await this.request({ + request: { method: 'DELETE', key, uploadId }, + }) + + const parsed = U.parseXml(xhr.responseText) as Record + if ( + parsed && + 'error' in parsed && + typeof parsed.error === 'object' && + parsed.error !== null && + 'message' in parsed.error + ) { + throw new Error( + `${C.ERROR_PREFIX}Failed to abort multipart upload: ${String( + parsed.error.message, + )}`, + ) + } + } + + private _buildCompleteMultipartUploadXml( + parts: Array, + ): string { + let xml = '' + for (const part of parts) { + xml += `${part.partNumber}${part.etag}` + } + xml += '' + return xml + } + + /** Deletes an object from the bucket. Returns true on success. */ + public override async deleteObject(key: string) { + const { xhr } = await this.request({ + request: { method: 'DELETE', key }, + }) + + if (xhr.status !== 200 && xhr.status !== 204) { + throw new Error( + `${C.ERROR_PREFIX}Failed to delete object. HTTP status: ${xhr.status}`, + ) + } + } + + /** + * Clears cached credentials. + * Call this method when you need to force a credential refresh on the next request. + */ + public clearCachedCredentials(): void { + this.cachedCredentials = undefined + this.cachedCredentialsPromise = undefined + } + + private _parseErrorXml( + getHeader: (name: string) => string | null, + body: string, + ): { svcCode?: string; errorMessage?: string } { + if (getHeader('content-type') !== 'application/xml') { + return {} + } + const parsedBody = U.parseXml(body) + if ( + !parsedBody || + typeof parsedBody !== 'object' || + !('Error' in parsedBody) || + !parsedBody.Error || + typeof parsedBody.Error !== 'object' + ) { + return {} + } + const error = parsedBody.Error + return { + svcCode: + 'Code' in error && typeof error.Code === 'string' + ? error.Code + : undefined, + errorMessage: + 'Message' in error && typeof error.Message === 'string' + ? error.Message + : undefined, + } + } +} + +export { S3mini } +export default S3mini diff --git a/packages/@uppy/aws-s3/src/s3-client/index.ts b/packages/@uppy/aws-s3/src/s3-client/index.ts index 719088756..cf6e14488 100644 --- a/packages/@uppy/aws-s3/src/s3-client/index.ts +++ b/packages/@uppy/aws-s3/src/s3-client/index.ts @@ -1,10 +1,8 @@ // Export the S3 class as default export and named export -export { S3mini } from './S3.js' +export { S3mini } from './S3mini.js' // Re-export types export type { - CompleteMultipartUploadResult, ErrorWithCode, S3Config, UploadPart, } from './types.js' -export { sanitizeETag } from './utils.js' diff --git a/packages/@uppy/aws-s3/src/s3-client/signer.ts b/packages/@uppy/aws-s3/src/s3-client/signer.ts index c976b7518..a5c3f58e9 100644 --- a/packages/@uppy/aws-s3/src/s3-client/signer.ts +++ b/packages/@uppy/aws-s3/src/s3-client/signer.ts @@ -5,7 +5,7 @@ */ import * as C from './consts.js' -import type { presignableRequest, presignedResponse } from './types.js' +import type { PresignableRequest, presignedResponse } from './types.js' import * as U from './utils.js' export interface SignerConfig { @@ -45,15 +45,9 @@ export function createSigV4Presigner(config: SignerConfig) { } return async function presign( - request: presignableRequest, + request: PresignableRequest, ): Promise { - const { - method, - key, - uploadId, - partNumber, - expiresIn = DEFAULT_EXPIRES_IN, - } = request + const { method, key, expiresIn = DEFAULT_EXPIRES_IN } = request // Build the URL - need to track encoded path separately because URL object decodes it const url = new URL(endpoint) @@ -92,15 +86,15 @@ export function createSigV4Presigner(config: SignerConfig) { } // Add multipart-specific params - if (uploadId) { - url.searchParams.set('uploadId', uploadId) + if ('uploadId' in request) { + url.searchParams.set('uploadId', request.uploadId) } - if (partNumber !== undefined) { - url.searchParams.set('partNumber', String(partNumber)) + if ('partNumber' in request) { + url.searchParams.set('partNumber', String(request.partNumber)) } // For CreateMultipartUpload, add uploads param - if (method === 'POST' && !uploadId) { + if (method === 'POST' && !('uploadId' in request)) { url.searchParams.set('uploads', '') } diff --git a/packages/@uppy/aws-s3/src/s3-client/types.ts b/packages/@uppy/aws-s3/src/s3-client/types.ts index da1082919..889dc465a 100644 --- a/packages/@uppy/aws-s3/src/s3-client/types.ts +++ b/packages/@uppy/aws-s3/src/s3-client/types.ts @@ -1,21 +1,53 @@ -/** Request data to be pre-signed */ -export type presignableRequest = { - method: string +export interface PresignableRequestBase { key: string - uploadId?: string - partNumber?: number expiresIn?: number - contentType?: string } +export interface PutObjectRequest extends PresignableRequestBase { + method: 'PUT' +} +export interface DeleteObjectRequest extends PresignableRequestBase { + method: 'DELETE' +} +export interface CreateMultipartUploadRequest extends PresignableRequestBase { + method: 'POST' +} +export interface CompleteMultipartUploadRequest extends PresignableRequestBase { + method: 'POST' + uploadId: string +} +export interface DeleteMultipartUploadRequest extends PresignableRequestBase { + method: 'DELETE' + uploadId: string +} +export interface ListPartsRequest extends PresignableRequestBase { + method: 'GET' + uploadId: string +} +export interface UploadPartRequest extends PresignableRequestBase { + method: 'PUT' + uploadId: string + partNumber: number +} + +/** Request data to be pre-signed */ +export type PresignableRequest = + | PutObjectRequest + | DeleteObjectRequest + | CreateMultipartUploadRequest + | CompleteMultipartUploadRequest + | DeleteMultipartUploadRequest + | ListPartsRequest + | UploadPartRequest + /** Response with the pre-signed URL */ export type presignedResponse = { url: string } /** Function that generates a pre-signed URL for a request */ -export type signRequestFn = ( - request: presignableRequest, +export type SignRequestFn = ( + request: PresignableRequest, ) => Promise /** @@ -40,36 +72,34 @@ export interface CredentialsResponse { } /** Function that retrieves temporary credentials */ -export type getCredentialsFn = (options?: { +export type GetCredentialsFn = (options?: { signal?: AbortSignal -}) => Promise +}) => CredentialsResponse | Promise /** Base configuration shared by both signing approaches */ type S3ConfigBase = { - /** Endpoint URL of the S3-compatible service (e.g., 'https://s3.amazonaws.com/bucket-name') */ - endpoint: string /** AWS region. Defaults to 'auto'. */ - region?: string + region?: string | undefined /** Request size in bytes for multipart uploads. Defaults to 8MB. */ - requestSizeInBytes?: number + requestSizeInBytes?: number | undefined /** Timeout in ms after which a request should be aborted. */ - requestAbortTimeout?: number + requestAbortTimeout?: number | undefined } /** Config when using signRequest callback (region optional) */ type S3ConfigWithSignRequest = S3ConfigBase & { /** Function to sign requests. Called for each S3 API request. */ - signRequest: signRequestFn - getCredentials?: never + signRequest: SignRequestFn } /** Config when using getCredentials callback (region required for signing) */ type S3ConfigWithGetCredentials = Omit & { - signRequest?: never /** Function to retrieve temporary credentials for client-side signing. */ - getCredentials: getCredentialsFn + getCredentials: GetCredentialsFn /** AWS region. Required for signing with getCredentials. */ - region: string + region?: string + /** Endpoint URL of the S3-compatible service (e.g., 'https://s3.amazonaws.com/bucket-name') */ + endpoint: string } /** Configuration options for S3mini client */ @@ -80,13 +110,6 @@ export interface UploadPart { etag: string } -export interface CompleteMultipartUploadResult { - location: string - bucket: string - key: string - etag: string -} - export interface ErrorWithCode { code?: string cause?: { code?: string } @@ -108,11 +131,3 @@ export type BinaryData = ArrayBuffer | Uint8Array | Blob /** Progress callback for upload operations */ export type OnProgressFn = (bytesUploaded: number, bytesTotal: number) => void - -export interface PutObjectResult { - status: number - ok: boolean - headers: { - get(name: string): string | null - } -} diff --git a/packages/@uppy/aws-s3/src/s3-client/utils.ts b/packages/@uppy/aws-s3/src/s3-client/utils.ts index 3c9dd4a0b..26beff568 100644 --- a/packages/@uppy/aws-s3/src/s3-client/utils.ts +++ b/packages/@uppy/aws-s3/src/s3-client/utils.ts @@ -1,25 +1,22 @@ -import type { ErrorWithCode, XmlMap, XmlValue } from './types.js' +import type { XmlMap, XmlValue } from './types.js' -const ENCODR = new TextEncoder() -const chunkSize = 0x8000 // 32KB chunks -const HEXS = '0123456789abcdef' +export const sanitizeXmlETag = (etag: string): string => + etag.replace(/^("|")+|("|")+$/g, '') -export const getByteSize = (data: unknown): number => { - if (typeof data === 'string') { - return ENCODR.encode(data).byteLength - } - if (data instanceof ArrayBuffer) { - return data.byteLength - } - if (ArrayBuffer.isView(data)) { - return data.byteLength - } - if (data instanceof Blob) { - return data.size - } - throw new Error('Unsupported data type') +export const sanitizeETag = (etag: string | null): string | undefined => + etag?.replace(/^"+|"+$/g, '') + +/** Strips query string and hash from a URL to derive the object location. */ +export function removeQueryString(urlString: string): string { + const urlObject = new URL(urlString) + urlObject.search = '' + urlObject.hash = '' + return urlObject.href } +const textEncoder = new TextEncoder() +const HEXS = '0123456789abcdef' + /** * Turn a raw ArrayBuffer into its hexadecimal representation. * @param {ArrayBuffer} buffer The raw bytes. @@ -34,28 +31,13 @@ export const hexFromBuffer = (buffer: ArrayBuffer): string => { return hex } -/** - * Turn a raw ArrayBuffer into its base64 representation. - * @param {ArrayBuffer} buffer The raw bytes. - * @returns {string} Base64 string - */ -export const base64FromBuffer = (buffer: ArrayBuffer): string => { - const bytes = new Uint8Array(buffer) - let result = '' - for (let i = 0; i < bytes.length; i += chunkSize) { - const chunk = bytes.subarray(i, i + chunkSize) - result += btoa(String.fromCodePoint(...chunk)) - } - return result -} - /** * Compute SHA-256 hash of arbitrary string data. * @param {string} content The content to be hashed. * @returns {ArrayBuffer} The raw hash */ export const sha256 = async (content: string): Promise => { - const data = ENCODR.encode(content) + const data = textEncoder.encode(content) return await globalThis.crypto.subtle.digest('SHA-256', data) } @@ -72,33 +54,16 @@ export const hmac = async ( ): Promise => { const secret = await globalThis.crypto.subtle.importKey( 'raw', - typeof key === 'string' ? ENCODR.encode(key) : key, + typeof key === 'string' ? textEncoder.encode(key) : key, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign'], ) - const data = ENCODR.encode(content) + const data = textEncoder.encode(content) return await globalThis.crypto.subtle.sign('HMAC', secret, data) } -/** - * Sanitize ETag value by removing quotes and XML entities - * @param etag ETag value to sanitize - * @returns Sanitized ETag - */ -export const sanitizeETag = (etag: string): string => { - const replaceChars: Record = { - '"': '', - '"': '', - '"': '', - } - return etag.replaceAll( - /(^("|"|"))|(("|"|")$)/g, - (m) => replaceChars[m] || '', - ) -} - const entityMap = { '"': '"', ''': "'", @@ -107,20 +72,6 @@ const entityMap = { '&': '&', } as const -/** - * Escape special characters for XML - * @param value String to escape - * @returns XML-escaped string - */ -export const escapeXml = (value: string): string => { - return value - .replaceAll('&', '&') - .replaceAll('<', '<') - .replaceAll('>', '>') - .replaceAll('"', '"') - .replaceAll("'", ''') -} - const unescapeXml = (value: string): string => value.replaceAll( /&(quot|apos|lt|gt|amp);/g, @@ -182,7 +133,7 @@ const encodeAsHex = (c: string): string => * @param uriStr URI string to escape * @returns Escaped URI string */ -export const uriEscape = (uriStr: string): string => { +const uriEscape = (uriStr: string): string => { return encodeURIComponent(uriStr).replace(/[!'()*]/g, encodeAsHex) } @@ -195,17 +146,6 @@ export const uriResourceEscape = (string: string): string => { return uriEscape(string).replaceAll('%2F', '/') } -export const extractErrCode = (e: unknown): string | undefined => { - if (typeof e !== 'object' || e === null) { - return undefined - } - const err = e as ErrorWithCode - if (typeof err.code === 'string') { - return err.code - } - return typeof err.cause?.code === 'string' ? err.cause.code : undefined -} - export class S3Error extends Error { readonly code?: string constructor(msg: string, code?: string, cause?: unknown) { diff --git a/packages/@uppy/aws-s3/src/utils.ts b/packages/@uppy/aws-s3/src/utils.ts index d5a2700b0..27804f894 100644 --- a/packages/@uppy/aws-s3/src/utils.ts +++ b/packages/@uppy/aws-s3/src/utils.ts @@ -1,4 +1,3 @@ -import type { Body } from '@uppy/utils' import { createAbortError } from '@uppy/utils' import type { AwsS3Part } from './index.js' @@ -22,9 +21,3 @@ export type UploadPartBytesResult = { ETag: string location?: string } - -export interface AwsBody extends Body { - location: string - key: string - bucket: string -} diff --git a/packages/@uppy/aws-s3/tests/index.test.ts b/packages/@uppy/aws-s3/tests/index.test.ts index d3c9490af..c52ebf176 100644 --- a/packages/@uppy/aws-s3/tests/index.test.ts +++ b/packages/@uppy/aws-s3/tests/index.test.ts @@ -1,4 +1,14 @@ -import { describe, expect, it, vi } from 'vitest' +import { HttpResponse, http } from 'msw' +import { setupServer } from 'msw/node' +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from 'vitest' import 'whatwg-fetch' import Core, { type Meta, type UppyFile } from '@uppy/core' @@ -7,15 +17,145 @@ import AwsS3, { type AwsBody, type AwsS3Options } from '../src/index.js' const KB = 1024 const MB = KB * KB +// --------------------------------------------------------------------------- +// Helpers for multipart upload tests +// --------------------------------------------------------------------------- + +/** Minimal XML responses that S3 returns for multipart operations */ +const s3Responses = { + createMultipart: (uploadId: string, key: string) => + ` + + ${uploadId} + ${key} + `, + + uploadPart: (etag: string) => '', + + listParts: (parts: { partNumber: number; etag: string }[]) => + ` + + ${parts.map((p) => `${p.partNumber}${p.etag}`).join('')} + `, + + completeMultipart: (location: string, key: string) => + ` + + ${location} + ${key} + test-bucket + `, + + abortMultipart: () => '', +} + +const server = setupServer() +const s3Url = 'https://test-bucket.s3.us-east-1.amazonaws.com/:key' + +/** + * Creates signRequest + MSW handler state for multipart upload tests. + */ +function createMultipartMocks(opts: { uploadId?: string; key?: string } = {}) { + const uploadId = opts.uploadId ?? 'test-upload-id' + const key = opts.key ?? 'test-key' + + // signRequest encodes operation details in the URL for MSW routing + const signRequest = vi.fn().mockImplementation(async (req: any) => { + const params = new URLSearchParams() + if (req.uploadId) params.set('uploadId', req.uploadId) + if (req.partNumber) params.set('partNumber', String(req.partNumber)) + params.set('method', req.method) + return { + url: `https://test-bucket.s3.us-east-1.amazonaws.com/${req.key || key}?${params}`, + } + }) + + const operations: string[] = [] + + const registerHandlers = ({ + hangNonCreate = false, + listParts = [] as { partNumber: number; etag: string }[], + } = {}) => { + const maybeHang = () => + hangNonCreate ? (new Promise(() => {}) as Promise) : null + + server.use( + http.post(s3Url, ({ request }) => { + const hasUploadId = new URL(request.url).searchParams.has('uploadId') + if (!hasUploadId) { + operations.push('createMultipart') + return new HttpResponse(s3Responses.createMultipart(uploadId, key), { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }) + } + + operations.push('completeMultipart') + const hung = maybeHang() + if (hung) return hung + + return new HttpResponse( + s3Responses.completeMultipart( + `https://test-bucket.s3.amazonaws.com/${key}`, + key, + ), + { status: 200, headers: { 'Content-Type': 'application/xml' } }, + ) + }), + http.put(s3Url, () => { + operations.push('uploadPart') + const hung = maybeHang() + if (hung) return hung + return new HttpResponse('', { + status: 200, + headers: { ETag: '"etag-1"' }, + }) + }), + http.get(s3Url, ({ request }) => { + const hasUploadId = new URL(request.url).searchParams.has('uploadId') + if (!hasUploadId) { + return new HttpResponse('Not Found', { status: 404 }) + } + + operations.push('listParts') + const hung = maybeHang() + if (hung) return hung + + return new HttpResponse(s3Responses.listParts(listParts), { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }) + }), + http.delete(s3Url, () => { + operations.push('abortMultipart') + return new HttpResponse('', { status: 204 }) + }), + ) + } + + return { signRequest, operations, uploadId, key, registerHandlers } +} + describe('AwsS3', () => { + beforeAll(() => { + server.listen({ onUnhandledRequest: 'error' }) + }) + + afterEach(() => { + server.resetHandlers() + }) + + afterAll(() => { + server.close() + }) + it('Registers AwsS3 upload plugin', () => { const core = new Core().use(AwsS3, { - bucket: 'test-bucket', region: 'us-east-1', - endpoint: 'https://companion.example.com', + s3Endpoint: 'https://companion.example.com', + companionEndpoint: 'https://companion.example.com', }) - // @ts-expect-error private property const pluginNames = core[Symbol.for('uppy test: getPlugins')]( 'uploader', ).map((plugin: AwsS3) => plugin.constructor.name) @@ -23,34 +163,25 @@ describe('AwsS3', () => { }) describe('configuration validation', () => { - it('throws if bucket is not provided', () => { - expect(() => { - const core = new Core() - // @ts-expect-error - testing missing required option - core.use(AwsS3, {}) - }).toThrow('`bucket` option is required') - }) - - it('throws if region is not provided', () => { - expect(() => { - const core = new Core() - core.use(AwsS3, { bucket: 'test-bucket' }) - }).toThrow('`region` option is required') - }) - it('throws if no signing method is provided', () => { expect(() => { const core = new Core() - core.use(AwsS3, { bucket: 'test-bucket', region: 'us-east-1' }) - }).toThrow('`endpoint`, `signRequest`, or `getCredentials` is required') + // @ts-expect-error - testing runtime validation, so omit required options + core.use(AwsS3, { + s3Endpoint: 'https://companion.example.com', + region: 'us-east-1', + }) + }).toThrow( + 'One of options `companionEndpoint`, `signRequest`, or `getCredentials` is required', + ) }) it('accepts endpoint option', () => { const core = new Core() core.use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', - endpoint: 'https://companion.example.com', + companionEndpoint: 'https://companion.example.com', }) expect(core.getPlugin('AwsS3')).toBeDefined() }) @@ -58,7 +189,7 @@ describe('AwsS3', () => { it('accepts signRequest option', () => { const core = new Core() core.use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', signRequest: vi.fn(), }) @@ -68,7 +199,7 @@ describe('AwsS3', () => { it('accepts getCredentials option', () => { const core = new Core() core.use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', getCredentials: vi.fn(), }) @@ -89,9 +220,9 @@ describe('AwsS3', () => { it('defaults to multipart for files > 100MB', () => { const core = new Core().use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', - endpoint: 'https://companion.example.com', + companionEndpoint: 'https://companion.example.com', }) const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options const shouldUseMultipart = opts.shouldUseMultipart as ( @@ -108,9 +239,9 @@ describe('AwsS3', () => { it('handles very large files', () => { const core = new Core().use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', - endpoint: 'https://companion.example.com', + companionEndpoint: 'https://companion.example.com', }) const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options const shouldUseMultipart = opts.shouldUseMultipart as ( @@ -127,7 +258,7 @@ describe('AwsS3', () => { const signRequest = vi.fn().mockRejectedValue(new Error('Test stop')) const core = new Core().use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', signRequest, shouldUseMultipart: false, @@ -156,7 +287,7 @@ describe('AwsS3', () => { const signRequest = vi.fn().mockRejectedValue(new Error('Sign failed')) const core = new Core().use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', signRequest, shouldUseMultipart: false, @@ -191,7 +322,7 @@ describe('AwsS3', () => { ) const core = new Core().use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', signRequest, shouldUseMultipart: false, @@ -222,7 +353,7 @@ describe('AwsS3', () => { ) const core = new Core().use(AwsS3, { - bucket: 'test-bucket', + s3Endpoint: 'https://companion.example.com', region: 'us-east-1', signRequest, shouldUseMultipart: false, @@ -244,4 +375,120 @@ describe('AwsS3', () => { expect(result?.successful).toHaveLength(0) }) }) + + describe('Golden Retriever resume state (s3Multipart)', () => { + it('persists s3Multipart on file state after creating multipart upload', async () => { + const { signRequest, uploadId, registerHandlers } = createMultipartMocks() + // After createMultipart succeeds, hang on subsequent requests so we can inspect state + registerHandlers({ hangNonCreate: true }) + + const core = new Core().use(AwsS3, { + s3Endpoint: 'https://companion.example.com', + region: 'us-east-1', + signRequest, + shouldUseMultipart: true, + }) + + const fileId = core.addFile({ + source: 'test', + name: 'big.dat', + type: 'application/octet-stream', + data: new File([new Uint8Array(6 * MB)], 'big.dat'), + }) + + const uploadPromise = core.upload() + + // Wait for createMultipart to complete and state to be persisted + await new Promise((resolve) => setTimeout(resolve, 100)) + + const file = core.getFile(fileId) + expect(file.s3Multipart).toBeDefined() + expect(file.s3Multipart?.uploadId).toBe(uploadId) + + // Clean up + core.cancelAll() + await uploadPromise + }) + + it('clears s3Multipart when upload is aborted via cancelAll', async () => { + const { signRequest, registerHandlers } = createMultipartMocks({ + uploadId: 'cancel-test-id', + key: 'cancel-key', + }) + registerHandlers({ hangNonCreate: true }) + + const core = new Core().use(AwsS3, { + s3Endpoint: 'https://companion.example.com', + region: 'us-east-1', + signRequest, + shouldUseMultipart: true, + }) + + const fileId = core.addFile({ + source: 'test', + name: 'big.dat', + type: 'application/octet-stream', + data: new File([new Uint8Array(6 * MB)], 'big.dat'), + }) + + const uploadPromise = core.upload() + + // Wait for createMultipart, then cancel + await new Promise((resolve) => setTimeout(resolve, 50)) + core.cancelAll() + + await uploadPromise + + const file = core.getFile(fileId) + // s3Multipart should be cleared so retries don't use a dead uploadId + expect(file?.s3Multipart).toBeUndefined() + }) + + it('uses persisted s3Multipart key for resume (listParts, not createMultipart)', async () => { + const persistedKey = 'persisted-object-key' + const persistedUploadId = 'persisted-upload-id' + const { signRequest, operations, registerHandlers } = + createMultipartMocks({ + uploadId: persistedUploadId, + key: persistedKey, + }) + registerHandlers() + + const core = new Core().use(AwsS3, { + s3Endpoint: 'https://companion.example.com', + region: 'us-east-1', + signRequest, + shouldUseMultipart: false, // Would normally be simple upload + }) + + const fileId = core.addFile({ + source: 'test', + name: 'big.dat', + type: 'application/octet-stream', + data: new File([new Uint8Array(6 * MB)], 'big.dat'), + }) + + // Simulate Golden Retriever restoring s3Multipart state + core.setFileState(fileId, { + s3Multipart: { uploadId: persistedUploadId, key: persistedKey }, + }) + + const uploadPromise = core.upload() + + // Wait for the resume flow to call listParts (via fetch), then cancel + await new Promise((resolve) => setTimeout(resolve, 100)) + + // Should have resumed (listParts) instead of creating a new multipart upload + expect(operations).toContain('listParts') + expect(operations).not.toContain('createMultipart') + + // The signRequest calls should use the persisted key, not a generated one + const signedKeys = signRequest.mock.calls.map((call: any) => call[0].key) + expect(signedKeys.every((k: string) => k === persistedKey)).toBe(true) + + // Clean up + core.cancelAll() + await uploadPromise + }) + }) }) diff --git a/packages/@uppy/aws-s3/tests/s3-client/_shared.test.js b/packages/@uppy/aws-s3/tests/s3-client/_shared.test.js deleted file mode 100644 index 6bfd44fd5..000000000 --- a/packages/@uppy/aws-s3/tests/s3-client/_shared.test.js +++ /dev/null @@ -1,273 +0,0 @@ -import { beforeAll, describe, expect, it, vi } from 'vitest' -import { S3mini } from '../../src/s3-client/index.js' -import { createSigV4Signer } from '../../src/s3-client/signer.js' -import { randomBytes } from '../test-utils/browser-crypto.js' - -let _providerName - -export const beforeRun = (raw, name, providerSpecific) => { - if (!raw) { - console.error( - 'No credentials found. Please set the BUCKET_ENV_ environment variables.', - ) - describe.skip(name, () => { - it('skipped', () => { - expect(true).toBe(true) - }) - }) - } else { - console.log('Running tests for bucket:', name) - const credentials = { - provider: raw[0], - accessKeyId: raw[1], - secretAccessKey: raw[2], - endpoint: raw[3], - region: raw[4], - } - describe(`:::: ${credentials.provider} ::::`, () => { - expect(credentials.provider).toBe(name) - _providerName = credentials.provider - expect(credentials.accessKeyId).toBeDefined() - expect(credentials.secretAccessKey).toBeDefined() - expect(credentials.endpoint).toBeDefined() - expect(credentials.region).toBeDefined() - testRunner(credentials) - if (providerSpecific) { - providerSpecific(credentials) - } - }) - } -} - -const EIGHT_MB = 8 * 1024 * 1024 -const key_bin = 'test-multipart.bin' - -const large_buffer = randomBytes(EIGHT_MB * 3.2) -const content = 'some content' -const key = 'first-test-object.txt' -const key_list_parts = 'test-list-parts.bin' -const key_abort_multipart = 'test-abort-multipart.bin' - -const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart] - -export const cleanupTestBeforeAll = (s3client) => { - beforeAll(async () => { - for (const key of FILE_KEYS) { - try { - await s3client.deleteObject(key) - } catch { - // intentionally ignore the errors as the objects don't exists. still feels hacky tbh - } - } - }) -} - -// --- 2 ■ A separate describe makes test output nicer ----------------------- -export const testRunner = (bucket) => { - vi.setConfig({ testTimeout: 120_000 }) - - const presigner = createSigV4Signer({ - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - endpoint: bucket.endpoint, - }) - - const s3client = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - signRequest: presigner, - }) - - cleanupTestBeforeAll(s3client) - - it('instantiates s3client', () => { - expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation - }) - - // we don't need an explicit eTag method as we already get eTag in the putOject response - it('putObject uploads successfully and returns ETag', async () => { - const response = await s3client.putObject(key, content, 'text/plain') - expect(response.status).toBe(200) - expect(response.headers.get('etag')).toBeDefined() - await s3client.deleteObject(key) - }) - - it('putObject handles binary data', async () => { - const binaryData = new Uint8Array(6).fill(0xff) - - const response = await s3client.putObject( - key, - binaryData, - 'application/octet-stream', - ) - - expect(response).toBeDefined() - expect(response.status).toBe(200) - expect(response.headers.get('etag')).toBeDefined() - - // cleanup - - await s3client.deleteObject(key) - }) - - // test getMultipartUploadId - - it('getMultipartUploadId returns a valid uploadId', async () => { - const uploadId = await s3client.getMultipartUploadId( - key_bin, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - expect(typeof uploadId).toBe('string') - expect(uploadId.length).toBeGreaterThan(0) - - // cleanup - await s3client.abortMultipartUpload(key, uploadId) - }) - - // test uploadPart - - it('uploadPart returns partNumber and Etag', async () => { - const partData = randomBytes(EIGHT_MB) - - const uploadId = await s3client.getMultipartUploadId( - key_bin, - 'application/octet-stream', - ) - - // upload part - const partResult = await s3client.uploadPart(key_bin, uploadId, partData, 1) - - expect(partResult).toBeDefined() - expect(partResult.partNumber).toBe(1) - expect(partResult.etag).toBeDefined() - expect(typeof partResult.etag).toBe('string') - expect(partResult.etag.length).toBe(32) - - // cleanup - await s3client.abortMultipartUpload(key, uploadId) - }) - - // end to end multipart flow - - it('completeMultipartUpload assembles parts correctly', async () => { - // key - key_bin - const partSize = EIGHT_MB - const totalParts = Math.ceil(large_buffer.length / partSize) - - const uploadId = await s3client.getMultipartUploadId( - key_bin, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - - // upload all parts - const uploadPromises = [] - for (let i = 0; i < totalParts; i++) { - const partBuffer = large_buffer.subarray(i * partSize, (i + 1) * partSize) - uploadPromises.push( - s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1), - ) - } - const uploadResponses = await Promise.all(uploadPromises) - - // verify all parts uploaded succesfully - - expect(uploadResponses.length).toBe(totalParts) - - uploadResponses.forEach((response, index) => { - expect(response.partNumber).toBe(index + 1) - expect(response.etag).toBeDefined() - }) - - // create multipart upload - - const parts = uploadResponses.map((response) => ({ - partNumber: response.partNumber, - etag: response.etag, - })) - - const completeResponse = await s3client.completeMultipartUpload( - key_bin, - uploadId, - parts, - ) - - expect(completeResponse).toBeDefined() - expect(typeof completeResponse).toBe('object') - expect(completeResponse.etag).toBeDefined() - expect(typeof completeResponse.etag).toBe('string') - expect(completeResponse.etag.length).toBe(32 + 2) - - // cleanup - - await s3client.deleteObject(key_bin) - }) - - it('abortMultipartUpload cancels upload successfully', async () => { - // start upload - const uploadId = await s3client.getMultipartUploadId( - key_abort_multipart, - 'application/octet-stream', - ) - - expect(uploadId).toBeDefined() - - const partData = randomBytes(EIGHT_MB) - await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1) - - // abort - const abortResult = await s3client.abortMultipartUpload( - key_abort_multipart, - uploadId, - ) - - expect(abortResult).toBeDefined() - expect(abortResult.status).toBe('Aborted') - expect(abortResult.key).toBe(key_abort_multipart) - expect(abortResult.uploadId).toBe(uploadId) - }) - - it('listParts returns uploaded parts correctly', async () => { - const partSize = EIGHT_MB - - const uploadId = await s3client.getMultipartUploadId( - key_list_parts, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - - const part1Data = randomBytes(partSize) - const part2Data = randomBytes(partSize) - - const part1Result = await s3client.uploadPart( - key_list_parts, - uploadId, - part1Data, - 1, - ) - const part2Result = await s3client.uploadPart( - key_list_parts, - uploadId, - part2Data, - 2, - ) - - const parts = await s3client.listParts(uploadId, key_list_parts) - - expect(parts).toBeInstanceOf(Array) - expect(parts.length).toBe(2) - - // verify part 1 - expect(parts[0].partNumber).toBe(1) - expect(parts[0].etag).toBe(part1Result.etag) - - // verify part 2 - expect(parts[1].partNumber).toBe(2) - expect(parts[1].etag).toBe(part2Result.etag) - - // cleanup abort upload - await s3client.abortMultipartUpload(key, uploadId) - }) -} diff --git a/packages/@uppy/aws-s3/tests/s3-client/config.ts b/packages/@uppy/aws-s3/tests/s3-client/config.ts new file mode 100644 index 000000000..4f1a93753 --- /dev/null +++ b/packages/@uppy/aws-s3/tests/s3-client/config.ts @@ -0,0 +1,14 @@ +export const accessKeyId = 'stsuser' +export const secretAccessKey = 'stspassword123' + +export function getConfig(env: Record) { + const config = env['VITE_MINIO_CONFIG'] + if (!config) { + return undefined + } + const [rootAccessKeyId, rootSecretAccessKey, endpoint, region] = + config.split(',') + // Use stsuser credentials for all tests (readwrite policy is sufficient) + // Root credentials are kept for Docker container startup + return { endpoint, region, rootAccessKeyId, rootSecretAccessKey } +} diff --git a/packages/@uppy/aws-s3/tests/s3-client/docker.js b/packages/@uppy/aws-s3/tests/s3-client/docker.ts similarity index 57% rename from packages/@uppy/aws-s3/tests/s3-client/docker.js rename to packages/@uppy/aws-s3/tests/s3-client/docker.ts index 4cb03c150..93ddaf276 100644 --- a/packages/@uppy/aws-s3/tests/s3-client/docker.js +++ b/packages/@uppy/aws-s3/tests/s3-client/docker.ts @@ -1,32 +1,23 @@ import { exec, spawn } from 'node:child_process' -import { resolve } from 'node:path' import { promisify } from 'node:util' -const CWD = resolve('.') - const execAsync = promisify(exec) -export async function getContainerName(serviceName) { - try { - // Find container by service name using docker ps - const { stdout } = await execAsync( - `docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`, - ) - const containerName = stdout.trim().split('\n')[0] // Get first matching container - if (!containerName) { - throw new Error(`No running container found for service: ${serviceName}`) - } - return containerName - } catch (error) { - throw new Error( - `Failed to find container for service ${serviceName}: ${error.message}`, - ) +export async function getContainerName(serviceName: string) { + // Find container by service name using docker ps + const { stdout } = await execAsync( + `docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`, + ) + const containerName = stdout.trim().split('\n')[0] // Get first matching container + if (!containerName) { + throw new Error(`No running container found for service: ${serviceName}`) } + return containerName } export async function execDockerCommand( - containerName, - command, + containerName: string, + command: string, timeoutMs = 10000, ) { // If it's a service name (like 'garage'), find the actual container name @@ -34,9 +25,7 @@ export async function execDockerCommand( if (['garage', 'minio', 'ceph'].includes(containerName)) { try { actualContainerName = await getContainerName(containerName) - console.log( - `Found container: ${actualContainerName} for service: ${containerName}`, - ) + // console.log(`Found container: ${actualContainerName} for service: ${containerName}`,) } catch (_error) { console.log(`Using container name as-is: ${containerName}`) } @@ -74,27 +63,27 @@ export async function execDockerCommand( } } -function run(cmd, args) { - return new Promise((res, rej) => { - const p = spawn(cmd, args, { cwd: CWD, stdio: 'inherit' }) +async function run(cmd: string, args: string[], env?: Record) { + // console.log('running command:', cmd, args.join(' ')) + return new Promise((res, rej) => { + const p = spawn(cmd, args, { + stdio: 'inherit', + env: { ...process.env, ...env }, + }) p.on('close', (code) => code === 0 ? res() : rej(new Error(`${cmd} ${args.join(' ')} exited ${code}`)), ) + p.on('error', (err) => rej(err)) }) } -export const composeUp = (file) => - run('docker', ['compose', '-f', file, 'up', '-d', '--force-recreate']) -export const composeUpWait = (file) => - run('docker', [ - 'compose', - '-f', - file, - 'up', - '-d', - '--force-recreate', - '--wait', - ]) -export const composeDown = (file) => + +export const composeUpWait = (file: string, env?: Record) => + run( + 'docker', + ['compose', '-f', file, 'up', '-d', '--force-recreate', '--wait'], + env, + ) +export const composeDown = (file: string) => run('docker', ['compose', '-f', file, 'down', '--remove-orphans', '-v']) diff --git a/packages/@uppy/aws-s3/tests/s3-client/minio.test.js b/packages/@uppy/aws-s3/tests/s3-client/minio.test.js deleted file mode 100644 index c524a830b..000000000 --- a/packages/@uppy/aws-s3/tests/s3-client/minio.test.js +++ /dev/null @@ -1,238 +0,0 @@ -import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts' -import { describe, expect, inject, it, vi } from 'vitest' -import { S3mini } from '../../src/s3-client/S3.js' -import { createSigV4Signer } from '../../src/s3-client/signer.js' -import { randomBytes } from '../test-utils/browser-crypto.js' -import { beforeRun, cleanupTestBeforeAll } from './_shared.test.js' - -const name = 'minio' - -// Get bucket configs from globalSetup via Vitest inject -const bucketConfigs = inject('bucketConfigs') || [] -const raw = bucketConfigs.find((c) => c.provider === name) - ? [ - bucketConfigs.find((c) => c.provider === name).provider, - bucketConfigs.find((c) => c.provider === name).accessKeyId, - bucketConfigs.find((c) => c.provider === name).secretAccessKey, - bucketConfigs.find((c) => c.provider === name).endpoint, - bucketConfigs.find((c) => c.provider === name).region, - ] - : null - -/** Create STS client using @aws-sdk/client-sts */ -function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) { - return new STSClient({ - region, - endpoint, - credentials: { accessKeyId, secretAccessKey }, - }) -} - -/** Get temporary credentials via AssumeRole */ -async function assumeRole(stsClient, { durationSeconds = 900 } = {}) { - const response = await stsClient.send( - new AssumeRoleCommand({ - RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN - RoleSessionName: 'uppy-test', - DurationSeconds: durationSeconds, - }), - ) - const creds = response.Credentials - return { - AccessKeyId: creds.AccessKeyId, - SecretAccessKey: creds.SecretAccessKey, - SessionToken: creds.SessionToken, - Expiration: creds.Expiration?.toISOString() || creds.Expiration, - } -} - -const minioSpecific = (bucket) => { - vi.setConfig({ testTimeout: 120_000 }) - - const presigner = createSigV4Signer({ - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - endpoint: bucket.endpoint, - }) - - const s3client = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - signRequest: presigner, - }) - - cleanupTestBeforeAll(s3client) - - // ===== signRequest tests ===== - - it('simple putObject upload', async () => { - const fileContents = new TextEncoder().encode( - 'Hello from pre-signed URL test.', - ) - - const result = await s3client.putObject( - 'presigned-test-file.txt', - fileContents, - 'text/plain', - ) - - expect(result.ok).toBe(true) - }) - - it('multipart upload with signRequest', async () => { - const key = `presigned-multipart-${Date.now()}.bin` - const partSize = 5 * 1024 * 1024 // 5MB - const part = randomBytes(partSize) - - const uploadId = await s3client.getMultipartUploadId( - key, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - - const uploaded = await s3client.uploadPart(key, uploadId, part, 1) - expect(uploaded.etag).toBeDefined() - - const result = await s3client.completeMultipartUpload(key, uploadId, [ - uploaded, - ]) - expect(result.etag).toBeDefined() - expect(result.location).toBeDefined() - expect(result.location).toContain(key) - expect(result.key).toBe(key) - - await s3client.deleteObject(key) - }) - - // ===== getCredentials tests (STS) ===== - - describe('getCredentials (STS)', () => { - const stsEndpoint = new URL(bucket.endpoint).origin - - it('should upload using getCredentials callback', async () => { - const testKey = `sts-getcreds-${Date.now()}.txt` - - const s3 = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - getCredentials: async () => { - const stsClient = createSTSClient({ - endpoint: stsEndpoint, - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - }) - const creds = await assumeRole(stsClient) - return { - credentials: { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }, - bucket: new URL(bucket.endpoint).pathname.slice(1), - region: bucket.region, - } - }, - }) - - try { - const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain') - expect(result.ok).toBe(true) - } finally { - await s3.deleteObject(testKey) - } - }) - - it('should cache credentials across multiple requests', async () => { - let fetchCount = 0 - - const s3 = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - getCredentials: async () => { - fetchCount++ - const stsClient = createSTSClient({ - endpoint: stsEndpoint, - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - }) - const creds = await assumeRole(stsClient) - return { - credentials: { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }, - bucket: new URL(bucket.endpoint).pathname.slice(1), - region: bucket.region, - } - }, - }) - - await s3.putObject( - `sts-cache-1-${Date.now()}.txt`, - 'File 1', - 'text/plain', - ) - await s3.putObject( - `sts-cache-2-${Date.now()}.txt`, - 'File 2', - 'text/plain', - ) - expect(fetchCount).toBe(1) - }) - - it('should perform multipart upload with getCredentials', async () => { - const s3 = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - getCredentials: async () => { - const stsClient = createSTSClient({ - endpoint: stsEndpoint, - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - }) - const creds = await assumeRole(stsClient) - return { - credentials: { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }, - bucket: new URL(bucket.endpoint).pathname.slice(1), - region: bucket.region, - } - }, - }) - - const key = `sts-multipart-${Date.now()}.bin` - const partSize = 5 * 1024 * 1024 // 5MB - const part = randomBytes(partSize) - - const uploadId = await s3.getMultipartUploadId( - key, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - - const uploaded = await s3.uploadPart(key, uploadId, part, 1) - expect(uploaded.etag).toBeDefined() - - const result = await s3.completeMultipartUpload(key, uploadId, [uploaded]) - expect(result.etag).toBeDefined() - expect(result.location).toBeDefined() - expect(result.location).toContain(key) - expect(result.key).toBe(key) - - await s3.deleteObject(key) - }) - }) -} - -beforeRun(raw, name, minioSpecific) diff --git a/packages/@uppy/aws-s3/tests/s3-client/minio.test.ts b/packages/@uppy/aws-s3/tests/s3-client/minio.test.ts new file mode 100644 index 000000000..190c13c15 --- /dev/null +++ b/packages/@uppy/aws-s3/tests/s3-client/minio.test.ts @@ -0,0 +1,456 @@ +import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts' +import { beforeAll, describe, expect, it } from 'vitest' +import { S3mini } from '../../src/s3-client/S3mini.js' +import { createSigV4Signer } from '../../src/s3-client/signer.js' +import { randomBytes } from '../test-utils/browser-crypto.js' +import { accessKeyId, getConfig, secretAccessKey } from './config.js' + +// @ts-expect-error todo +const config = getConfig(import.meta.env) + +const suiteName = 'MinIO S3 client tests' + +if (config) { + const { endpoint, region } = config + const presigner = createSigV4Signer({ + accessKeyId, + secretAccessKey, + region, + endpoint, + }) + + const s3client = new S3mini({ + endpoint, + region, + signRequest: presigner, + }) + + const EIGHT_MB = 8 * 1024 * 1024 + const key_bin = 'test-multipart.bin' + + const large_buffer = randomBytes(EIGHT_MB * 3.2) + const content = 'some content' + const key = 'first-test-object.txt' + const key_list_parts = 'test-list-parts.bin' + const key_abort_multipart = 'test-abort-multipart.bin' + + const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart] + + beforeAll(async () => { + for (const key of FILE_KEYS) { + try { + await s3client.deleteObject(key) + } catch { + // intentionally ignore the errors as the objects don't exists. still feels hacky tbh + } + } + }) + + /** Create STS client using @aws-sdk/client-sts */ + function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) { + return new STSClient({ + region, + endpoint, + credentials: { accessKeyId, secretAccessKey }, + }) + } + + /** Get temporary credentials via AssumeRole */ + async function assumeRole( + stsClient: STSClient, + { durationSeconds = 900 } = {}, + ) { + const response = await stsClient.send( + new AssumeRoleCommand({ + RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN + RoleSessionName: 'uppy-test', + DurationSeconds: durationSeconds, + }), + ) + const creds = response.Credentials! + return { + AccessKeyId: creds.AccessKeyId, + SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, + Expiration: creds.Expiration?.toISOString() || creds.Expiration, + } + } + + // ===== signRequest tests ===== + + describe(suiteName, () => { + it('simple putObject upload', async () => { + const key = 'presigned-test-file.txt' + const fileContents = new TextEncoder().encode( + 'Hello from pre-signed URL test.', + ) + + const result = await s3client.putObject(key, fileContents, 'text/plain') + + expect(result.location).toBeDefined() + expect(result.location).toContain(key) + // location should be a clean URL without query string (no signing params) + expect(result.location).not.toContain('X-Amz-Signature') + expect(result.location).not.toContain('?') + }) + + it('multipart upload with signRequest', async () => { + const key = `presigned-multipart-${Date.now()}.bin` + const partSize = 5 * 1024 * 1024 // 5MB + const part = randomBytes(partSize) + + const { uploadId } = await s3client.createMultipartUpload( + key, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + + const partNumber = 1 + const { etag } = await s3client.uploadPart( + key, + uploadId, + part, + partNumber, + ) + expect(etag).toBeDefined() + + const result = await s3client.completeMultipartUpload(key, uploadId, [ + { etag, partNumber }, + ]) + expect(result.etag).toBeDefined() + expect(result.location).toBeDefined() + expect(result.location).toContain(key) + expect(result.key).toBe(key) + + await s3client.deleteObject(key) + }) + + // ===== getCredentials tests (STS) ===== + + describe('getCredentials (STS)', () => { + const stsEndpoint = new URL(endpoint).origin + + it('should upload using getCredentials callback', async () => { + const testKey = `sts-getcreds-${Date.now()}.txt` + + const s3 = new S3mini({ + endpoint, + region, + getCredentials: async () => { + const stsClient = createSTSClient({ + endpoint: stsEndpoint, + accessKeyId, + secretAccessKey, + region, + }) + const creds = await assumeRole(stsClient) + return { + credentials: { + accessKeyId: creds.AccessKeyId!, + secretAccessKey: creds.SecretAccessKey!, + sessionToken: creds.SessionToken!, + expiration: creds.Expiration as string, + }, + bucket: new URL(endpoint).pathname.slice(1), + region, + } + }, + }) + + try { + const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain') + expect(result.location).toBeDefined() + expect(result.location).toContain(testKey) + expect(result.location).not.toContain('X-Amz-Signature') + expect(result.location).not.toContain('?') + } finally { + await s3.deleteObject(testKey) + } + }) + + it('should cache credentials across multiple requests', async () => { + let fetchCount = 0 + + const s3 = new S3mini({ + endpoint, + region, + getCredentials: async () => { + fetchCount++ + const stsClient = createSTSClient({ + endpoint: stsEndpoint, + accessKeyId, + secretAccessKey, + region, + }) + const creds = await assumeRole(stsClient) + return { + credentials: { + accessKeyId: creds.AccessKeyId!, + secretAccessKey: creds.SecretAccessKey!, + sessionToken: creds.SessionToken!, + expiration: creds.Expiration as string, + }, + bucket: new URL(endpoint).pathname.slice(1), + region, + } + }, + }) + + await s3.putObject( + `sts-cache-1-${Date.now()}.txt`, + 'File 1', + 'text/plain', + ) + await s3.putObject( + `sts-cache-2-${Date.now()}.txt`, + 'File 2', + 'text/plain', + ) + expect(fetchCount).toBe(1) + }) + + it('should perform multipart upload with getCredentials', async () => { + const s3 = new S3mini({ + endpoint, + region, + getCredentials: async () => { + const stsClient = createSTSClient({ + endpoint: stsEndpoint, + accessKeyId, + secretAccessKey, + region, + }) + const creds = await assumeRole(stsClient) + return { + credentials: { + accessKeyId: creds.AccessKeyId!, + secretAccessKey: creds.SecretAccessKey!, + sessionToken: creds.SessionToken!, + expiration: creds.Expiration as string, + }, + bucket: new URL(endpoint).pathname.slice(1), + region, + } + }, + }) + + const key = `sts-multipart-${Date.now()}.bin` + const partSize = 5 * 1024 * 1024 // 5MB + const part = randomBytes(partSize) + + const { uploadId } = await s3.createMultipartUpload( + key, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + + const partNumber = 1 + const { etag } = await s3.uploadPart(key, uploadId, part, partNumber) + expect(etag).toBeDefined() + + const result = await s3.completeMultipartUpload(key, uploadId, [ + { etag, partNumber }, + ]) + expect(result.etag).toBeDefined() + expect(result.location).toBeDefined() + expect(result.location).toContain(key) + expect(result.key).toBe(key) + + await s3.deleteObject(key) + }) + }) + + it('instantiates s3client', () => { + expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation + }) + + // we don't need an explicit eTag method as we already get eTag in the putOject response + it('putObject uploads successfully and returns ETag', async () => { + const response = await s3client.putObject(key, content, 'text/plain') + expect(response.etag).toBeDefined() + await s3client.deleteObject(key) + }) + + it('putObject handles binary data', async () => { + const binaryData = new Uint8Array(6).fill(0xff) + + const response = await s3client.putObject( + key, + binaryData, + 'application/octet-stream', + ) + + expect(response).toBeDefined() + expect(response.etag).toBeDefined() + + // cleanup + + await s3client.deleteObject(key) + }) + + // test createMultipartUpload + + it('createMultipartUpload returns a valid uploadId', async () => { + const { uploadId } = await s3client.createMultipartUpload( + key_bin, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + expect(typeof uploadId).toBe('string') + expect(uploadId.length).toBeGreaterThan(0) + + // cleanup + await s3client.abortMultipartUpload(key, uploadId) + }) + + // test uploadPart + + it('uploadPart returns partNumber and Etag', async () => { + const partData = randomBytes(EIGHT_MB) + + const { uploadId } = await s3client.createMultipartUpload( + key_bin, + 'application/octet-stream', + ) + + // upload part + const partResult = await s3client.uploadPart( + key_bin, + uploadId, + partData, + 1, + ) + + expect(partResult).toBeDefined() + expect(partResult.etag).toBeDefined() + expect(typeof partResult.etag).toBe('string') + expect(partResult.etag.length).toBe(32) + + // cleanup + await s3client.abortMultipartUpload(key, uploadId) + }) + + // end to end multipart flow + + it('completeMultipartUpload assembles parts correctly', async () => { + // key - key_bin + const partSize = EIGHT_MB + const totalParts = Math.ceil(large_buffer.byteLength / partSize) + + const { uploadId } = await s3client.createMultipartUpload( + key_bin, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + + // upload all parts + const uploadPromises: Promise<{ etag: string }>[] = [] + for (let i = 0; i < totalParts; i++) { + const partBuffer = large_buffer.subarray( + i * partSize, + (i + 1) * partSize, + ) + uploadPromises.push( + s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1), + ) + } + const uploadResponses = await Promise.all(uploadPromises) + + // verify all parts uploaded succesfully + + expect(uploadResponses.length).toBe(totalParts) + + uploadResponses.forEach((response) => { + expect(response.etag).toBeDefined() + }) + + // create multipart upload + + const parts = uploadResponses.map((response, index) => ({ + partNumber: index + 1, + etag: response.etag, + })) + + const completeResponse = await s3client.completeMultipartUpload( + key_bin, + uploadId, + parts, + ) + + expect(completeResponse).toBeDefined() + expect(typeof completeResponse).toBe('object') + expect(completeResponse.etag).toBeDefined() + expect(typeof completeResponse.etag).toBe('string') + expect(completeResponse.etag!.length).toBe(32 + 2) + + // cleanup + + await s3client.deleteObject(key_bin) + }) + + it('abortMultipartUpload cancels upload successfully', async () => { + // start upload + const { uploadId } = await s3client.createMultipartUpload( + key_abort_multipart, + 'application/octet-stream', + ) + + expect(uploadId).toBeDefined() + + const partData = randomBytes(EIGHT_MB) + await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1) + + // abort + await s3client.abortMultipartUpload(key_abort_multipart, uploadId) + }) + + it('listParts returns uploaded parts correctly', async () => { + const partSize = EIGHT_MB + + const { uploadId } = await s3client.createMultipartUpload( + key_list_parts, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + + const part1Data = randomBytes(partSize) + const part2Data = randomBytes(partSize) + + const part1Result = await s3client.uploadPart( + key_list_parts, + uploadId, + part1Data, + 1, + ) + const part2Result = await s3client.uploadPart( + key_list_parts, + uploadId, + part2Data, + 2, + ) + + const parts = await s3client.listParts(uploadId, key_list_parts) + + expect(parts).toBeInstanceOf(Array) + expect(parts.length).toBe(2) + + // verify part 1 + expect(parts[0].partNumber).toBe(1) + expect(parts[0].etag).toBe(part1Result.etag) + + // verify part 2 + expect(parts[1].partNumber).toBe(2) + expect(parts[1].etag).toBe(part2Result.etag) + + // cleanup abort upload + await s3client.abortMultipartUpload(key, uploadId) + }) + }) +} else { + console.warn( + 'Skipping MinIO S3 client tests: missing env variable VITE_MINIO_CONFIG', + ) + describe.skip(suiteName, () => { + it('noop', () => {}) + }) +} diff --git a/packages/@uppy/aws-s3/tests/s3-client/setup.js b/packages/@uppy/aws-s3/tests/s3-client/setup.js deleted file mode 100644 index fbcf4a030..000000000 --- a/packages/@uppy/aws-s3/tests/s3-client/setup.js +++ /dev/null @@ -1,75 +0,0 @@ -import * as dotenv from 'dotenv' - -dotenv.config() - -import { join } from 'node:path' - -import { composeUp, composeUpWait, execDockerCommand } from './docker.js' - -const composeFiles = { - minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'), -} - -const bucketConfigs = Object.keys(process.env) - .filter((k) => k.startsWith('BUCKET_ENV_')) - .map((k) => { - const [provider, rootAccessKeyId, rootSecretAccessKey, endpoint, region] = - process.env[k].split(',') - // Use stsuser credentials for all tests (readwrite policy is sufficient) - // Root credentials are kept for Docker container startup - return { - provider, - accessKeyId: 'stsuser', - secretAccessKey: 'stspassword123', - rootAccessKeyId, - rootSecretAccessKey, - endpoint, - region, - } - }) - -export default async function setup({ provide }) { - // Provide bucket configs to browser tests via Vitest's inject mechanism - provide('bucketConfigs', bucketConfigs) - - for (const cfg of bucketConfigs) { - const composeFile = composeFiles[cfg.provider] - if (!composeFile) continue - console.log(`⏫ starting ${cfg.provider} image …`) - switch (cfg.provider) { - case 'minio': - process.env.MINIO_ROOT_USER = cfg.rootAccessKeyId - process.env.MINIO_ROOT_PASSWORD = cfg.rootSecretAccessKey - await composeUpWait(composeFile) - // biome-ignore lint/correctness/noSwitchDeclarations: no other switch blocks - const bucketName = new URL(cfg.endpoint).pathname.split('/')[1] - if (bucketName) { - await execDockerCommand( - 'minio', - `mc mb local/${bucketName} --ignore-existing`, - 30000, - ) - // Create a user with readwrite policy for STS testing - // The root user cannot AssumeRole for itself - needs a regular user - try { - await execDockerCommand( - 'minio', - `mc admin user add local stsuser stspassword123`, - 15000, - ) - await execDockerCommand( - 'minio', - `mc admin policy attach local readwrite --user stsuser`, - 15000, - ) - } catch (err) { - // User may already exist, that's fine - console.log('STS user setup:', err.message || 'error occurred') - } - } - break - default: - await composeUp(composeFile) - } - } -} diff --git a/packages/@uppy/aws-s3/tests/s3-client/setup.ts b/packages/@uppy/aws-s3/tests/s3-client/setup.ts new file mode 100644 index 000000000..2f52e586d --- /dev/null +++ b/packages/@uppy/aws-s3/tests/s3-client/setup.ts @@ -0,0 +1,48 @@ +import path from 'node:path' +import { getConfig } from './config' +import { composeDown, composeUpWait, execDockerCommand } from './docker' + +const composeFile = path.join(__dirname, 'compose.minio.yaml') + +const config = getConfig(process.env) + +export async function setup() { + if (config == null) return + + const { endpoint, rootAccessKeyId, rootSecretAccessKey } = config + + console.log('⏫ starting minio image', composeFile) + process.env.MINIO_ROOT_USER = rootAccessKeyId + process.env.MINIO_ROOT_PASSWORD = rootSecretAccessKey + await composeUpWait(composeFile) + const bucketName = new URL(endpoint).pathname.split('/')[1] + await execDockerCommand( + 'minio', + `mc mb local/${bucketName} --ignore-existing`, + 30000, + ) + // Create a user with readwrite policy for STS testing + // The root user cannot AssumeRole for itself - needs a regular user + try { + await execDockerCommand( + 'minio', + `mc admin user add local stsuser stspassword123`, + ) + await execDockerCommand( + 'minio', + `mc admin policy attach local readwrite --user stsuser`, + ) + } catch (err) { + // User may already exist, that's fine + console.log('STS user setup:', err.message || 'error occurred') + } + console.log(`✅ minio is ready`) +} + +export async function teardown() { + if (config == null) return + + console.log(`⏬ stopping minio image …`) + await composeDown(composeFile) + console.log(`✅ minio stopped and cleaned up`) +} diff --git a/packages/@uppy/aws-s3/tests/s3-client/teardown.js b/packages/@uppy/aws-s3/tests/s3-client/teardown.js deleted file mode 100644 index 88ca71663..000000000 --- a/packages/@uppy/aws-s3/tests/s3-client/teardown.js +++ /dev/null @@ -1,28 +0,0 @@ -import * as dotenv from 'dotenv' - -dotenv.config() - -import { join } from 'node:path' -import { composeDown } from './docker.js' - -const composeFiles = { - minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'), -} - -const bucketConfigs = Object.keys(process.env) - .filter((k) => k.startsWith('BUCKET_ENV_')) - .map((k) => { - const [provider, accessKeyId, secretAccessKey, endpoint, region] = - process.env[k].split(',') - return { provider, accessKeyId, secretAccessKey, endpoint, region } - }) - -export default async () => { - for (const cfg of bucketConfigs) { - const composeFile = composeFiles[cfg.provider] - if (!composeFile) continue // ignore unknown providers - - console.log(`⏬ stopping ${cfg.provider} …`) - await composeDown(composeFile) // `docker compose -f … down` - } -} diff --git a/packages/@uppy/aws-s3/tests/test-utils/browser-crypto.js b/packages/@uppy/aws-s3/tests/test-utils/browser-crypto.js deleted file mode 100644 index 4a8ef003a..000000000 --- a/packages/@uppy/aws-s3/tests/test-utils/browser-crypto.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Generate random bytes using Web Crypto API. - * Web Crypto has a 65536 byte limit per getRandomValues call, - * so we chunk large requests. - * @param {number} size - Number of random bytes to generate - * @returns {Uint8Array} Random bytes - */ -export function randomBytes(size) { - const buffer = new Uint8Array(size) - const maxChunk = 65536 // Web Crypto API limit - - for (let offset = 0; offset < size; offset += maxChunk) { - const chunkSize = Math.min(maxChunk, size - offset) - const chunk = buffer.subarray(offset, offset + chunkSize) - globalThis.crypto.getRandomValues(chunk) - } - - return buffer -} - -/** - * Compute SHA-1 hash and return base64 string. - * @param {Uint8Array | ArrayBuffer} data - Data to hash - * @returns {Promise} Base64-encoded SHA-1 hash - */ -export async function sha1Base64(data) { - const hash = await globalThis.crypto.subtle.digest('SHA-1', data) - const bytes = new Uint8Array(hash) - // Convert to base64 in chunks to avoid call stack issues with large data - const chunkSize = 0x8000 - let result = '' - for (let i = 0; i < bytes.length; i += chunkSize) { - const chunk = bytes.subarray(i, i + chunkSize) - result += String.fromCharCode(...chunk) - } - return btoa(result) -} diff --git a/packages/@uppy/aws-s3/tests/test-utils/browser-crypto.ts b/packages/@uppy/aws-s3/tests/test-utils/browser-crypto.ts new file mode 100644 index 000000000..2aa5aee66 --- /dev/null +++ b/packages/@uppy/aws-s3/tests/test-utils/browser-crypto.ts @@ -0,0 +1,17 @@ +/** + * Generate random bytes using Web Crypto API. + * Web Crypto has a 65536 byte limit per getRandomValues call, + * so we chunk large requests. + */ +export function randomBytes(size: number) { + const buffer = new Uint8Array(size) + const maxChunk = 65536 // Web Crypto API limit + + for (let offset = 0; offset < size; offset += maxChunk) { + const chunkSize = Math.min(maxChunk, size - offset) + const chunk = buffer.subarray(offset, offset + chunkSize) + globalThis.crypto.getRandomValues(chunk) + } + + return buffer +} diff --git a/packages/@uppy/aws-s3/vitest.config.ts b/packages/@uppy/aws-s3/vitest.config.ts index 74e01e7ea..37b4bd06a 100644 --- a/packages/@uppy/aws-s3/vitest.config.ts +++ b/packages/@uppy/aws-s3/vitest.config.ts @@ -3,21 +3,19 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { testTimeout: 120_000, - globalSetup: ['./tests/s3-client/setup.js'], - setupFiles: ['./test-setup.mjs'], + globalSetup: ['tests/s3-client/setup.ts'], projects: [ { test: { - name: 'aws-s3-node', - include: ['tests/index.test.ts', 'tests/createSignedURL.test.ts'], + name: 's3-jsdom', + include: ['**/*.test.ts'], environment: 'jsdom', }, }, { test: { - name: 's3-client-browser', - include: ['tests/s3-client/*.test.{js,ts}'], - exclude: ['tests/s3-client/_shared.test.js'], + name: 's3-browser', + include: ['**/minio.test.ts'], browser: { enabled: true, provider: 'playwright', diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index 6f13065b2..5ced71801 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -662,11 +662,12 @@ export class Uppy< ...files[fileID].progress, ...defaultProgress, }, - // @ts-expect-error these typed are inserted + // @ts-expect-error these types are inserted // into the namespace in their respective packages - // but core isn't ware of those + // but core isn't aware of those tus: undefined, transloadit: undefined, + s3Multipart: undefined, } }) diff --git a/packages/@uppy/utils/src/fetcher.ts b/packages/@uppy/utils/src/fetcher.ts index 87e0540db..c550aecef 100644 --- a/packages/@uppy/utils/src/fetcher.ts +++ b/packages/@uppy/utils/src/fetcher.ts @@ -61,7 +61,7 @@ export function fetcher( options: FetcherOptions = {}, ): Promise { const { - body = null, + body, headers = {}, method = 'GET', onBeforeRequest = noop, diff --git a/private/dev/Dashboard.js b/private/dev/Dashboard.js index f3b7ffdd5..16b73f68a 100644 --- a/private/dev/Dashboard.js +++ b/private/dev/Dashboard.js @@ -208,13 +208,13 @@ export default () => { break case 's3': uppyDashboard.use(AwsS3, { - endpoint: COMPANION_URL, + companionEndpoint: COMPANION_URL, shouldUseMultipart: false, }) break case 's3-multipart': uppyDashboard.use(AwsS3, { - endpoint: COMPANION_URL, + companionEndpoint: COMPANION_URL, shouldUseMultipart: true, }) break diff --git a/turbo.json b/turbo.json index c36552ab0..a3386bfe3 100644 --- a/turbo.json +++ b/turbo.json @@ -42,7 +42,7 @@ "test": { "dependsOn": ["^test"], "cache": false, - "passThroughEnv": ["BUCKET_ENV_MINIO"] + "passThroughEnv": ["VITE_MINIO_CONFIG"] }, "test:e2e": { "dependsOn": ["^test:e2e"],