@uppy/aws-s3: remote uploads and golden retriever (#6186)

- add support for remote uploads
- add support for restore using golden-retriever
- update examples
- #6181 still persists

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Mikael Finstad <finstaden@gmail.com>
This commit is contained in:
Prakash 2026-05-01 00:00:01 +05:30 committed by prakash
parent da46951771
commit 2d429a8aef
No known key found for this signature in database
31 changed files with 2478 additions and 2244 deletions

@ -0,0 +1 @@
Subproject commit e603096c56d853b429ccb06a585ffb6d0aa7539e

View file

@ -47,6 +47,10 @@ yarn build
- `yarn typecheck` - Run TypeScript type checking - `yarn typecheck` - Run TypeScript type checking
- `yarn check` - Run Biome linting and formatting - `yarn check` - Run Biome linting and formatting
### Running minio tests
Tests for `@uppy/aws-s3` need a minio server running inside Docker, and in order to run those tests you must set the environment variable `VITE_MINIO_CONFIG="test_user,test_password,http://localhost:9002/test-bucket,us-east-1"` when running the tests.
### Headless components ### Headless components
When adding a new component to `@uppy/components`, you have to run `yarn migrate:components` from root When adding a new component to `@uppy/components`, you have to run `yarn migrate:components` from root

View file

@ -59,7 +59,7 @@ jobs:
COMPANION_DOMAIN: localhost:3020 COMPANION_DOMAIN: localhost:3020
COMPANION_PROTOCOL: http COMPANION_PROTOCOL: http
COMPANION_REDIS_URL: redis://localhost:6379 COMPANION_REDIS_URL: redis://localhost:6379
BUCKET_ENV_MINIO: "minio,test_user,test_password,http://localhost:9002/test-bucket,us-east-1" VITE_MINIO_CONFIG: "test_user,test_password,http://localhost:9002/test-bucket,us-east-1"
types: types:
name: Types name: Types

View file

@ -23,5 +23,5 @@ uppy.use(Dashboard, {
plugins: ['GoogleDrive', 'Webcam'], plugins: ['GoogleDrive', 'Webcam'],
}) })
uppy.use(AwsS3, { uppy.use(AwsS3, {
endpoint: 'http://localhost:3020', companionEndpoint: 'http://localhost:3020',
}) })

View file

@ -0,0 +1,421 @@
import { EventManager, type Uppy } from '@uppy/core'
import type { Body, LocalUppyFile, Meta } from '@uppy/utils'
import type S3Client from './s3-client/S3Client.js'
/** Persisted S3 multipart state for Golden Retriever resume support */
interface S3MultipartState {
uploadId: string
key: string
}
declare module '@uppy/utils' {
// biome-ignore lint/correctness/noUnusedVariables: must match existing interface signature
export interface LocalUppyFile<M extends Meta, B extends Body> {
s3Multipart?: S3MultipartState
}
// biome-ignore lint/correctness/noUnusedVariables: must match existing interface signature
export interface RemoteUppyFile<M extends Meta, B extends Body> {
s3Multipart?: S3MultipartState
}
}
// ============================================================================
// Constants
// ============================================================================
const MB = 1024 * 1024
/** Minimum chunk size required by S3 (5MB) */
const MIN_CHUNK_SIZE = 5 * MB
/** Maximum number of parts allowed by S3 */
const MAX_PARTS = 10000
// ============================================================================
// S3Uploader Types
// ============================================================================
interface S3UploaderOptions<M extends Meta, B extends Body> {
uppy: Uppy<M, B>
s3Client: S3Client
file: LocalUppyFile<M, B>
metadata: Record<string, unknown>
key: string
shouldUseMultipart?: boolean
getChunkSize?: (file: { size: number }) => number
onProgress?: (bytesUploaded: number, bytesTotal: number) => void
onPartComplete?: (part: { PartNumber: number; ETag: string }) => void
onSuccess?: (result: UploadResult) => void
onError?: (err: Error) => void
onAbort?: () => void
log?: Uppy['log']
}
export interface UploadResult {
location: string
key: string
/** Only returned for multipart uploads */
uploadId?: string
}
interface Chunk {
index: number
start: number
end: number
size: number
}
interface ChunkState {
uploaded: number
etag?: string
}
export default class S3Uploader<M extends Meta, B extends Body> {
readonly #data: NonNullable<LocalUppyFile<M, B>['data']>
#key: string | undefined
readonly #options: S3UploaderOptions<M, B>
readonly #eventManager: EventManager<M, B>
#chunks: Chunk[] = []
#chunkState: ChunkState[] = []
#shouldUseMultipart: boolean = false
#uploadId?: string
#uploadHasStarted: boolean = false
#abortController: AbortController | undefined
constructor(options: S3UploaderOptions<M, B>) {
if (options.file.data == null) {
throw new Error(`File data is missing for file ${options.file.id}`)
}
this.#options = options
this.#data = options.file.data
this.#eventManager = new EventManager(options.uppy)
// Detect resume state from file (persisted by Golden Retriever across page refreshes).
// Must run before #initChunks so it can force multipart mode for resumed uploads.
const resumeState = options.file.s3Multipart
if (resumeState) {
this.#key = resumeState.key
this.#uploadId = resumeState.uploadId
this.#uploadHasStarted = true
}
const fileSize = options.file.data.size
// Determine if we should use multipart
// If we're resuming a multipart upload, force multipart. Otherwise use
// the boolean option (true/false) and ensure the file is larger than
// S3's minimum chunk size when enabling multipart.
this.#shouldUseMultipart =
Boolean(resumeState) ||
(this.#options.shouldUseMultipart === true && fileSize > MIN_CHUNK_SIZE)
// Create chunks based on upload strategy
if (this.#shouldUseMultipart) {
// Calculate chunk size: at least MIN_CHUNK_SIZE, but may be larger for huge files
let chunkSize = this.#getChunkSize(fileSize)
chunkSize = Math.max(chunkSize, MIN_CHUNK_SIZE)
// Ensure we don't exceed MAX_PARTS (S3 limit: 10,000 parts)
if (Math.ceil(fileSize / chunkSize) > MAX_PARTS) {
chunkSize = Math.ceil(fileSize / MAX_PARTS)
}
// Create chunk definitions
for (
let offset = 0, index = 0;
offset < fileSize;
offset += chunkSize, index++
) {
const end = Math.min(offset + chunkSize, fileSize)
this.#chunks.push({ index, start: offset, end, size: end - offset })
}
} else {
// Simple upload: single chunk for the entire file
this.#chunks = [{ index: 0, start: 0, end: fileSize, size: fileSize }]
}
this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
// Setup events:
const fileId = this.#options.file.id
this.#eventManager.onFileRemove(fileId, () => {
this.abort()
this.#options.onAbort?.()
})
this.#eventManager.onCancelAll(fileId, () => {
this.abort()
this.#options.onAbort?.()
})
this.#eventManager.onFilePause(fileId, (isPaused) => {
if (isPaused) {
this.pause()
} else {
this.start()
}
})
this.#eventManager.onPauseAll(fileId, () => {
this.pause()
})
this.#eventManager.onResumeAll(fileId, () => {
this.start()
})
this.#eventManager.onRetry(fileId, () => {
this.start()
})
this.#eventManager.onRetryAll(fileId, () => {
this.start()
})
}
#getChunkSize(fileSize: number): number {
if (this.#options.getChunkSize) {
return this.#options.getChunkSize({ size: fileSize })
}
return Math.ceil(fileSize / MAX_PARTS)
}
async start(): Promise<void> {
// Abort any pending operations (if not already aborted)
this.#abortController?.abort()
// Always create a fresh AbortController (also for resume)
this.#abortController = new AbortController()
try {
if (this.#uploadHasStarted) {
await this.#resumeUpload()
} else {
await this.#createUpload()
}
} catch (err) {
this.#onError(err as Error)
}
}
pause(): void {
this.#abortController?.abort()
}
/**
*
* @param opts - `abortInS3`: Whether to also abort the multipart upload in S3. Default: true. Set to false to keep the multipart upload in S3 active, allowing for manual cleanup later and preventing accidental data loss if the user later tries to resume the upload.
*/
abort(opts?: { abortInS3?: boolean }): void {
this.#abortController?.abort()
// Clean up event listeners
this.#eventManager.remove()
if (opts?.abortInS3 !== false && this.#uploadId) {
if (!this.#key) {
throw new Error('Missing S3 object key for aborting upload')
}
this.#options.s3Client
.abortMultipartUpload(this.#key, this.#uploadId)
.catch((abortErr) => {
this.#options.log?.(abortErr, 'warning')
})
}
}
async #createUpload(): Promise<void> {
this.#uploadHasStarted = true
if (this.#shouldUseMultipart) {
await this.#uploadMultipart()
} else {
await this.#uploadNonMultipart()
}
}
async #resumeUpload(): Promise<void> {
if (!this.#uploadId) {
await this.#createUpload()
return
}
if (!this.#key) {
throw new Error('Missing S3 object key for resuming upload')
}
const existingParts = await this.#options.s3Client.listParts(
this.#uploadId,
this.#key,
)
// Sync local state with S3 - mark already-uploaded parts
for (const part of existingParts) {
const chunkIndex = part.partNumber - 1
if (chunkIndex >= 0 && chunkIndex < this.#chunkState.length) {
this.#chunkState[chunkIndex].uploaded = this.#chunks[chunkIndex].size
this.#chunkState[chunkIndex].etag = part.etag
}
}
// Emit progress update to reflect already-uploaded parts
this.#onProgress()
await this.#uploadRemainingParts()
}
async #uploadNonMultipart(): Promise<void> {
const signal = this.#abortController?.signal
signal?.throwIfAborted()
const { location, key } = await this.#options.s3Client.putObject(
this.#options.key,
this.#data,
this.#options.file.type || 'application/octet-stream',
this.#options.metadata,
(bytesUploaded: number) => {
this.#chunkState[0].uploaded = bytesUploaded
this.#onProgress()
},
signal,
)
this.#onSuccess({
location,
key,
})
}
async #uploadMultipart(): Promise<void> {
const signal = this.#abortController?.signal
signal?.throwIfAborted()
const { uploadId, key } =
await this.#options.s3Client.createMultipartUpload(
this.#options.key,
this.#options.file.type || 'application/octet-stream',
this.#options.metadata,
)
if (key == null) {
throw new Error(
'S3 client did not return object key for multipart upload',
)
}
this.#key = key
// Persist resume state so Golden Retriever can restore it after page refresh
this.#options.uppy.setFileState(this.#options.file.id, {
s3Multipart: { uploadId, key },
})
this.#uploadId = uploadId
await this.#uploadRemainingParts()
}
async #uploadRemainingParts(): Promise<void> {
const signal = this.#abortController?.signal
for (let i = 0; i < this.#chunks.length; i++) {
signal?.throwIfAborted()
if (this.#chunkState[i].etag) continue // already uploaded
const chunk = this.#chunks[i]
const partNumber = i + 1
const chunkData = this.#data.slice(chunk.start, chunk.end)
const chunkIndex = i // Capture for closure (cannot use for-loop variable i directly in a closure)
if (this.#key == null) {
throw new Error('Missing S3 object key for uploading part')
}
const { etag } = await this.#options.s3Client.uploadPart(
this.#key,
this.#uploadId!,
chunkData,
partNumber,
(bytesUploaded: number) => {
this.#chunkState[chunkIndex].uploaded = bytesUploaded
this.#onProgress()
},
signal,
)
// after part finished uploading, update chunk state
this.#chunkState[i].uploaded = chunk.size
this.#chunkState[i].etag = etag
this.#onProgress()
if (this.#options.onPartComplete) {
this.#options.onPartComplete({
PartNumber: partNumber,
ETag: etag,
})
}
}
signal?.throwIfAborted()
const parts = this.#chunkState.flatMap((state, i) =>
state.etag ? [{ partNumber: i + 1, etag: state.etag }] : [],
)
if (this.#key == null) {
throw new Error('Missing S3 object key for completing multipart upload')
}
const { location, key } =
await this.#options.s3Client.completeMultipartUpload(
this.#key,
this.#uploadId!,
parts,
)
this.#onSuccess({
location,
key,
uploadId: this.#uploadId,
})
}
#onProgress(): void {
if (!this.#options.onProgress) return
const bytesUploaded = this.#chunkState.reduce(
(sum, state) => sum + state.uploaded,
0,
)
this.#options.onProgress(bytesUploaded, this.#data.size)
}
#onSuccess(result: UploadResult): void {
// If the upload was aborted (file removed mid-upload), the network request
// may still complete successfully. Don't emit success in this case since
// the file no longer exists in Uppy's state.
this.#eventManager.remove()
if (this.#abortController?.signal.aborted) {
return
}
// Clear persisted resume state — upload completed successfully.
this.#options.uppy.setFileState(this.#options.file.id, {
s3Multipart: undefined,
})
this.#options.onSuccess?.(result)
}
#onError(err: Error): void {
// ignore abort signals from intentional cancellation
if (err.name === 'AbortError') return
// If we intentionally aborted, don't report any subsequent errors
// (e.g., S3 returning 404 NoSuchUpload after we aborted the upload)
if (this.#abortController?.signal.aborted) return
// Clean up event listeners so this uploader doesn't become a "zombie"
// that reacts to future retry/pause/resume events after the error.
// Without this, each failed retry leaves an orphaned uploader that
// still listens for retry-all, causing duplicate uploads on the next
// successful retry.
this.#eventManager.remove()
// NOTE: We intentionally do NOT abort the multipart upload in S3 here.
// This allows the user to retry and resume from where they left off.
// The multipart upload is only aborted when the user cancels via the
// `abort()` method. By default `abort()` will also abort the multipart
// upload in S3 (abortInS3 = true). Pass { abortInS3: false } to keep the
// multipart upload in S3 so it can be cleaned up manually or resumed later.
this.#options.onError?.(err)
}
}

View file

@ -1,18 +1,28 @@
import type { RequestClient } from '@uppy/companion-client'
import { import {
BasePlugin, BasePlugin,
type DefinePluginOpts, type DefinePluginOpts,
EventManager,
type PluginOpts, type PluginOpts,
type Uppy, type Uppy,
} from '@uppy/core' } from '@uppy/core'
import type { Body, Meta, UppyFile } from '@uppy/utils' import type {
Body,
LocalUppyFile,
Meta,
RemoteUppyFile,
UppyFile,
} from '@uppy/utils'
import { import {
filterFilesToEmitUploadStarted, filterFilesToEmitUploadStarted,
filterFilesToUpload, filterFilesToUpload,
getAllowedMetaFields,
TaskQueue, TaskQueue,
} from '@uppy/utils' } from '@uppy/utils'
import packageJson from '../package.json' with { type: 'json' } import packageJson from '../package.json' with { type: 'json' }
import S3mini from './s3-client/S3.js' import S3Uploader, { type UploadResult } from './S3Uploader.js'
import S3Companion from './s3-client/CompanionS3.js'
import type S3Client from './s3-client/S3Client.js'
import S3mini from './s3-client/S3mini.js'
import type * as IT from './s3-client/types.js' import type * as IT from './s3-client/types.js'
// ============================================================================ // ============================================================================
@ -37,26 +47,7 @@ declare module '@uppy/core' {
} }
} }
export interface AwsS3Options<M extends Meta, B extends Body> export type AwsS3Options<M extends Meta, B extends Body> = PluginOpts & {
extends PluginOpts {
bucket: string
region?: string
endpoint?: string
/**
* Custom function to sign requests.
* Called with request details, should return signed headers.
* Alternative to using Companion endpoint.
*/
signRequest?: IT.signRequestFn
/**
* Function to retrieve temporary credentials for client-side signing.
* When provided, S3mini handles signing internally using SigV4.
* Alternative to signRequest or endpoint.
*/
getCredentials?: IT.getCredentialsFn
/** /**
* Whether to use multipart uploads. * Whether to use multipart uploads.
* - `true`: Always use multipart * - `true`: Always use multipart
@ -83,7 +74,34 @@ export interface AwsS3Options<M extends Meta, B extends Body>
* Default: `{randomId}-{filename}` * Default: `{randomId}-{filename}`
*/ */
generateObjectKey?: (file: UppyFile<M, B>) => string generateObjectKey?: (file: UppyFile<M, B>) => string
} } & (
| {
/** S3 upload endpoint */
s3Endpoint: string
/** AWS region, required for signing */
region?: string
/**
* Function to retrieve temporary credentials for client-side signing.
* When provided, S3mini handles signing internally using SigV4.
* Alternative to signRequest or endpoint.
*/
getCredentials: IT.GetCredentialsFn
}
| {
/**
* Custom function to sign requests.
* Called with request details, should return signed headers.
* Alternative to using Companion endpoint.
*/
signRequest: IT.SignRequestFn
}
| {
/** Companion URL if you want to use Companion for signing */
companionEndpoint: string
}
)
// ============================================================================ // ============================================================================
// Constants // Constants
@ -91,12 +109,6 @@ export interface AwsS3Options<M extends Meta, B extends Body>
const MB = 1024 * 1024 const MB = 1024 * 1024
/** Minimum chunk size required by S3 (5MB) */
const MIN_CHUNK_SIZE = 5 * MB
/** Maximum number of parts allowed by S3 */
const MAX_PARTS = 10000
const defaultOptions = { const defaultOptions = {
shouldUseMultipart: (file: UppyFile<any, any>) => (file.size || 0) > 100 * MB, shouldUseMultipart: (file: UppyFile<any, any>) => (file.size || 0) > 100 * MB,
allowedMetaFields: true, allowedMetaFields: true,
@ -108,381 +120,6 @@ const defaultOptions = {
// S3Uploader Types // S3Uploader Types
// ============================================================================ // ============================================================================
interface S3UploaderOptions<M extends Meta, B extends Body> {
uppy: Uppy<M, B>
s3Client: S3mini
file: UppyFile<M, B>
key: string
shouldUseMultipart?: boolean
getChunkSize?: (file: { size: number }) => number
onProgress?: (bytesUploaded: number, bytesTotal: number) => void
onPartComplete?: (part: { PartNumber: number; ETag: string }) => void
onSuccess?: (result: UploadResult) => void
onError?: (err: Error) => void
onAbort?: () => void
log?: (message: string | Error, type?: 'error' | 'warning') => void
}
interface UploadResult {
location: string
key: string
bucket?: string
uploadId?: string
}
interface Chunk {
index: number
start: number
end: number
size: number
}
interface ChunkState {
uploaded: number
etag?: string
}
/** Reason for pausing (not a real error) */
const pausingUploadReason = Symbol('pausing upload, not an actual error')
/** Type guard to check if an error is from pausing (not a real failure) */
const isPauseError = (err: unknown): boolean =>
err instanceof Error &&
(err as { cause?: unknown }).cause === pausingUploadReason
// ============================================================================
// S3Uploader Class
// ============================================================================
class S3Uploader<M extends Meta, B extends Body> {
readonly #s3Client: S3mini
readonly #file: UppyFile<M, B>
readonly #data: Blob
readonly #key: string
readonly #options: S3UploaderOptions<M, B>
readonly #eventManager: EventManager<M, B>
#chunks: Chunk[] = []
#chunkState: ChunkState[] = []
#shouldUseMultipart: boolean = false
#uploadId?: string
#uploadHasStarted: boolean = false
#abortController: AbortController = new AbortController()
constructor(data: Blob, options: S3UploaderOptions<M, B>) {
this.#s3Client = options.s3Client
this.#file = options.file
this.#data = data
this.#key = options.key
this.#options = options
this.#eventManager = new EventManager(options.uppy)
this.#initChunks()
this.#setupEvents()
}
#setupEvents(): void {
const fileId = this.#file.id
this.#eventManager.onFileRemove(fileId, () => {
this.abort()
this.#options.onAbort?.()
})
this.#eventManager.onCancelAll(fileId, () => {
this.abort()
this.#options.onAbort?.()
})
this.#eventManager.onFilePause(fileId, (isPaused) => {
if (isPaused) {
this.pause()
} else {
this.start()
}
})
this.#eventManager.onPauseAll(fileId, () => {
this.pause()
})
this.#eventManager.onResumeAll(fileId, () => {
this.start()
})
this.#eventManager.onRetry(fileId, () => {
this.start()
})
this.#eventManager.onRetryAll(fileId, () => {
this.start()
})
}
#initChunks(): void {
const fileSize = this.#data.size
// Step 1: Determine if we should use multipart
// - If explicitly set to boolean, use that
// - Otherwise, use multipart for files larger than MIN_CHUNK_SIZE (5MB)
if (typeof this.#options.shouldUseMultipart === 'boolean') {
this.#shouldUseMultipart = this.#options.shouldUseMultipart
} else {
this.#shouldUseMultipart = fileSize > MIN_CHUNK_SIZE
}
// Step 2: Force simple upload if file is too small for multipart
// (S3 requires minimum 5MB parts, except for the last part)
if (fileSize <= MIN_CHUNK_SIZE) {
this.#shouldUseMultipart = false
}
// Step 3: Create chunks based on upload strategy
if (this.#shouldUseMultipart) {
// Calculate chunk size: at least MIN_CHUNK_SIZE, but may be larger for huge files
let chunkSize = this.#getChunkSize(fileSize)
chunkSize = Math.max(chunkSize, MIN_CHUNK_SIZE)
// Ensure we don't exceed MAX_PARTS (S3 limit: 10,000 parts)
if (Math.ceil(fileSize / chunkSize) > MAX_PARTS) {
chunkSize = Math.ceil(fileSize / MAX_PARTS)
}
// Create chunk definitions
for (
let offset = 0, index = 0;
offset < fileSize;
offset += chunkSize, index++
) {
const end = Math.min(offset + chunkSize, fileSize)
this.#chunks.push({ index, start: offset, end, size: end - offset })
}
} else {
// Simple upload: single chunk for the entire file
this.#chunks = [{ index: 0, start: 0, end: fileSize, size: fileSize }]
}
this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
}
#getChunkSize(fileSize: number): number {
if (this.#options.getChunkSize) {
return this.#options.getChunkSize({ size: fileSize })
}
return Math.max(MIN_CHUNK_SIZE, Math.ceil(fileSize / MAX_PARTS))
}
start(): void {
if (this.#uploadHasStarted) {
// Abort any pending operations (if not already aborted)
if (!this.#abortController.signal.aborted) {
this.#abortController.abort(pausingUploadReason)
}
// Always create a fresh AbortController for resume
this.#abortController = new AbortController()
this.#resumeUpload()
} else {
this.#createUpload()
}
}
pause(): void {
this.#abortController.abort(pausingUploadReason)
this.#abortController = new AbortController()
}
abort(opts?: { abortOnS3?: boolean }): void {
this.#abortController.abort()
// Clean up event listeners
this.#eventManager.remove()
if (opts?.abortOnS3 !== false && this.#uploadId) {
this.#s3Client
.abortMultipartUpload(this.#key, this.#uploadId)
.catch((abortErr) => {
this.#options.log?.(abortErr, 'warning')
})
}
}
async #createUpload(): Promise<void> {
this.#uploadHasStarted = true
try {
if (this.#shouldUseMultipart) {
await this.#multipartUpload()
} else {
await this.#simpleUpload()
}
} catch (err) {
this.#onError(err as Error)
}
}
async #resumeUpload(): Promise<void> {
if (!this.#uploadId) {
await this.#createUpload()
return
}
try {
const existingParts = await this.#s3Client.listParts(
this.#uploadId,
this.#key,
)
// Sync local state with S3 - mark already-uploaded parts
for (const part of existingParts) {
const chunkIndex = part.partNumber - 1
if (chunkIndex >= 0 && chunkIndex < this.#chunkState.length) {
this.#chunkState[chunkIndex].uploaded = this.#chunks[chunkIndex].size
this.#chunkState[chunkIndex].etag = part.etag
}
}
// Emit progress update to reflect already-uploaded parts
this.#onProgress()
await this.#uploadRemainingParts()
} catch (err) {
this.#onError(err as Error)
}
}
async #simpleUpload(): Promise<void> {
const signal = this.#abortController.signal
if (signal.aborted) {
throw new Error('Upload aborted', { cause: signal.reason })
}
await this.#s3Client.putObject(
this.#key,
this.#data,
this.#file.type || 'application/octet-stream',
(bytesUploaded: number) => {
this.#chunkState[0].uploaded = bytesUploaded
this.#onProgress()
},
signal,
)
this.#onSuccess({
location: `${this.#s3Client.endpoint}/${this.#key}`,
key: this.#key,
})
}
async #multipartUpload(): Promise<void> {
const signal = this.#abortController.signal
if (signal.aborted) {
throw new Error('Upload aborted', { cause: signal.reason })
}
this.#uploadId = await this.#s3Client.getMultipartUploadId(
this.#key,
this.#file.type || 'application/octet-stream',
)
await this.#uploadRemainingParts()
}
async #uploadRemainingParts(): Promise<void> {
const signal = this.#abortController.signal
// Collect already-uploaded parts (from resume)
const parts = this.#chunkState
.map((state, i) =>
state.etag ? { partNumber: i + 1, etag: state.etag } : null,
)
.filter((p): p is NonNullable<typeof p> => p !== null)
for (let i = 0; i < this.#chunks.length; i++) {
if (signal.aborted) {
throw new Error('Upload aborted', { cause: signal.reason })
}
if (this.#chunkState[i].etag) continue
const chunk = this.#chunks[i]
const partNumber = i + 1
const chunkData = this.#data.slice(chunk.start, chunk.end)
const chunkIndex = i // Capture for closure
const part = await this.#s3Client.uploadPart(
this.#key,
this.#uploadId!,
chunkData,
partNumber,
(bytesUploaded: number) => {
this.#chunkState[chunkIndex].uploaded = bytesUploaded
this.#onProgress()
},
signal,
)
this.#chunkState[i].uploaded = chunk.size
this.#chunkState[i].etag = part.etag
parts.push({ partNumber: part.partNumber, etag: part.etag })
this.#onProgress()
if (this.#options.onPartComplete) {
this.#options.onPartComplete({
PartNumber: part.partNumber,
ETag: part.etag,
})
}
}
if (signal.aborted) {
throw new Error('Upload aborted', { cause: signal.reason })
}
const result = await this.#s3Client.completeMultipartUpload(
this.#key,
this.#uploadId!,
parts,
)
this.#onSuccess({
location: result.location,
key: result.key,
bucket: result.bucket,
uploadId: this.#uploadId,
})
}
#onProgress(): void {
if (!this.#options.onProgress) return
const bytesUploaded = this.#chunkState.reduce(
(sum, state) => sum + state.uploaded,
0,
)
this.#options.onProgress(bytesUploaded, this.#data.size)
}
#onSuccess(result: UploadResult): void {
// If the upload was aborted (file removed mid-upload), the network request
// may still complete successfully. Don't emit success in this case since
// the file no longer exists in Uppy's state.
if (this.#abortController.signal.aborted) {
this.#eventManager.remove()
return
}
this.#eventManager.remove()
this.#options.onSuccess?.(result)
}
#onError(err: Error): void {
if (isPauseError(err)) return
// Also ignore abort signals from intentional cancellation
if (err.name === 'AbortError') return
// If we intentionally aborted, don't report any subsequent errors
// (e.g., S3 returning 404 NoSuchUpload after we aborted the upload)
if (this.#abortController.signal.aborted) return
// NOTE: We intentionally do NOT abort the multipart upload on S3 here.
// This allows the user to retry and resume from where they left off.
// The multipart upload is only aborted when the user explicitly cancels
// (via the abort() method with abortOnS3: true).
this.#options.onError?.(err)
}
}
// ============================================================================
// Plugin Class
// ============================================================================
export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin< export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
DefinePluginOpts<AwsS3Options<M, B>, keyof typeof defaultOptions>, DefinePluginOpts<AwsS3Options<M, B>, keyof typeof defaultOptions>,
M, M,
@ -490,7 +127,7 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
> { > {
static VERSION = packageJson.version static VERSION = packageJson.version
#s3Client!: S3mini #s3Client!: S3Client
#queue!: TaskQueue #queue!: TaskQueue
#uploaders: Record<string, S3Uploader<M, B> | null> = {} #uploaders: Record<string, S3Uploader<M, B> | null> = {}
@ -522,10 +159,6 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
} }
} }
// --------------------------------------------------------------------------
// Resumable Uploads Capability
// --------------------------------------------------------------------------
#setResumableUploadsCapability = (value: boolean): void => { #setResumableUploadsCapability = (value: boolean): void => {
const { capabilities } = this.uppy.getState() const { capabilities } = this.uppy.getState()
this.uppy.setState({ this.uppy.setState({
@ -546,68 +179,42 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
// -------------------------------------------------------------------------- // --------------------------------------------------------------------------
#initS3Client(): void { #initS3Client(): void {
const { endpoint, signRequest, getCredentials, bucket, region } = this.opts if ('companionEndpoint' in this.opts) {
if (typeof this.opts.companionEndpoint !== 'string') {
throw new TypeError('companionEndpoint must be a string')
}
this.#s3Client = new S3Companion({
companionEndpoint: this.opts.companionEndpoint,
})
} else if ('getCredentials' in this.opts) {
if (typeof this.opts.s3Endpoint !== 'string') {
throw new TypeError('s3Endpoint must be a string')
}
if (typeof this.opts.getCredentials !== 'function') {
throw new TypeError('getCredentials must be a function')
}
if (this.opts.region != null && typeof this.opts.region !== 'string') {
throw new TypeError('region must be a string')
}
if (typeof bucket !== 'string' || bucket.trim() === '') {
throw new Error(
'AwsS3: `bucket` option is required and must be a non-empty string',
)
}
if (typeof region !== 'string' || region.trim() === '') {
throw new Error(
'AwsS3: `region` option is required and must be a non-empty string',
)
}
const bucketName = bucket.trim()
if (!signRequest && !getCredentials && !endpoint) {
throw new Error(
'AwsS3: `endpoint`, `signRequest`, or `getCredentials` is required',
)
}
const s3Endpoint = `https://${bucketName}.s3.${region}.amazonaws.com`
if (getCredentials) {
// Mode: Temporary credentials (client-side signing) // Mode: Temporary credentials (client-side signing)
this.#s3Client = new S3mini({ this.#s3Client = new S3mini({
endpoint: s3Endpoint, endpoint: this.opts.s3Endpoint,
getCredentials, getCredentials: this.opts.getCredentials,
region, region: this.opts.region,
}) })
} else if (signRequest) { } else if ('signRequest' in this.opts) {
if (typeof this.opts.signRequest !== 'function') {
throw new TypeError('signRequest must be a function')
}
// Mode: Custom signing function // Mode: Custom signing function
this.#s3Client = new S3mini({ this.#s3Client = new S3mini({
endpoint: s3Endpoint, signRequest: this.opts.signRequest,
signRequest,
region,
}) })
} else { } else {
// Mode: Companion signing (endpoint is guaranteed to be set here) throw new TypeError(
this.#s3Client = new S3mini({ 'One of options `companionEndpoint`, `signRequest`, or `getCredentials` is required',
endpoint: s3Endpoint, )
signRequest: this.#createCompanionSigner(endpoint!),
region,
})
}
}
/**
* Creates a signing function that calls Companion's /s3/sign endpoint.
*/
#createCompanionSigner(companionUrl: string): IT.signRequestFn {
return async (request) => {
const response = await fetch(`${companionUrl}/s3/sign`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(request),
})
if (!response.ok) {
throw new Error(`Failed to sign request: ${response.statusText}`)
}
return response.json()
} }
} }
@ -626,10 +233,9 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
const promises = filesToUpload.map((file) => { const promises = filesToUpload.map((file) => {
if (file.isRemote) { if (file.isRemote) {
// Remote files not yet supported in this minimal implementation // Remote uploads are queued internally by RequestClient.uploadRemoteFile()
return Promise.reject( // via getQueue(), so no outer queue wrapping is needed here.
new Error('Remote file uploads not yet supported'), return this.#uploadRemoteFile(file)
)
} }
return this.#queue.add(async () => { return this.#queue.add(async () => {
// File may have been removed while waiting in the queue. // File may have been removed while waiting in the queue.
@ -640,31 +246,32 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
if (!currentFile) { if (!currentFile) {
return return
} }
return this.#uploadLocalFile(currentFile) return this.#uploadLocalFile(currentFile as LocalUppyFile<M, B>) // assume it's still a local file since remote files aren't queued
}) })
}) })
await Promise.allSettled(promises) await Promise.allSettled(promises)
// After the upload batch is done, restore resumable uploads capability.
// It may have been set to false if there were remote files in this batch.
this.#setResumableUploadsCapability(true)
} }
// -------------------------------------------------------------------------- // --------------------------------------------------------------------------
// Local File Upload // Local File Upload
// -------------------------------------------------------------------------- // --------------------------------------------------------------------------
async #uploadLocalFile(file: UppyFile<M, B>): Promise<void> { async #uploadLocalFile(file: LocalUppyFile<M, B>): Promise<void> {
return new Promise((resolve, reject) => { try {
const data = file.data as Blob return await new Promise((resolve, reject) => {
const key = this.#generateKey(file) // Create uploader (events are wired internally).
const shouldMultipart = this.#shouldUseMultipart(file) // S3Uploader detects resume state from file.s3Multipart internally.
const uploader = new S3Uploader<M, B>({
try {
// Create uploader (events are wired internally)
const uploader = new S3Uploader<M, B>(data, {
uppy: this.uppy, uppy: this.uppy,
s3Client: this.#s3Client, s3Client: this.#s3Client,
file, file,
key, metadata: this.#getAllowedMeta(file),
shouldUseMultipart: shouldMultipart, key: this.#generateKey(file),
shouldUseMultipart: this.#shouldUseMultipart(file),
getChunkSize: this.opts.getChunkSize, getChunkSize: this.opts.getChunkSize,
log: (...args) => this.uppy.log(...args), log: (...args) => this.uppy.log(...args),
@ -686,22 +293,18 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
body: { body: {
location: result.location, location: result.location,
key: result.key, key: result.key,
bucket: result.bucket, } satisfies AwsBody as unknown as B,
} as unknown as B,
uploadURL: result.location, uploadURL: result.location,
}) })
this.#cleanup(file.id)
resolve() resolve()
}, },
onError: (err) => { onError: (err) => {
this.uppy.emit('upload-error', file, err) this.uppy.emit('upload-error', file, err)
this.#cleanup(file.id)
reject(err) reject(err)
}, },
onAbort: () => { onAbort: () => {
this.#cleanup(file.id)
resolve() // Normal completion, not an error resolve() // Normal completion, not an error
}, },
}) })
@ -711,12 +314,11 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
// Start the upload // Start the upload
uploader.start() uploader.start()
} catch (err) { })
// Cleanup on synchronous failure during setup } finally {
this.#cleanup(file.id) // Clean up uploader instance after upload completes or fails
reject(err) delete this.#uploaders[file.id]
} }
})
} }
#shouldUseMultipart(file: UppyFile<M, B>): boolean { #shouldUseMultipart(file: UppyFile<M, B>): boolean {
@ -728,21 +330,59 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
return shouldUseMultipart return shouldUseMultipart
} }
// Default: multipart for files > 100MB // Default: multipart for files > 100MB
return (file.size || 0) > 100 * MB return (file.size ?? 0) > 100 * MB
} }
#generateKey(file: UppyFile<M, B>): string { #generateKey(file: UppyFile<M, B>): string {
if (this.opts.generateObjectKey) { return this.opts.generateObjectKey?.(file) ?? file.name
return this.opts.generateObjectKey(file)
}
// Default: {randomId}-{filename}
const randomId = crypto.randomUUID()
return `${randomId}-${file.name}`
} }
#cleanup(fileId: string): void { // --------------------------------------------------------------------------
if (this.#uploaders[fileId]) { // Remote File Upload
delete this.#uploaders[fileId] // --------------------------------------------------------------------------
#getAllowedMeta(file: UppyFile<M, B>) {
const allowedMetaFields = getAllowedMetaFields(
this.opts.allowedMetaFields,
file.meta,
)
return Object.fromEntries(
allowedMetaFields.map((key) => [key, file.meta[key]]),
)
}
/**
* Builds the request body sent to Companion's provider get endpoint.
* Tells Companion to use its server-side S3 upload path.
*/
#getCompanionClientArgs(file: RemoteUppyFile<M, B>): Record<string, unknown> {
return {
...file.remote.body,
protocol: 's3-multipart',
size: file.data.size,
metadata: this.#getAllowedMeta(file),
}
}
async #uploadRemoteFile(file: RemoteUppyFile<M, B>): Promise<void> {
this.#setResumableUploadsCapability(false)
const controller = new AbortController()
const removedHandler = (removedFile: UppyFile<M, B>) => {
if (removedFile.id === file.id) controller.abort()
}
this.uppy.on('file-removed', removedHandler)
try {
await this.uppy
.getRequestClientForFile<RequestClient<M, B>>(file)
.uploadRemoteFile(file, this.#getCompanionClientArgs(file), {
signal: controller.signal,
getQueue: () => this.#queue,
})
} finally {
this.uppy.off('file-removed', removedHandler)
} }
} }
} }
@ -753,5 +393,4 @@ export type { AwsS3Options as AwsS3MultipartOptions }
export interface AwsBody extends Body { export interface AwsBody extends Body {
location: string location: string
key: string key: string
bucket?: string
} }

View file

@ -0,0 +1,212 @@
import * as C from './consts.js'
import S3Client from './S3Client.js'
import type * as IT from './types.js'
import * as U from './utils.js'
/**
* S3Companion is an S3Client that interacts with a Companion server to perform S3 operations.
*/
class S3Companion extends S3Client {
readonly companionEndpoint: string
constructor({
companionEndpoint,
requestAbortTimeout,
}: { companionEndpoint: string; requestAbortTimeout?: number | undefined }) {
super({ requestAbortTimeout })
this.companionEndpoint = companionEndpoint
}
private async _fetch(path: string, opts?: RequestInit) {
const response = await fetch(`${this.companionEndpoint}/s3${path}`, opts)
if (!response.ok) {
throw new Error(`Companion request failed: ${response.statusText}`)
}
return response
}
private async _request({
url,
method,
data,
onProgress,
signal,
contentType,
}: {
url: string
method: IT.HttpMethod
data?: XMLHttpRequestBodyInit
onProgress?: IT.OnProgressFn
signal?: AbortSignal
contentType?: string
}): Promise<XMLHttpRequest> {
// Wait for online before starting
await this.waitForOnline(signal)
return this.xhr({ url, method, data, onProgress, signal, contentType })
}
public override async putObject(
keyIn: string,
data: Blob | File,
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
metadata: Record<string, unknown>,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
) {
const response = await this._fetch(
`/params?${new URLSearchParams({ filename: keyIn, type: fileType, ...Object.fromEntries(Object.entries(metadata).map(([k, v]) => [`metadata[${k}]`, String(v)])) })}`,
)
const { url, fields }: { url: string; fields: Record<string, string> } =
await response.json()
const formData = new FormData()
Object.entries(fields).forEach(([key, value]) => formData.set(key, value))
formData.set('file', data)
const xhr = await this._request({
url,
method: 'POST',
data: formData,
onProgress,
signal,
})
return {
location: `${url}${fields.key}`, // `url` is returned by the signer as the bucket URL without any path, but trailing slash, so we need to add the key (path) to get the full object URL
etag: U.sanitizeETag(xhr.getResponseHeader('etag')),
key: fields.key,
}
}
public override async createMultipartUpload(
keyIn: string,
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
metadata: Record<string, unknown>,
) {
if (typeof fileType !== 'string') {
throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`)
}
const method = 'POST'
const response = await this._fetch('/multipart', {
method,
body: JSON.stringify({ filename: keyIn, metadata, type: fileType }),
headers: { 'content-type': 'application/json' },
})
const {
key,
uploadId,
}: { key?: string; uploadId?: string; bucket?: string } =
await response.json()
if (uploadId == null) throw new Error('No uploadId returned')
if (key == null) throw new Error('No key returned')
return { uploadId, key }
}
public override async uploadPart(
key: string,
uploadId: string,
data: XMLHttpRequestBodyInit,
partNumber: number,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
) {
const response = await this._fetch(
`/multipart/${encodeURIComponent(uploadId)}/${encodeURIComponent(partNumber)}?${new URLSearchParams({ key })}`,
{
method: 'GET',
},
)
const { url }: { url: string } = await response.json()
const xhr = await this._request({
url,
method: 'PUT',
data,
onProgress,
signal,
})
const etag = U.sanitizeETag(xhr.getResponseHeader('etag'))
if (etag == null) {
throw new Error(
`${C.ERROR_PREFIX}Missing ETag in uploadPart response headers`,
)
}
return { etag }
}
public override async listParts(
uploadId: string,
key: string,
): Promise<IT.UploadPart[]> {
if (!uploadId) {
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
}
const response = await this._fetch(
`/multipart/${encodeURIComponent(uploadId)}?${new URLSearchParams({ key })}`,
{
method: 'GET',
},
)
const parts: { PartNumber: string; ETag: string }[] = await response.json()
return parts.map((p) => ({
partNumber: parseInt(String(p.PartNumber), 10),
etag: String(p.ETag),
}))
}
public override async completeMultipartUpload(
key: string,
uploadId: string,
parts: Array<IT.UploadPart>,
) {
const response = await this._fetch(
`/multipart/${encodeURIComponent(uploadId)}/complete?${new URLSearchParams({ key })}`,
{
method: 'POST',
body: JSON.stringify({
parts: parts.map((part) => ({
PartNumber: part.partNumber,
ETag: part.etag,
})),
}),
headers: { 'content-type': 'application/json' },
},
)
const {
location,
bucket,
key: resultKey,
}: {
location: string
bucket?: string
key: string
} = await response.json()
return {
location,
bucket,
key: resultKey,
}
}
public override async abortMultipartUpload(key: string, uploadId: string) {
await this._fetch(
`/multipart/${encodeURIComponent(uploadId)}?${new URLSearchParams({ key })}`,
{
method: 'DELETE',
},
)
}
}
export default S3Companion

View file

@ -1,866 +0,0 @@
/**
* Taken from https://github.com/good-lly/s3mini.git, by Jølly Good, under MIT license.
* Modified to make it work with Uppy.
*/
import { fetcher, NetworkError } from '@uppy/utils'
import * as C from './consts.js'
import { createSigV4Signer } from './signer.js'
import type * as IT from './types.js'
import * as U from './utils.js'
/**
* S3 client for browser-compatible interaction with S3-compatible storage.
* Supports simple uploads, multipart uploads, and object deletion.
*
* @example
* // Option 1: With signRequest callback
* const s3 = new S3mini({
* endpoint: 'https://s3.amazonaws.com/my-bucket',
* signRequest: async ({ method, url, headers }) => {
* return await fetchSignedHeaders(method, url, headers);
* },
* });
*
* // Option 2: With getCredentials callback (client-side signing)
* const s3 = new S3mini({
* endpoint: 'https://s3.amazonaws.com/my-bucket',
* getCredentials: async () => {
* const resp = await fetch('/api/s3/credentials');
* return resp.json(); // { credentials, bucket, region }
* },
* });
*
* await s3.putObject('file.txt', 'Hello, World!');
*/
class S3mini {
readonly endpoint: URL
readonly region: string
readonly requestSizeInBytes: number
readonly requestAbortTimeout?: number
private readonly getCredentials?: IT.getCredentialsFn
private cachedCredentials?: IT.CredentialsResponse
private cachedCredentialsPromise?: Promise<IT.CredentialsResponse>
private signRequest!: IT.signRequestFn
constructor({
endpoint,
signRequest,
getCredentials,
region = 'auto',
requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES,
requestAbortTimeout = undefined,
}: IT.S3Config) {
this._validateConstructorParams(endpoint, signRequest, getCredentials)
this.endpoint = new URL(this._ensureValidUrl(endpoint))
this.region = region
this.requestSizeInBytes = requestSizeInBytes
this.requestAbortTimeout = requestAbortTimeout
if (signRequest) {
this.signRequest = signRequest
} else if (getCredentials) {
this.getCredentials = getCredentials
this.signRequest = this._createCredentialBasedSigner()
}
}
/** Creates a presigner that fetches/caches credentials and generates pre-signed URLs. */
private _createCredentialBasedSigner(): IT.signRequestFn {
return async (
request: IT.presignableRequest,
): Promise<IT.presignedResponse> => {
const creds = await this._getCachedCredentials()
const presigner = createSigV4Signer({
accessKeyId: creds.credentials.accessKeyId,
secretAccessKey: creds.credentials.secretAccessKey,
sessionToken: creds.credentials.sessionToken,
region: creds.region || this.region,
endpoint: this.endpoint.toString(),
})
return presigner(request)
}
}
/** Gets cached credentials or fetches new ones. */
private async _getCachedCredentials(): Promise<IT.CredentialsResponse> {
// Return Cached Credentials if available
if (this.cachedCredentials != null) {
return this.cachedCredentials
}
// Cache the promise so concurrent calls wait for the same fetch
if (this.cachedCredentialsPromise == null) {
this.cachedCredentialsPromise = this.getCredentials!({})
.then((creds) => {
this.cachedCredentials = creds
return creds
})
.finally(() => {
// Clear promise cache after resolution to allow future retries
this.cachedCredentialsPromise = undefined
})
}
return this.cachedCredentialsPromise
}
private _validateConstructorParams(
endpoint: string,
signRequest?: IT.signRequestFn,
getCredentials?: IT.getCredentialsFn,
): void {
if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
throw new TypeError(C.ERROR_ENDPOINT_REQUIRED)
}
if (!signRequest && !getCredentials) {
throw new TypeError(
'Either signRequest or getCredentials must be provided',
)
}
if (signRequest && typeof signRequest !== 'function') {
throw new TypeError('signRequest must be a function')
}
if (getCredentials && typeof getCredentials !== 'function') {
throw new TypeError('getCredentials must be a function')
}
}
private _ensureValidUrl(raw: string): string {
const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}`
try {
new URL(candidate)
// Find the last non-slash character
let endIndex = candidate.length
while (endIndex > 0 && candidate[endIndex - 1] === '/') {
endIndex--
}
return endIndex === candidate.length
? candidate
: candidate.substring(0, endIndex)
} catch {
const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"`
throw new TypeError(msg)
}
}
private _checkKey(key: string): void {
if (typeof key !== 'string' || key.trim().length === 0) {
throw new TypeError(C.ERROR_KEY_REQUIRED)
}
}
private _checkOpts(opts: object): void {
if (typeof opts !== 'object') {
throw new TypeError(`${C.ERROR_PREFIX}opts must be an object`)
}
}
private _filterIfHeaders(opts: Record<string, unknown>): {
filteredOpts: Record<string, string>
conditionalHeaders: Record<string, unknown>
} {
const filteredOpts: Record<string, string> = {}
const conditionalHeaders: Record<string, unknown> = {}
for (const [key, value] of Object.entries(opts)) {
if (
C.IFHEADERS.has(
key.toLowerCase() as typeof C.IFHEADERS extends Set<infer T>
? T
: never,
)
) {
conditionalHeaders[key] = value
} else {
filteredOpts[key] = value as string
}
}
return { filteredOpts, conditionalHeaders }
}
private _validateData(data: unknown): BodyInit {
if (
typeof data === 'string' ||
data instanceof ArrayBuffer ||
ArrayBuffer.isView(data) ||
data instanceof Blob
) {
return data as BodyInit
}
throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED)
}
private _validateUploadPartParams(
key: string,
uploadId: string,
partNumber: number,
): void {
this._checkKey(key)
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
}
if (!Number.isInteger(partNumber) || partNumber <= 0) {
throw new TypeError(
`${C.ERROR_PREFIX}partNumber must be a positive integer`,
)
}
}
private async _presignedRequest(
method: IT.HttpMethod, // 'GET' | 'HEAD' | 'PUT' | 'POST' | 'DELETE'
key: string, // '' allowed for bucketlevel ops
{
uploadId,
partNumber,
body = '',
contentType,
tolerated = [],
}: {
uploadId?: string
partNumber?: number
body?: BodyInit
contentType?: string
tolerated?: number[]
} = {},
): Promise<Response> {
// Get pre-signed URL from callback
const { url } = await this.signRequest({
method,
key,
uploadId,
partNumber,
contentType,
})
// Build request headers
const requestHeaders: Record<string, string> = contentType
? { 'Content-Type': contentType }
: {}
try {
return await this._sendRequest(
url,
method,
requestHeaders,
body,
tolerated,
)
} catch (err) {
// If expired token error and using getCredentials, clear cache and retry once
if (
this.getCredentials &&
err instanceof U.S3ServiceError &&
err.code &&
['ExpiredToken', 'InvalidAccessKeyId'].includes(err.code)
) {
// Clear cache
this.clearCachedCredentials()
// Retry with fresh credentials
const fresh = await this.signRequest({
method,
key,
uploadId,
partNumber,
})
return this._sendRequest(
fresh.url,
method,
contentType ? { 'Content-Type': contentType } : {},
body,
tolerated,
)
}
throw err
}
}
/**
* Uploads an object to S3 using XHR for progress tracking.
* @param key - Object key
* @param data - Data to upload (Blob, ArrayBuffer, Uint8Array, or string)
* @param fileType - Content type
* @param onProgress - Optional progress callback
* @param signal - Optional abort signal
*/
public async putObject(
key: string,
data: IT.BinaryData | string,
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
): Promise<IT.PutObjectResult> {
this._checkKey(key)
const attemptUpload = async (): Promise<IT.PutObjectResult> => {
const { url } = await this.signRequest({ method: 'PUT', key })
return this._xhrUpload(url, data, onProgress, signal, fileType)
}
try {
return await attemptUpload()
} catch (err) {
if (this._isExpiredTokenError(err)) {
this.clearCachedCredentials()
return attemptUpload()
}
throw err
}
}
/** Initiates a multipart upload and returns the upload ID. */
public async getMultipartUploadId(
key: string,
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
): Promise<string> {
this._checkKey(key)
if (typeof fileType !== 'string') {
throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`)
}
const res = await this._presignedRequest('POST', key, {
contentType: fileType,
})
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
if (parsed && typeof parsed === 'object') {
// Check for both cases of InitiateMultipartUploadResult
const uploadResult =
(parsed.initiateMultipartUploadResult as Record<string, unknown>) ||
(parsed.InitiateMultipartUploadResult as Record<string, unknown>)
if (uploadResult && typeof uploadResult === 'object') {
// Check for both cases of uploadId
const uploadId = uploadResult.uploadId || uploadResult.UploadId
if (uploadId && typeof uploadId === 'string') {
return uploadId
}
}
}
throw new Error(
`${C.ERROR_PREFIX}Failed to create multipart upload: ${JSON.stringify(
parsed,
)}`,
)
}
public async uploadPart(
key: string,
uploadId: string,
data: IT.BinaryData | string,
partNumber: number,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
): Promise<IT.UploadPart> {
this._validateUploadPartParams(key, uploadId, partNumber)
const attemptUpload = async (): Promise<IT.UploadPart> => {
const { url } = await this.signRequest({
method: 'PUT',
key,
uploadId,
partNumber,
})
const result = await this._xhrUpload(url, data, onProgress, signal)
return {
partNumber,
etag: result.headers.get('etag')
? U.sanitizeETag(result.headers.get('etag')!)
: '',
}
}
try {
return await attemptUpload()
} catch (err) {
if (this._isExpiredTokenError(err)) {
this.clearCachedCredentials()
return attemptUpload()
}
throw err
}
}
/**
* Helper to check if we're currently offline in a browser context.
*/
private _isOffline(): boolean {
return typeof navigator !== 'undefined' && navigator.onLine === false
}
/** Checks if error is an expired/invalid token that can be retried with fresh credentials */
private _isExpiredTokenError(err: unknown): boolean {
return (
this.getCredentials != null &&
err instanceof U.S3ServiceError &&
err.code != null &&
['ExpiredToken', 'InvalidAccessKeyId'].includes(err.code)
)
}
/**
* Core XHR upload implementation using @uppy/utils/fetcher.
*
* Features:
* - Automatic retry with exponential backoff (3 attempts)
* - Offline detection with automatic resume on reconnect
* - Stall detection via ProgressTimeout
*/
private async _xhrUpload(
url: string,
data: IT.BinaryData | string,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
contentType?: string,
): Promise<IT.PutObjectResult> {
// Wait for online before starting
if (this._isOffline()) {
await this._waitForOnline(signal)
}
try {
const xhr = await fetcher(url, {
method: 'PUT',
// XHR natively supports ArrayBuffer, Uint8Array, Blob, and string
body: data as XMLHttpRequestBodyInit,
headers: contentType ? { 'Content-Type': contentType } : {},
signal,
timeout: this.requestAbortTimeout || 30_000,
retries: 3,
/**
* Retry logic:
* - Retries: 5xx server errors, 429 rate limiting
* - Skips: 4xx client errors (except 429), offline (handled separately)
*/
shouldRetry: (xhr) => {
// If offline, don't retry via fetcher - our handler will resume
if (this._isOffline()) return false
// Don't retry client errors (except 429 rate limit)
if (xhr.status >= 400 && xhr.status < 500 && xhr.status !== 429) {
return false
}
return true
},
onUploadProgress: (event) => {
if (event.lengthComputable && onProgress) {
onProgress(event.loaded, event.total)
}
},
onTimeout: (timeout) => {
// Log stall detection - upload will continue but may be slow
console.warn(
`[S3mini] Upload stalled - no progress for ${Math.ceil(timeout / 1000)}s`,
)
},
})
// Return Response-like object for test compatibility
return {
status: xhr.status,
ok: xhr.status >= 200 && xhr.status < 300,
headers: {
get: (name: string) => xhr.getResponseHeader(name),
},
}
} catch (err: unknown) {
return this._handleUploadError(
err,
url,
data,
onProgress,
signal,
contentType,
)
}
}
/**
* Handles errors from _xhrUpload, including offline recovery and error mapping.
*/
private async _handleUploadError(
err: unknown,
url: string,
data: IT.BinaryData | string,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
contentType?: string,
): Promise<IT.PutObjectResult> {
// Offline during request - wait and retry
if (this._isOffline()) {
await this._waitForOnline(signal)
// Check if aborted while waiting for online
if (signal?.aborted) {
throw new DOMException('Upload aborted', 'AbortError')
}
return this._xhrUpload(url, data, onProgress, signal, contentType)
}
// Abort errors pass through
if (err instanceof DOMException && err.name === 'AbortError') {
throw err
}
// Network errors (from fetcher's NetworkError)
if (
err instanceof NetworkError ||
(err instanceof Error && 'isNetworkError' in err && err.isNetworkError)
) {
const xhr = 'request' in err ? (err as NetworkError).request : null
if (!xhr?.status) {
throw new U.S3NetworkError(
'Network error during upload',
'NETWORK',
err,
)
}
// HTTP errors (non-2xx responses from NetworkError)
const headerCode = xhr.getResponseHeader('x-amz-error-code')
const parsedBody = this._parseErrorXml(
(name: string) => xhr.getResponseHeader(name),
xhr.responseText,
)
const serviceCode = headerCode ?? parsedBody.svcCode
throw new U.S3ServiceError(
`S3 returned ${xhr.status}${serviceCode ? ` ${serviceCode}` : ''}`,
xhr.status,
serviceCode,
xhr.responseText,
)
}
// Handle errors with attached XHR (from onAfterResponse throws)
if (
err instanceof Error &&
'request' in err &&
err.request instanceof XMLHttpRequest
) {
const xhr = err.request
if (!xhr.status) {
throw new U.S3NetworkError(
'Network error during upload',
'NETWORK',
err,
)
}
const headerCode = xhr.getResponseHeader('x-amz-error-code')
const parsedBody = this._parseErrorXml(
(name: string) => xhr.getResponseHeader(name),
xhr.responseText,
)
const serviceCode = headerCode ?? parsedBody.svcCode
throw new U.S3ServiceError(
`S3 returned ${xhr.status}${serviceCode ? ` ${serviceCode}` : ''}`,
xhr.status,
serviceCode,
xhr.responseText,
)
}
throw err
}
/** Lists uploaded parts for a multipart upload. */
public async listParts(
uploadId: string,
key: string,
): Promise<IT.UploadPart[]> {
this._checkKey(key)
if (!uploadId) {
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
}
const res = await this._presignedRequest('GET', key, {
uploadId,
})
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
const result = (parsed.listPartsResult ||
parsed.ListPartsResult ||
parsed) as Record<string, unknown>
if (result && typeof result === 'object') {
const parts = result.Part || result.part || []
const partsArray = Array.isArray(parts) ? parts : [parts]
return partsArray
.filter(
(p): p is Record<string, unknown> =>
p != null &&
typeof p === 'object' &&
'PartNumber' in p &&
'ETag' in p,
)
.map((p) => ({
partNumber: parseInt(String(p.PartNumber), 10),
etag: U.sanitizeETag(String(p.ETag)),
}))
}
return []
}
/** Completes a multipart upload by combining all uploaded parts. */
public async completeMultipartUpload(
key: string,
uploadId: string,
parts: Array<IT.UploadPart>,
): Promise<IT.CompleteMultipartUploadResult> {
const xmlBody = this._buildCompleteMultipartUploadXml(parts)
const res = await this._presignedRequest('POST', key, {
uploadId,
body: xmlBody,
contentType: C.XML_CONTENT_TYPE,
})
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
if (parsed && typeof parsed === 'object') {
// Check for both cases (camelCase from our parser, PascalCase from S3)
const result =
parsed.completeMultipartUploadResult ||
parsed.CompleteMultipartUploadResult ||
parsed
if (result && typeof result === 'object') {
const r = result as Record<string, unknown>
// S3 returns PascalCase (Location, Bucket, Key, ETag).
// Normalize to lowercase for our type interface.
const resultLocation = (r.Location || r.location) as string | undefined
const resultBucket = (r.Bucket || r.bucket) as string | undefined
const resultKey = (r.Key || r.key) as string | undefined
const rawEtag = (r.ETag || r.eTag || r.etag) as string | undefined
if (!resultLocation || !resultKey) {
throw new Error(
`${C.ERROR_PREFIX}CompleteMultipartUpload response missing Location or Key: ${JSON.stringify(r)}`,
)
}
const etag = rawEtag ? U.sanitizeETag(rawEtag) : ''
return {
location: resultLocation,
bucket: resultBucket ?? '',
key: resultKey,
etag,
} satisfies IT.CompleteMultipartUploadResult
}
}
throw new Error(
`${C.ERROR_PREFIX}Failed to complete multipart upload: ${JSON.stringify(
parsed,
)}`,
)
}
/** Aborts a multipart upload and removes all uploaded parts. */
public async abortMultipartUpload(
key: string,
uploadId: string,
): Promise<object> {
this._checkKey(key)
if (!uploadId) {
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
}
const res = await this._presignedRequest('DELETE', key, {
uploadId,
})
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
if (
parsed &&
'error' in parsed &&
typeof parsed.error === 'object' &&
parsed.error !== null &&
'message' in parsed.error
) {
throw new Error(
`${C.ERROR_PREFIX}Failed to abort multipart upload: ${String(
parsed.error.message,
)}`,
)
}
return { status: 'Aborted', key, uploadId, response: parsed }
}
private _buildCompleteMultipartUploadXml(
parts: Array<IT.UploadPart>,
): string {
let xml = '<CompleteMultipartUpload>'
for (const part of parts) {
xml += `<Part><PartNumber>${part.partNumber}</PartNumber><ETag>${part.etag}</ETag></Part>`
}
xml += '</CompleteMultipartUpload>'
return xml
}
/** Deletes an object from the bucket. Returns true on success. */
public async deleteObject(key: string): Promise<boolean> {
const res = await this._presignedRequest('DELETE', key, {
tolerated: [200, 204],
})
return res.status === 200 || res.status === 204
}
/**
* Clears cached credentials.
* Call this method when you need to force a credential refresh on the next request.
*/
public clearCachedCredentials(): void {
this.cachedCredentials = undefined
this.cachedCredentialsPromise = undefined
}
/**
* Waits for the browser to come back online.
* Returns a promise that resolves when the 'online' event fires,
* or rejects if the abort signal is triggered.
*/
private _waitForOnline(signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
// Already online or not in browser
if (
typeof navigator === 'undefined' ||
navigator.onLine === true ||
navigator.onLine === undefined
) {
resolve()
return
}
// Already aborted
if (signal?.aborted) {
reject(new DOMException('Upload aborted', 'AbortError'))
return
}
const cleanup = () => {
window.removeEventListener('online', onOnline)
signal?.removeEventListener('abort', onAbort)
}
const onOnline = () => {
cleanup()
resolve()
}
const onAbort = () => {
cleanup()
reject(new DOMException('Upload aborted', 'AbortError'))
}
window.addEventListener('online', onOnline)
signal?.addEventListener('abort', onAbort)
})
}
private async _sendRequest(
url: string,
method: IT.HttpMethod,
headers: Record<string, string>,
body?: BodyInit,
toleratedStatusCodes: number[] = [],
): Promise<Response> {
// Wait for online if currently offline
if (this._isOffline()) {
await this._waitForOnline()
}
try {
const res = await fetch(url, {
method,
headers,
body: ['GET', 'HEAD'].includes(method) ? undefined : body,
signal: this.requestAbortTimeout
? AbortSignal.timeout(this.requestAbortTimeout)
: undefined,
})
if (res.ok || toleratedStatusCodes.includes(res.status)) {
return res
}
await this._handleErrorResponse(res)
return res
} catch (err: unknown) {
// Check if we're offline - if so, wait and retry
if (this._isOffline()) {
await this._waitForOnline()
// Retry the request
return this._sendRequest(
url,
method,
headers,
body,
toleratedStatusCodes,
)
}
const code = U.extractErrCode(err)
if (
code &&
['ENOTFOUND', 'EAI_AGAIN', 'ETIMEDOUT', 'ECONNREFUSED'].includes(code)
) {
throw new U.S3NetworkError(`S3 network error: ${code}`, code, err)
}
throw err
}
}
private _parseErrorXml(
getHeader: (name: string) => string | null,
body: string,
): { svcCode?: string; errorMessage?: string } {
if (getHeader('content-type') !== 'application/xml') {
return {}
}
const parsedBody = U.parseXml(body)
if (
!parsedBody ||
typeof parsedBody !== 'object' ||
!('Error' in parsedBody) ||
!parsedBody.Error ||
typeof parsedBody.Error !== 'object'
) {
return {}
}
const error = parsedBody.Error
return {
svcCode:
'Code' in error && typeof error.Code === 'string'
? error.Code
: undefined,
errorMessage:
'Message' in error && typeof error.Message === 'string'
? error.Message
: undefined,
}
}
private async _handleErrorResponse(res: Response): Promise<void> {
const errorBody = await res.text()
const parsedErrorBody = this._parseErrorXml(
(n) => res.headers.get(n),
errorBody,
)
const svcCode =
res.headers.get('x-amz-error-code') ??
parsedErrorBody.svcCode ??
'Unknown'
throw new U.S3ServiceError(
`S3 returned ${res.status} ${svcCode}`,
res.status,
svcCode,
errorBody,
)
}
}
export { S3mini }
export default S3mini

View file

@ -0,0 +1,197 @@
import { fetcher } from '@uppy/utils'
import * as C from './consts.js'
import type * as IT from './types.js'
class S3Client {
readonly requestAbortTimeout?: number
constructor({
requestAbortTimeout,
...rest
}: { requestAbortTimeout?: number | undefined }) {
this.requestAbortTimeout = requestAbortTimeout
}
/**
* Helper to check if we're currently offline in a browser context.
*/
protected isOffline(): boolean {
return typeof navigator !== 'undefined' && navigator.onLine === false
}
/**
* Waits for the browser to come back online.
* Returns a promise that resolves when the 'online' event fires,
* or rejects if the abort signal is triggered.
*/
protected waitForOnline(signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (!this.isOffline()) {
resolve()
return
}
// Already online or not in browser
if (
typeof navigator === 'undefined' ||
navigator.onLine === true ||
navigator.onLine === undefined
) {
resolve()
return
}
// Already aborted
if (signal?.aborted) {
reject(new DOMException('Upload aborted', 'AbortError'))
return
}
const cleanup = () => {
window.removeEventListener('online', onOnline)
signal?.removeEventListener('abort', onAbort)
}
const onOnline = () => {
cleanup()
resolve()
}
const onAbort = () => {
cleanup()
reject(new DOMException('Upload aborted', 'AbortError'))
}
window.addEventListener('online', onOnline)
signal?.addEventListener('abort', onAbort)
})
}
protected async xhr({
url,
method,
data,
onProgress,
signal,
contentType,
}: {
url: string
method: IT.HttpMethod
data?: XMLHttpRequestBodyInit
onProgress?: IT.OnProgressFn
signal?: AbortSignal
contentType?: string
}) {
// Check if aborted while waiting for online
if (signal?.aborted) {
throw new DOMException('Request aborted', 'AbortError')
}
return fetcher(url, {
method,
// XHR natively supports ArrayBuffer, Uint8Array, Blob, and string
body: ['GET', 'HEAD'].includes(method) ? undefined : data,
headers: contentType ? { 'Content-Type': contentType } : {},
signal,
timeout: this.requestAbortTimeout,
retries: 3,
/**
* Retry logic:
* - Retries: 5xx server errors, 429 rate limiting
* - Skips: 4xx client errors (except 429), offline (handled separately)
*/
shouldRetry: (xhr) => {
// If offline, don't retry via fetcher - our handler will resume
if (this.isOffline()) return false
// Don't retry client errors (except 429 rate limit)
if (xhr.status >= 400 && xhr.status < 500 && xhr.status !== 429) {
return false
}
return true
},
onUploadProgress: (event) => {
if (event.lengthComputable && onProgress) {
onProgress(event.loaded, event.total)
}
},
onTimeout: (timeout) => {
// Log stall detection - upload will continue but may be slow
console.warn(
`[S3mini] Upload stalled - no progress for ${Math.ceil(timeout / 1000)}s`,
)
},
})
}
public async putObject(
key: string,
data: XMLHttpRequestBodyInit,
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
metadata: Record<string, unknown>,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
): Promise<{
location: string
key: string
etag: string | undefined
}> {
throw new Error('Not implemented')
}
public async createMultipartUpload(
key: string,
fileType?: string,
// @ts-expect-error unused
metadata: Record<string, unknown>,
): Promise<{
uploadId: string
key: string
}> {
throw new Error('Not implemented')
}
public async uploadPart(
key: string,
uploadId: string,
data: XMLHttpRequestBodyInit,
partNumber: number,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
): Promise<{
etag: string
}> {
throw new Error('Not implemented')
}
public async listParts(
uploadId: string,
key: string,
): Promise<IT.UploadPart[]> {
throw new Error('Not implemented')
}
public async completeMultipartUpload(
key: string,
uploadId: string,
parts: IT.UploadPart[],
): Promise<{
location: string
bucket: string | undefined
key: string
etag?: string | undefined
}> {
throw new Error('Not implemented')
}
public async abortMultipartUpload(
key: string,
uploadId: string,
): Promise<void> {
throw new Error('Not implemented')
}
public async deleteObject(key: string): Promise<void> {
throw new Error('Not implemented')
}
}
export default S3Client

View file

@ -0,0 +1,567 @@
/**
* Taken from https://github.com/good-lly/s3mini.git, by Jølly Good, under MIT license.
* Modified to make it work with Uppy.
*/
import * as C from './consts.js'
import S3Client from './S3Client.js'
import { createSigV4Signer } from './signer.js'
import type * as IT from './types.js'
import * as U from './utils.js'
/**
* S3 client for browser-compatible interaction with S3-compatible storage.
* Supports simple uploads, multipart uploads, and object deletion.
*
* @example
* // Option 1: With signRequest callback (no endpoint needed)
* const s3 = new S3mini({
* signRequest: async ({ method, key, uploadId, partNumber }) => {
* const resp = await fetch('/api/s3/sign', {
* method: 'POST',
* body: JSON.stringify({ method, key, uploadId, partNumber }),
* });
* return resp.json(); // { url }
* },
* });
*
* // Option 2: With getCredentials callback (endpoint required for client-side signing)
* const s3 = new S3mini({
* endpoint: 'https://s3.us-east-1.amazonaws.com/my-bucket',
* getCredentials: async () => {
* const resp = await fetch('/api/s3/credentials');
* return resp.json(); // { credentials, region }
* },
* });
*
* await s3.putObject('file.txt', 'Hello, World!');
*/
class S3mini extends S3Client {
readonly endpoint?: URL
readonly region: string
readonly requestSizeInBytes: number
private readonly getCredentials?: IT.GetCredentialsFn
private cachedCredentials?: IT.CredentialsResponse
private cachedCredentialsPromise?: Promise<IT.CredentialsResponse>
private signRequest!: IT.SignRequestFn
constructor({
region = 'auto',
requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES,
requestAbortTimeout,
...rest
}: IT.S3Config) {
super({ requestAbortTimeout })
if ('signRequest' in rest) {
const { signRequest } = rest
if (!signRequest) {
throw new TypeError(
'Either signRequest or getCredentials must be provided',
)
}
if (signRequest && typeof signRequest !== 'function') {
throw new TypeError('signRequest must be a function')
}
this.signRequest = signRequest
} else if ('getCredentials' in rest) {
const { getCredentials, endpoint } = rest
if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
throw new TypeError(C.ERROR_ENDPOINT_REQUIRED)
}
if (getCredentials && typeof getCredentials !== 'function') {
throw new TypeError('getCredentials must be a function')
}
this.endpoint = new URL(this._ensureValidUrl(endpoint))
this.getCredentials = getCredentials
this.signRequest = this._createCredentialBasedSigner()
} else {
throw new TypeError(
'Either signRequest or getCredentials must be provided',
)
}
this.region = region
this.requestSizeInBytes = requestSizeInBytes
}
/** Creates a presigner that fetches/caches credentials and generates pre-signed URLs. */
private _createCredentialBasedSigner(): IT.SignRequestFn {
return async (
request: IT.PresignableRequest,
): Promise<IT.presignedResponse> => {
const creds = await this._getCachedCredentials()
if (this.endpoint == null) {
throw new Error('Endpoint is required for credential-based signing')
}
const presigner = createSigV4Signer({
accessKeyId: creds.credentials.accessKeyId,
secretAccessKey: creds.credentials.secretAccessKey,
sessionToken: creds.credentials.sessionToken,
region: creds.region || this.region,
endpoint: this.endpoint.toString(),
})
return presigner(request)
}
}
/** Gets cached credentials or fetches new ones. */
private async _getCachedCredentials(): Promise<IT.CredentialsResponse> {
// Return Cached Credentials if available
if (this.cachedCredentials != null) {
return this.cachedCredentials
}
// Cache the promise so concurrent calls wait for the same fetch
if (this.cachedCredentialsPromise == null) {
this.cachedCredentialsPromise = (async () => {
try {
const creds = await this.getCredentials!({})
this.cachedCredentials = creds
return creds
} finally {
// Clear promise cache after resolution to allow future retries
this.cachedCredentialsPromise = undefined
}
})()
}
return this.cachedCredentialsPromise
}
private _ensureValidUrl(raw: string): string {
const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}`
try {
new URL(candidate)
// Find the last non-slash character
let endIndex = candidate.length
while (endIndex > 0 && candidate[endIndex - 1] === '/') {
endIndex--
}
return endIndex === candidate.length
? candidate
: candidate.substring(0, endIndex)
} catch {
const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"`
throw new TypeError(msg)
}
}
private _checkKey(key: string): void {
if (typeof key !== 'string' || key.trim().length === 0) {
throw new TypeError(C.ERROR_KEY_REQUIRED)
}
}
private _validateUploadPartParams(
key: string,
uploadId: string,
partNumber: number,
): void {
this._checkKey(key)
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
}
if (!Number.isInteger(partNumber) || partNumber <= 0) {
throw new TypeError(
`${C.ERROR_PREFIX}partNumber must be a positive integer`,
)
}
}
/**
* Uploads an object to S3 using XHR for progress tracking.
* @param key - Object key
* @param data - Data to upload (Blob, ArrayBuffer, Uint8Array, or string)
* @param fileType - Content type
* @param onProgress - Optional progress callback
* @param signal - Optional abort signal
*/
public override async putObject(
key: string,
data: XMLHttpRequestBodyInit,
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
metadata?: Record<string, unknown>,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
) {
this._checkKey(key)
const { xhr, url } = await this.request({
request: { method: 'PUT', key },
data,
onProgress,
signal,
contentType: fileType,
})
return {
location: U.removeQueryString(url),
etag: U.sanitizeETag(xhr.getResponseHeader('etag')),
key,
}
}
/** Initiates a multipart upload and returns the upload ID. */
public override async createMultipartUpload(
key: string,
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
metadata?: Record<string, unknown>, // todo support metadata here too?
) {
this._checkKey(key)
if (typeof fileType !== 'string') {
throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`)
}
const { xhr } = await this.request({
request: { method: 'POST', key },
contentType: fileType,
})
const parsed = U.parseXml(xhr.responseText) as Record<string, unknown>
if (parsed && typeof parsed === 'object') {
// Check for both cases of InitiateMultipartUploadResult
const uploadResult =
(parsed.initiateMultipartUploadResult as Record<string, unknown>) ||
(parsed.InitiateMultipartUploadResult as Record<string, unknown>)
if (uploadResult && typeof uploadResult === 'object') {
// Check for both cases of uploadId
const uploadId = uploadResult.uploadId || uploadResult.UploadId
if (uploadId && typeof uploadId === 'string') {
return { uploadId, key }
}
}
}
throw new Error(
`${C.ERROR_PREFIX}Failed to create multipart upload: ${JSON.stringify(
parsed,
)}`,
)
}
public override async uploadPart(
key: string,
uploadId: string,
data: XMLHttpRequestBodyInit,
partNumber: number,
onProgress?: IT.OnProgressFn,
signal?: AbortSignal,
) {
this._validateUploadPartParams(key, uploadId, partNumber)
const { xhr } = await this.request({
request: {
method: 'PUT',
key,
uploadId,
partNumber,
},
data,
onProgress,
signal,
})
const etag = U.sanitizeETag(xhr.getResponseHeader('etag'))
if (etag == null) {
throw new Error(
`${C.ERROR_PREFIX}Missing ETag in uploadPart response headers`,
)
}
return { etag }
}
/**
* Core XHR upload implementation using @uppy/utils/fetcher.
*
* Features:
* - Automatic retry with exponential backoff (3 attempts)
* - Offline detection with automatic resume on reconnect
* - Stall detection via ProgressTimeout
*/
private async request({
request,
data,
onProgress,
signal,
contentType,
shouldRetryCredentials = true,
}: {
request: IT.PresignableRequest
data?: XMLHttpRequestBodyInit
onProgress?: IT.OnProgressFn
signal?: AbortSignal
contentType?: string
shouldRetryCredentials?: boolean
}): Promise<{ xhr: XMLHttpRequest; url: string }> {
// Wait for online before starting
await this.waitForOnline(signal)
// Check if aborted while waiting for online
if (signal?.aborted) {
throw new DOMException('Request aborted', 'AbortError')
}
try {
const { url } = await this.signRequest(request)
const xhr = await this.xhr({
url,
method: request.method,
data,
onProgress,
signal,
contentType,
})
return { xhr, url }
} catch (err: unknown) {
// NetworkError or errors with attached XHR (from onAfterResponse throws)
if (
err instanceof Error &&
'request' in err &&
err.request instanceof XMLHttpRequest
) {
const xhr = err.request as XMLHttpRequest
if (xhr.status === 0) {
throw new U.S3NetworkError(
'Network error during S3 request',
'NETWORK',
err,
)
}
// HTTP errors (non-2xx responses from NetworkError)
const parsedBody = this._parseErrorXml(
(name: string) => xhr.getResponseHeader(name),
xhr.responseText,
)
const serviceCode =
xhr.getResponseHeader('x-amz-error-code') ?? parsedBody.svcCode
// If expired token error and using getCredentials, clear cache and retry once
if (
shouldRetryCredentials &&
this.getCredentials != null &&
serviceCode != null &&
['ExpiredToken', 'InvalidAccessKeyId'].includes(serviceCode)
) {
this.clearCachedCredentials()
// Retry with fresh credentials
return this.request({
request,
data,
onProgress,
signal,
contentType,
shouldRetryCredentials: false, // prevent infinite recursion
})
}
throw new U.S3ServiceError(
`S3 returned ${xhr.status}${serviceCode ? ` ${serviceCode}` : ''}`,
xhr.status,
serviceCode,
xhr.responseText,
)
}
throw err
}
}
/** Lists uploaded parts for a multipart upload. */
public override async listParts(
uploadId: string,
key: string,
): Promise<IT.UploadPart[]> {
this._checkKey(key)
if (!uploadId) {
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
}
const { xhr } = await this.request({
request: { method: 'GET', key, uploadId },
})
const parsed = U.parseXml(xhr.responseText) as Record<string, unknown>
const result = (parsed.listPartsResult ||
parsed.ListPartsResult ||
parsed) as Record<string, unknown>
if (result && typeof result === 'object') {
const parts = result.Part || result.part || []
const partsArray = Array.isArray(parts) ? parts : [parts]
return partsArray
.filter(
(p): p is Record<string, unknown> =>
p != null &&
typeof p === 'object' &&
'PartNumber' in p &&
'ETag' in p,
)
.map((p) => ({
partNumber: parseInt(String(p.PartNumber), 10),
etag: U.sanitizeXmlETag(String(p.ETag)),
}))
}
return []
}
/** Completes a multipart upload by combining all uploaded parts. */
public override async completeMultipartUpload(
key: string,
uploadId: string,
parts: Array<IT.UploadPart>,
) {
const xmlBody = this._buildCompleteMultipartUploadXml(parts)
const { xhr } = await this.request({
request: { method: 'POST', key, uploadId },
contentType: C.XML_CONTENT_TYPE,
data: xmlBody,
})
const parsed = U.parseXml(xhr.responseText)
if (parsed && typeof parsed === 'object') {
// Check for both cases (camelCase from our parser, PascalCase from S3)
const result =
parsed.completeMultipartUploadResult ||
parsed.CompleteMultipartUploadResult ||
parsed
if (result && typeof result === 'object') {
const r = result as Record<string, unknown>
// S3 returns PascalCase (Location, Bucket, Key, ETag).
// Normalize to lowercase for our type interface.
const resultLocation = (r.Location || r.location) as string | undefined
const resultBucket = (r.Bucket || r.bucket) as string | undefined
const resultKey = (r.Key || r.key) as string | undefined
const rawEtag = (r.ETag || r.eTag || r.etag) as string | undefined
if (!resultLocation || !resultKey) {
throw new Error(
`${C.ERROR_PREFIX}CompleteMultipartUpload response missing Location or Key: ${JSON.stringify(r)}`,
)
}
const etag = rawEtag ? U.sanitizeXmlETag(rawEtag) : undefined
return {
location: resultLocation,
bucket: resultBucket,
key: resultKey,
etag,
}
}
}
throw new Error(
`${C.ERROR_PREFIX}Failed to complete multipart upload: ${JSON.stringify(
parsed,
)}`,
)
}
/** Aborts a multipart upload and removes all uploaded parts. */
public override async abortMultipartUpload(key: string, uploadId: string) {
this._checkKey(key)
if (!uploadId) {
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
}
const { xhr } = await this.request({
request: { method: 'DELETE', key, uploadId },
})
const parsed = U.parseXml(xhr.responseText) as Record<string, unknown>
if (
parsed &&
'error' in parsed &&
typeof parsed.error === 'object' &&
parsed.error !== null &&
'message' in parsed.error
) {
throw new Error(
`${C.ERROR_PREFIX}Failed to abort multipart upload: ${String(
parsed.error.message,
)}`,
)
}
}
private _buildCompleteMultipartUploadXml(
parts: Array<IT.UploadPart>,
): string {
let xml = '<CompleteMultipartUpload>'
for (const part of parts) {
xml += `<Part><PartNumber>${part.partNumber}</PartNumber><ETag>${part.etag}</ETag></Part>`
}
xml += '</CompleteMultipartUpload>'
return xml
}
/** Deletes an object from the bucket. Returns true on success. */
public override async deleteObject(key: string) {
const { xhr } = await this.request({
request: { method: 'DELETE', key },
})
if (xhr.status !== 200 && xhr.status !== 204) {
throw new Error(
`${C.ERROR_PREFIX}Failed to delete object. HTTP status: ${xhr.status}`,
)
}
}
/**
* Clears cached credentials.
* Call this method when you need to force a credential refresh on the next request.
*/
public clearCachedCredentials(): void {
this.cachedCredentials = undefined
this.cachedCredentialsPromise = undefined
}
private _parseErrorXml(
getHeader: (name: string) => string | null,
body: string,
): { svcCode?: string; errorMessage?: string } {
if (getHeader('content-type') !== 'application/xml') {
return {}
}
const parsedBody = U.parseXml(body)
if (
!parsedBody ||
typeof parsedBody !== 'object' ||
!('Error' in parsedBody) ||
!parsedBody.Error ||
typeof parsedBody.Error !== 'object'
) {
return {}
}
const error = parsedBody.Error
return {
svcCode:
'Code' in error && typeof error.Code === 'string'
? error.Code
: undefined,
errorMessage:
'Message' in error && typeof error.Message === 'string'
? error.Message
: undefined,
}
}
}
export { S3mini }
export default S3mini

View file

@ -1,10 +1,8 @@
// Export the S3 class as default export and named export // Export the S3 class as default export and named export
export { S3mini } from './S3.js' export { S3mini } from './S3mini.js'
// Re-export types // Re-export types
export type { export type {
CompleteMultipartUploadResult,
ErrorWithCode, ErrorWithCode,
S3Config, S3Config,
UploadPart, UploadPart,
} from './types.js' } from './types.js'
export { sanitizeETag } from './utils.js'

View file

@ -5,7 +5,7 @@
*/ */
import * as C from './consts.js' import * as C from './consts.js'
import type { presignableRequest, presignedResponse } from './types.js' import type { PresignableRequest, presignedResponse } from './types.js'
import * as U from './utils.js' import * as U from './utils.js'
export interface SignerConfig { export interface SignerConfig {
@ -45,15 +45,9 @@ export function createSigV4Presigner(config: SignerConfig) {
} }
return async function presign( return async function presign(
request: presignableRequest, request: PresignableRequest,
): Promise<presignedResponse> { ): Promise<presignedResponse> {
const { const { method, key, expiresIn = DEFAULT_EXPIRES_IN } = request
method,
key,
uploadId,
partNumber,
expiresIn = DEFAULT_EXPIRES_IN,
} = request
// Build the URL - need to track encoded path separately because URL object decodes it // Build the URL - need to track encoded path separately because URL object decodes it
const url = new URL(endpoint) const url = new URL(endpoint)
@ -92,15 +86,15 @@ export function createSigV4Presigner(config: SignerConfig) {
} }
// Add multipart-specific params // Add multipart-specific params
if (uploadId) { if ('uploadId' in request) {
url.searchParams.set('uploadId', uploadId) url.searchParams.set('uploadId', request.uploadId)
} }
if (partNumber !== undefined) { if ('partNumber' in request) {
url.searchParams.set('partNumber', String(partNumber)) url.searchParams.set('partNumber', String(request.partNumber))
} }
// For CreateMultipartUpload, add uploads param // For CreateMultipartUpload, add uploads param
if (method === 'POST' && !uploadId) { if (method === 'POST' && !('uploadId' in request)) {
url.searchParams.set('uploads', '') url.searchParams.set('uploads', '')
} }

View file

@ -1,21 +1,53 @@
/** Request data to be pre-signed */ export interface PresignableRequestBase {
export type presignableRequest = {
method: string
key: string key: string
uploadId?: string
partNumber?: number
expiresIn?: number expiresIn?: number
contentType?: string
} }
export interface PutObjectRequest extends PresignableRequestBase {
method: 'PUT'
}
export interface DeleteObjectRequest extends PresignableRequestBase {
method: 'DELETE'
}
export interface CreateMultipartUploadRequest extends PresignableRequestBase {
method: 'POST'
}
export interface CompleteMultipartUploadRequest extends PresignableRequestBase {
method: 'POST'
uploadId: string
}
export interface DeleteMultipartUploadRequest extends PresignableRequestBase {
method: 'DELETE'
uploadId: string
}
export interface ListPartsRequest extends PresignableRequestBase {
method: 'GET'
uploadId: string
}
export interface UploadPartRequest extends PresignableRequestBase {
method: 'PUT'
uploadId: string
partNumber: number
}
/** Request data to be pre-signed */
export type PresignableRequest =
| PutObjectRequest
| DeleteObjectRequest
| CreateMultipartUploadRequest
| CompleteMultipartUploadRequest
| DeleteMultipartUploadRequest
| ListPartsRequest
| UploadPartRequest
/** Response with the pre-signed URL */ /** Response with the pre-signed URL */
export type presignedResponse = { export type presignedResponse = {
url: string url: string
} }
/** Function that generates a pre-signed URL for a request */ /** Function that generates a pre-signed URL for a request */
export type signRequestFn = ( export type SignRequestFn = (
request: presignableRequest, request: PresignableRequest,
) => Promise<presignedResponse> ) => Promise<presignedResponse>
/** /**
@ -40,36 +72,34 @@ export interface CredentialsResponse {
} }
/** Function that retrieves temporary credentials */ /** Function that retrieves temporary credentials */
export type getCredentialsFn = (options?: { export type GetCredentialsFn = (options?: {
signal?: AbortSignal signal?: AbortSignal
}) => Promise<CredentialsResponse> }) => CredentialsResponse | Promise<CredentialsResponse>
/** Base configuration shared by both signing approaches */ /** Base configuration shared by both signing approaches */
type S3ConfigBase = { type S3ConfigBase = {
/** Endpoint URL of the S3-compatible service (e.g., 'https://s3.amazonaws.com/bucket-name') */
endpoint: string
/** AWS region. Defaults to 'auto'. */ /** AWS region. Defaults to 'auto'. */
region?: string region?: string | undefined
/** Request size in bytes for multipart uploads. Defaults to 8MB. */ /** Request size in bytes for multipart uploads. Defaults to 8MB. */
requestSizeInBytes?: number requestSizeInBytes?: number | undefined
/** Timeout in ms after which a request should be aborted. */ /** Timeout in ms after which a request should be aborted. */
requestAbortTimeout?: number requestAbortTimeout?: number | undefined
} }
/** Config when using signRequest callback (region optional) */ /** Config when using signRequest callback (region optional) */
type S3ConfigWithSignRequest = S3ConfigBase & { type S3ConfigWithSignRequest = S3ConfigBase & {
/** Function to sign requests. Called for each S3 API request. */ /** Function to sign requests. Called for each S3 API request. */
signRequest: signRequestFn signRequest: SignRequestFn
getCredentials?: never
} }
/** Config when using getCredentials callback (region required for signing) */ /** Config when using getCredentials callback (region required for signing) */
type S3ConfigWithGetCredentials = Omit<S3ConfigBase, 'region'> & { type S3ConfigWithGetCredentials = Omit<S3ConfigBase, 'region'> & {
signRequest?: never
/** Function to retrieve temporary credentials for client-side signing. */ /** Function to retrieve temporary credentials for client-side signing. */
getCredentials: getCredentialsFn getCredentials: GetCredentialsFn
/** AWS region. Required for signing with getCredentials. */ /** AWS region. Required for signing with getCredentials. */
region: string region?: string
/** Endpoint URL of the S3-compatible service (e.g., 'https://s3.amazonaws.com/bucket-name') */
endpoint: string
} }
/** Configuration options for S3mini client */ /** Configuration options for S3mini client */
@ -80,13 +110,6 @@ export interface UploadPart {
etag: string etag: string
} }
export interface CompleteMultipartUploadResult {
location: string
bucket: string
key: string
etag: string
}
export interface ErrorWithCode { export interface ErrorWithCode {
code?: string code?: string
cause?: { code?: string } cause?: { code?: string }
@ -108,11 +131,3 @@ export type BinaryData = ArrayBuffer | Uint8Array | Blob
/** Progress callback for upload operations */ /** Progress callback for upload operations */
export type OnProgressFn = (bytesUploaded: number, bytesTotal: number) => void export type OnProgressFn = (bytesUploaded: number, bytesTotal: number) => void
export interface PutObjectResult {
status: number
ok: boolean
headers: {
get(name: string): string | null
}
}

View file

@ -1,25 +1,22 @@
import type { ErrorWithCode, XmlMap, XmlValue } from './types.js' import type { XmlMap, XmlValue } from './types.js'
const ENCODR = new TextEncoder() export const sanitizeXmlETag = (etag: string): string =>
const chunkSize = 0x8000 // 32KB chunks etag.replace(/^(&quot;|&#34;)+|(&quot;|&#34;)+$/g, '')
const HEXS = '0123456789abcdef'
export const getByteSize = (data: unknown): number => { export const sanitizeETag = (etag: string | null): string | undefined =>
if (typeof data === 'string') { etag?.replace(/^"+|"+$/g, '')
return ENCODR.encode(data).byteLength
} /** Strips query string and hash from a URL to derive the object location. */
if (data instanceof ArrayBuffer) { export function removeQueryString(urlString: string): string {
return data.byteLength const urlObject = new URL(urlString)
} urlObject.search = ''
if (ArrayBuffer.isView(data)) { urlObject.hash = ''
return data.byteLength return urlObject.href
}
if (data instanceof Blob) {
return data.size
}
throw new Error('Unsupported data type')
} }
const textEncoder = new TextEncoder()
const HEXS = '0123456789abcdef'
/** /**
* Turn a raw ArrayBuffer into its hexadecimal representation. * Turn a raw ArrayBuffer into its hexadecimal representation.
* @param {ArrayBuffer} buffer The raw bytes. * @param {ArrayBuffer} buffer The raw bytes.
@ -34,28 +31,13 @@ export const hexFromBuffer = (buffer: ArrayBuffer): string => {
return hex return hex
} }
/**
* Turn a raw ArrayBuffer into its base64 representation.
* @param {ArrayBuffer} buffer The raw bytes.
* @returns {string} Base64 string
*/
export const base64FromBuffer = (buffer: ArrayBuffer): string => {
const bytes = new Uint8Array(buffer)
let result = ''
for (let i = 0; i < bytes.length; i += chunkSize) {
const chunk = bytes.subarray(i, i + chunkSize)
result += btoa(String.fromCodePoint(...chunk))
}
return result
}
/** /**
* Compute SHA-256 hash of arbitrary string data. * Compute SHA-256 hash of arbitrary string data.
* @param {string} content The content to be hashed. * @param {string} content The content to be hashed.
* @returns {ArrayBuffer} The raw hash * @returns {ArrayBuffer} The raw hash
*/ */
export const sha256 = async (content: string): Promise<ArrayBuffer> => { export const sha256 = async (content: string): Promise<ArrayBuffer> => {
const data = ENCODR.encode(content) const data = textEncoder.encode(content)
return await globalThis.crypto.subtle.digest('SHA-256', data) return await globalThis.crypto.subtle.digest('SHA-256', data)
} }
@ -72,33 +54,16 @@ export const hmac = async (
): Promise<ArrayBuffer> => { ): Promise<ArrayBuffer> => {
const secret = await globalThis.crypto.subtle.importKey( const secret = await globalThis.crypto.subtle.importKey(
'raw', 'raw',
typeof key === 'string' ? ENCODR.encode(key) : key, typeof key === 'string' ? textEncoder.encode(key) : key,
{ name: 'HMAC', hash: 'SHA-256' }, { name: 'HMAC', hash: 'SHA-256' },
false, false,
['sign'], ['sign'],
) )
const data = ENCODR.encode(content) const data = textEncoder.encode(content)
return await globalThis.crypto.subtle.sign('HMAC', secret, data) return await globalThis.crypto.subtle.sign('HMAC', secret, data)
} }
/**
* Sanitize ETag value by removing quotes and XML entities
* @param etag ETag value to sanitize
* @returns Sanitized ETag
*/
export const sanitizeETag = (etag: string): string => {
const replaceChars: Record<string, string> = {
'"': '',
'&quot;': '',
'&#34;': '',
}
return etag.replaceAll(
/(^("|&quot;|&#34;))|(("|&quot;|&#34;)$)/g,
(m) => replaceChars[m] || '',
)
}
const entityMap = { const entityMap = {
'&quot;': '"', '&quot;': '"',
'&apos;': "'", '&apos;': "'",
@ -107,20 +72,6 @@ const entityMap = {
'&amp;': '&', '&amp;': '&',
} as const } as const
/**
* Escape special characters for XML
* @param value String to escape
* @returns XML-escaped string
*/
export const escapeXml = (value: string): string => {
return value
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
.replaceAll('"', '&quot;')
.replaceAll("'", '&apos;')
}
const unescapeXml = (value: string): string => const unescapeXml = (value: string): string =>
value.replaceAll( value.replaceAll(
/&(quot|apos|lt|gt|amp);/g, /&(quot|apos|lt|gt|amp);/g,
@ -182,7 +133,7 @@ const encodeAsHex = (c: string): string =>
* @param uriStr URI string to escape * @param uriStr URI string to escape
* @returns Escaped URI string * @returns Escaped URI string
*/ */
export const uriEscape = (uriStr: string): string => { const uriEscape = (uriStr: string): string => {
return encodeURIComponent(uriStr).replace(/[!'()*]/g, encodeAsHex) return encodeURIComponent(uriStr).replace(/[!'()*]/g, encodeAsHex)
} }
@ -195,17 +146,6 @@ export const uriResourceEscape = (string: string): string => {
return uriEscape(string).replaceAll('%2F', '/') return uriEscape(string).replaceAll('%2F', '/')
} }
export const extractErrCode = (e: unknown): string | undefined => {
if (typeof e !== 'object' || e === null) {
return undefined
}
const err = e as ErrorWithCode
if (typeof err.code === 'string') {
return err.code
}
return typeof err.cause?.code === 'string' ? err.cause.code : undefined
}
export class S3Error extends Error { export class S3Error extends Error {
readonly code?: string readonly code?: string
constructor(msg: string, code?: string, cause?: unknown) { constructor(msg: string, code?: string, cause?: unknown) {

View file

@ -1,4 +1,3 @@
import type { Body } from '@uppy/utils'
import { createAbortError } from '@uppy/utils' import { createAbortError } from '@uppy/utils'
import type { AwsS3Part } from './index.js' import type { AwsS3Part } from './index.js'
@ -22,9 +21,3 @@ export type UploadPartBytesResult = {
ETag: string ETag: string
location?: string location?: string
} }
export interface AwsBody extends Body {
location: string
key: string
bucket: string
}

View file

@ -1,4 +1,14 @@
import { describe, expect, it, vi } from 'vitest' import { HttpResponse, http } from 'msw'
import { setupServer } from 'msw/node'
import {
afterAll,
afterEach,
beforeAll,
describe,
expect,
it,
vi,
} from 'vitest'
import 'whatwg-fetch' import 'whatwg-fetch'
import Core, { type Meta, type UppyFile } from '@uppy/core' import Core, { type Meta, type UppyFile } from '@uppy/core'
@ -7,15 +17,145 @@ import AwsS3, { type AwsBody, type AwsS3Options } from '../src/index.js'
const KB = 1024 const KB = 1024
const MB = KB * KB const MB = KB * KB
// ---------------------------------------------------------------------------
// Helpers for multipart upload tests
// ---------------------------------------------------------------------------
/** Minimal XML responses that S3 returns for multipart operations */
const s3Responses = {
createMultipart: (uploadId: string, key: string) =>
`<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult>
<UploadId>${uploadId}</UploadId>
<Key>${key}</Key>
</InitiateMultipartUploadResult>`,
uploadPart: (etag: string) => '',
listParts: (parts: { partNumber: number; etag: string }[]) =>
`<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult>
${parts.map((p) => `<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${p.etag}</ETag></Part>`).join('')}
</ListPartsResult>`,
completeMultipart: (location: string, key: string) =>
`<?xml version="1.0" encoding="UTF-8"?>
<CompleteMultipartUploadResult>
<Location>${location}</Location>
<Key>${key}</Key>
<Bucket>test-bucket</Bucket>
</CompleteMultipartUploadResult>`,
abortMultipart: () => '',
}
const server = setupServer()
const s3Url = 'https://test-bucket.s3.us-east-1.amazonaws.com/:key'
/**
* Creates signRequest + MSW handler state for multipart upload tests.
*/
function createMultipartMocks(opts: { uploadId?: string; key?: string } = {}) {
const uploadId = opts.uploadId ?? 'test-upload-id'
const key = opts.key ?? 'test-key'
// signRequest encodes operation details in the URL for MSW routing
const signRequest = vi.fn().mockImplementation(async (req: any) => {
const params = new URLSearchParams()
if (req.uploadId) params.set('uploadId', req.uploadId)
if (req.partNumber) params.set('partNumber', String(req.partNumber))
params.set('method', req.method)
return {
url: `https://test-bucket.s3.us-east-1.amazonaws.com/${req.key || key}?${params}`,
}
})
const operations: string[] = []
const registerHandlers = ({
hangNonCreate = false,
listParts = [] as { partNumber: number; etag: string }[],
} = {}) => {
const maybeHang = () =>
hangNonCreate ? (new Promise(() => {}) as Promise<any>) : null
server.use(
http.post(s3Url, ({ request }) => {
const hasUploadId = new URL(request.url).searchParams.has('uploadId')
if (!hasUploadId) {
operations.push('createMultipart')
return new HttpResponse(s3Responses.createMultipart(uploadId, key), {
status: 200,
headers: { 'Content-Type': 'application/xml' },
})
}
operations.push('completeMultipart')
const hung = maybeHang()
if (hung) return hung
return new HttpResponse(
s3Responses.completeMultipart(
`https://test-bucket.s3.amazonaws.com/${key}`,
key,
),
{ status: 200, headers: { 'Content-Type': 'application/xml' } },
)
}),
http.put(s3Url, () => {
operations.push('uploadPart')
const hung = maybeHang()
if (hung) return hung
return new HttpResponse('', {
status: 200,
headers: { ETag: '"etag-1"' },
})
}),
http.get(s3Url, ({ request }) => {
const hasUploadId = new URL(request.url).searchParams.has('uploadId')
if (!hasUploadId) {
return new HttpResponse('Not Found', { status: 404 })
}
operations.push('listParts')
const hung = maybeHang()
if (hung) return hung
return new HttpResponse(s3Responses.listParts(listParts), {
status: 200,
headers: { 'Content-Type': 'application/xml' },
})
}),
http.delete(s3Url, () => {
operations.push('abortMultipart')
return new HttpResponse('', { status: 204 })
}),
)
}
return { signRequest, operations, uploadId, key, registerHandlers }
}
describe('AwsS3', () => { describe('AwsS3', () => {
beforeAll(() => {
server.listen({ onUnhandledRequest: 'error' })
})
afterEach(() => {
server.resetHandlers()
})
afterAll(() => {
server.close()
})
it('Registers AwsS3 upload plugin', () => { it('Registers AwsS3 upload plugin', () => {
const core = new Core().use(AwsS3, { const core = new Core().use(AwsS3, {
bucket: 'test-bucket',
region: 'us-east-1', region: 'us-east-1',
endpoint: 'https://companion.example.com', s3Endpoint: 'https://companion.example.com',
companionEndpoint: 'https://companion.example.com',
}) })
// @ts-expect-error private property
const pluginNames = core[Symbol.for('uppy test: getPlugins')]( const pluginNames = core[Symbol.for('uppy test: getPlugins')](
'uploader', 'uploader',
).map((plugin: AwsS3<Meta, AwsBody>) => plugin.constructor.name) ).map((plugin: AwsS3<Meta, AwsBody>) => plugin.constructor.name)
@ -23,34 +163,25 @@ describe('AwsS3', () => {
}) })
describe('configuration validation', () => { describe('configuration validation', () => {
it('throws if bucket is not provided', () => {
expect(() => {
const core = new Core()
// @ts-expect-error - testing missing required option
core.use(AwsS3, {})
}).toThrow('`bucket` option is required')
})
it('throws if region is not provided', () => {
expect(() => {
const core = new Core()
core.use(AwsS3, { bucket: 'test-bucket' })
}).toThrow('`region` option is required')
})
it('throws if no signing method is provided', () => { it('throws if no signing method is provided', () => {
expect(() => { expect(() => {
const core = new Core() const core = new Core()
core.use(AwsS3, { bucket: 'test-bucket', region: 'us-east-1' }) // @ts-expect-error - testing runtime validation, so omit required options
}).toThrow('`endpoint`, `signRequest`, or `getCredentials` is required') core.use(AwsS3, {
s3Endpoint: 'https://companion.example.com',
region: 'us-east-1',
})
}).toThrow(
'One of options `companionEndpoint`, `signRequest`, or `getCredentials` is required',
)
}) })
it('accepts endpoint option', () => { it('accepts endpoint option', () => {
const core = new Core() const core = new Core()
core.use(AwsS3, { core.use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
endpoint: 'https://companion.example.com', companionEndpoint: 'https://companion.example.com',
}) })
expect(core.getPlugin('AwsS3')).toBeDefined() expect(core.getPlugin('AwsS3')).toBeDefined()
}) })
@ -58,7 +189,7 @@ describe('AwsS3', () => {
it('accepts signRequest option', () => { it('accepts signRequest option', () => {
const core = new Core() const core = new Core()
core.use(AwsS3, { core.use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
signRequest: vi.fn(), signRequest: vi.fn(),
}) })
@ -68,7 +199,7 @@ describe('AwsS3', () => {
it('accepts getCredentials option', () => { it('accepts getCredentials option', () => {
const core = new Core() const core = new Core()
core.use(AwsS3, { core.use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
getCredentials: vi.fn(), getCredentials: vi.fn(),
}) })
@ -89,9 +220,9 @@ describe('AwsS3', () => {
it('defaults to multipart for files > 100MB', () => { it('defaults to multipart for files > 100MB', () => {
const core = new Core<Meta, AwsBody>().use(AwsS3, { const core = new Core<Meta, AwsBody>().use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
endpoint: 'https://companion.example.com', companionEndpoint: 'https://companion.example.com',
}) })
const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody> const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody>
const shouldUseMultipart = opts.shouldUseMultipart as ( const shouldUseMultipart = opts.shouldUseMultipart as (
@ -108,9 +239,9 @@ describe('AwsS3', () => {
it('handles very large files', () => { it('handles very large files', () => {
const core = new Core<Meta, AwsBody>().use(AwsS3, { const core = new Core<Meta, AwsBody>().use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
endpoint: 'https://companion.example.com', companionEndpoint: 'https://companion.example.com',
}) })
const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody> const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody>
const shouldUseMultipart = opts.shouldUseMultipart as ( const shouldUseMultipart = opts.shouldUseMultipart as (
@ -127,7 +258,7 @@ describe('AwsS3', () => {
const signRequest = vi.fn().mockRejectedValue(new Error('Test stop')) const signRequest = vi.fn().mockRejectedValue(new Error('Test stop'))
const core = new Core().use(AwsS3, { const core = new Core().use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
signRequest, signRequest,
shouldUseMultipart: false, shouldUseMultipart: false,
@ -156,7 +287,7 @@ describe('AwsS3', () => {
const signRequest = vi.fn().mockRejectedValue(new Error('Sign failed')) const signRequest = vi.fn().mockRejectedValue(new Error('Sign failed'))
const core = new Core().use(AwsS3, { const core = new Core().use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
signRequest, signRequest,
shouldUseMultipart: false, shouldUseMultipart: false,
@ -191,7 +322,7 @@ describe('AwsS3', () => {
) )
const core = new Core().use(AwsS3, { const core = new Core().use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
signRequest, signRequest,
shouldUseMultipart: false, shouldUseMultipart: false,
@ -222,7 +353,7 @@ describe('AwsS3', () => {
) )
const core = new Core().use(AwsS3, { const core = new Core().use(AwsS3, {
bucket: 'test-bucket', s3Endpoint: 'https://companion.example.com',
region: 'us-east-1', region: 'us-east-1',
signRequest, signRequest,
shouldUseMultipart: false, shouldUseMultipart: false,
@ -244,4 +375,120 @@ describe('AwsS3', () => {
expect(result?.successful).toHaveLength(0) expect(result?.successful).toHaveLength(0)
}) })
}) })
describe('Golden Retriever resume state (s3Multipart)', () => {
it('persists s3Multipart on file state after creating multipart upload', async () => {
const { signRequest, uploadId, registerHandlers } = createMultipartMocks()
// After createMultipart succeeds, hang on subsequent requests so we can inspect state
registerHandlers({ hangNonCreate: true })
const core = new Core().use(AwsS3, {
s3Endpoint: 'https://companion.example.com',
region: 'us-east-1',
signRequest,
shouldUseMultipart: true,
})
const fileId = core.addFile({
source: 'test',
name: 'big.dat',
type: 'application/octet-stream',
data: new File([new Uint8Array(6 * MB)], 'big.dat'),
})
const uploadPromise = core.upload()
// Wait for createMultipart to complete and state to be persisted
await new Promise((resolve) => setTimeout(resolve, 100))
const file = core.getFile(fileId)
expect(file.s3Multipart).toBeDefined()
expect(file.s3Multipart?.uploadId).toBe(uploadId)
// Clean up
core.cancelAll()
await uploadPromise
})
it('clears s3Multipart when upload is aborted via cancelAll', async () => {
const { signRequest, registerHandlers } = createMultipartMocks({
uploadId: 'cancel-test-id',
key: 'cancel-key',
})
registerHandlers({ hangNonCreate: true })
const core = new Core().use(AwsS3, {
s3Endpoint: 'https://companion.example.com',
region: 'us-east-1',
signRequest,
shouldUseMultipart: true,
})
const fileId = core.addFile({
source: 'test',
name: 'big.dat',
type: 'application/octet-stream',
data: new File([new Uint8Array(6 * MB)], 'big.dat'),
})
const uploadPromise = core.upload()
// Wait for createMultipart, then cancel
await new Promise((resolve) => setTimeout(resolve, 50))
core.cancelAll()
await uploadPromise
const file = core.getFile(fileId)
// s3Multipart should be cleared so retries don't use a dead uploadId
expect(file?.s3Multipart).toBeUndefined()
})
it('uses persisted s3Multipart key for resume (listParts, not createMultipart)', async () => {
const persistedKey = 'persisted-object-key'
const persistedUploadId = 'persisted-upload-id'
const { signRequest, operations, registerHandlers } =
createMultipartMocks({
uploadId: persistedUploadId,
key: persistedKey,
})
registerHandlers()
const core = new Core().use(AwsS3, {
s3Endpoint: 'https://companion.example.com',
region: 'us-east-1',
signRequest,
shouldUseMultipart: false, // Would normally be simple upload
})
const fileId = core.addFile({
source: 'test',
name: 'big.dat',
type: 'application/octet-stream',
data: new File([new Uint8Array(6 * MB)], 'big.dat'),
})
// Simulate Golden Retriever restoring s3Multipart state
core.setFileState(fileId, {
s3Multipart: { uploadId: persistedUploadId, key: persistedKey },
})
const uploadPromise = core.upload()
// Wait for the resume flow to call listParts (via fetch), then cancel
await new Promise((resolve) => setTimeout(resolve, 100))
// Should have resumed (listParts) instead of creating a new multipart upload
expect(operations).toContain('listParts')
expect(operations).not.toContain('createMultipart')
// The signRequest calls should use the persisted key, not a generated one
const signedKeys = signRequest.mock.calls.map((call: any) => call[0].key)
expect(signedKeys.every((k: string) => k === persistedKey)).toBe(true)
// Clean up
core.cancelAll()
await uploadPromise
})
})
}) })

View file

@ -1,273 +0,0 @@
import { beforeAll, describe, expect, it, vi } from 'vitest'
import { S3mini } from '../../src/s3-client/index.js'
import { createSigV4Signer } from '../../src/s3-client/signer.js'
import { randomBytes } from '../test-utils/browser-crypto.js'
let _providerName
export const beforeRun = (raw, name, providerSpecific) => {
if (!raw) {
console.error(
'No credentials found. Please set the BUCKET_ENV_ environment variables.',
)
describe.skip(name, () => {
it('skipped', () => {
expect(true).toBe(true)
})
})
} else {
console.log('Running tests for bucket:', name)
const credentials = {
provider: raw[0],
accessKeyId: raw[1],
secretAccessKey: raw[2],
endpoint: raw[3],
region: raw[4],
}
describe(`:::: ${credentials.provider} ::::`, () => {
expect(credentials.provider).toBe(name)
_providerName = credentials.provider
expect(credentials.accessKeyId).toBeDefined()
expect(credentials.secretAccessKey).toBeDefined()
expect(credentials.endpoint).toBeDefined()
expect(credentials.region).toBeDefined()
testRunner(credentials)
if (providerSpecific) {
providerSpecific(credentials)
}
})
}
}
const EIGHT_MB = 8 * 1024 * 1024
const key_bin = 'test-multipart.bin'
const large_buffer = randomBytes(EIGHT_MB * 3.2)
const content = 'some content'
const key = 'first-test-object.txt'
const key_list_parts = 'test-list-parts.bin'
const key_abort_multipart = 'test-abort-multipart.bin'
const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart]
export const cleanupTestBeforeAll = (s3client) => {
beforeAll(async () => {
for (const key of FILE_KEYS) {
try {
await s3client.deleteObject(key)
} catch {
// intentionally ignore the errors as the objects don't exists. still feels hacky tbh
}
}
})
}
// --- 2 ■ A separate describe makes test output nicer -----------------------
export const testRunner = (bucket) => {
vi.setConfig({ testTimeout: 120_000 })
const presigner = createSigV4Signer({
accessKeyId: bucket.accessKeyId,
secretAccessKey: bucket.secretAccessKey,
region: bucket.region,
endpoint: bucket.endpoint,
})
const s3client = new S3mini({
endpoint: bucket.endpoint,
region: bucket.region,
signRequest: presigner,
})
cleanupTestBeforeAll(s3client)
it('instantiates s3client', () => {
expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation
})
// we don't need an explicit eTag method as we already get eTag in the putOject response
it('putObject uploads successfully and returns ETag', async () => {
const response = await s3client.putObject(key, content, 'text/plain')
expect(response.status).toBe(200)
expect(response.headers.get('etag')).toBeDefined()
await s3client.deleteObject(key)
})
it('putObject handles binary data', async () => {
const binaryData = new Uint8Array(6).fill(0xff)
const response = await s3client.putObject(
key,
binaryData,
'application/octet-stream',
)
expect(response).toBeDefined()
expect(response.status).toBe(200)
expect(response.headers.get('etag')).toBeDefined()
// cleanup
await s3client.deleteObject(key)
})
// test getMultipartUploadId
it('getMultipartUploadId returns a valid uploadId', async () => {
const uploadId = await s3client.getMultipartUploadId(
key_bin,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
expect(typeof uploadId).toBe('string')
expect(uploadId.length).toBeGreaterThan(0)
// cleanup
await s3client.abortMultipartUpload(key, uploadId)
})
// test uploadPart
it('uploadPart returns partNumber and Etag', async () => {
const partData = randomBytes(EIGHT_MB)
const uploadId = await s3client.getMultipartUploadId(
key_bin,
'application/octet-stream',
)
// upload part
const partResult = await s3client.uploadPart(key_bin, uploadId, partData, 1)
expect(partResult).toBeDefined()
expect(partResult.partNumber).toBe(1)
expect(partResult.etag).toBeDefined()
expect(typeof partResult.etag).toBe('string')
expect(partResult.etag.length).toBe(32)
// cleanup
await s3client.abortMultipartUpload(key, uploadId)
})
// end to end multipart flow
it('completeMultipartUpload assembles parts correctly', async () => {
// key - key_bin
const partSize = EIGHT_MB
const totalParts = Math.ceil(large_buffer.length / partSize)
const uploadId = await s3client.getMultipartUploadId(
key_bin,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
// upload all parts
const uploadPromises = []
for (let i = 0; i < totalParts; i++) {
const partBuffer = large_buffer.subarray(i * partSize, (i + 1) * partSize)
uploadPromises.push(
s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1),
)
}
const uploadResponses = await Promise.all(uploadPromises)
// verify all parts uploaded succesfully
expect(uploadResponses.length).toBe(totalParts)
uploadResponses.forEach((response, index) => {
expect(response.partNumber).toBe(index + 1)
expect(response.etag).toBeDefined()
})
// create multipart upload
const parts = uploadResponses.map((response) => ({
partNumber: response.partNumber,
etag: response.etag,
}))
const completeResponse = await s3client.completeMultipartUpload(
key_bin,
uploadId,
parts,
)
expect(completeResponse).toBeDefined()
expect(typeof completeResponse).toBe('object')
expect(completeResponse.etag).toBeDefined()
expect(typeof completeResponse.etag).toBe('string')
expect(completeResponse.etag.length).toBe(32 + 2)
// cleanup
await s3client.deleteObject(key_bin)
})
it('abortMultipartUpload cancels upload successfully', async () => {
// start upload
const uploadId = await s3client.getMultipartUploadId(
key_abort_multipart,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const partData = randomBytes(EIGHT_MB)
await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1)
// abort
const abortResult = await s3client.abortMultipartUpload(
key_abort_multipart,
uploadId,
)
expect(abortResult).toBeDefined()
expect(abortResult.status).toBe('Aborted')
expect(abortResult.key).toBe(key_abort_multipart)
expect(abortResult.uploadId).toBe(uploadId)
})
it('listParts returns uploaded parts correctly', async () => {
const partSize = EIGHT_MB
const uploadId = await s3client.getMultipartUploadId(
key_list_parts,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const part1Data = randomBytes(partSize)
const part2Data = randomBytes(partSize)
const part1Result = await s3client.uploadPart(
key_list_parts,
uploadId,
part1Data,
1,
)
const part2Result = await s3client.uploadPart(
key_list_parts,
uploadId,
part2Data,
2,
)
const parts = await s3client.listParts(uploadId, key_list_parts)
expect(parts).toBeInstanceOf(Array)
expect(parts.length).toBe(2)
// verify part 1
expect(parts[0].partNumber).toBe(1)
expect(parts[0].etag).toBe(part1Result.etag)
// verify part 2
expect(parts[1].partNumber).toBe(2)
expect(parts[1].etag).toBe(part2Result.etag)
// cleanup abort upload
await s3client.abortMultipartUpload(key, uploadId)
})
}

View file

@ -0,0 +1,14 @@
export const accessKeyId = 'stsuser'
export const secretAccessKey = 'stspassword123'
export function getConfig(env: Record<string, string | undefined>) {
const config = env['VITE_MINIO_CONFIG']
if (!config) {
return undefined
}
const [rootAccessKeyId, rootSecretAccessKey, endpoint, region] =
config.split(',')
// Use stsuser credentials for all tests (readwrite policy is sufficient)
// Root credentials are kept for Docker container startup
return { endpoint, region, rootAccessKeyId, rootSecretAccessKey }
}

View file

@ -1,32 +1,23 @@
import { exec, spawn } from 'node:child_process' import { exec, spawn } from 'node:child_process'
import { resolve } from 'node:path'
import { promisify } from 'node:util' import { promisify } from 'node:util'
const CWD = resolve('.')
const execAsync = promisify(exec) const execAsync = promisify(exec)
export async function getContainerName(serviceName) { export async function getContainerName(serviceName: string) {
try { // Find container by service name using docker ps
// Find container by service name using docker ps const { stdout } = await execAsync(
const { stdout } = await execAsync( `docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`,
`docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`, )
) const containerName = stdout.trim().split('\n')[0] // Get first matching container
const containerName = stdout.trim().split('\n')[0] // Get first matching container if (!containerName) {
if (!containerName) { throw new Error(`No running container found for service: ${serviceName}`)
throw new Error(`No running container found for service: ${serviceName}`)
}
return containerName
} catch (error) {
throw new Error(
`Failed to find container for service ${serviceName}: ${error.message}`,
)
} }
return containerName
} }
export async function execDockerCommand( export async function execDockerCommand(
containerName, containerName: string,
command, command: string,
timeoutMs = 10000, timeoutMs = 10000,
) { ) {
// If it's a service name (like 'garage'), find the actual container name // If it's a service name (like 'garage'), find the actual container name
@ -34,9 +25,7 @@ export async function execDockerCommand(
if (['garage', 'minio', 'ceph'].includes(containerName)) { if (['garage', 'minio', 'ceph'].includes(containerName)) {
try { try {
actualContainerName = await getContainerName(containerName) actualContainerName = await getContainerName(containerName)
console.log( // console.log(`Found container: ${actualContainerName} for service: ${containerName}`,)
`Found container: ${actualContainerName} for service: ${containerName}`,
)
} catch (_error) { } catch (_error) {
console.log(`Using container name as-is: ${containerName}`) console.log(`Using container name as-is: ${containerName}`)
} }
@ -74,27 +63,27 @@ export async function execDockerCommand(
} }
} }
function run(cmd, args) { async function run(cmd: string, args: string[], env?: Record<string, string>) {
return new Promise((res, rej) => { // console.log('running command:', cmd, args.join(' '))
const p = spawn(cmd, args, { cwd: CWD, stdio: 'inherit' }) return new Promise<void>((res, rej) => {
const p = spawn(cmd, args, {
stdio: 'inherit',
env: { ...process.env, ...env },
})
p.on('close', (code) => p.on('close', (code) =>
code === 0 code === 0
? res() ? res()
: rej(new Error(`${cmd} ${args.join(' ')} exited ${code}`)), : rej(new Error(`${cmd} ${args.join(' ')} exited ${code}`)),
) )
p.on('error', (err) => rej(err))
}) })
} }
export const composeUp = (file) =>
run('docker', ['compose', '-f', file, 'up', '-d', '--force-recreate']) export const composeUpWait = (file: string, env?: Record<string, string>) =>
export const composeUpWait = (file) => run(
run('docker', [ 'docker',
'compose', ['compose', '-f', file, 'up', '-d', '--force-recreate', '--wait'],
'-f', env,
file, )
'up', export const composeDown = (file: string) =>
'-d',
'--force-recreate',
'--wait',
])
export const composeDown = (file) =>
run('docker', ['compose', '-f', file, 'down', '--remove-orphans', '-v']) run('docker', ['compose', '-f', file, 'down', '--remove-orphans', '-v'])

View file

@ -1,238 +0,0 @@
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts'
import { describe, expect, inject, it, vi } from 'vitest'
import { S3mini } from '../../src/s3-client/S3.js'
import { createSigV4Signer } from '../../src/s3-client/signer.js'
import { randomBytes } from '../test-utils/browser-crypto.js'
import { beforeRun, cleanupTestBeforeAll } from './_shared.test.js'
const name = 'minio'
// Get bucket configs from globalSetup via Vitest inject
const bucketConfigs = inject('bucketConfigs') || []
const raw = bucketConfigs.find((c) => c.provider === name)
? [
bucketConfigs.find((c) => c.provider === name).provider,
bucketConfigs.find((c) => c.provider === name).accessKeyId,
bucketConfigs.find((c) => c.provider === name).secretAccessKey,
bucketConfigs.find((c) => c.provider === name).endpoint,
bucketConfigs.find((c) => c.provider === name).region,
]
: null
/** Create STS client using @aws-sdk/client-sts */
function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) {
return new STSClient({
region,
endpoint,
credentials: { accessKeyId, secretAccessKey },
})
}
/** Get temporary credentials via AssumeRole */
async function assumeRole(stsClient, { durationSeconds = 900 } = {}) {
const response = await stsClient.send(
new AssumeRoleCommand({
RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN
RoleSessionName: 'uppy-test',
DurationSeconds: durationSeconds,
}),
)
const creds = response.Credentials
return {
AccessKeyId: creds.AccessKeyId,
SecretAccessKey: creds.SecretAccessKey,
SessionToken: creds.SessionToken,
Expiration: creds.Expiration?.toISOString() || creds.Expiration,
}
}
const minioSpecific = (bucket) => {
vi.setConfig({ testTimeout: 120_000 })
const presigner = createSigV4Signer({
accessKeyId: bucket.accessKeyId,
secretAccessKey: bucket.secretAccessKey,
region: bucket.region,
endpoint: bucket.endpoint,
})
const s3client = new S3mini({
endpoint: bucket.endpoint,
region: bucket.region,
signRequest: presigner,
})
cleanupTestBeforeAll(s3client)
// ===== signRequest tests =====
it('simple putObject upload', async () => {
const fileContents = new TextEncoder().encode(
'Hello from pre-signed URL test.',
)
const result = await s3client.putObject(
'presigned-test-file.txt',
fileContents,
'text/plain',
)
expect(result.ok).toBe(true)
})
it('multipart upload with signRequest', async () => {
const key = `presigned-multipart-${Date.now()}.bin`
const partSize = 5 * 1024 * 1024 // 5MB
const part = randomBytes(partSize)
const uploadId = await s3client.getMultipartUploadId(
key,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const uploaded = await s3client.uploadPart(key, uploadId, part, 1)
expect(uploaded.etag).toBeDefined()
const result = await s3client.completeMultipartUpload(key, uploadId, [
uploaded,
])
expect(result.etag).toBeDefined()
expect(result.location).toBeDefined()
expect(result.location).toContain(key)
expect(result.key).toBe(key)
await s3client.deleteObject(key)
})
// ===== getCredentials tests (STS) =====
describe('getCredentials (STS)', () => {
const stsEndpoint = new URL(bucket.endpoint).origin
it('should upload using getCredentials callback', async () => {
const testKey = `sts-getcreds-${Date.now()}.txt`
const s3 = new S3mini({
endpoint: bucket.endpoint,
region: bucket.region,
getCredentials: async () => {
const stsClient = createSTSClient({
endpoint: stsEndpoint,
accessKeyId: bucket.accessKeyId,
secretAccessKey: bucket.secretAccessKey,
region: bucket.region,
})
const creds = await assumeRole(stsClient)
return {
credentials: {
accessKeyId: creds.AccessKeyId,
secretAccessKey: creds.SecretAccessKey,
sessionToken: creds.SessionToken,
expiration: creds.Expiration,
},
bucket: new URL(bucket.endpoint).pathname.slice(1),
region: bucket.region,
}
},
})
try {
const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain')
expect(result.ok).toBe(true)
} finally {
await s3.deleteObject(testKey)
}
})
it('should cache credentials across multiple requests', async () => {
let fetchCount = 0
const s3 = new S3mini({
endpoint: bucket.endpoint,
region: bucket.region,
getCredentials: async () => {
fetchCount++
const stsClient = createSTSClient({
endpoint: stsEndpoint,
accessKeyId: bucket.accessKeyId,
secretAccessKey: bucket.secretAccessKey,
region: bucket.region,
})
const creds = await assumeRole(stsClient)
return {
credentials: {
accessKeyId: creds.AccessKeyId,
secretAccessKey: creds.SecretAccessKey,
sessionToken: creds.SessionToken,
expiration: creds.Expiration,
},
bucket: new URL(bucket.endpoint).pathname.slice(1),
region: bucket.region,
}
},
})
await s3.putObject(
`sts-cache-1-${Date.now()}.txt`,
'File 1',
'text/plain',
)
await s3.putObject(
`sts-cache-2-${Date.now()}.txt`,
'File 2',
'text/plain',
)
expect(fetchCount).toBe(1)
})
it('should perform multipart upload with getCredentials', async () => {
const s3 = new S3mini({
endpoint: bucket.endpoint,
region: bucket.region,
getCredentials: async () => {
const stsClient = createSTSClient({
endpoint: stsEndpoint,
accessKeyId: bucket.accessKeyId,
secretAccessKey: bucket.secretAccessKey,
region: bucket.region,
})
const creds = await assumeRole(stsClient)
return {
credentials: {
accessKeyId: creds.AccessKeyId,
secretAccessKey: creds.SecretAccessKey,
sessionToken: creds.SessionToken,
expiration: creds.Expiration,
},
bucket: new URL(bucket.endpoint).pathname.slice(1),
region: bucket.region,
}
},
})
const key = `sts-multipart-${Date.now()}.bin`
const partSize = 5 * 1024 * 1024 // 5MB
const part = randomBytes(partSize)
const uploadId = await s3.getMultipartUploadId(
key,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const uploaded = await s3.uploadPart(key, uploadId, part, 1)
expect(uploaded.etag).toBeDefined()
const result = await s3.completeMultipartUpload(key, uploadId, [uploaded])
expect(result.etag).toBeDefined()
expect(result.location).toBeDefined()
expect(result.location).toContain(key)
expect(result.key).toBe(key)
await s3.deleteObject(key)
})
})
}
beforeRun(raw, name, minioSpecific)

View file

@ -0,0 +1,456 @@
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts'
import { beforeAll, describe, expect, it } from 'vitest'
import { S3mini } from '../../src/s3-client/S3mini.js'
import { createSigV4Signer } from '../../src/s3-client/signer.js'
import { randomBytes } from '../test-utils/browser-crypto.js'
import { accessKeyId, getConfig, secretAccessKey } from './config.js'
// @ts-expect-error todo
const config = getConfig(import.meta.env)
const suiteName = 'MinIO S3 client tests'
if (config) {
const { endpoint, region } = config
const presigner = createSigV4Signer({
accessKeyId,
secretAccessKey,
region,
endpoint,
})
const s3client = new S3mini({
endpoint,
region,
signRequest: presigner,
})
const EIGHT_MB = 8 * 1024 * 1024
const key_bin = 'test-multipart.bin'
const large_buffer = randomBytes(EIGHT_MB * 3.2)
const content = 'some content'
const key = 'first-test-object.txt'
const key_list_parts = 'test-list-parts.bin'
const key_abort_multipart = 'test-abort-multipart.bin'
const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart]
beforeAll(async () => {
for (const key of FILE_KEYS) {
try {
await s3client.deleteObject(key)
} catch {
// intentionally ignore the errors as the objects don't exists. still feels hacky tbh
}
}
})
/** Create STS client using @aws-sdk/client-sts */
function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) {
return new STSClient({
region,
endpoint,
credentials: { accessKeyId, secretAccessKey },
})
}
/** Get temporary credentials via AssumeRole */
async function assumeRole(
stsClient: STSClient,
{ durationSeconds = 900 } = {},
) {
const response = await stsClient.send(
new AssumeRoleCommand({
RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN
RoleSessionName: 'uppy-test',
DurationSeconds: durationSeconds,
}),
)
const creds = response.Credentials!
return {
AccessKeyId: creds.AccessKeyId,
SecretAccessKey: creds.SecretAccessKey,
SessionToken: creds.SessionToken,
Expiration: creds.Expiration?.toISOString() || creds.Expiration,
}
}
// ===== signRequest tests =====
describe(suiteName, () => {
it('simple putObject upload', async () => {
const key = 'presigned-test-file.txt'
const fileContents = new TextEncoder().encode(
'Hello from pre-signed URL test.',
)
const result = await s3client.putObject(key, fileContents, 'text/plain')
expect(result.location).toBeDefined()
expect(result.location).toContain(key)
// location should be a clean URL without query string (no signing params)
expect(result.location).not.toContain('X-Amz-Signature')
expect(result.location).not.toContain('?')
})
it('multipart upload with signRequest', async () => {
const key = `presigned-multipart-${Date.now()}.bin`
const partSize = 5 * 1024 * 1024 // 5MB
const part = randomBytes(partSize)
const { uploadId } = await s3client.createMultipartUpload(
key,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const partNumber = 1
const { etag } = await s3client.uploadPart(
key,
uploadId,
part,
partNumber,
)
expect(etag).toBeDefined()
const result = await s3client.completeMultipartUpload(key, uploadId, [
{ etag, partNumber },
])
expect(result.etag).toBeDefined()
expect(result.location).toBeDefined()
expect(result.location).toContain(key)
expect(result.key).toBe(key)
await s3client.deleteObject(key)
})
// ===== getCredentials tests (STS) =====
describe('getCredentials (STS)', () => {
const stsEndpoint = new URL(endpoint).origin
it('should upload using getCredentials callback', async () => {
const testKey = `sts-getcreds-${Date.now()}.txt`
const s3 = new S3mini({
endpoint,
region,
getCredentials: async () => {
const stsClient = createSTSClient({
endpoint: stsEndpoint,
accessKeyId,
secretAccessKey,
region,
})
const creds = await assumeRole(stsClient)
return {
credentials: {
accessKeyId: creds.AccessKeyId!,
secretAccessKey: creds.SecretAccessKey!,
sessionToken: creds.SessionToken!,
expiration: creds.Expiration as string,
},
bucket: new URL(endpoint).pathname.slice(1),
region,
}
},
})
try {
const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain')
expect(result.location).toBeDefined()
expect(result.location).toContain(testKey)
expect(result.location).not.toContain('X-Amz-Signature')
expect(result.location).not.toContain('?')
} finally {
await s3.deleteObject(testKey)
}
})
it('should cache credentials across multiple requests', async () => {
let fetchCount = 0
const s3 = new S3mini({
endpoint,
region,
getCredentials: async () => {
fetchCount++
const stsClient = createSTSClient({
endpoint: stsEndpoint,
accessKeyId,
secretAccessKey,
region,
})
const creds = await assumeRole(stsClient)
return {
credentials: {
accessKeyId: creds.AccessKeyId!,
secretAccessKey: creds.SecretAccessKey!,
sessionToken: creds.SessionToken!,
expiration: creds.Expiration as string,
},
bucket: new URL(endpoint).pathname.slice(1),
region,
}
},
})
await s3.putObject(
`sts-cache-1-${Date.now()}.txt`,
'File 1',
'text/plain',
)
await s3.putObject(
`sts-cache-2-${Date.now()}.txt`,
'File 2',
'text/plain',
)
expect(fetchCount).toBe(1)
})
it('should perform multipart upload with getCredentials', async () => {
const s3 = new S3mini({
endpoint,
region,
getCredentials: async () => {
const stsClient = createSTSClient({
endpoint: stsEndpoint,
accessKeyId,
secretAccessKey,
region,
})
const creds = await assumeRole(stsClient)
return {
credentials: {
accessKeyId: creds.AccessKeyId!,
secretAccessKey: creds.SecretAccessKey!,
sessionToken: creds.SessionToken!,
expiration: creds.Expiration as string,
},
bucket: new URL(endpoint).pathname.slice(1),
region,
}
},
})
const key = `sts-multipart-${Date.now()}.bin`
const partSize = 5 * 1024 * 1024 // 5MB
const part = randomBytes(partSize)
const { uploadId } = await s3.createMultipartUpload(
key,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const partNumber = 1
const { etag } = await s3.uploadPart(key, uploadId, part, partNumber)
expect(etag).toBeDefined()
const result = await s3.completeMultipartUpload(key, uploadId, [
{ etag, partNumber },
])
expect(result.etag).toBeDefined()
expect(result.location).toBeDefined()
expect(result.location).toContain(key)
expect(result.key).toBe(key)
await s3.deleteObject(key)
})
})
it('instantiates s3client', () => {
expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation
})
// we don't need an explicit eTag method as we already get eTag in the putOject response
it('putObject uploads successfully and returns ETag', async () => {
const response = await s3client.putObject(key, content, 'text/plain')
expect(response.etag).toBeDefined()
await s3client.deleteObject(key)
})
it('putObject handles binary data', async () => {
const binaryData = new Uint8Array(6).fill(0xff)
const response = await s3client.putObject(
key,
binaryData,
'application/octet-stream',
)
expect(response).toBeDefined()
expect(response.etag).toBeDefined()
// cleanup
await s3client.deleteObject(key)
})
// test createMultipartUpload
it('createMultipartUpload returns a valid uploadId', async () => {
const { uploadId } = await s3client.createMultipartUpload(
key_bin,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
expect(typeof uploadId).toBe('string')
expect(uploadId.length).toBeGreaterThan(0)
// cleanup
await s3client.abortMultipartUpload(key, uploadId)
})
// test uploadPart
it('uploadPart returns partNumber and Etag', async () => {
const partData = randomBytes(EIGHT_MB)
const { uploadId } = await s3client.createMultipartUpload(
key_bin,
'application/octet-stream',
)
// upload part
const partResult = await s3client.uploadPart(
key_bin,
uploadId,
partData,
1,
)
expect(partResult).toBeDefined()
expect(partResult.etag).toBeDefined()
expect(typeof partResult.etag).toBe('string')
expect(partResult.etag.length).toBe(32)
// cleanup
await s3client.abortMultipartUpload(key, uploadId)
})
// end to end multipart flow
it('completeMultipartUpload assembles parts correctly', async () => {
// key - key_bin
const partSize = EIGHT_MB
const totalParts = Math.ceil(large_buffer.byteLength / partSize)
const { uploadId } = await s3client.createMultipartUpload(
key_bin,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
// upload all parts
const uploadPromises: Promise<{ etag: string }>[] = []
for (let i = 0; i < totalParts; i++) {
const partBuffer = large_buffer.subarray(
i * partSize,
(i + 1) * partSize,
)
uploadPromises.push(
s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1),
)
}
const uploadResponses = await Promise.all(uploadPromises)
// verify all parts uploaded succesfully
expect(uploadResponses.length).toBe(totalParts)
uploadResponses.forEach((response) => {
expect(response.etag).toBeDefined()
})
// create multipart upload
const parts = uploadResponses.map((response, index) => ({
partNumber: index + 1,
etag: response.etag,
}))
const completeResponse = await s3client.completeMultipartUpload(
key_bin,
uploadId,
parts,
)
expect(completeResponse).toBeDefined()
expect(typeof completeResponse).toBe('object')
expect(completeResponse.etag).toBeDefined()
expect(typeof completeResponse.etag).toBe('string')
expect(completeResponse.etag!.length).toBe(32 + 2)
// cleanup
await s3client.deleteObject(key_bin)
})
it('abortMultipartUpload cancels upload successfully', async () => {
// start upload
const { uploadId } = await s3client.createMultipartUpload(
key_abort_multipart,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const partData = randomBytes(EIGHT_MB)
await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1)
// abort
await s3client.abortMultipartUpload(key_abort_multipart, uploadId)
})
it('listParts returns uploaded parts correctly', async () => {
const partSize = EIGHT_MB
const { uploadId } = await s3client.createMultipartUpload(
key_list_parts,
'application/octet-stream',
)
expect(uploadId).toBeDefined()
const part1Data = randomBytes(partSize)
const part2Data = randomBytes(partSize)
const part1Result = await s3client.uploadPart(
key_list_parts,
uploadId,
part1Data,
1,
)
const part2Result = await s3client.uploadPart(
key_list_parts,
uploadId,
part2Data,
2,
)
const parts = await s3client.listParts(uploadId, key_list_parts)
expect(parts).toBeInstanceOf(Array)
expect(parts.length).toBe(2)
// verify part 1
expect(parts[0].partNumber).toBe(1)
expect(parts[0].etag).toBe(part1Result.etag)
// verify part 2
expect(parts[1].partNumber).toBe(2)
expect(parts[1].etag).toBe(part2Result.etag)
// cleanup abort upload
await s3client.abortMultipartUpload(key, uploadId)
})
})
} else {
console.warn(
'Skipping MinIO S3 client tests: missing env variable VITE_MINIO_CONFIG',
)
describe.skip(suiteName, () => {
it('noop', () => {})
})
}

View file

@ -1,75 +0,0 @@
import * as dotenv from 'dotenv'
dotenv.config()
import { join } from 'node:path'
import { composeUp, composeUpWait, execDockerCommand } from './docker.js'
const composeFiles = {
minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'),
}
const bucketConfigs = Object.keys(process.env)
.filter((k) => k.startsWith('BUCKET_ENV_'))
.map((k) => {
const [provider, rootAccessKeyId, rootSecretAccessKey, endpoint, region] =
process.env[k].split(',')
// Use stsuser credentials for all tests (readwrite policy is sufficient)
// Root credentials are kept for Docker container startup
return {
provider,
accessKeyId: 'stsuser',
secretAccessKey: 'stspassword123',
rootAccessKeyId,
rootSecretAccessKey,
endpoint,
region,
}
})
export default async function setup({ provide }) {
// Provide bucket configs to browser tests via Vitest's inject mechanism
provide('bucketConfigs', bucketConfigs)
for (const cfg of bucketConfigs) {
const composeFile = composeFiles[cfg.provider]
if (!composeFile) continue
console.log(`⏫ starting ${cfg.provider} image …`)
switch (cfg.provider) {
case 'minio':
process.env.MINIO_ROOT_USER = cfg.rootAccessKeyId
process.env.MINIO_ROOT_PASSWORD = cfg.rootSecretAccessKey
await composeUpWait(composeFile)
// biome-ignore lint/correctness/noSwitchDeclarations: no other switch blocks
const bucketName = new URL(cfg.endpoint).pathname.split('/')[1]
if (bucketName) {
await execDockerCommand(
'minio',
`mc mb local/${bucketName} --ignore-existing`,
30000,
)
// Create a user with readwrite policy for STS testing
// The root user cannot AssumeRole for itself - needs a regular user
try {
await execDockerCommand(
'minio',
`mc admin user add local stsuser stspassword123`,
15000,
)
await execDockerCommand(
'minio',
`mc admin policy attach local readwrite --user stsuser`,
15000,
)
} catch (err) {
// User may already exist, that's fine
console.log('STS user setup:', err.message || 'error occurred')
}
}
break
default:
await composeUp(composeFile)
}
}
}

View file

@ -0,0 +1,48 @@
import path from 'node:path'
import { getConfig } from './config'
import { composeDown, composeUpWait, execDockerCommand } from './docker'
const composeFile = path.join(__dirname, 'compose.minio.yaml')
const config = getConfig(process.env)
export async function setup() {
if (config == null) return
const { endpoint, rootAccessKeyId, rootSecretAccessKey } = config
console.log('⏫ starting minio image', composeFile)
process.env.MINIO_ROOT_USER = rootAccessKeyId
process.env.MINIO_ROOT_PASSWORD = rootSecretAccessKey
await composeUpWait(composeFile)
const bucketName = new URL(endpoint).pathname.split('/')[1]
await execDockerCommand(
'minio',
`mc mb local/${bucketName} --ignore-existing`,
30000,
)
// Create a user with readwrite policy for STS testing
// The root user cannot AssumeRole for itself - needs a regular user
try {
await execDockerCommand(
'minio',
`mc admin user add local stsuser stspassword123`,
)
await execDockerCommand(
'minio',
`mc admin policy attach local readwrite --user stsuser`,
)
} catch (err) {
// User may already exist, that's fine
console.log('STS user setup:', err.message || 'error occurred')
}
console.log(`✅ minio is ready`)
}
export async function teardown() {
if (config == null) return
console.log(`⏬ stopping minio image …`)
await composeDown(composeFile)
console.log(`✅ minio stopped and cleaned up`)
}

View file

@ -1,28 +0,0 @@
import * as dotenv from 'dotenv'
dotenv.config()
import { join } from 'node:path'
import { composeDown } from './docker.js'
const composeFiles = {
minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'),
}
const bucketConfigs = Object.keys(process.env)
.filter((k) => k.startsWith('BUCKET_ENV_'))
.map((k) => {
const [provider, accessKeyId, secretAccessKey, endpoint, region] =
process.env[k].split(',')
return { provider, accessKeyId, secretAccessKey, endpoint, region }
})
export default async () => {
for (const cfg of bucketConfigs) {
const composeFile = composeFiles[cfg.provider]
if (!composeFile) continue // ignore unknown providers
console.log(`⏬ stopping ${cfg.provider}`)
await composeDown(composeFile) // `docker compose -f … down`
}
}

View file

@ -1,37 +0,0 @@
/**
* Generate random bytes using Web Crypto API.
* Web Crypto has a 65536 byte limit per getRandomValues call,
* so we chunk large requests.
* @param {number} size - Number of random bytes to generate
* @returns {Uint8Array} Random bytes
*/
export function randomBytes(size) {
const buffer = new Uint8Array(size)
const maxChunk = 65536 // Web Crypto API limit
for (let offset = 0; offset < size; offset += maxChunk) {
const chunkSize = Math.min(maxChunk, size - offset)
const chunk = buffer.subarray(offset, offset + chunkSize)
globalThis.crypto.getRandomValues(chunk)
}
return buffer
}
/**
* Compute SHA-1 hash and return base64 string.
* @param {Uint8Array | ArrayBuffer} data - Data to hash
* @returns {Promise<string>} Base64-encoded SHA-1 hash
*/
export async function sha1Base64(data) {
const hash = await globalThis.crypto.subtle.digest('SHA-1', data)
const bytes = new Uint8Array(hash)
// Convert to base64 in chunks to avoid call stack issues with large data
const chunkSize = 0x8000
let result = ''
for (let i = 0; i < bytes.length; i += chunkSize) {
const chunk = bytes.subarray(i, i + chunkSize)
result += String.fromCharCode(...chunk)
}
return btoa(result)
}

View file

@ -0,0 +1,17 @@
/**
* Generate random bytes using Web Crypto API.
* Web Crypto has a 65536 byte limit per getRandomValues call,
* so we chunk large requests.
*/
export function randomBytes(size: number) {
const buffer = new Uint8Array(size)
const maxChunk = 65536 // Web Crypto API limit
for (let offset = 0; offset < size; offset += maxChunk) {
const chunkSize = Math.min(maxChunk, size - offset)
const chunk = buffer.subarray(offset, offset + chunkSize)
globalThis.crypto.getRandomValues(chunk)
}
return buffer
}

View file

@ -3,21 +3,19 @@ import { defineConfig } from 'vitest/config'
export default defineConfig({ export default defineConfig({
test: { test: {
testTimeout: 120_000, testTimeout: 120_000,
globalSetup: ['./tests/s3-client/setup.js'], globalSetup: ['tests/s3-client/setup.ts'],
setupFiles: ['./test-setup.mjs'],
projects: [ projects: [
{ {
test: { test: {
name: 'aws-s3-node', name: 's3-jsdom',
include: ['tests/index.test.ts', 'tests/createSignedURL.test.ts'], include: ['**/*.test.ts'],
environment: 'jsdom', environment: 'jsdom',
}, },
}, },
{ {
test: { test: {
name: 's3-client-browser', name: 's3-browser',
include: ['tests/s3-client/*.test.{js,ts}'], include: ['**/minio.test.ts'],
exclude: ['tests/s3-client/_shared.test.js'],
browser: { browser: {
enabled: true, enabled: true,
provider: 'playwright', provider: 'playwright',

View file

@ -662,11 +662,12 @@ export class Uppy<
...files[fileID].progress, ...files[fileID].progress,
...defaultProgress, ...defaultProgress,
}, },
// @ts-expect-error these typed are inserted // @ts-expect-error these types are inserted
// into the namespace in their respective packages // into the namespace in their respective packages
// but core isn't ware of those // but core isn't aware of those
tus: undefined, tus: undefined,
transloadit: undefined, transloadit: undefined,
s3Multipart: undefined,
} }
}) })

View file

@ -61,7 +61,7 @@ export function fetcher(
options: FetcherOptions = {}, options: FetcherOptions = {},
): Promise<XMLHttpRequest> { ): Promise<XMLHttpRequest> {
const { const {
body = null, body,
headers = {}, headers = {},
method = 'GET', method = 'GET',
onBeforeRequest = noop, onBeforeRequest = noop,

View file

@ -208,13 +208,13 @@ export default () => {
break break
case 's3': case 's3':
uppyDashboard.use(AwsS3, { uppyDashboard.use(AwsS3, {
endpoint: COMPANION_URL, companionEndpoint: COMPANION_URL,
shouldUseMultipart: false, shouldUseMultipart: false,
}) })
break break
case 's3-multipart': case 's3-multipart':
uppyDashboard.use(AwsS3, { uppyDashboard.use(AwsS3, {
endpoint: COMPANION_URL, companionEndpoint: COMPANION_URL,
shouldUseMultipart: true, shouldUseMultipart: true,
}) })
break break

View file

@ -42,7 +42,7 @@
"test": { "test": {
"dependsOn": ["^test"], "dependsOn": ["^test"],
"cache": false, "cache": false,
"passThroughEnv": ["BUCKET_ENV_MINIO"] "passThroughEnv": ["VITE_MINIO_CONFIG"]
}, },
"test:e2e": { "test:e2e": {
"dependsOn": ["^test:e2e"], "dependsOn": ["^test:e2e"],