mirror of
https://github.com/transloadit/uppy.git
synced 2026-07-17 16:50:22 +00:00
@uppy/aws-s3: remote uploads and golden retriever (#6186)
- add support for remote uploads - add support for restore using golden-retriever - update examples - #6181 still persists --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Mikael Finstad <finstaden@gmail.com>
This commit is contained in:
parent
da46951771
commit
2d429a8aef
31 changed files with 2478 additions and 2244 deletions
1
.claude/worktrees/upbeat-tharp
Submodule
1
.claude/worktrees/upbeat-tharp
Submodule
|
|
@ -0,0 +1 @@
|
||||||
|
Subproject commit e603096c56d853b429ccb06a585ffb6d0aa7539e
|
||||||
4
.github/CONTRIBUTING.md
vendored
4
.github/CONTRIBUTING.md
vendored
|
|
@ -47,6 +47,10 @@ yarn build
|
||||||
- `yarn typecheck` - Run TypeScript type checking
|
- `yarn typecheck` - Run TypeScript type checking
|
||||||
- `yarn check` - Run Biome linting and formatting
|
- `yarn check` - Run Biome linting and formatting
|
||||||
|
|
||||||
|
### Running minio tests
|
||||||
|
|
||||||
|
Tests for `@uppy/aws-s3` need a minio server running inside Docker, and in order to run those tests you must set the environment variable `VITE_MINIO_CONFIG="test_user,test_password,http://localhost:9002/test-bucket,us-east-1"` when running the tests.
|
||||||
|
|
||||||
### Headless components
|
### Headless components
|
||||||
|
|
||||||
When adding a new component to `@uppy/components`, you have to run `yarn migrate:components` from root
|
When adding a new component to `@uppy/components`, you have to run `yarn migrate:components` from root
|
||||||
|
|
|
||||||
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
|
|
@ -59,7 +59,7 @@ jobs:
|
||||||
COMPANION_DOMAIN: localhost:3020
|
COMPANION_DOMAIN: localhost:3020
|
||||||
COMPANION_PROTOCOL: http
|
COMPANION_PROTOCOL: http
|
||||||
COMPANION_REDIS_URL: redis://localhost:6379
|
COMPANION_REDIS_URL: redis://localhost:6379
|
||||||
BUCKET_ENV_MINIO: "minio,test_user,test_password,http://localhost:9002/test-bucket,us-east-1"
|
VITE_MINIO_CONFIG: "test_user,test_password,http://localhost:9002/test-bucket,us-east-1"
|
||||||
|
|
||||||
types:
|
types:
|
||||||
name: Types
|
name: Types
|
||||||
|
|
|
||||||
|
|
@ -23,5 +23,5 @@ uppy.use(Dashboard, {
|
||||||
plugins: ['GoogleDrive', 'Webcam'],
|
plugins: ['GoogleDrive', 'Webcam'],
|
||||||
})
|
})
|
||||||
uppy.use(AwsS3, {
|
uppy.use(AwsS3, {
|
||||||
endpoint: 'http://localhost:3020',
|
companionEndpoint: 'http://localhost:3020',
|
||||||
})
|
})
|
||||||
|
|
|
||||||
421
packages/@uppy/aws-s3/src/S3Uploader.ts
Normal file
421
packages/@uppy/aws-s3/src/S3Uploader.ts
Normal file
|
|
@ -0,0 +1,421 @@
|
||||||
|
import { EventManager, type Uppy } from '@uppy/core'
|
||||||
|
import type { Body, LocalUppyFile, Meta } from '@uppy/utils'
|
||||||
|
import type S3Client from './s3-client/S3Client.js'
|
||||||
|
|
||||||
|
/** Persisted S3 multipart state for Golden Retriever resume support */
|
||||||
|
interface S3MultipartState {
|
||||||
|
uploadId: string
|
||||||
|
key: string
|
||||||
|
}
|
||||||
|
|
||||||
|
declare module '@uppy/utils' {
|
||||||
|
// biome-ignore lint/correctness/noUnusedVariables: must match existing interface signature
|
||||||
|
export interface LocalUppyFile<M extends Meta, B extends Body> {
|
||||||
|
s3Multipart?: S3MultipartState
|
||||||
|
}
|
||||||
|
// biome-ignore lint/correctness/noUnusedVariables: must match existing interface signature
|
||||||
|
export interface RemoteUppyFile<M extends Meta, B extends Body> {
|
||||||
|
s3Multipart?: S3MultipartState
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Constants
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
const MB = 1024 * 1024
|
||||||
|
|
||||||
|
/** Minimum chunk size required by S3 (5MB) */
|
||||||
|
const MIN_CHUNK_SIZE = 5 * MB
|
||||||
|
|
||||||
|
/** Maximum number of parts allowed by S3 */
|
||||||
|
const MAX_PARTS = 10000
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// S3Uploader Types
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
interface S3UploaderOptions<M extends Meta, B extends Body> {
|
||||||
|
uppy: Uppy<M, B>
|
||||||
|
s3Client: S3Client
|
||||||
|
file: LocalUppyFile<M, B>
|
||||||
|
metadata: Record<string, unknown>
|
||||||
|
key: string
|
||||||
|
shouldUseMultipart?: boolean
|
||||||
|
getChunkSize?: (file: { size: number }) => number
|
||||||
|
onProgress?: (bytesUploaded: number, bytesTotal: number) => void
|
||||||
|
onPartComplete?: (part: { PartNumber: number; ETag: string }) => void
|
||||||
|
onSuccess?: (result: UploadResult) => void
|
||||||
|
onError?: (err: Error) => void
|
||||||
|
onAbort?: () => void
|
||||||
|
log?: Uppy['log']
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UploadResult {
|
||||||
|
location: string
|
||||||
|
key: string
|
||||||
|
/** Only returned for multipart uploads */
|
||||||
|
uploadId?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Chunk {
|
||||||
|
index: number
|
||||||
|
start: number
|
||||||
|
end: number
|
||||||
|
size: number
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ChunkState {
|
||||||
|
uploaded: number
|
||||||
|
etag?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export default class S3Uploader<M extends Meta, B extends Body> {
|
||||||
|
readonly #data: NonNullable<LocalUppyFile<M, B>['data']>
|
||||||
|
#key: string | undefined
|
||||||
|
readonly #options: S3UploaderOptions<M, B>
|
||||||
|
readonly #eventManager: EventManager<M, B>
|
||||||
|
|
||||||
|
#chunks: Chunk[] = []
|
||||||
|
#chunkState: ChunkState[] = []
|
||||||
|
#shouldUseMultipart: boolean = false
|
||||||
|
#uploadId?: string
|
||||||
|
#uploadHasStarted: boolean = false
|
||||||
|
#abortController: AbortController | undefined
|
||||||
|
|
||||||
|
constructor(options: S3UploaderOptions<M, B>) {
|
||||||
|
if (options.file.data == null) {
|
||||||
|
throw new Error(`File data is missing for file ${options.file.id}`)
|
||||||
|
}
|
||||||
|
this.#options = options
|
||||||
|
this.#data = options.file.data
|
||||||
|
this.#eventManager = new EventManager(options.uppy)
|
||||||
|
|
||||||
|
// Detect resume state from file (persisted by Golden Retriever across page refreshes).
|
||||||
|
// Must run before #initChunks so it can force multipart mode for resumed uploads.
|
||||||
|
const resumeState = options.file.s3Multipart
|
||||||
|
if (resumeState) {
|
||||||
|
this.#key = resumeState.key
|
||||||
|
this.#uploadId = resumeState.uploadId
|
||||||
|
this.#uploadHasStarted = true
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileSize = options.file.data.size
|
||||||
|
|
||||||
|
// Determine if we should use multipart
|
||||||
|
// If we're resuming a multipart upload, force multipart. Otherwise use
|
||||||
|
// the boolean option (true/false) and ensure the file is larger than
|
||||||
|
// S3's minimum chunk size when enabling multipart.
|
||||||
|
this.#shouldUseMultipart =
|
||||||
|
Boolean(resumeState) ||
|
||||||
|
(this.#options.shouldUseMultipart === true && fileSize > MIN_CHUNK_SIZE)
|
||||||
|
|
||||||
|
// Create chunks based on upload strategy
|
||||||
|
if (this.#shouldUseMultipart) {
|
||||||
|
// Calculate chunk size: at least MIN_CHUNK_SIZE, but may be larger for huge files
|
||||||
|
let chunkSize = this.#getChunkSize(fileSize)
|
||||||
|
chunkSize = Math.max(chunkSize, MIN_CHUNK_SIZE)
|
||||||
|
|
||||||
|
// Ensure we don't exceed MAX_PARTS (S3 limit: 10,000 parts)
|
||||||
|
if (Math.ceil(fileSize / chunkSize) > MAX_PARTS) {
|
||||||
|
chunkSize = Math.ceil(fileSize / MAX_PARTS)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create chunk definitions
|
||||||
|
for (
|
||||||
|
let offset = 0, index = 0;
|
||||||
|
offset < fileSize;
|
||||||
|
offset += chunkSize, index++
|
||||||
|
) {
|
||||||
|
const end = Math.min(offset + chunkSize, fileSize)
|
||||||
|
this.#chunks.push({ index, start: offset, end, size: end - offset })
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Simple upload: single chunk for the entire file
|
||||||
|
this.#chunks = [{ index: 0, start: 0, end: fileSize, size: fileSize }]
|
||||||
|
}
|
||||||
|
|
||||||
|
this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
|
||||||
|
|
||||||
|
// Setup events:
|
||||||
|
const fileId = this.#options.file.id
|
||||||
|
|
||||||
|
this.#eventManager.onFileRemove(fileId, () => {
|
||||||
|
this.abort()
|
||||||
|
this.#options.onAbort?.()
|
||||||
|
})
|
||||||
|
|
||||||
|
this.#eventManager.onCancelAll(fileId, () => {
|
||||||
|
this.abort()
|
||||||
|
this.#options.onAbort?.()
|
||||||
|
})
|
||||||
|
|
||||||
|
this.#eventManager.onFilePause(fileId, (isPaused) => {
|
||||||
|
if (isPaused) {
|
||||||
|
this.pause()
|
||||||
|
} else {
|
||||||
|
this.start()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
this.#eventManager.onPauseAll(fileId, () => {
|
||||||
|
this.pause()
|
||||||
|
})
|
||||||
|
|
||||||
|
this.#eventManager.onResumeAll(fileId, () => {
|
||||||
|
this.start()
|
||||||
|
})
|
||||||
|
|
||||||
|
this.#eventManager.onRetry(fileId, () => {
|
||||||
|
this.start()
|
||||||
|
})
|
||||||
|
|
||||||
|
this.#eventManager.onRetryAll(fileId, () => {
|
||||||
|
this.start()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#getChunkSize(fileSize: number): number {
|
||||||
|
if (this.#options.getChunkSize) {
|
||||||
|
return this.#options.getChunkSize({ size: fileSize })
|
||||||
|
}
|
||||||
|
return Math.ceil(fileSize / MAX_PARTS)
|
||||||
|
}
|
||||||
|
|
||||||
|
async start(): Promise<void> {
|
||||||
|
// Abort any pending operations (if not already aborted)
|
||||||
|
this.#abortController?.abort()
|
||||||
|
// Always create a fresh AbortController (also for resume)
|
||||||
|
this.#abortController = new AbortController()
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (this.#uploadHasStarted) {
|
||||||
|
await this.#resumeUpload()
|
||||||
|
} else {
|
||||||
|
await this.#createUpload()
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
this.#onError(err as Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pause(): void {
|
||||||
|
this.#abortController?.abort()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param opts - `abortInS3`: Whether to also abort the multipart upload in S3. Default: true. Set to false to keep the multipart upload in S3 active, allowing for manual cleanup later and preventing accidental data loss if the user later tries to resume the upload.
|
||||||
|
*/
|
||||||
|
abort(opts?: { abortInS3?: boolean }): void {
|
||||||
|
this.#abortController?.abort()
|
||||||
|
// Clean up event listeners
|
||||||
|
this.#eventManager.remove()
|
||||||
|
if (opts?.abortInS3 !== false && this.#uploadId) {
|
||||||
|
if (!this.#key) {
|
||||||
|
throw new Error('Missing S3 object key for aborting upload')
|
||||||
|
}
|
||||||
|
this.#options.s3Client
|
||||||
|
.abortMultipartUpload(this.#key, this.#uploadId)
|
||||||
|
.catch((abortErr) => {
|
||||||
|
this.#options.log?.(abortErr, 'warning')
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async #createUpload(): Promise<void> {
|
||||||
|
this.#uploadHasStarted = true
|
||||||
|
if (this.#shouldUseMultipart) {
|
||||||
|
await this.#uploadMultipart()
|
||||||
|
} else {
|
||||||
|
await this.#uploadNonMultipart()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async #resumeUpload(): Promise<void> {
|
||||||
|
if (!this.#uploadId) {
|
||||||
|
await this.#createUpload()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (!this.#key) {
|
||||||
|
throw new Error('Missing S3 object key for resuming upload')
|
||||||
|
}
|
||||||
|
const existingParts = await this.#options.s3Client.listParts(
|
||||||
|
this.#uploadId,
|
||||||
|
this.#key,
|
||||||
|
)
|
||||||
|
// Sync local state with S3 - mark already-uploaded parts
|
||||||
|
for (const part of existingParts) {
|
||||||
|
const chunkIndex = part.partNumber - 1
|
||||||
|
if (chunkIndex >= 0 && chunkIndex < this.#chunkState.length) {
|
||||||
|
this.#chunkState[chunkIndex].uploaded = this.#chunks[chunkIndex].size
|
||||||
|
this.#chunkState[chunkIndex].etag = part.etag
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Emit progress update to reflect already-uploaded parts
|
||||||
|
this.#onProgress()
|
||||||
|
await this.#uploadRemainingParts()
|
||||||
|
}
|
||||||
|
|
||||||
|
async #uploadNonMultipart(): Promise<void> {
|
||||||
|
const signal = this.#abortController?.signal
|
||||||
|
signal?.throwIfAborted()
|
||||||
|
|
||||||
|
const { location, key } = await this.#options.s3Client.putObject(
|
||||||
|
this.#options.key,
|
||||||
|
this.#data,
|
||||||
|
this.#options.file.type || 'application/octet-stream',
|
||||||
|
this.#options.metadata,
|
||||||
|
(bytesUploaded: number) => {
|
||||||
|
this.#chunkState[0].uploaded = bytesUploaded
|
||||||
|
this.#onProgress()
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
)
|
||||||
|
|
||||||
|
this.#onSuccess({
|
||||||
|
location,
|
||||||
|
key,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async #uploadMultipart(): Promise<void> {
|
||||||
|
const signal = this.#abortController?.signal
|
||||||
|
signal?.throwIfAborted()
|
||||||
|
|
||||||
|
const { uploadId, key } =
|
||||||
|
await this.#options.s3Client.createMultipartUpload(
|
||||||
|
this.#options.key,
|
||||||
|
this.#options.file.type || 'application/octet-stream',
|
||||||
|
this.#options.metadata,
|
||||||
|
)
|
||||||
|
if (key == null) {
|
||||||
|
throw new Error(
|
||||||
|
'S3 client did not return object key for multipart upload',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
this.#key = key
|
||||||
|
|
||||||
|
// Persist resume state so Golden Retriever can restore it after page refresh
|
||||||
|
this.#options.uppy.setFileState(this.#options.file.id, {
|
||||||
|
s3Multipart: { uploadId, key },
|
||||||
|
})
|
||||||
|
|
||||||
|
this.#uploadId = uploadId
|
||||||
|
|
||||||
|
await this.#uploadRemainingParts()
|
||||||
|
}
|
||||||
|
|
||||||
|
async #uploadRemainingParts(): Promise<void> {
|
||||||
|
const signal = this.#abortController?.signal
|
||||||
|
|
||||||
|
for (let i = 0; i < this.#chunks.length; i++) {
|
||||||
|
signal?.throwIfAborted()
|
||||||
|
if (this.#chunkState[i].etag) continue // already uploaded
|
||||||
|
|
||||||
|
const chunk = this.#chunks[i]
|
||||||
|
const partNumber = i + 1
|
||||||
|
const chunkData = this.#data.slice(chunk.start, chunk.end)
|
||||||
|
const chunkIndex = i // Capture for closure (cannot use for-loop variable i directly in a closure)
|
||||||
|
|
||||||
|
if (this.#key == null) {
|
||||||
|
throw new Error('Missing S3 object key for uploading part')
|
||||||
|
}
|
||||||
|
const { etag } = await this.#options.s3Client.uploadPart(
|
||||||
|
this.#key,
|
||||||
|
this.#uploadId!,
|
||||||
|
chunkData,
|
||||||
|
partNumber,
|
||||||
|
(bytesUploaded: number) => {
|
||||||
|
this.#chunkState[chunkIndex].uploaded = bytesUploaded
|
||||||
|
this.#onProgress()
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
)
|
||||||
|
|
||||||
|
// after part finished uploading, update chunk state
|
||||||
|
this.#chunkState[i].uploaded = chunk.size
|
||||||
|
this.#chunkState[i].etag = etag
|
||||||
|
this.#onProgress()
|
||||||
|
|
||||||
|
if (this.#options.onPartComplete) {
|
||||||
|
this.#options.onPartComplete({
|
||||||
|
PartNumber: partNumber,
|
||||||
|
ETag: etag,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
signal?.throwIfAborted()
|
||||||
|
|
||||||
|
const parts = this.#chunkState.flatMap((state, i) =>
|
||||||
|
state.etag ? [{ partNumber: i + 1, etag: state.etag }] : [],
|
||||||
|
)
|
||||||
|
|
||||||
|
if (this.#key == null) {
|
||||||
|
throw new Error('Missing S3 object key for completing multipart upload')
|
||||||
|
}
|
||||||
|
|
||||||
|
const { location, key } =
|
||||||
|
await this.#options.s3Client.completeMultipartUpload(
|
||||||
|
this.#key,
|
||||||
|
this.#uploadId!,
|
||||||
|
parts,
|
||||||
|
)
|
||||||
|
|
||||||
|
this.#onSuccess({
|
||||||
|
location,
|
||||||
|
key,
|
||||||
|
uploadId: this.#uploadId,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#onProgress(): void {
|
||||||
|
if (!this.#options.onProgress) return
|
||||||
|
const bytesUploaded = this.#chunkState.reduce(
|
||||||
|
(sum, state) => sum + state.uploaded,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
this.#options.onProgress(bytesUploaded, this.#data.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
#onSuccess(result: UploadResult): void {
|
||||||
|
// If the upload was aborted (file removed mid-upload), the network request
|
||||||
|
// may still complete successfully. Don't emit success in this case since
|
||||||
|
// the file no longer exists in Uppy's state.
|
||||||
|
this.#eventManager.remove()
|
||||||
|
if (this.#abortController?.signal.aborted) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Clear persisted resume state — upload completed successfully.
|
||||||
|
this.#options.uppy.setFileState(this.#options.file.id, {
|
||||||
|
s3Multipart: undefined,
|
||||||
|
})
|
||||||
|
this.#options.onSuccess?.(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
#onError(err: Error): void {
|
||||||
|
// ignore abort signals from intentional cancellation
|
||||||
|
if (err.name === 'AbortError') return
|
||||||
|
// If we intentionally aborted, don't report any subsequent errors
|
||||||
|
// (e.g., S3 returning 404 NoSuchUpload after we aborted the upload)
|
||||||
|
if (this.#abortController?.signal.aborted) return
|
||||||
|
|
||||||
|
// Clean up event listeners so this uploader doesn't become a "zombie"
|
||||||
|
// that reacts to future retry/pause/resume events after the error.
|
||||||
|
// Without this, each failed retry leaves an orphaned uploader that
|
||||||
|
// still listens for retry-all, causing duplicate uploads on the next
|
||||||
|
// successful retry.
|
||||||
|
this.#eventManager.remove()
|
||||||
|
|
||||||
|
// NOTE: We intentionally do NOT abort the multipart upload in S3 here.
|
||||||
|
// This allows the user to retry and resume from where they left off.
|
||||||
|
// The multipart upload is only aborted when the user cancels via the
|
||||||
|
// `abort()` method. By default `abort()` will also abort the multipart
|
||||||
|
// upload in S3 (abortInS3 = true). Pass { abortInS3: false } to keep the
|
||||||
|
// multipart upload in S3 so it can be cleaned up manually or resumed later.
|
||||||
|
|
||||||
|
this.#options.onError?.(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,18 +1,28 @@
|
||||||
|
import type { RequestClient } from '@uppy/companion-client'
|
||||||
import {
|
import {
|
||||||
BasePlugin,
|
BasePlugin,
|
||||||
type DefinePluginOpts,
|
type DefinePluginOpts,
|
||||||
EventManager,
|
|
||||||
type PluginOpts,
|
type PluginOpts,
|
||||||
type Uppy,
|
type Uppy,
|
||||||
} from '@uppy/core'
|
} from '@uppy/core'
|
||||||
import type { Body, Meta, UppyFile } from '@uppy/utils'
|
import type {
|
||||||
|
Body,
|
||||||
|
LocalUppyFile,
|
||||||
|
Meta,
|
||||||
|
RemoteUppyFile,
|
||||||
|
UppyFile,
|
||||||
|
} from '@uppy/utils'
|
||||||
import {
|
import {
|
||||||
filterFilesToEmitUploadStarted,
|
filterFilesToEmitUploadStarted,
|
||||||
filterFilesToUpload,
|
filterFilesToUpload,
|
||||||
|
getAllowedMetaFields,
|
||||||
TaskQueue,
|
TaskQueue,
|
||||||
} from '@uppy/utils'
|
} from '@uppy/utils'
|
||||||
import packageJson from '../package.json' with { type: 'json' }
|
import packageJson from '../package.json' with { type: 'json' }
|
||||||
import S3mini from './s3-client/S3.js'
|
import S3Uploader, { type UploadResult } from './S3Uploader.js'
|
||||||
|
import S3Companion from './s3-client/CompanionS3.js'
|
||||||
|
import type S3Client from './s3-client/S3Client.js'
|
||||||
|
import S3mini from './s3-client/S3mini.js'
|
||||||
import type * as IT from './s3-client/types.js'
|
import type * as IT from './s3-client/types.js'
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
@ -37,26 +47,7 @@ declare module '@uppy/core' {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AwsS3Options<M extends Meta, B extends Body>
|
export type AwsS3Options<M extends Meta, B extends Body> = PluginOpts & {
|
||||||
extends PluginOpts {
|
|
||||||
bucket: string
|
|
||||||
region?: string
|
|
||||||
endpoint?: string
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Custom function to sign requests.
|
|
||||||
* Called with request details, should return signed headers.
|
|
||||||
* Alternative to using Companion endpoint.
|
|
||||||
*/
|
|
||||||
signRequest?: IT.signRequestFn
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Function to retrieve temporary credentials for client-side signing.
|
|
||||||
* When provided, S3mini handles signing internally using SigV4.
|
|
||||||
* Alternative to signRequest or endpoint.
|
|
||||||
*/
|
|
||||||
getCredentials?: IT.getCredentialsFn
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether to use multipart uploads.
|
* Whether to use multipart uploads.
|
||||||
* - `true`: Always use multipart
|
* - `true`: Always use multipart
|
||||||
|
|
@ -83,7 +74,34 @@ export interface AwsS3Options<M extends Meta, B extends Body>
|
||||||
* Default: `{randomId}-{filename}`
|
* Default: `{randomId}-{filename}`
|
||||||
*/
|
*/
|
||||||
generateObjectKey?: (file: UppyFile<M, B>) => string
|
generateObjectKey?: (file: UppyFile<M, B>) => string
|
||||||
}
|
} & (
|
||||||
|
| {
|
||||||
|
/** S3 upload endpoint */
|
||||||
|
s3Endpoint: string
|
||||||
|
|
||||||
|
/** AWS region, required for signing */
|
||||||
|
region?: string
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Function to retrieve temporary credentials for client-side signing.
|
||||||
|
* When provided, S3mini handles signing internally using SigV4.
|
||||||
|
* Alternative to signRequest or endpoint.
|
||||||
|
*/
|
||||||
|
getCredentials: IT.GetCredentialsFn
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
/**
|
||||||
|
* Custom function to sign requests.
|
||||||
|
* Called with request details, should return signed headers.
|
||||||
|
* Alternative to using Companion endpoint.
|
||||||
|
*/
|
||||||
|
signRequest: IT.SignRequestFn
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
/** Companion URL if you want to use Companion for signing */
|
||||||
|
companionEndpoint: string
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Constants
|
// Constants
|
||||||
|
|
@ -91,12 +109,6 @@ export interface AwsS3Options<M extends Meta, B extends Body>
|
||||||
|
|
||||||
const MB = 1024 * 1024
|
const MB = 1024 * 1024
|
||||||
|
|
||||||
/** Minimum chunk size required by S3 (5MB) */
|
|
||||||
const MIN_CHUNK_SIZE = 5 * MB
|
|
||||||
|
|
||||||
/** Maximum number of parts allowed by S3 */
|
|
||||||
const MAX_PARTS = 10000
|
|
||||||
|
|
||||||
const defaultOptions = {
|
const defaultOptions = {
|
||||||
shouldUseMultipart: (file: UppyFile<any, any>) => (file.size || 0) > 100 * MB,
|
shouldUseMultipart: (file: UppyFile<any, any>) => (file.size || 0) > 100 * MB,
|
||||||
allowedMetaFields: true,
|
allowedMetaFields: true,
|
||||||
|
|
@ -108,381 +120,6 @@ const defaultOptions = {
|
||||||
// S3Uploader Types
|
// S3Uploader Types
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
interface S3UploaderOptions<M extends Meta, B extends Body> {
|
|
||||||
uppy: Uppy<M, B>
|
|
||||||
s3Client: S3mini
|
|
||||||
file: UppyFile<M, B>
|
|
||||||
key: string
|
|
||||||
shouldUseMultipart?: boolean
|
|
||||||
getChunkSize?: (file: { size: number }) => number
|
|
||||||
onProgress?: (bytesUploaded: number, bytesTotal: number) => void
|
|
||||||
onPartComplete?: (part: { PartNumber: number; ETag: string }) => void
|
|
||||||
onSuccess?: (result: UploadResult) => void
|
|
||||||
onError?: (err: Error) => void
|
|
||||||
onAbort?: () => void
|
|
||||||
log?: (message: string | Error, type?: 'error' | 'warning') => void
|
|
||||||
}
|
|
||||||
|
|
||||||
interface UploadResult {
|
|
||||||
location: string
|
|
||||||
key: string
|
|
||||||
bucket?: string
|
|
||||||
uploadId?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Chunk {
|
|
||||||
index: number
|
|
||||||
start: number
|
|
||||||
end: number
|
|
||||||
size: number
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ChunkState {
|
|
||||||
uploaded: number
|
|
||||||
etag?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Reason for pausing (not a real error) */
|
|
||||||
const pausingUploadReason = Symbol('pausing upload, not an actual error')
|
|
||||||
|
|
||||||
/** Type guard to check if an error is from pausing (not a real failure) */
|
|
||||||
const isPauseError = (err: unknown): boolean =>
|
|
||||||
err instanceof Error &&
|
|
||||||
(err as { cause?: unknown }).cause === pausingUploadReason
|
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// S3Uploader Class
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
class S3Uploader<M extends Meta, B extends Body> {
|
|
||||||
readonly #s3Client: S3mini
|
|
||||||
readonly #file: UppyFile<M, B>
|
|
||||||
readonly #data: Blob
|
|
||||||
readonly #key: string
|
|
||||||
readonly #options: S3UploaderOptions<M, B>
|
|
||||||
readonly #eventManager: EventManager<M, B>
|
|
||||||
|
|
||||||
#chunks: Chunk[] = []
|
|
||||||
#chunkState: ChunkState[] = []
|
|
||||||
#shouldUseMultipart: boolean = false
|
|
||||||
#uploadId?: string
|
|
||||||
#uploadHasStarted: boolean = false
|
|
||||||
#abortController: AbortController = new AbortController()
|
|
||||||
|
|
||||||
constructor(data: Blob, options: S3UploaderOptions<M, B>) {
|
|
||||||
this.#s3Client = options.s3Client
|
|
||||||
this.#file = options.file
|
|
||||||
this.#data = data
|
|
||||||
this.#key = options.key
|
|
||||||
this.#options = options
|
|
||||||
this.#eventManager = new EventManager(options.uppy)
|
|
||||||
this.#initChunks()
|
|
||||||
this.#setupEvents()
|
|
||||||
}
|
|
||||||
|
|
||||||
#setupEvents(): void {
|
|
||||||
const fileId = this.#file.id
|
|
||||||
|
|
||||||
this.#eventManager.onFileRemove(fileId, () => {
|
|
||||||
this.abort()
|
|
||||||
this.#options.onAbort?.()
|
|
||||||
})
|
|
||||||
|
|
||||||
this.#eventManager.onCancelAll(fileId, () => {
|
|
||||||
this.abort()
|
|
||||||
this.#options.onAbort?.()
|
|
||||||
})
|
|
||||||
|
|
||||||
this.#eventManager.onFilePause(fileId, (isPaused) => {
|
|
||||||
if (isPaused) {
|
|
||||||
this.pause()
|
|
||||||
} else {
|
|
||||||
this.start()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
this.#eventManager.onPauseAll(fileId, () => {
|
|
||||||
this.pause()
|
|
||||||
})
|
|
||||||
|
|
||||||
this.#eventManager.onResumeAll(fileId, () => {
|
|
||||||
this.start()
|
|
||||||
})
|
|
||||||
|
|
||||||
this.#eventManager.onRetry(fileId, () => {
|
|
||||||
this.start()
|
|
||||||
})
|
|
||||||
|
|
||||||
this.#eventManager.onRetryAll(fileId, () => {
|
|
||||||
this.start()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#initChunks(): void {
|
|
||||||
const fileSize = this.#data.size
|
|
||||||
|
|
||||||
// Step 1: Determine if we should use multipart
|
|
||||||
// - If explicitly set to boolean, use that
|
|
||||||
// - Otherwise, use multipart for files larger than MIN_CHUNK_SIZE (5MB)
|
|
||||||
if (typeof this.#options.shouldUseMultipart === 'boolean') {
|
|
||||||
this.#shouldUseMultipart = this.#options.shouldUseMultipart
|
|
||||||
} else {
|
|
||||||
this.#shouldUseMultipart = fileSize > MIN_CHUNK_SIZE
|
|
||||||
}
|
|
||||||
|
|
||||||
// Step 2: Force simple upload if file is too small for multipart
|
|
||||||
// (S3 requires minimum 5MB parts, except for the last part)
|
|
||||||
if (fileSize <= MIN_CHUNK_SIZE) {
|
|
||||||
this.#shouldUseMultipart = false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Step 3: Create chunks based on upload strategy
|
|
||||||
if (this.#shouldUseMultipart) {
|
|
||||||
// Calculate chunk size: at least MIN_CHUNK_SIZE, but may be larger for huge files
|
|
||||||
let chunkSize = this.#getChunkSize(fileSize)
|
|
||||||
chunkSize = Math.max(chunkSize, MIN_CHUNK_SIZE)
|
|
||||||
|
|
||||||
// Ensure we don't exceed MAX_PARTS (S3 limit: 10,000 parts)
|
|
||||||
if (Math.ceil(fileSize / chunkSize) > MAX_PARTS) {
|
|
||||||
chunkSize = Math.ceil(fileSize / MAX_PARTS)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create chunk definitions
|
|
||||||
for (
|
|
||||||
let offset = 0, index = 0;
|
|
||||||
offset < fileSize;
|
|
||||||
offset += chunkSize, index++
|
|
||||||
) {
|
|
||||||
const end = Math.min(offset + chunkSize, fileSize)
|
|
||||||
this.#chunks.push({ index, start: offset, end, size: end - offset })
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Simple upload: single chunk for the entire file
|
|
||||||
this.#chunks = [{ index: 0, start: 0, end: fileSize, size: fileSize }]
|
|
||||||
}
|
|
||||||
|
|
||||||
this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
|
|
||||||
}
|
|
||||||
|
|
||||||
#getChunkSize(fileSize: number): number {
|
|
||||||
if (this.#options.getChunkSize) {
|
|
||||||
return this.#options.getChunkSize({ size: fileSize })
|
|
||||||
}
|
|
||||||
return Math.max(MIN_CHUNK_SIZE, Math.ceil(fileSize / MAX_PARTS))
|
|
||||||
}
|
|
||||||
|
|
||||||
start(): void {
|
|
||||||
if (this.#uploadHasStarted) {
|
|
||||||
// Abort any pending operations (if not already aborted)
|
|
||||||
if (!this.#abortController.signal.aborted) {
|
|
||||||
this.#abortController.abort(pausingUploadReason)
|
|
||||||
}
|
|
||||||
// Always create a fresh AbortController for resume
|
|
||||||
this.#abortController = new AbortController()
|
|
||||||
this.#resumeUpload()
|
|
||||||
} else {
|
|
||||||
this.#createUpload()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pause(): void {
|
|
||||||
this.#abortController.abort(pausingUploadReason)
|
|
||||||
this.#abortController = new AbortController()
|
|
||||||
}
|
|
||||||
|
|
||||||
abort(opts?: { abortOnS3?: boolean }): void {
|
|
||||||
this.#abortController.abort()
|
|
||||||
// Clean up event listeners
|
|
||||||
this.#eventManager.remove()
|
|
||||||
if (opts?.abortOnS3 !== false && this.#uploadId) {
|
|
||||||
this.#s3Client
|
|
||||||
.abortMultipartUpload(this.#key, this.#uploadId)
|
|
||||||
.catch((abortErr) => {
|
|
||||||
this.#options.log?.(abortErr, 'warning')
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async #createUpload(): Promise<void> {
|
|
||||||
this.#uploadHasStarted = true
|
|
||||||
try {
|
|
||||||
if (this.#shouldUseMultipart) {
|
|
||||||
await this.#multipartUpload()
|
|
||||||
} else {
|
|
||||||
await this.#simpleUpload()
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
this.#onError(err as Error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async #resumeUpload(): Promise<void> {
|
|
||||||
if (!this.#uploadId) {
|
|
||||||
await this.#createUpload()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const existingParts = await this.#s3Client.listParts(
|
|
||||||
this.#uploadId,
|
|
||||||
this.#key,
|
|
||||||
)
|
|
||||||
// Sync local state with S3 - mark already-uploaded parts
|
|
||||||
for (const part of existingParts) {
|
|
||||||
const chunkIndex = part.partNumber - 1
|
|
||||||
if (chunkIndex >= 0 && chunkIndex < this.#chunkState.length) {
|
|
||||||
this.#chunkState[chunkIndex].uploaded = this.#chunks[chunkIndex].size
|
|
||||||
this.#chunkState[chunkIndex].etag = part.etag
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Emit progress update to reflect already-uploaded parts
|
|
||||||
this.#onProgress()
|
|
||||||
await this.#uploadRemainingParts()
|
|
||||||
} catch (err) {
|
|
||||||
this.#onError(err as Error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async #simpleUpload(): Promise<void> {
|
|
||||||
const signal = this.#abortController.signal
|
|
||||||
if (signal.aborted) {
|
|
||||||
throw new Error('Upload aborted', { cause: signal.reason })
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.#s3Client.putObject(
|
|
||||||
this.#key,
|
|
||||||
this.#data,
|
|
||||||
this.#file.type || 'application/octet-stream',
|
|
||||||
(bytesUploaded: number) => {
|
|
||||||
this.#chunkState[0].uploaded = bytesUploaded
|
|
||||||
this.#onProgress()
|
|
||||||
},
|
|
||||||
signal,
|
|
||||||
)
|
|
||||||
|
|
||||||
this.#onSuccess({
|
|
||||||
location: `${this.#s3Client.endpoint}/${this.#key}`,
|
|
||||||
key: this.#key,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async #multipartUpload(): Promise<void> {
|
|
||||||
const signal = this.#abortController.signal
|
|
||||||
if (signal.aborted) {
|
|
||||||
throw new Error('Upload aborted', { cause: signal.reason })
|
|
||||||
}
|
|
||||||
|
|
||||||
this.#uploadId = await this.#s3Client.getMultipartUploadId(
|
|
||||||
this.#key,
|
|
||||||
this.#file.type || 'application/octet-stream',
|
|
||||||
)
|
|
||||||
await this.#uploadRemainingParts()
|
|
||||||
}
|
|
||||||
|
|
||||||
async #uploadRemainingParts(): Promise<void> {
|
|
||||||
const signal = this.#abortController.signal
|
|
||||||
// Collect already-uploaded parts (from resume)
|
|
||||||
const parts = this.#chunkState
|
|
||||||
.map((state, i) =>
|
|
||||||
state.etag ? { partNumber: i + 1, etag: state.etag } : null,
|
|
||||||
)
|
|
||||||
.filter((p): p is NonNullable<typeof p> => p !== null)
|
|
||||||
|
|
||||||
for (let i = 0; i < this.#chunks.length; i++) {
|
|
||||||
if (signal.aborted) {
|
|
||||||
throw new Error('Upload aborted', { cause: signal.reason })
|
|
||||||
}
|
|
||||||
if (this.#chunkState[i].etag) continue
|
|
||||||
|
|
||||||
const chunk = this.#chunks[i]
|
|
||||||
const partNumber = i + 1
|
|
||||||
const chunkData = this.#data.slice(chunk.start, chunk.end)
|
|
||||||
const chunkIndex = i // Capture for closure
|
|
||||||
|
|
||||||
const part = await this.#s3Client.uploadPart(
|
|
||||||
this.#key,
|
|
||||||
this.#uploadId!,
|
|
||||||
chunkData,
|
|
||||||
partNumber,
|
|
||||||
(bytesUploaded: number) => {
|
|
||||||
this.#chunkState[chunkIndex].uploaded = bytesUploaded
|
|
||||||
this.#onProgress()
|
|
||||||
},
|
|
||||||
signal,
|
|
||||||
)
|
|
||||||
|
|
||||||
this.#chunkState[i].uploaded = chunk.size
|
|
||||||
this.#chunkState[i].etag = part.etag
|
|
||||||
parts.push({ partNumber: part.partNumber, etag: part.etag })
|
|
||||||
this.#onProgress()
|
|
||||||
|
|
||||||
if (this.#options.onPartComplete) {
|
|
||||||
this.#options.onPartComplete({
|
|
||||||
PartNumber: part.partNumber,
|
|
||||||
ETag: part.etag,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (signal.aborted) {
|
|
||||||
throw new Error('Upload aborted', { cause: signal.reason })
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await this.#s3Client.completeMultipartUpload(
|
|
||||||
this.#key,
|
|
||||||
this.#uploadId!,
|
|
||||||
parts,
|
|
||||||
)
|
|
||||||
|
|
||||||
this.#onSuccess({
|
|
||||||
location: result.location,
|
|
||||||
key: result.key,
|
|
||||||
bucket: result.bucket,
|
|
||||||
uploadId: this.#uploadId,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#onProgress(): void {
|
|
||||||
if (!this.#options.onProgress) return
|
|
||||||
const bytesUploaded = this.#chunkState.reduce(
|
|
||||||
(sum, state) => sum + state.uploaded,
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
this.#options.onProgress(bytesUploaded, this.#data.size)
|
|
||||||
}
|
|
||||||
|
|
||||||
#onSuccess(result: UploadResult): void {
|
|
||||||
// If the upload was aborted (file removed mid-upload), the network request
|
|
||||||
// may still complete successfully. Don't emit success in this case since
|
|
||||||
// the file no longer exists in Uppy's state.
|
|
||||||
if (this.#abortController.signal.aborted) {
|
|
||||||
this.#eventManager.remove()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
this.#eventManager.remove()
|
|
||||||
this.#options.onSuccess?.(result)
|
|
||||||
}
|
|
||||||
|
|
||||||
#onError(err: Error): void {
|
|
||||||
if (isPauseError(err)) return
|
|
||||||
// Also ignore abort signals from intentional cancellation
|
|
||||||
if (err.name === 'AbortError') return
|
|
||||||
// If we intentionally aborted, don't report any subsequent errors
|
|
||||||
// (e.g., S3 returning 404 NoSuchUpload after we aborted the upload)
|
|
||||||
if (this.#abortController.signal.aborted) return
|
|
||||||
|
|
||||||
// NOTE: We intentionally do NOT abort the multipart upload on S3 here.
|
|
||||||
// This allows the user to retry and resume from where they left off.
|
|
||||||
// The multipart upload is only aborted when the user explicitly cancels
|
|
||||||
// (via the abort() method with abortOnS3: true).
|
|
||||||
|
|
||||||
this.#options.onError?.(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Plugin Class
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
DefinePluginOpts<AwsS3Options<M, B>, keyof typeof defaultOptions>,
|
DefinePluginOpts<AwsS3Options<M, B>, keyof typeof defaultOptions>,
|
||||||
M,
|
M,
|
||||||
|
|
@ -490,7 +127,7 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
> {
|
> {
|
||||||
static VERSION = packageJson.version
|
static VERSION = packageJson.version
|
||||||
|
|
||||||
#s3Client!: S3mini
|
#s3Client!: S3Client
|
||||||
#queue!: TaskQueue
|
#queue!: TaskQueue
|
||||||
#uploaders: Record<string, S3Uploader<M, B> | null> = {}
|
#uploaders: Record<string, S3Uploader<M, B> | null> = {}
|
||||||
|
|
||||||
|
|
@ -522,10 +159,6 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --------------------------------------------------------------------------
|
|
||||||
// Resumable Uploads Capability
|
|
||||||
// --------------------------------------------------------------------------
|
|
||||||
|
|
||||||
#setResumableUploadsCapability = (value: boolean): void => {
|
#setResumableUploadsCapability = (value: boolean): void => {
|
||||||
const { capabilities } = this.uppy.getState()
|
const { capabilities } = this.uppy.getState()
|
||||||
this.uppy.setState({
|
this.uppy.setState({
|
||||||
|
|
@ -546,68 +179,42 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
|
|
||||||
#initS3Client(): void {
|
#initS3Client(): void {
|
||||||
const { endpoint, signRequest, getCredentials, bucket, region } = this.opts
|
if ('companionEndpoint' in this.opts) {
|
||||||
|
if (typeof this.opts.companionEndpoint !== 'string') {
|
||||||
|
throw new TypeError('companionEndpoint must be a string')
|
||||||
|
}
|
||||||
|
this.#s3Client = new S3Companion({
|
||||||
|
companionEndpoint: this.opts.companionEndpoint,
|
||||||
|
})
|
||||||
|
} else if ('getCredentials' in this.opts) {
|
||||||
|
if (typeof this.opts.s3Endpoint !== 'string') {
|
||||||
|
throw new TypeError('s3Endpoint must be a string')
|
||||||
|
}
|
||||||
|
if (typeof this.opts.getCredentials !== 'function') {
|
||||||
|
throw new TypeError('getCredentials must be a function')
|
||||||
|
}
|
||||||
|
if (this.opts.region != null && typeof this.opts.region !== 'string') {
|
||||||
|
throw new TypeError('region must be a string')
|
||||||
|
}
|
||||||
|
|
||||||
if (typeof bucket !== 'string' || bucket.trim() === '') {
|
|
||||||
throw new Error(
|
|
||||||
'AwsS3: `bucket` option is required and must be a non-empty string',
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof region !== 'string' || region.trim() === '') {
|
|
||||||
throw new Error(
|
|
||||||
'AwsS3: `region` option is required and must be a non-empty string',
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const bucketName = bucket.trim()
|
|
||||||
|
|
||||||
if (!signRequest && !getCredentials && !endpoint) {
|
|
||||||
throw new Error(
|
|
||||||
'AwsS3: `endpoint`, `signRequest`, or `getCredentials` is required',
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const s3Endpoint = `https://${bucketName}.s3.${region}.amazonaws.com`
|
|
||||||
|
|
||||||
if (getCredentials) {
|
|
||||||
// Mode: Temporary credentials (client-side signing)
|
// Mode: Temporary credentials (client-side signing)
|
||||||
this.#s3Client = new S3mini({
|
this.#s3Client = new S3mini({
|
||||||
endpoint: s3Endpoint,
|
endpoint: this.opts.s3Endpoint,
|
||||||
getCredentials,
|
getCredentials: this.opts.getCredentials,
|
||||||
region,
|
region: this.opts.region,
|
||||||
})
|
})
|
||||||
} else if (signRequest) {
|
} else if ('signRequest' in this.opts) {
|
||||||
|
if (typeof this.opts.signRequest !== 'function') {
|
||||||
|
throw new TypeError('signRequest must be a function')
|
||||||
|
}
|
||||||
// Mode: Custom signing function
|
// Mode: Custom signing function
|
||||||
this.#s3Client = new S3mini({
|
this.#s3Client = new S3mini({
|
||||||
endpoint: s3Endpoint,
|
signRequest: this.opts.signRequest,
|
||||||
signRequest,
|
|
||||||
region,
|
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
// Mode: Companion signing (endpoint is guaranteed to be set here)
|
throw new TypeError(
|
||||||
this.#s3Client = new S3mini({
|
'One of options `companionEndpoint`, `signRequest`, or `getCredentials` is required',
|
||||||
endpoint: s3Endpoint,
|
)
|
||||||
signRequest: this.#createCompanionSigner(endpoint!),
|
|
||||||
region,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a signing function that calls Companion's /s3/sign endpoint.
|
|
||||||
*/
|
|
||||||
#createCompanionSigner(companionUrl: string): IT.signRequestFn {
|
|
||||||
return async (request) => {
|
|
||||||
const response = await fetch(`${companionUrl}/s3/sign`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: { 'Content-Type': 'application/json' },
|
|
||||||
body: JSON.stringify(request),
|
|
||||||
})
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to sign request: ${response.statusText}`)
|
|
||||||
}
|
|
||||||
return response.json()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -626,10 +233,9 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
|
|
||||||
const promises = filesToUpload.map((file) => {
|
const promises = filesToUpload.map((file) => {
|
||||||
if (file.isRemote) {
|
if (file.isRemote) {
|
||||||
// Remote files not yet supported in this minimal implementation
|
// Remote uploads are queued internally by RequestClient.uploadRemoteFile()
|
||||||
return Promise.reject(
|
// via getQueue(), so no outer queue wrapping is needed here.
|
||||||
new Error('Remote file uploads not yet supported'),
|
return this.#uploadRemoteFile(file)
|
||||||
)
|
|
||||||
}
|
}
|
||||||
return this.#queue.add(async () => {
|
return this.#queue.add(async () => {
|
||||||
// File may have been removed while waiting in the queue.
|
// File may have been removed while waiting in the queue.
|
||||||
|
|
@ -640,31 +246,32 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
if (!currentFile) {
|
if (!currentFile) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
return this.#uploadLocalFile(currentFile)
|
return this.#uploadLocalFile(currentFile as LocalUppyFile<M, B>) // assume it's still a local file since remote files aren't queued
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
await Promise.allSettled(promises)
|
await Promise.allSettled(promises)
|
||||||
|
// After the upload batch is done, restore resumable uploads capability.
|
||||||
|
// It may have been set to false if there were remote files in this batch.
|
||||||
|
this.#setResumableUploadsCapability(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
// Local File Upload
|
// Local File Upload
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
|
|
||||||
async #uploadLocalFile(file: UppyFile<M, B>): Promise<void> {
|
async #uploadLocalFile(file: LocalUppyFile<M, B>): Promise<void> {
|
||||||
return new Promise((resolve, reject) => {
|
try {
|
||||||
const data = file.data as Blob
|
return await new Promise((resolve, reject) => {
|
||||||
const key = this.#generateKey(file)
|
// Create uploader (events are wired internally).
|
||||||
const shouldMultipart = this.#shouldUseMultipart(file)
|
// S3Uploader detects resume state from file.s3Multipart internally.
|
||||||
|
const uploader = new S3Uploader<M, B>({
|
||||||
try {
|
|
||||||
// Create uploader (events are wired internally)
|
|
||||||
const uploader = new S3Uploader<M, B>(data, {
|
|
||||||
uppy: this.uppy,
|
uppy: this.uppy,
|
||||||
s3Client: this.#s3Client,
|
s3Client: this.#s3Client,
|
||||||
file,
|
file,
|
||||||
key,
|
metadata: this.#getAllowedMeta(file),
|
||||||
shouldUseMultipart: shouldMultipart,
|
key: this.#generateKey(file),
|
||||||
|
shouldUseMultipart: this.#shouldUseMultipart(file),
|
||||||
getChunkSize: this.opts.getChunkSize,
|
getChunkSize: this.opts.getChunkSize,
|
||||||
log: (...args) => this.uppy.log(...args),
|
log: (...args) => this.uppy.log(...args),
|
||||||
|
|
||||||
|
|
@ -686,22 +293,18 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
body: {
|
body: {
|
||||||
location: result.location,
|
location: result.location,
|
||||||
key: result.key,
|
key: result.key,
|
||||||
bucket: result.bucket,
|
} satisfies AwsBody as unknown as B,
|
||||||
} as unknown as B,
|
|
||||||
uploadURL: result.location,
|
uploadURL: result.location,
|
||||||
})
|
})
|
||||||
this.#cleanup(file.id)
|
|
||||||
resolve()
|
resolve()
|
||||||
},
|
},
|
||||||
|
|
||||||
onError: (err) => {
|
onError: (err) => {
|
||||||
this.uppy.emit('upload-error', file, err)
|
this.uppy.emit('upload-error', file, err)
|
||||||
this.#cleanup(file.id)
|
|
||||||
reject(err)
|
reject(err)
|
||||||
},
|
},
|
||||||
|
|
||||||
onAbort: () => {
|
onAbort: () => {
|
||||||
this.#cleanup(file.id)
|
|
||||||
resolve() // Normal completion, not an error
|
resolve() // Normal completion, not an error
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
@ -711,12 +314,11 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
|
|
||||||
// Start the upload
|
// Start the upload
|
||||||
uploader.start()
|
uploader.start()
|
||||||
} catch (err) {
|
})
|
||||||
// Cleanup on synchronous failure during setup
|
} finally {
|
||||||
this.#cleanup(file.id)
|
// Clean up uploader instance after upload completes or fails
|
||||||
reject(err)
|
delete this.#uploaders[file.id]
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#shouldUseMultipart(file: UppyFile<M, B>): boolean {
|
#shouldUseMultipart(file: UppyFile<M, B>): boolean {
|
||||||
|
|
@ -728,21 +330,59 @@ export default class AwsS3<M extends Meta, B extends Body> extends BasePlugin<
|
||||||
return shouldUseMultipart
|
return shouldUseMultipart
|
||||||
}
|
}
|
||||||
// Default: multipart for files > 100MB
|
// Default: multipart for files > 100MB
|
||||||
return (file.size || 0) > 100 * MB
|
return (file.size ?? 0) > 100 * MB
|
||||||
}
|
}
|
||||||
|
|
||||||
#generateKey(file: UppyFile<M, B>): string {
|
#generateKey(file: UppyFile<M, B>): string {
|
||||||
if (this.opts.generateObjectKey) {
|
return this.opts.generateObjectKey?.(file) ?? file.name
|
||||||
return this.opts.generateObjectKey(file)
|
|
||||||
}
|
|
||||||
// Default: {randomId}-{filename}
|
|
||||||
const randomId = crypto.randomUUID()
|
|
||||||
return `${randomId}-${file.name}`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#cleanup(fileId: string): void {
|
// --------------------------------------------------------------------------
|
||||||
if (this.#uploaders[fileId]) {
|
// Remote File Upload
|
||||||
delete this.#uploaders[fileId]
|
// --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#getAllowedMeta(file: UppyFile<M, B>) {
|
||||||
|
const allowedMetaFields = getAllowedMetaFields(
|
||||||
|
this.opts.allowedMetaFields,
|
||||||
|
file.meta,
|
||||||
|
)
|
||||||
|
return Object.fromEntries(
|
||||||
|
allowedMetaFields.map((key) => [key, file.meta[key]]),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds the request body sent to Companion's provider get endpoint.
|
||||||
|
* Tells Companion to use its server-side S3 upload path.
|
||||||
|
*/
|
||||||
|
#getCompanionClientArgs(file: RemoteUppyFile<M, B>): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
...file.remote.body,
|
||||||
|
protocol: 's3-multipart',
|
||||||
|
size: file.data.size,
|
||||||
|
metadata: this.#getAllowedMeta(file),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async #uploadRemoteFile(file: RemoteUppyFile<M, B>): Promise<void> {
|
||||||
|
this.#setResumableUploadsCapability(false)
|
||||||
|
|
||||||
|
const controller = new AbortController()
|
||||||
|
|
||||||
|
const removedHandler = (removedFile: UppyFile<M, B>) => {
|
||||||
|
if (removedFile.id === file.id) controller.abort()
|
||||||
|
}
|
||||||
|
this.uppy.on('file-removed', removedHandler)
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.uppy
|
||||||
|
.getRequestClientForFile<RequestClient<M, B>>(file)
|
||||||
|
.uploadRemoteFile(file, this.#getCompanionClientArgs(file), {
|
||||||
|
signal: controller.signal,
|
||||||
|
getQueue: () => this.#queue,
|
||||||
|
})
|
||||||
|
} finally {
|
||||||
|
this.uppy.off('file-removed', removedHandler)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -753,5 +393,4 @@ export type { AwsS3Options as AwsS3MultipartOptions }
|
||||||
export interface AwsBody extends Body {
|
export interface AwsBody extends Body {
|
||||||
location: string
|
location: string
|
||||||
key: string
|
key: string
|
||||||
bucket?: string
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
212
packages/@uppy/aws-s3/src/s3-client/CompanionS3.ts
Normal file
212
packages/@uppy/aws-s3/src/s3-client/CompanionS3.ts
Normal file
|
|
@ -0,0 +1,212 @@
|
||||||
|
import * as C from './consts.js'
|
||||||
|
import S3Client from './S3Client.js'
|
||||||
|
import type * as IT from './types.js'
|
||||||
|
import * as U from './utils.js'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* S3Companion is an S3Client that interacts with a Companion server to perform S3 operations.
|
||||||
|
*/
|
||||||
|
class S3Companion extends S3Client {
|
||||||
|
readonly companionEndpoint: string
|
||||||
|
|
||||||
|
constructor({
|
||||||
|
companionEndpoint,
|
||||||
|
requestAbortTimeout,
|
||||||
|
}: { companionEndpoint: string; requestAbortTimeout?: number | undefined }) {
|
||||||
|
super({ requestAbortTimeout })
|
||||||
|
this.companionEndpoint = companionEndpoint
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _fetch(path: string, opts?: RequestInit) {
|
||||||
|
const response = await fetch(`${this.companionEndpoint}/s3${path}`, opts)
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`Companion request failed: ${response.statusText}`)
|
||||||
|
}
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _request({
|
||||||
|
url,
|
||||||
|
method,
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
contentType,
|
||||||
|
}: {
|
||||||
|
url: string
|
||||||
|
method: IT.HttpMethod
|
||||||
|
data?: XMLHttpRequestBodyInit
|
||||||
|
onProgress?: IT.OnProgressFn
|
||||||
|
signal?: AbortSignal
|
||||||
|
contentType?: string
|
||||||
|
}): Promise<XMLHttpRequest> {
|
||||||
|
// Wait for online before starting
|
||||||
|
await this.waitForOnline(signal)
|
||||||
|
|
||||||
|
return this.xhr({ url, method, data, onProgress, signal, contentType })
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async putObject(
|
||||||
|
keyIn: string,
|
||||||
|
data: Blob | File,
|
||||||
|
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
|
||||||
|
metadata: Record<string, unknown>,
|
||||||
|
onProgress?: IT.OnProgressFn,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
) {
|
||||||
|
const response = await this._fetch(
|
||||||
|
`/params?${new URLSearchParams({ filename: keyIn, type: fileType, ...Object.fromEntries(Object.entries(metadata).map(([k, v]) => [`metadata[${k}]`, String(v)])) })}`,
|
||||||
|
)
|
||||||
|
const { url, fields }: { url: string; fields: Record<string, string> } =
|
||||||
|
await response.json()
|
||||||
|
|
||||||
|
const formData = new FormData()
|
||||||
|
Object.entries(fields).forEach(([key, value]) => formData.set(key, value))
|
||||||
|
formData.set('file', data)
|
||||||
|
|
||||||
|
const xhr = await this._request({
|
||||||
|
url,
|
||||||
|
method: 'POST',
|
||||||
|
data: formData,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
location: `${url}${fields.key}`, // `url` is returned by the signer as the bucket URL without any path, but trailing slash, so we need to add the key (path) to get the full object URL
|
||||||
|
etag: U.sanitizeETag(xhr.getResponseHeader('etag')),
|
||||||
|
key: fields.key,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async createMultipartUpload(
|
||||||
|
keyIn: string,
|
||||||
|
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
|
||||||
|
metadata: Record<string, unknown>,
|
||||||
|
) {
|
||||||
|
if (typeof fileType !== 'string') {
|
||||||
|
throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const method = 'POST'
|
||||||
|
const response = await this._fetch('/multipart', {
|
||||||
|
method,
|
||||||
|
body: JSON.stringify({ filename: keyIn, metadata, type: fileType }),
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
})
|
||||||
|
const {
|
||||||
|
key,
|
||||||
|
uploadId,
|
||||||
|
}: { key?: string; uploadId?: string; bucket?: string } =
|
||||||
|
await response.json()
|
||||||
|
if (uploadId == null) throw new Error('No uploadId returned')
|
||||||
|
if (key == null) throw new Error('No key returned')
|
||||||
|
|
||||||
|
return { uploadId, key }
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async uploadPart(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
data: XMLHttpRequestBodyInit,
|
||||||
|
partNumber: number,
|
||||||
|
onProgress?: IT.OnProgressFn,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
) {
|
||||||
|
const response = await this._fetch(
|
||||||
|
`/multipart/${encodeURIComponent(uploadId)}/${encodeURIComponent(partNumber)}?${new URLSearchParams({ key })}`,
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
const { url }: { url: string } = await response.json()
|
||||||
|
|
||||||
|
const xhr = await this._request({
|
||||||
|
url,
|
||||||
|
method: 'PUT',
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
})
|
||||||
|
|
||||||
|
const etag = U.sanitizeETag(xhr.getResponseHeader('etag'))
|
||||||
|
if (etag == null) {
|
||||||
|
throw new Error(
|
||||||
|
`${C.ERROR_PREFIX}Missing ETag in uploadPart response headers`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return { etag }
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async listParts(
|
||||||
|
uploadId: string,
|
||||||
|
key: string,
|
||||||
|
): Promise<IT.UploadPart[]> {
|
||||||
|
if (!uploadId) {
|
||||||
|
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await this._fetch(
|
||||||
|
`/multipart/${encodeURIComponent(uploadId)}?${new URLSearchParams({ key })}`,
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
const parts: { PartNumber: string; ETag: string }[] = await response.json()
|
||||||
|
|
||||||
|
return parts.map((p) => ({
|
||||||
|
partNumber: parseInt(String(p.PartNumber), 10),
|
||||||
|
etag: String(p.ETag),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async completeMultipartUpload(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
parts: Array<IT.UploadPart>,
|
||||||
|
) {
|
||||||
|
const response = await this._fetch(
|
||||||
|
`/multipart/${encodeURIComponent(uploadId)}/complete?${new URLSearchParams({ key })}`,
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({
|
||||||
|
parts: parts.map((part) => ({
|
||||||
|
PartNumber: part.partNumber,
|
||||||
|
ETag: part.etag,
|
||||||
|
})),
|
||||||
|
}),
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
const {
|
||||||
|
location,
|
||||||
|
bucket,
|
||||||
|
key: resultKey,
|
||||||
|
}: {
|
||||||
|
location: string
|
||||||
|
bucket?: string
|
||||||
|
key: string
|
||||||
|
} = await response.json()
|
||||||
|
|
||||||
|
return {
|
||||||
|
location,
|
||||||
|
bucket,
|
||||||
|
key: resultKey,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async abortMultipartUpload(key: string, uploadId: string) {
|
||||||
|
await this._fetch(
|
||||||
|
`/multipart/${encodeURIComponent(uploadId)}?${new URLSearchParams({ key })}`,
|
||||||
|
{
|
||||||
|
method: 'DELETE',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default S3Companion
|
||||||
|
|
@ -1,866 +0,0 @@
|
||||||
/**
|
|
||||||
* Taken from https://github.com/good-lly/s3mini.git, by Jølly Good, under MIT license.
|
|
||||||
* Modified to make it work with Uppy.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { fetcher, NetworkError } from '@uppy/utils'
|
|
||||||
import * as C from './consts.js'
|
|
||||||
import { createSigV4Signer } from './signer.js'
|
|
||||||
import type * as IT from './types.js'
|
|
||||||
import * as U from './utils.js'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* S3 client for browser-compatible interaction with S3-compatible storage.
|
|
||||||
* Supports simple uploads, multipart uploads, and object deletion.
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* // Option 1: With signRequest callback
|
|
||||||
* const s3 = new S3mini({
|
|
||||||
* endpoint: 'https://s3.amazonaws.com/my-bucket',
|
|
||||||
* signRequest: async ({ method, url, headers }) => {
|
|
||||||
* return await fetchSignedHeaders(method, url, headers);
|
|
||||||
* },
|
|
||||||
* });
|
|
||||||
*
|
|
||||||
* // Option 2: With getCredentials callback (client-side signing)
|
|
||||||
* const s3 = new S3mini({
|
|
||||||
* endpoint: 'https://s3.amazonaws.com/my-bucket',
|
|
||||||
* getCredentials: async () => {
|
|
||||||
* const resp = await fetch('/api/s3/credentials');
|
|
||||||
* return resp.json(); // { credentials, bucket, region }
|
|
||||||
* },
|
|
||||||
* });
|
|
||||||
*
|
|
||||||
* await s3.putObject('file.txt', 'Hello, World!');
|
|
||||||
*/
|
|
||||||
class S3mini {
|
|
||||||
readonly endpoint: URL
|
|
||||||
readonly region: string
|
|
||||||
readonly requestSizeInBytes: number
|
|
||||||
readonly requestAbortTimeout?: number
|
|
||||||
|
|
||||||
private readonly getCredentials?: IT.getCredentialsFn
|
|
||||||
private cachedCredentials?: IT.CredentialsResponse
|
|
||||||
private cachedCredentialsPromise?: Promise<IT.CredentialsResponse>
|
|
||||||
private signRequest!: IT.signRequestFn
|
|
||||||
|
|
||||||
constructor({
|
|
||||||
endpoint,
|
|
||||||
signRequest,
|
|
||||||
getCredentials,
|
|
||||||
region = 'auto',
|
|
||||||
requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES,
|
|
||||||
requestAbortTimeout = undefined,
|
|
||||||
}: IT.S3Config) {
|
|
||||||
this._validateConstructorParams(endpoint, signRequest, getCredentials)
|
|
||||||
this.endpoint = new URL(this._ensureValidUrl(endpoint))
|
|
||||||
this.region = region
|
|
||||||
this.requestSizeInBytes = requestSizeInBytes
|
|
||||||
this.requestAbortTimeout = requestAbortTimeout
|
|
||||||
|
|
||||||
if (signRequest) {
|
|
||||||
this.signRequest = signRequest
|
|
||||||
} else if (getCredentials) {
|
|
||||||
this.getCredentials = getCredentials
|
|
||||||
this.signRequest = this._createCredentialBasedSigner()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Creates a presigner that fetches/caches credentials and generates pre-signed URLs. */
|
|
||||||
private _createCredentialBasedSigner(): IT.signRequestFn {
|
|
||||||
return async (
|
|
||||||
request: IT.presignableRequest,
|
|
||||||
): Promise<IT.presignedResponse> => {
|
|
||||||
const creds = await this._getCachedCredentials()
|
|
||||||
const presigner = createSigV4Signer({
|
|
||||||
accessKeyId: creds.credentials.accessKeyId,
|
|
||||||
secretAccessKey: creds.credentials.secretAccessKey,
|
|
||||||
sessionToken: creds.credentials.sessionToken,
|
|
||||||
region: creds.region || this.region,
|
|
||||||
endpoint: this.endpoint.toString(),
|
|
||||||
})
|
|
||||||
return presigner(request)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Gets cached credentials or fetches new ones. */
|
|
||||||
private async _getCachedCredentials(): Promise<IT.CredentialsResponse> {
|
|
||||||
// Return Cached Credentials if available
|
|
||||||
if (this.cachedCredentials != null) {
|
|
||||||
return this.cachedCredentials
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cache the promise so concurrent calls wait for the same fetch
|
|
||||||
if (this.cachedCredentialsPromise == null) {
|
|
||||||
this.cachedCredentialsPromise = this.getCredentials!({})
|
|
||||||
.then((creds) => {
|
|
||||||
this.cachedCredentials = creds
|
|
||||||
return creds
|
|
||||||
})
|
|
||||||
.finally(() => {
|
|
||||||
// Clear promise cache after resolution to allow future retries
|
|
||||||
this.cachedCredentialsPromise = undefined
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.cachedCredentialsPromise
|
|
||||||
}
|
|
||||||
|
|
||||||
private _validateConstructorParams(
|
|
||||||
endpoint: string,
|
|
||||||
signRequest?: IT.signRequestFn,
|
|
||||||
getCredentials?: IT.getCredentialsFn,
|
|
||||||
): void {
|
|
||||||
if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
|
|
||||||
throw new TypeError(C.ERROR_ENDPOINT_REQUIRED)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!signRequest && !getCredentials) {
|
|
||||||
throw new TypeError(
|
|
||||||
'Either signRequest or getCredentials must be provided',
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (signRequest && typeof signRequest !== 'function') {
|
|
||||||
throw new TypeError('signRequest must be a function')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (getCredentials && typeof getCredentials !== 'function') {
|
|
||||||
throw new TypeError('getCredentials must be a function')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _ensureValidUrl(raw: string): string {
|
|
||||||
const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}`
|
|
||||||
try {
|
|
||||||
new URL(candidate)
|
|
||||||
|
|
||||||
// Find the last non-slash character
|
|
||||||
let endIndex = candidate.length
|
|
||||||
while (endIndex > 0 && candidate[endIndex - 1] === '/') {
|
|
||||||
endIndex--
|
|
||||||
}
|
|
||||||
return endIndex === candidate.length
|
|
||||||
? candidate
|
|
||||||
: candidate.substring(0, endIndex)
|
|
||||||
} catch {
|
|
||||||
const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"`
|
|
||||||
throw new TypeError(msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _checkKey(key: string): void {
|
|
||||||
if (typeof key !== 'string' || key.trim().length === 0) {
|
|
||||||
throw new TypeError(C.ERROR_KEY_REQUIRED)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _checkOpts(opts: object): void {
|
|
||||||
if (typeof opts !== 'object') {
|
|
||||||
throw new TypeError(`${C.ERROR_PREFIX}opts must be an object`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _filterIfHeaders(opts: Record<string, unknown>): {
|
|
||||||
filteredOpts: Record<string, string>
|
|
||||||
conditionalHeaders: Record<string, unknown>
|
|
||||||
} {
|
|
||||||
const filteredOpts: Record<string, string> = {}
|
|
||||||
const conditionalHeaders: Record<string, unknown> = {}
|
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(opts)) {
|
|
||||||
if (
|
|
||||||
C.IFHEADERS.has(
|
|
||||||
key.toLowerCase() as typeof C.IFHEADERS extends Set<infer T>
|
|
||||||
? T
|
|
||||||
: never,
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
conditionalHeaders[key] = value
|
|
||||||
} else {
|
|
||||||
filteredOpts[key] = value as string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { filteredOpts, conditionalHeaders }
|
|
||||||
}
|
|
||||||
|
|
||||||
private _validateData(data: unknown): BodyInit {
|
|
||||||
if (
|
|
||||||
typeof data === 'string' ||
|
|
||||||
data instanceof ArrayBuffer ||
|
|
||||||
ArrayBuffer.isView(data) ||
|
|
||||||
data instanceof Blob
|
|
||||||
) {
|
|
||||||
return data as BodyInit
|
|
||||||
}
|
|
||||||
throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED)
|
|
||||||
}
|
|
||||||
|
|
||||||
private _validateUploadPartParams(
|
|
||||||
key: string,
|
|
||||||
uploadId: string,
|
|
||||||
partNumber: number,
|
|
||||||
): void {
|
|
||||||
this._checkKey(key)
|
|
||||||
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
|
|
||||||
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
|
|
||||||
}
|
|
||||||
if (!Number.isInteger(partNumber) || partNumber <= 0) {
|
|
||||||
throw new TypeError(
|
|
||||||
`${C.ERROR_PREFIX}partNumber must be a positive integer`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async _presignedRequest(
|
|
||||||
method: IT.HttpMethod, // 'GET' | 'HEAD' | 'PUT' | 'POST' | 'DELETE'
|
|
||||||
key: string, // '' allowed for bucket‑level ops
|
|
||||||
{
|
|
||||||
uploadId,
|
|
||||||
partNumber,
|
|
||||||
body = '',
|
|
||||||
contentType,
|
|
||||||
tolerated = [],
|
|
||||||
}: {
|
|
||||||
uploadId?: string
|
|
||||||
partNumber?: number
|
|
||||||
body?: BodyInit
|
|
||||||
contentType?: string
|
|
||||||
tolerated?: number[]
|
|
||||||
} = {},
|
|
||||||
): Promise<Response> {
|
|
||||||
// Get pre-signed URL from callback
|
|
||||||
const { url } = await this.signRequest({
|
|
||||||
method,
|
|
||||||
key,
|
|
||||||
uploadId,
|
|
||||||
partNumber,
|
|
||||||
contentType,
|
|
||||||
})
|
|
||||||
|
|
||||||
// Build request headers
|
|
||||||
const requestHeaders: Record<string, string> = contentType
|
|
||||||
? { 'Content-Type': contentType }
|
|
||||||
: {}
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await this._sendRequest(
|
|
||||||
url,
|
|
||||||
method,
|
|
||||||
requestHeaders,
|
|
||||||
body,
|
|
||||||
tolerated,
|
|
||||||
)
|
|
||||||
} catch (err) {
|
|
||||||
// If expired token error and using getCredentials, clear cache and retry once
|
|
||||||
if (
|
|
||||||
this.getCredentials &&
|
|
||||||
err instanceof U.S3ServiceError &&
|
|
||||||
err.code &&
|
|
||||||
['ExpiredToken', 'InvalidAccessKeyId'].includes(err.code)
|
|
||||||
) {
|
|
||||||
// Clear cache
|
|
||||||
this.clearCachedCredentials()
|
|
||||||
|
|
||||||
// Retry with fresh credentials
|
|
||||||
const fresh = await this.signRequest({
|
|
||||||
method,
|
|
||||||
key,
|
|
||||||
uploadId,
|
|
||||||
partNumber,
|
|
||||||
})
|
|
||||||
return this._sendRequest(
|
|
||||||
fresh.url,
|
|
||||||
method,
|
|
||||||
contentType ? { 'Content-Type': contentType } : {},
|
|
||||||
body,
|
|
||||||
tolerated,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Uploads an object to S3 using XHR for progress tracking.
|
|
||||||
* @param key - Object key
|
|
||||||
* @param data - Data to upload (Blob, ArrayBuffer, Uint8Array, or string)
|
|
||||||
* @param fileType - Content type
|
|
||||||
* @param onProgress - Optional progress callback
|
|
||||||
* @param signal - Optional abort signal
|
|
||||||
*/
|
|
||||||
public async putObject(
|
|
||||||
key: string,
|
|
||||||
data: IT.BinaryData | string,
|
|
||||||
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
|
|
||||||
onProgress?: IT.OnProgressFn,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
): Promise<IT.PutObjectResult> {
|
|
||||||
this._checkKey(key)
|
|
||||||
|
|
||||||
const attemptUpload = async (): Promise<IT.PutObjectResult> => {
|
|
||||||
const { url } = await this.signRequest({ method: 'PUT', key })
|
|
||||||
return this._xhrUpload(url, data, onProgress, signal, fileType)
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await attemptUpload()
|
|
||||||
} catch (err) {
|
|
||||||
if (this._isExpiredTokenError(err)) {
|
|
||||||
this.clearCachedCredentials()
|
|
||||||
return attemptUpload()
|
|
||||||
}
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Initiates a multipart upload and returns the upload ID. */
|
|
||||||
public async getMultipartUploadId(
|
|
||||||
key: string,
|
|
||||||
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
|
|
||||||
): Promise<string> {
|
|
||||||
this._checkKey(key)
|
|
||||||
if (typeof fileType !== 'string') {
|
|
||||||
throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`)
|
|
||||||
}
|
|
||||||
const res = await this._presignedRequest('POST', key, {
|
|
||||||
contentType: fileType,
|
|
||||||
})
|
|
||||||
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
|
|
||||||
|
|
||||||
if (parsed && typeof parsed === 'object') {
|
|
||||||
// Check for both cases of InitiateMultipartUploadResult
|
|
||||||
const uploadResult =
|
|
||||||
(parsed.initiateMultipartUploadResult as Record<string, unknown>) ||
|
|
||||||
(parsed.InitiateMultipartUploadResult as Record<string, unknown>)
|
|
||||||
|
|
||||||
if (uploadResult && typeof uploadResult === 'object') {
|
|
||||||
// Check for both cases of uploadId
|
|
||||||
const uploadId = uploadResult.uploadId || uploadResult.UploadId
|
|
||||||
|
|
||||||
if (uploadId && typeof uploadId === 'string') {
|
|
||||||
return uploadId
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(
|
|
||||||
`${C.ERROR_PREFIX}Failed to create multipart upload: ${JSON.stringify(
|
|
||||||
parsed,
|
|
||||||
)}`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
public async uploadPart(
|
|
||||||
key: string,
|
|
||||||
uploadId: string,
|
|
||||||
data: IT.BinaryData | string,
|
|
||||||
partNumber: number,
|
|
||||||
onProgress?: IT.OnProgressFn,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
): Promise<IT.UploadPart> {
|
|
||||||
this._validateUploadPartParams(key, uploadId, partNumber)
|
|
||||||
|
|
||||||
const attemptUpload = async (): Promise<IT.UploadPart> => {
|
|
||||||
const { url } = await this.signRequest({
|
|
||||||
method: 'PUT',
|
|
||||||
key,
|
|
||||||
uploadId,
|
|
||||||
partNumber,
|
|
||||||
})
|
|
||||||
const result = await this._xhrUpload(url, data, onProgress, signal)
|
|
||||||
return {
|
|
||||||
partNumber,
|
|
||||||
etag: result.headers.get('etag')
|
|
||||||
? U.sanitizeETag(result.headers.get('etag')!)
|
|
||||||
: '',
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await attemptUpload()
|
|
||||||
} catch (err) {
|
|
||||||
if (this._isExpiredTokenError(err)) {
|
|
||||||
this.clearCachedCredentials()
|
|
||||||
return attemptUpload()
|
|
||||||
}
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Helper to check if we're currently offline in a browser context.
|
|
||||||
*/
|
|
||||||
private _isOffline(): boolean {
|
|
||||||
return typeof navigator !== 'undefined' && navigator.onLine === false
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Checks if error is an expired/invalid token that can be retried with fresh credentials */
|
|
||||||
private _isExpiredTokenError(err: unknown): boolean {
|
|
||||||
return (
|
|
||||||
this.getCredentials != null &&
|
|
||||||
err instanceof U.S3ServiceError &&
|
|
||||||
err.code != null &&
|
|
||||||
['ExpiredToken', 'InvalidAccessKeyId'].includes(err.code)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Core XHR upload implementation using @uppy/utils/fetcher.
|
|
||||||
*
|
|
||||||
* Features:
|
|
||||||
* - Automatic retry with exponential backoff (3 attempts)
|
|
||||||
* - Offline detection with automatic resume on reconnect
|
|
||||||
* - Stall detection via ProgressTimeout
|
|
||||||
*/
|
|
||||||
private async _xhrUpload(
|
|
||||||
url: string,
|
|
||||||
data: IT.BinaryData | string,
|
|
||||||
onProgress?: IT.OnProgressFn,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
contentType?: string,
|
|
||||||
): Promise<IT.PutObjectResult> {
|
|
||||||
// Wait for online before starting
|
|
||||||
if (this._isOffline()) {
|
|
||||||
await this._waitForOnline(signal)
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const xhr = await fetcher(url, {
|
|
||||||
method: 'PUT',
|
|
||||||
// XHR natively supports ArrayBuffer, Uint8Array, Blob, and string
|
|
||||||
body: data as XMLHttpRequestBodyInit,
|
|
||||||
headers: contentType ? { 'Content-Type': contentType } : {},
|
|
||||||
signal,
|
|
||||||
timeout: this.requestAbortTimeout || 30_000,
|
|
||||||
retries: 3,
|
|
||||||
/**
|
|
||||||
* Retry logic:
|
|
||||||
* - Retries: 5xx server errors, 429 rate limiting
|
|
||||||
* - Skips: 4xx client errors (except 429), offline (handled separately)
|
|
||||||
*/
|
|
||||||
shouldRetry: (xhr) => {
|
|
||||||
// If offline, don't retry via fetcher - our handler will resume
|
|
||||||
if (this._isOffline()) return false
|
|
||||||
// Don't retry client errors (except 429 rate limit)
|
|
||||||
if (xhr.status >= 400 && xhr.status < 500 && xhr.status !== 429) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
},
|
|
||||||
onUploadProgress: (event) => {
|
|
||||||
if (event.lengthComputable && onProgress) {
|
|
||||||
onProgress(event.loaded, event.total)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
onTimeout: (timeout) => {
|
|
||||||
// Log stall detection - upload will continue but may be slow
|
|
||||||
console.warn(
|
|
||||||
`[S3mini] Upload stalled - no progress for ${Math.ceil(timeout / 1000)}s`,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
// Return Response-like object for test compatibility
|
|
||||||
return {
|
|
||||||
status: xhr.status,
|
|
||||||
ok: xhr.status >= 200 && xhr.status < 300,
|
|
||||||
headers: {
|
|
||||||
get: (name: string) => xhr.getResponseHeader(name),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
} catch (err: unknown) {
|
|
||||||
return this._handleUploadError(
|
|
||||||
err,
|
|
||||||
url,
|
|
||||||
data,
|
|
||||||
onProgress,
|
|
||||||
signal,
|
|
||||||
contentType,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handles errors from _xhrUpload, including offline recovery and error mapping.
|
|
||||||
*/
|
|
||||||
private async _handleUploadError(
|
|
||||||
err: unknown,
|
|
||||||
url: string,
|
|
||||||
data: IT.BinaryData | string,
|
|
||||||
onProgress?: IT.OnProgressFn,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
contentType?: string,
|
|
||||||
): Promise<IT.PutObjectResult> {
|
|
||||||
// Offline during request - wait and retry
|
|
||||||
if (this._isOffline()) {
|
|
||||||
await this._waitForOnline(signal)
|
|
||||||
// Check if aborted while waiting for online
|
|
||||||
if (signal?.aborted) {
|
|
||||||
throw new DOMException('Upload aborted', 'AbortError')
|
|
||||||
}
|
|
||||||
return this._xhrUpload(url, data, onProgress, signal, contentType)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Abort errors pass through
|
|
||||||
if (err instanceof DOMException && err.name === 'AbortError') {
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Network errors (from fetcher's NetworkError)
|
|
||||||
if (
|
|
||||||
err instanceof NetworkError ||
|
|
||||||
(err instanceof Error && 'isNetworkError' in err && err.isNetworkError)
|
|
||||||
) {
|
|
||||||
const xhr = 'request' in err ? (err as NetworkError).request : null
|
|
||||||
if (!xhr?.status) {
|
|
||||||
throw new U.S3NetworkError(
|
|
||||||
'Network error during upload',
|
|
||||||
'NETWORK',
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
// HTTP errors (non-2xx responses from NetworkError)
|
|
||||||
const headerCode = xhr.getResponseHeader('x-amz-error-code')
|
|
||||||
const parsedBody = this._parseErrorXml(
|
|
||||||
(name: string) => xhr.getResponseHeader(name),
|
|
||||||
xhr.responseText,
|
|
||||||
)
|
|
||||||
const serviceCode = headerCode ?? parsedBody.svcCode
|
|
||||||
throw new U.S3ServiceError(
|
|
||||||
`S3 returned ${xhr.status}${serviceCode ? ` – ${serviceCode}` : ''}`,
|
|
||||||
xhr.status,
|
|
||||||
serviceCode,
|
|
||||||
xhr.responseText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle errors with attached XHR (from onAfterResponse throws)
|
|
||||||
if (
|
|
||||||
err instanceof Error &&
|
|
||||||
'request' in err &&
|
|
||||||
err.request instanceof XMLHttpRequest
|
|
||||||
) {
|
|
||||||
const xhr = err.request
|
|
||||||
if (!xhr.status) {
|
|
||||||
throw new U.S3NetworkError(
|
|
||||||
'Network error during upload',
|
|
||||||
'NETWORK',
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
const headerCode = xhr.getResponseHeader('x-amz-error-code')
|
|
||||||
const parsedBody = this._parseErrorXml(
|
|
||||||
(name: string) => xhr.getResponseHeader(name),
|
|
||||||
xhr.responseText,
|
|
||||||
)
|
|
||||||
const serviceCode = headerCode ?? parsedBody.svcCode
|
|
||||||
throw new U.S3ServiceError(
|
|
||||||
`S3 returned ${xhr.status}${serviceCode ? ` – ${serviceCode}` : ''}`,
|
|
||||||
xhr.status,
|
|
||||||
serviceCode,
|
|
||||||
xhr.responseText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Lists uploaded parts for a multipart upload. */
|
|
||||||
public async listParts(
|
|
||||||
uploadId: string,
|
|
||||||
key: string,
|
|
||||||
): Promise<IT.UploadPart[]> {
|
|
||||||
this._checkKey(key)
|
|
||||||
if (!uploadId) {
|
|
||||||
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
|
|
||||||
}
|
|
||||||
const res = await this._presignedRequest('GET', key, {
|
|
||||||
uploadId,
|
|
||||||
})
|
|
||||||
|
|
||||||
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
|
|
||||||
const result = (parsed.listPartsResult ||
|
|
||||||
parsed.ListPartsResult ||
|
|
||||||
parsed) as Record<string, unknown>
|
|
||||||
|
|
||||||
if (result && typeof result === 'object') {
|
|
||||||
const parts = result.Part || result.part || []
|
|
||||||
const partsArray = Array.isArray(parts) ? parts : [parts]
|
|
||||||
|
|
||||||
return partsArray
|
|
||||||
.filter(
|
|
||||||
(p): p is Record<string, unknown> =>
|
|
||||||
p != null &&
|
|
||||||
typeof p === 'object' &&
|
|
||||||
'PartNumber' in p &&
|
|
||||||
'ETag' in p,
|
|
||||||
)
|
|
||||||
.map((p) => ({
|
|
||||||
partNumber: parseInt(String(p.PartNumber), 10),
|
|
||||||
etag: U.sanitizeETag(String(p.ETag)),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Completes a multipart upload by combining all uploaded parts. */
|
|
||||||
public async completeMultipartUpload(
|
|
||||||
key: string,
|
|
||||||
uploadId: string,
|
|
||||||
parts: Array<IT.UploadPart>,
|
|
||||||
): Promise<IT.CompleteMultipartUploadResult> {
|
|
||||||
const xmlBody = this._buildCompleteMultipartUploadXml(parts)
|
|
||||||
|
|
||||||
const res = await this._presignedRequest('POST', key, {
|
|
||||||
uploadId,
|
|
||||||
body: xmlBody,
|
|
||||||
contentType: C.XML_CONTENT_TYPE,
|
|
||||||
})
|
|
||||||
|
|
||||||
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
|
|
||||||
if (parsed && typeof parsed === 'object') {
|
|
||||||
// Check for both cases (camelCase from our parser, PascalCase from S3)
|
|
||||||
const result =
|
|
||||||
parsed.completeMultipartUploadResult ||
|
|
||||||
parsed.CompleteMultipartUploadResult ||
|
|
||||||
parsed
|
|
||||||
|
|
||||||
if (result && typeof result === 'object') {
|
|
||||||
const r = result as Record<string, unknown>
|
|
||||||
|
|
||||||
// S3 returns PascalCase (Location, Bucket, Key, ETag).
|
|
||||||
// Normalize to lowercase for our type interface.
|
|
||||||
const resultLocation = (r.Location || r.location) as string | undefined
|
|
||||||
const resultBucket = (r.Bucket || r.bucket) as string | undefined
|
|
||||||
const resultKey = (r.Key || r.key) as string | undefined
|
|
||||||
const rawEtag = (r.ETag || r.eTag || r.etag) as string | undefined
|
|
||||||
|
|
||||||
if (!resultLocation || !resultKey) {
|
|
||||||
throw new Error(
|
|
||||||
`${C.ERROR_PREFIX}CompleteMultipartUpload response missing Location or Key: ${JSON.stringify(r)}`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const etag = rawEtag ? U.sanitizeETag(rawEtag) : ''
|
|
||||||
|
|
||||||
return {
|
|
||||||
location: resultLocation,
|
|
||||||
bucket: resultBucket ?? '',
|
|
||||||
key: resultKey,
|
|
||||||
etag,
|
|
||||||
} satisfies IT.CompleteMultipartUploadResult
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(
|
|
||||||
`${C.ERROR_PREFIX}Failed to complete multipart upload: ${JSON.stringify(
|
|
||||||
parsed,
|
|
||||||
)}`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Aborts a multipart upload and removes all uploaded parts. */
|
|
||||||
public async abortMultipartUpload(
|
|
||||||
key: string,
|
|
||||||
uploadId: string,
|
|
||||||
): Promise<object> {
|
|
||||||
this._checkKey(key)
|
|
||||||
if (!uploadId) {
|
|
||||||
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
|
|
||||||
}
|
|
||||||
|
|
||||||
const res = await this._presignedRequest('DELETE', key, {
|
|
||||||
uploadId,
|
|
||||||
})
|
|
||||||
const parsed = U.parseXml(await res.text()) as Record<string, unknown>
|
|
||||||
if (
|
|
||||||
parsed &&
|
|
||||||
'error' in parsed &&
|
|
||||||
typeof parsed.error === 'object' &&
|
|
||||||
parsed.error !== null &&
|
|
||||||
'message' in parsed.error
|
|
||||||
) {
|
|
||||||
throw new Error(
|
|
||||||
`${C.ERROR_PREFIX}Failed to abort multipart upload: ${String(
|
|
||||||
parsed.error.message,
|
|
||||||
)}`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return { status: 'Aborted', key, uploadId, response: parsed }
|
|
||||||
}
|
|
||||||
|
|
||||||
private _buildCompleteMultipartUploadXml(
|
|
||||||
parts: Array<IT.UploadPart>,
|
|
||||||
): string {
|
|
||||||
let xml = '<CompleteMultipartUpload>'
|
|
||||||
for (const part of parts) {
|
|
||||||
xml += `<Part><PartNumber>${part.partNumber}</PartNumber><ETag>${part.etag}</ETag></Part>`
|
|
||||||
}
|
|
||||||
xml += '</CompleteMultipartUpload>'
|
|
||||||
return xml
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Deletes an object from the bucket. Returns true on success. */
|
|
||||||
public async deleteObject(key: string): Promise<boolean> {
|
|
||||||
const res = await this._presignedRequest('DELETE', key, {
|
|
||||||
tolerated: [200, 204],
|
|
||||||
})
|
|
||||||
return res.status === 200 || res.status === 204
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clears cached credentials.
|
|
||||||
* Call this method when you need to force a credential refresh on the next request.
|
|
||||||
*/
|
|
||||||
public clearCachedCredentials(): void {
|
|
||||||
this.cachedCredentials = undefined
|
|
||||||
this.cachedCredentialsPromise = undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Waits for the browser to come back online.
|
|
||||||
* Returns a promise that resolves when the 'online' event fires,
|
|
||||||
* or rejects if the abort signal is triggered.
|
|
||||||
*/
|
|
||||||
private _waitForOnline(signal?: AbortSignal): Promise<void> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
// Already online or not in browser
|
|
||||||
if (
|
|
||||||
typeof navigator === 'undefined' ||
|
|
||||||
navigator.onLine === true ||
|
|
||||||
navigator.onLine === undefined
|
|
||||||
) {
|
|
||||||
resolve()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Already aborted
|
|
||||||
if (signal?.aborted) {
|
|
||||||
reject(new DOMException('Upload aborted', 'AbortError'))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const cleanup = () => {
|
|
||||||
window.removeEventListener('online', onOnline)
|
|
||||||
signal?.removeEventListener('abort', onAbort)
|
|
||||||
}
|
|
||||||
|
|
||||||
const onOnline = () => {
|
|
||||||
cleanup()
|
|
||||||
resolve()
|
|
||||||
}
|
|
||||||
|
|
||||||
const onAbort = () => {
|
|
||||||
cleanup()
|
|
||||||
reject(new DOMException('Upload aborted', 'AbortError'))
|
|
||||||
}
|
|
||||||
|
|
||||||
window.addEventListener('online', onOnline)
|
|
||||||
signal?.addEventListener('abort', onAbort)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
private async _sendRequest(
|
|
||||||
url: string,
|
|
||||||
method: IT.HttpMethod,
|
|
||||||
headers: Record<string, string>,
|
|
||||||
body?: BodyInit,
|
|
||||||
toleratedStatusCodes: number[] = [],
|
|
||||||
): Promise<Response> {
|
|
||||||
// Wait for online if currently offline
|
|
||||||
if (this._isOffline()) {
|
|
||||||
await this._waitForOnline()
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const res = await fetch(url, {
|
|
||||||
method,
|
|
||||||
headers,
|
|
||||||
body: ['GET', 'HEAD'].includes(method) ? undefined : body,
|
|
||||||
signal: this.requestAbortTimeout
|
|
||||||
? AbortSignal.timeout(this.requestAbortTimeout)
|
|
||||||
: undefined,
|
|
||||||
})
|
|
||||||
if (res.ok || toleratedStatusCodes.includes(res.status)) {
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
await this._handleErrorResponse(res)
|
|
||||||
return res
|
|
||||||
} catch (err: unknown) {
|
|
||||||
// Check if we're offline - if so, wait and retry
|
|
||||||
if (this._isOffline()) {
|
|
||||||
await this._waitForOnline()
|
|
||||||
// Retry the request
|
|
||||||
return this._sendRequest(
|
|
||||||
url,
|
|
||||||
method,
|
|
||||||
headers,
|
|
||||||
body,
|
|
||||||
toleratedStatusCodes,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const code = U.extractErrCode(err)
|
|
||||||
if (
|
|
||||||
code &&
|
|
||||||
['ENOTFOUND', 'EAI_AGAIN', 'ETIMEDOUT', 'ECONNREFUSED'].includes(code)
|
|
||||||
) {
|
|
||||||
throw new U.S3NetworkError(`S3 network error: ${code}`, code, err)
|
|
||||||
}
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private _parseErrorXml(
|
|
||||||
getHeader: (name: string) => string | null,
|
|
||||||
body: string,
|
|
||||||
): { svcCode?: string; errorMessage?: string } {
|
|
||||||
if (getHeader('content-type') !== 'application/xml') {
|
|
||||||
return {}
|
|
||||||
}
|
|
||||||
const parsedBody = U.parseXml(body)
|
|
||||||
if (
|
|
||||||
!parsedBody ||
|
|
||||||
typeof parsedBody !== 'object' ||
|
|
||||||
!('Error' in parsedBody) ||
|
|
||||||
!parsedBody.Error ||
|
|
||||||
typeof parsedBody.Error !== 'object'
|
|
||||||
) {
|
|
||||||
return {}
|
|
||||||
}
|
|
||||||
const error = parsedBody.Error
|
|
||||||
return {
|
|
||||||
svcCode:
|
|
||||||
'Code' in error && typeof error.Code === 'string'
|
|
||||||
? error.Code
|
|
||||||
: undefined,
|
|
||||||
errorMessage:
|
|
||||||
'Message' in error && typeof error.Message === 'string'
|
|
||||||
? error.Message
|
|
||||||
: undefined,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async _handleErrorResponse(res: Response): Promise<void> {
|
|
||||||
const errorBody = await res.text()
|
|
||||||
const parsedErrorBody = this._parseErrorXml(
|
|
||||||
(n) => res.headers.get(n),
|
|
||||||
errorBody,
|
|
||||||
)
|
|
||||||
const svcCode =
|
|
||||||
res.headers.get('x-amz-error-code') ??
|
|
||||||
parsedErrorBody.svcCode ??
|
|
||||||
'Unknown'
|
|
||||||
throw new U.S3ServiceError(
|
|
||||||
`S3 returned ${res.status} – ${svcCode}`,
|
|
||||||
res.status,
|
|
||||||
svcCode,
|
|
||||||
errorBody,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export { S3mini }
|
|
||||||
export default S3mini
|
|
||||||
197
packages/@uppy/aws-s3/src/s3-client/S3Client.ts
Normal file
197
packages/@uppy/aws-s3/src/s3-client/S3Client.ts
Normal file
|
|
@ -0,0 +1,197 @@
|
||||||
|
import { fetcher } from '@uppy/utils'
|
||||||
|
import * as C from './consts.js'
|
||||||
|
import type * as IT from './types.js'
|
||||||
|
|
||||||
|
class S3Client {
|
||||||
|
readonly requestAbortTimeout?: number
|
||||||
|
|
||||||
|
constructor({
|
||||||
|
requestAbortTimeout,
|
||||||
|
...rest
|
||||||
|
}: { requestAbortTimeout?: number | undefined }) {
|
||||||
|
this.requestAbortTimeout = requestAbortTimeout
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper to check if we're currently offline in a browser context.
|
||||||
|
*/
|
||||||
|
protected isOffline(): boolean {
|
||||||
|
return typeof navigator !== 'undefined' && navigator.onLine === false
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Waits for the browser to come back online.
|
||||||
|
* Returns a promise that resolves when the 'online' event fires,
|
||||||
|
* or rejects if the abort signal is triggered.
|
||||||
|
*/
|
||||||
|
protected waitForOnline(signal?: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
if (!this.isOffline()) {
|
||||||
|
resolve()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Already online or not in browser
|
||||||
|
if (
|
||||||
|
typeof navigator === 'undefined' ||
|
||||||
|
navigator.onLine === true ||
|
||||||
|
navigator.onLine === undefined
|
||||||
|
) {
|
||||||
|
resolve()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Already aborted
|
||||||
|
if (signal?.aborted) {
|
||||||
|
reject(new DOMException('Upload aborted', 'AbortError'))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const cleanup = () => {
|
||||||
|
window.removeEventListener('online', onOnline)
|
||||||
|
signal?.removeEventListener('abort', onAbort)
|
||||||
|
}
|
||||||
|
|
||||||
|
const onOnline = () => {
|
||||||
|
cleanup()
|
||||||
|
resolve()
|
||||||
|
}
|
||||||
|
|
||||||
|
const onAbort = () => {
|
||||||
|
cleanup()
|
||||||
|
reject(new DOMException('Upload aborted', 'AbortError'))
|
||||||
|
}
|
||||||
|
|
||||||
|
window.addEventListener('online', onOnline)
|
||||||
|
signal?.addEventListener('abort', onAbort)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
protected async xhr({
|
||||||
|
url,
|
||||||
|
method,
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
contentType,
|
||||||
|
}: {
|
||||||
|
url: string
|
||||||
|
method: IT.HttpMethod
|
||||||
|
data?: XMLHttpRequestBodyInit
|
||||||
|
onProgress?: IT.OnProgressFn
|
||||||
|
signal?: AbortSignal
|
||||||
|
contentType?: string
|
||||||
|
}) {
|
||||||
|
// Check if aborted while waiting for online
|
||||||
|
if (signal?.aborted) {
|
||||||
|
throw new DOMException('Request aborted', 'AbortError')
|
||||||
|
}
|
||||||
|
|
||||||
|
return fetcher(url, {
|
||||||
|
method,
|
||||||
|
// XHR natively supports ArrayBuffer, Uint8Array, Blob, and string
|
||||||
|
body: ['GET', 'HEAD'].includes(method) ? undefined : data,
|
||||||
|
headers: contentType ? { 'Content-Type': contentType } : {},
|
||||||
|
signal,
|
||||||
|
timeout: this.requestAbortTimeout,
|
||||||
|
retries: 3,
|
||||||
|
/**
|
||||||
|
* Retry logic:
|
||||||
|
* - Retries: 5xx server errors, 429 rate limiting
|
||||||
|
* - Skips: 4xx client errors (except 429), offline (handled separately)
|
||||||
|
*/
|
||||||
|
shouldRetry: (xhr) => {
|
||||||
|
// If offline, don't retry via fetcher - our handler will resume
|
||||||
|
if (this.isOffline()) return false
|
||||||
|
// Don't retry client errors (except 429 rate limit)
|
||||||
|
if (xhr.status >= 400 && xhr.status < 500 && xhr.status !== 429) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
},
|
||||||
|
onUploadProgress: (event) => {
|
||||||
|
if (event.lengthComputable && onProgress) {
|
||||||
|
onProgress(event.loaded, event.total)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onTimeout: (timeout) => {
|
||||||
|
// Log stall detection - upload will continue but may be slow
|
||||||
|
console.warn(
|
||||||
|
`[S3mini] Upload stalled - no progress for ${Math.ceil(timeout / 1000)}s`,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
public async putObject(
|
||||||
|
key: string,
|
||||||
|
data: XMLHttpRequestBodyInit,
|
||||||
|
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
|
||||||
|
metadata: Record<string, unknown>,
|
||||||
|
onProgress?: IT.OnProgressFn,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<{
|
||||||
|
location: string
|
||||||
|
key: string
|
||||||
|
etag: string | undefined
|
||||||
|
}> {
|
||||||
|
throw new Error('Not implemented')
|
||||||
|
}
|
||||||
|
|
||||||
|
public async createMultipartUpload(
|
||||||
|
key: string,
|
||||||
|
fileType?: string,
|
||||||
|
// @ts-expect-error unused
|
||||||
|
metadata: Record<string, unknown>,
|
||||||
|
): Promise<{
|
||||||
|
uploadId: string
|
||||||
|
key: string
|
||||||
|
}> {
|
||||||
|
throw new Error('Not implemented')
|
||||||
|
}
|
||||||
|
|
||||||
|
public async uploadPart(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
data: XMLHttpRequestBodyInit,
|
||||||
|
partNumber: number,
|
||||||
|
onProgress?: IT.OnProgressFn,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<{
|
||||||
|
etag: string
|
||||||
|
}> {
|
||||||
|
throw new Error('Not implemented')
|
||||||
|
}
|
||||||
|
|
||||||
|
public async listParts(
|
||||||
|
uploadId: string,
|
||||||
|
key: string,
|
||||||
|
): Promise<IT.UploadPart[]> {
|
||||||
|
throw new Error('Not implemented')
|
||||||
|
}
|
||||||
|
|
||||||
|
public async completeMultipartUpload(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
parts: IT.UploadPart[],
|
||||||
|
): Promise<{
|
||||||
|
location: string
|
||||||
|
bucket: string | undefined
|
||||||
|
key: string
|
||||||
|
etag?: string | undefined
|
||||||
|
}> {
|
||||||
|
throw new Error('Not implemented')
|
||||||
|
}
|
||||||
|
|
||||||
|
public async abortMultipartUpload(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
throw new Error('Not implemented')
|
||||||
|
}
|
||||||
|
|
||||||
|
public async deleteObject(key: string): Promise<void> {
|
||||||
|
throw new Error('Not implemented')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default S3Client
|
||||||
567
packages/@uppy/aws-s3/src/s3-client/S3mini.ts
Normal file
567
packages/@uppy/aws-s3/src/s3-client/S3mini.ts
Normal file
|
|
@ -0,0 +1,567 @@
|
||||||
|
/**
|
||||||
|
* Taken from https://github.com/good-lly/s3mini.git, by Jølly Good, under MIT license.
|
||||||
|
* Modified to make it work with Uppy.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as C from './consts.js'
|
||||||
|
import S3Client from './S3Client.js'
|
||||||
|
import { createSigV4Signer } from './signer.js'
|
||||||
|
import type * as IT from './types.js'
|
||||||
|
import * as U from './utils.js'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* S3 client for browser-compatible interaction with S3-compatible storage.
|
||||||
|
* Supports simple uploads, multipart uploads, and object deletion.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* // Option 1: With signRequest callback (no endpoint needed)
|
||||||
|
* const s3 = new S3mini({
|
||||||
|
* signRequest: async ({ method, key, uploadId, partNumber }) => {
|
||||||
|
* const resp = await fetch('/api/s3/sign', {
|
||||||
|
* method: 'POST',
|
||||||
|
* body: JSON.stringify({ method, key, uploadId, partNumber }),
|
||||||
|
* });
|
||||||
|
* return resp.json(); // { url }
|
||||||
|
* },
|
||||||
|
* });
|
||||||
|
*
|
||||||
|
* // Option 2: With getCredentials callback (endpoint required for client-side signing)
|
||||||
|
* const s3 = new S3mini({
|
||||||
|
* endpoint: 'https://s3.us-east-1.amazonaws.com/my-bucket',
|
||||||
|
* getCredentials: async () => {
|
||||||
|
* const resp = await fetch('/api/s3/credentials');
|
||||||
|
* return resp.json(); // { credentials, region }
|
||||||
|
* },
|
||||||
|
* });
|
||||||
|
*
|
||||||
|
* await s3.putObject('file.txt', 'Hello, World!');
|
||||||
|
*/
|
||||||
|
class S3mini extends S3Client {
|
||||||
|
readonly endpoint?: URL
|
||||||
|
readonly region: string
|
||||||
|
readonly requestSizeInBytes: number
|
||||||
|
|
||||||
|
private readonly getCredentials?: IT.GetCredentialsFn
|
||||||
|
private cachedCredentials?: IT.CredentialsResponse
|
||||||
|
private cachedCredentialsPromise?: Promise<IT.CredentialsResponse>
|
||||||
|
private signRequest!: IT.SignRequestFn
|
||||||
|
|
||||||
|
constructor({
|
||||||
|
region = 'auto',
|
||||||
|
requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES,
|
||||||
|
requestAbortTimeout,
|
||||||
|
...rest
|
||||||
|
}: IT.S3Config) {
|
||||||
|
super({ requestAbortTimeout })
|
||||||
|
if ('signRequest' in rest) {
|
||||||
|
const { signRequest } = rest
|
||||||
|
if (!signRequest) {
|
||||||
|
throw new TypeError(
|
||||||
|
'Either signRequest or getCredentials must be provided',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (signRequest && typeof signRequest !== 'function') {
|
||||||
|
throw new TypeError('signRequest must be a function')
|
||||||
|
}
|
||||||
|
|
||||||
|
this.signRequest = signRequest
|
||||||
|
} else if ('getCredentials' in rest) {
|
||||||
|
const { getCredentials, endpoint } = rest
|
||||||
|
if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
|
||||||
|
throw new TypeError(C.ERROR_ENDPOINT_REQUIRED)
|
||||||
|
}
|
||||||
|
if (getCredentials && typeof getCredentials !== 'function') {
|
||||||
|
throw new TypeError('getCredentials must be a function')
|
||||||
|
}
|
||||||
|
this.endpoint = new URL(this._ensureValidUrl(endpoint))
|
||||||
|
|
||||||
|
this.getCredentials = getCredentials
|
||||||
|
this.signRequest = this._createCredentialBasedSigner()
|
||||||
|
} else {
|
||||||
|
throw new TypeError(
|
||||||
|
'Either signRequest or getCredentials must be provided',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
this.region = region
|
||||||
|
this.requestSizeInBytes = requestSizeInBytes
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Creates a presigner that fetches/caches credentials and generates pre-signed URLs. */
|
||||||
|
private _createCredentialBasedSigner(): IT.SignRequestFn {
|
||||||
|
return async (
|
||||||
|
request: IT.PresignableRequest,
|
||||||
|
): Promise<IT.presignedResponse> => {
|
||||||
|
const creds = await this._getCachedCredentials()
|
||||||
|
if (this.endpoint == null) {
|
||||||
|
throw new Error('Endpoint is required for credential-based signing')
|
||||||
|
}
|
||||||
|
const presigner = createSigV4Signer({
|
||||||
|
accessKeyId: creds.credentials.accessKeyId,
|
||||||
|
secretAccessKey: creds.credentials.secretAccessKey,
|
||||||
|
sessionToken: creds.credentials.sessionToken,
|
||||||
|
region: creds.region || this.region,
|
||||||
|
endpoint: this.endpoint.toString(),
|
||||||
|
})
|
||||||
|
return presigner(request)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Gets cached credentials or fetches new ones. */
|
||||||
|
private async _getCachedCredentials(): Promise<IT.CredentialsResponse> {
|
||||||
|
// Return Cached Credentials if available
|
||||||
|
if (this.cachedCredentials != null) {
|
||||||
|
return this.cachedCredentials
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cache the promise so concurrent calls wait for the same fetch
|
||||||
|
if (this.cachedCredentialsPromise == null) {
|
||||||
|
this.cachedCredentialsPromise = (async () => {
|
||||||
|
try {
|
||||||
|
const creds = await this.getCredentials!({})
|
||||||
|
this.cachedCredentials = creds
|
||||||
|
return creds
|
||||||
|
} finally {
|
||||||
|
// Clear promise cache after resolution to allow future retries
|
||||||
|
this.cachedCredentialsPromise = undefined
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.cachedCredentialsPromise
|
||||||
|
}
|
||||||
|
|
||||||
|
private _ensureValidUrl(raw: string): string {
|
||||||
|
const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}`
|
||||||
|
try {
|
||||||
|
new URL(candidate)
|
||||||
|
|
||||||
|
// Find the last non-slash character
|
||||||
|
let endIndex = candidate.length
|
||||||
|
while (endIndex > 0 && candidate[endIndex - 1] === '/') {
|
||||||
|
endIndex--
|
||||||
|
}
|
||||||
|
return endIndex === candidate.length
|
||||||
|
? candidate
|
||||||
|
: candidate.substring(0, endIndex)
|
||||||
|
} catch {
|
||||||
|
const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"`
|
||||||
|
throw new TypeError(msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private _checkKey(key: string): void {
|
||||||
|
if (typeof key !== 'string' || key.trim().length === 0) {
|
||||||
|
throw new TypeError(C.ERROR_KEY_REQUIRED)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private _validateUploadPartParams(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
partNumber: number,
|
||||||
|
): void {
|
||||||
|
this._checkKey(key)
|
||||||
|
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
|
||||||
|
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(partNumber) || partNumber <= 0) {
|
||||||
|
throw new TypeError(
|
||||||
|
`${C.ERROR_PREFIX}partNumber must be a positive integer`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Uploads an object to S3 using XHR for progress tracking.
|
||||||
|
* @param key - Object key
|
||||||
|
* @param data - Data to upload (Blob, ArrayBuffer, Uint8Array, or string)
|
||||||
|
* @param fileType - Content type
|
||||||
|
* @param onProgress - Optional progress callback
|
||||||
|
* @param signal - Optional abort signal
|
||||||
|
*/
|
||||||
|
public override async putObject(
|
||||||
|
key: string,
|
||||||
|
data: XMLHttpRequestBodyInit,
|
||||||
|
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
|
||||||
|
metadata?: Record<string, unknown>,
|
||||||
|
onProgress?: IT.OnProgressFn,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
) {
|
||||||
|
this._checkKey(key)
|
||||||
|
|
||||||
|
const { xhr, url } = await this.request({
|
||||||
|
request: { method: 'PUT', key },
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
contentType: fileType,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
location: U.removeQueryString(url),
|
||||||
|
etag: U.sanitizeETag(xhr.getResponseHeader('etag')),
|
||||||
|
key,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Initiates a multipart upload and returns the upload ID. */
|
||||||
|
public override async createMultipartUpload(
|
||||||
|
key: string,
|
||||||
|
fileType: string = C.DEFAULT_STREAM_CONTENT_TYPE,
|
||||||
|
metadata?: Record<string, unknown>, // todo support metadata here too?
|
||||||
|
) {
|
||||||
|
this._checkKey(key)
|
||||||
|
if (typeof fileType !== 'string') {
|
||||||
|
throw new TypeError(`${C.ERROR_PREFIX}fileType must be a string`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const { xhr } = await this.request({
|
||||||
|
request: { method: 'POST', key },
|
||||||
|
contentType: fileType,
|
||||||
|
})
|
||||||
|
|
||||||
|
const parsed = U.parseXml(xhr.responseText) as Record<string, unknown>
|
||||||
|
|
||||||
|
if (parsed && typeof parsed === 'object') {
|
||||||
|
// Check for both cases of InitiateMultipartUploadResult
|
||||||
|
const uploadResult =
|
||||||
|
(parsed.initiateMultipartUploadResult as Record<string, unknown>) ||
|
||||||
|
(parsed.InitiateMultipartUploadResult as Record<string, unknown>)
|
||||||
|
|
||||||
|
if (uploadResult && typeof uploadResult === 'object') {
|
||||||
|
// Check for both cases of uploadId
|
||||||
|
const uploadId = uploadResult.uploadId || uploadResult.UploadId
|
||||||
|
|
||||||
|
if (uploadId && typeof uploadId === 'string') {
|
||||||
|
return { uploadId, key }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`${C.ERROR_PREFIX}Failed to create multipart upload: ${JSON.stringify(
|
||||||
|
parsed,
|
||||||
|
)}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async uploadPart(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
data: XMLHttpRequestBodyInit,
|
||||||
|
partNumber: number,
|
||||||
|
onProgress?: IT.OnProgressFn,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
) {
|
||||||
|
this._validateUploadPartParams(key, uploadId, partNumber)
|
||||||
|
|
||||||
|
const { xhr } = await this.request({
|
||||||
|
request: {
|
||||||
|
method: 'PUT',
|
||||||
|
key,
|
||||||
|
uploadId,
|
||||||
|
partNumber,
|
||||||
|
},
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
})
|
||||||
|
|
||||||
|
const etag = U.sanitizeETag(xhr.getResponseHeader('etag'))
|
||||||
|
if (etag == null) {
|
||||||
|
throw new Error(
|
||||||
|
`${C.ERROR_PREFIX}Missing ETag in uploadPart response headers`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return { etag }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Core XHR upload implementation using @uppy/utils/fetcher.
|
||||||
|
*
|
||||||
|
* Features:
|
||||||
|
* - Automatic retry with exponential backoff (3 attempts)
|
||||||
|
* - Offline detection with automatic resume on reconnect
|
||||||
|
* - Stall detection via ProgressTimeout
|
||||||
|
*/
|
||||||
|
private async request({
|
||||||
|
request,
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
contentType,
|
||||||
|
shouldRetryCredentials = true,
|
||||||
|
}: {
|
||||||
|
request: IT.PresignableRequest
|
||||||
|
data?: XMLHttpRequestBodyInit
|
||||||
|
onProgress?: IT.OnProgressFn
|
||||||
|
signal?: AbortSignal
|
||||||
|
contentType?: string
|
||||||
|
shouldRetryCredentials?: boolean
|
||||||
|
}): Promise<{ xhr: XMLHttpRequest; url: string }> {
|
||||||
|
// Wait for online before starting
|
||||||
|
await this.waitForOnline(signal)
|
||||||
|
|
||||||
|
// Check if aborted while waiting for online
|
||||||
|
if (signal?.aborted) {
|
||||||
|
throw new DOMException('Request aborted', 'AbortError')
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { url } = await this.signRequest(request)
|
||||||
|
|
||||||
|
const xhr = await this.xhr({
|
||||||
|
url,
|
||||||
|
method: request.method,
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
contentType,
|
||||||
|
})
|
||||||
|
|
||||||
|
return { xhr, url }
|
||||||
|
} catch (err: unknown) {
|
||||||
|
// NetworkError or errors with attached XHR (from onAfterResponse throws)
|
||||||
|
if (
|
||||||
|
err instanceof Error &&
|
||||||
|
'request' in err &&
|
||||||
|
err.request instanceof XMLHttpRequest
|
||||||
|
) {
|
||||||
|
const xhr = err.request as XMLHttpRequest
|
||||||
|
if (xhr.status === 0) {
|
||||||
|
throw new U.S3NetworkError(
|
||||||
|
'Network error during S3 request',
|
||||||
|
'NETWORK',
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
// HTTP errors (non-2xx responses from NetworkError)
|
||||||
|
const parsedBody = this._parseErrorXml(
|
||||||
|
(name: string) => xhr.getResponseHeader(name),
|
||||||
|
xhr.responseText,
|
||||||
|
)
|
||||||
|
const serviceCode =
|
||||||
|
xhr.getResponseHeader('x-amz-error-code') ?? parsedBody.svcCode
|
||||||
|
|
||||||
|
// If expired token error and using getCredentials, clear cache and retry once
|
||||||
|
if (
|
||||||
|
shouldRetryCredentials &&
|
||||||
|
this.getCredentials != null &&
|
||||||
|
serviceCode != null &&
|
||||||
|
['ExpiredToken', 'InvalidAccessKeyId'].includes(serviceCode)
|
||||||
|
) {
|
||||||
|
this.clearCachedCredentials()
|
||||||
|
|
||||||
|
// Retry with fresh credentials
|
||||||
|
return this.request({
|
||||||
|
request,
|
||||||
|
data,
|
||||||
|
onProgress,
|
||||||
|
signal,
|
||||||
|
contentType,
|
||||||
|
shouldRetryCredentials: false, // prevent infinite recursion
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new U.S3ServiceError(
|
||||||
|
`S3 returned ${xhr.status}${serviceCode ? ` – ${serviceCode}` : ''}`,
|
||||||
|
xhr.status,
|
||||||
|
serviceCode,
|
||||||
|
xhr.responseText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Lists uploaded parts for a multipart upload. */
|
||||||
|
public override async listParts(
|
||||||
|
uploadId: string,
|
||||||
|
key: string,
|
||||||
|
): Promise<IT.UploadPart[]> {
|
||||||
|
this._checkKey(key)
|
||||||
|
if (!uploadId) {
|
||||||
|
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
|
||||||
|
}
|
||||||
|
const { xhr } = await this.request({
|
||||||
|
request: { method: 'GET', key, uploadId },
|
||||||
|
})
|
||||||
|
|
||||||
|
const parsed = U.parseXml(xhr.responseText) as Record<string, unknown>
|
||||||
|
const result = (parsed.listPartsResult ||
|
||||||
|
parsed.ListPartsResult ||
|
||||||
|
parsed) as Record<string, unknown>
|
||||||
|
|
||||||
|
if (result && typeof result === 'object') {
|
||||||
|
const parts = result.Part || result.part || []
|
||||||
|
const partsArray = Array.isArray(parts) ? parts : [parts]
|
||||||
|
|
||||||
|
return partsArray
|
||||||
|
.filter(
|
||||||
|
(p): p is Record<string, unknown> =>
|
||||||
|
p != null &&
|
||||||
|
typeof p === 'object' &&
|
||||||
|
'PartNumber' in p &&
|
||||||
|
'ETag' in p,
|
||||||
|
)
|
||||||
|
.map((p) => ({
|
||||||
|
partNumber: parseInt(String(p.PartNumber), 10),
|
||||||
|
etag: U.sanitizeXmlETag(String(p.ETag)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Completes a multipart upload by combining all uploaded parts. */
|
||||||
|
public override async completeMultipartUpload(
|
||||||
|
key: string,
|
||||||
|
uploadId: string,
|
||||||
|
parts: Array<IT.UploadPart>,
|
||||||
|
) {
|
||||||
|
const xmlBody = this._buildCompleteMultipartUploadXml(parts)
|
||||||
|
|
||||||
|
const { xhr } = await this.request({
|
||||||
|
request: { method: 'POST', key, uploadId },
|
||||||
|
contentType: C.XML_CONTENT_TYPE,
|
||||||
|
data: xmlBody,
|
||||||
|
})
|
||||||
|
|
||||||
|
const parsed = U.parseXml(xhr.responseText)
|
||||||
|
if (parsed && typeof parsed === 'object') {
|
||||||
|
// Check for both cases (camelCase from our parser, PascalCase from S3)
|
||||||
|
const result =
|
||||||
|
parsed.completeMultipartUploadResult ||
|
||||||
|
parsed.CompleteMultipartUploadResult ||
|
||||||
|
parsed
|
||||||
|
|
||||||
|
if (result && typeof result === 'object') {
|
||||||
|
const r = result as Record<string, unknown>
|
||||||
|
|
||||||
|
// S3 returns PascalCase (Location, Bucket, Key, ETag).
|
||||||
|
// Normalize to lowercase for our type interface.
|
||||||
|
const resultLocation = (r.Location || r.location) as string | undefined
|
||||||
|
const resultBucket = (r.Bucket || r.bucket) as string | undefined
|
||||||
|
const resultKey = (r.Key || r.key) as string | undefined
|
||||||
|
const rawEtag = (r.ETag || r.eTag || r.etag) as string | undefined
|
||||||
|
|
||||||
|
if (!resultLocation || !resultKey) {
|
||||||
|
throw new Error(
|
||||||
|
`${C.ERROR_PREFIX}CompleteMultipartUpload response missing Location or Key: ${JSON.stringify(r)}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const etag = rawEtag ? U.sanitizeXmlETag(rawEtag) : undefined
|
||||||
|
|
||||||
|
return {
|
||||||
|
location: resultLocation,
|
||||||
|
bucket: resultBucket,
|
||||||
|
key: resultKey,
|
||||||
|
etag,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`${C.ERROR_PREFIX}Failed to complete multipart upload: ${JSON.stringify(
|
||||||
|
parsed,
|
||||||
|
)}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Aborts a multipart upload and removes all uploaded parts. */
|
||||||
|
public override async abortMultipartUpload(key: string, uploadId: string) {
|
||||||
|
this._checkKey(key)
|
||||||
|
if (!uploadId) {
|
||||||
|
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED)
|
||||||
|
}
|
||||||
|
|
||||||
|
const { xhr } = await this.request({
|
||||||
|
request: { method: 'DELETE', key, uploadId },
|
||||||
|
})
|
||||||
|
|
||||||
|
const parsed = U.parseXml(xhr.responseText) as Record<string, unknown>
|
||||||
|
if (
|
||||||
|
parsed &&
|
||||||
|
'error' in parsed &&
|
||||||
|
typeof parsed.error === 'object' &&
|
||||||
|
parsed.error !== null &&
|
||||||
|
'message' in parsed.error
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
`${C.ERROR_PREFIX}Failed to abort multipart upload: ${String(
|
||||||
|
parsed.error.message,
|
||||||
|
)}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private _buildCompleteMultipartUploadXml(
|
||||||
|
parts: Array<IT.UploadPart>,
|
||||||
|
): string {
|
||||||
|
let xml = '<CompleteMultipartUpload>'
|
||||||
|
for (const part of parts) {
|
||||||
|
xml += `<Part><PartNumber>${part.partNumber}</PartNumber><ETag>${part.etag}</ETag></Part>`
|
||||||
|
}
|
||||||
|
xml += '</CompleteMultipartUpload>'
|
||||||
|
return xml
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Deletes an object from the bucket. Returns true on success. */
|
||||||
|
public override async deleteObject(key: string) {
|
||||||
|
const { xhr } = await this.request({
|
||||||
|
request: { method: 'DELETE', key },
|
||||||
|
})
|
||||||
|
|
||||||
|
if (xhr.status !== 200 && xhr.status !== 204) {
|
||||||
|
throw new Error(
|
||||||
|
`${C.ERROR_PREFIX}Failed to delete object. HTTP status: ${xhr.status}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clears cached credentials.
|
||||||
|
* Call this method when you need to force a credential refresh on the next request.
|
||||||
|
*/
|
||||||
|
public clearCachedCredentials(): void {
|
||||||
|
this.cachedCredentials = undefined
|
||||||
|
this.cachedCredentialsPromise = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
private _parseErrorXml(
|
||||||
|
getHeader: (name: string) => string | null,
|
||||||
|
body: string,
|
||||||
|
): { svcCode?: string; errorMessage?: string } {
|
||||||
|
if (getHeader('content-type') !== 'application/xml') {
|
||||||
|
return {}
|
||||||
|
}
|
||||||
|
const parsedBody = U.parseXml(body)
|
||||||
|
if (
|
||||||
|
!parsedBody ||
|
||||||
|
typeof parsedBody !== 'object' ||
|
||||||
|
!('Error' in parsedBody) ||
|
||||||
|
!parsedBody.Error ||
|
||||||
|
typeof parsedBody.Error !== 'object'
|
||||||
|
) {
|
||||||
|
return {}
|
||||||
|
}
|
||||||
|
const error = parsedBody.Error
|
||||||
|
return {
|
||||||
|
svcCode:
|
||||||
|
'Code' in error && typeof error.Code === 'string'
|
||||||
|
? error.Code
|
||||||
|
: undefined,
|
||||||
|
errorMessage:
|
||||||
|
'Message' in error && typeof error.Message === 'string'
|
||||||
|
? error.Message
|
||||||
|
: undefined,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export { S3mini }
|
||||||
|
export default S3mini
|
||||||
|
|
@ -1,10 +1,8 @@
|
||||||
// Export the S3 class as default export and named export
|
// Export the S3 class as default export and named export
|
||||||
export { S3mini } from './S3.js'
|
export { S3mini } from './S3mini.js'
|
||||||
// Re-export types
|
// Re-export types
|
||||||
export type {
|
export type {
|
||||||
CompleteMultipartUploadResult,
|
|
||||||
ErrorWithCode,
|
ErrorWithCode,
|
||||||
S3Config,
|
S3Config,
|
||||||
UploadPart,
|
UploadPart,
|
||||||
} from './types.js'
|
} from './types.js'
|
||||||
export { sanitizeETag } from './utils.js'
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import * as C from './consts.js'
|
import * as C from './consts.js'
|
||||||
import type { presignableRequest, presignedResponse } from './types.js'
|
import type { PresignableRequest, presignedResponse } from './types.js'
|
||||||
import * as U from './utils.js'
|
import * as U from './utils.js'
|
||||||
|
|
||||||
export interface SignerConfig {
|
export interface SignerConfig {
|
||||||
|
|
@ -45,15 +45,9 @@ export function createSigV4Presigner(config: SignerConfig) {
|
||||||
}
|
}
|
||||||
|
|
||||||
return async function presign(
|
return async function presign(
|
||||||
request: presignableRequest,
|
request: PresignableRequest,
|
||||||
): Promise<presignedResponse> {
|
): Promise<presignedResponse> {
|
||||||
const {
|
const { method, key, expiresIn = DEFAULT_EXPIRES_IN } = request
|
||||||
method,
|
|
||||||
key,
|
|
||||||
uploadId,
|
|
||||||
partNumber,
|
|
||||||
expiresIn = DEFAULT_EXPIRES_IN,
|
|
||||||
} = request
|
|
||||||
|
|
||||||
// Build the URL - need to track encoded path separately because URL object decodes it
|
// Build the URL - need to track encoded path separately because URL object decodes it
|
||||||
const url = new URL(endpoint)
|
const url = new URL(endpoint)
|
||||||
|
|
@ -92,15 +86,15 @@ export function createSigV4Presigner(config: SignerConfig) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add multipart-specific params
|
// Add multipart-specific params
|
||||||
if (uploadId) {
|
if ('uploadId' in request) {
|
||||||
url.searchParams.set('uploadId', uploadId)
|
url.searchParams.set('uploadId', request.uploadId)
|
||||||
}
|
}
|
||||||
if (partNumber !== undefined) {
|
if ('partNumber' in request) {
|
||||||
url.searchParams.set('partNumber', String(partNumber))
|
url.searchParams.set('partNumber', String(request.partNumber))
|
||||||
}
|
}
|
||||||
|
|
||||||
// For CreateMultipartUpload, add uploads param
|
// For CreateMultipartUpload, add uploads param
|
||||||
if (method === 'POST' && !uploadId) {
|
if (method === 'POST' && !('uploadId' in request)) {
|
||||||
url.searchParams.set('uploads', '')
|
url.searchParams.set('uploads', '')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,53 @@
|
||||||
/** Request data to be pre-signed */
|
export interface PresignableRequestBase {
|
||||||
export type presignableRequest = {
|
|
||||||
method: string
|
|
||||||
key: string
|
key: string
|
||||||
uploadId?: string
|
|
||||||
partNumber?: number
|
|
||||||
expiresIn?: number
|
expiresIn?: number
|
||||||
contentType?: string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface PutObjectRequest extends PresignableRequestBase {
|
||||||
|
method: 'PUT'
|
||||||
|
}
|
||||||
|
export interface DeleteObjectRequest extends PresignableRequestBase {
|
||||||
|
method: 'DELETE'
|
||||||
|
}
|
||||||
|
export interface CreateMultipartUploadRequest extends PresignableRequestBase {
|
||||||
|
method: 'POST'
|
||||||
|
}
|
||||||
|
export interface CompleteMultipartUploadRequest extends PresignableRequestBase {
|
||||||
|
method: 'POST'
|
||||||
|
uploadId: string
|
||||||
|
}
|
||||||
|
export interface DeleteMultipartUploadRequest extends PresignableRequestBase {
|
||||||
|
method: 'DELETE'
|
||||||
|
uploadId: string
|
||||||
|
}
|
||||||
|
export interface ListPartsRequest extends PresignableRequestBase {
|
||||||
|
method: 'GET'
|
||||||
|
uploadId: string
|
||||||
|
}
|
||||||
|
export interface UploadPartRequest extends PresignableRequestBase {
|
||||||
|
method: 'PUT'
|
||||||
|
uploadId: string
|
||||||
|
partNumber: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Request data to be pre-signed */
|
||||||
|
export type PresignableRequest =
|
||||||
|
| PutObjectRequest
|
||||||
|
| DeleteObjectRequest
|
||||||
|
| CreateMultipartUploadRequest
|
||||||
|
| CompleteMultipartUploadRequest
|
||||||
|
| DeleteMultipartUploadRequest
|
||||||
|
| ListPartsRequest
|
||||||
|
| UploadPartRequest
|
||||||
|
|
||||||
/** Response with the pre-signed URL */
|
/** Response with the pre-signed URL */
|
||||||
export type presignedResponse = {
|
export type presignedResponse = {
|
||||||
url: string
|
url: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Function that generates a pre-signed URL for a request */
|
/** Function that generates a pre-signed URL for a request */
|
||||||
export type signRequestFn = (
|
export type SignRequestFn = (
|
||||||
request: presignableRequest,
|
request: PresignableRequest,
|
||||||
) => Promise<presignedResponse>
|
) => Promise<presignedResponse>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -40,36 +72,34 @@ export interface CredentialsResponse {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Function that retrieves temporary credentials */
|
/** Function that retrieves temporary credentials */
|
||||||
export type getCredentialsFn = (options?: {
|
export type GetCredentialsFn = (options?: {
|
||||||
signal?: AbortSignal
|
signal?: AbortSignal
|
||||||
}) => Promise<CredentialsResponse>
|
}) => CredentialsResponse | Promise<CredentialsResponse>
|
||||||
|
|
||||||
/** Base configuration shared by both signing approaches */
|
/** Base configuration shared by both signing approaches */
|
||||||
type S3ConfigBase = {
|
type S3ConfigBase = {
|
||||||
/** Endpoint URL of the S3-compatible service (e.g., 'https://s3.amazonaws.com/bucket-name') */
|
|
||||||
endpoint: string
|
|
||||||
/** AWS region. Defaults to 'auto'. */
|
/** AWS region. Defaults to 'auto'. */
|
||||||
region?: string
|
region?: string | undefined
|
||||||
/** Request size in bytes for multipart uploads. Defaults to 8MB. */
|
/** Request size in bytes for multipart uploads. Defaults to 8MB. */
|
||||||
requestSizeInBytes?: number
|
requestSizeInBytes?: number | undefined
|
||||||
/** Timeout in ms after which a request should be aborted. */
|
/** Timeout in ms after which a request should be aborted. */
|
||||||
requestAbortTimeout?: number
|
requestAbortTimeout?: number | undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Config when using signRequest callback (region optional) */
|
/** Config when using signRequest callback (region optional) */
|
||||||
type S3ConfigWithSignRequest = S3ConfigBase & {
|
type S3ConfigWithSignRequest = S3ConfigBase & {
|
||||||
/** Function to sign requests. Called for each S3 API request. */
|
/** Function to sign requests. Called for each S3 API request. */
|
||||||
signRequest: signRequestFn
|
signRequest: SignRequestFn
|
||||||
getCredentials?: never
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Config when using getCredentials callback (region required for signing) */
|
/** Config when using getCredentials callback (region required for signing) */
|
||||||
type S3ConfigWithGetCredentials = Omit<S3ConfigBase, 'region'> & {
|
type S3ConfigWithGetCredentials = Omit<S3ConfigBase, 'region'> & {
|
||||||
signRequest?: never
|
|
||||||
/** Function to retrieve temporary credentials for client-side signing. */
|
/** Function to retrieve temporary credentials for client-side signing. */
|
||||||
getCredentials: getCredentialsFn
|
getCredentials: GetCredentialsFn
|
||||||
/** AWS region. Required for signing with getCredentials. */
|
/** AWS region. Required for signing with getCredentials. */
|
||||||
region: string
|
region?: string
|
||||||
|
/** Endpoint URL of the S3-compatible service (e.g., 'https://s3.amazonaws.com/bucket-name') */
|
||||||
|
endpoint: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Configuration options for S3mini client */
|
/** Configuration options for S3mini client */
|
||||||
|
|
@ -80,13 +110,6 @@ export interface UploadPart {
|
||||||
etag: string
|
etag: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CompleteMultipartUploadResult {
|
|
||||||
location: string
|
|
||||||
bucket: string
|
|
||||||
key: string
|
|
||||||
etag: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ErrorWithCode {
|
export interface ErrorWithCode {
|
||||||
code?: string
|
code?: string
|
||||||
cause?: { code?: string }
|
cause?: { code?: string }
|
||||||
|
|
@ -108,11 +131,3 @@ export type BinaryData = ArrayBuffer | Uint8Array | Blob
|
||||||
|
|
||||||
/** Progress callback for upload operations */
|
/** Progress callback for upload operations */
|
||||||
export type OnProgressFn = (bytesUploaded: number, bytesTotal: number) => void
|
export type OnProgressFn = (bytesUploaded: number, bytesTotal: number) => void
|
||||||
|
|
||||||
export interface PutObjectResult {
|
|
||||||
status: number
|
|
||||||
ok: boolean
|
|
||||||
headers: {
|
|
||||||
get(name: string): string | null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,22 @@
|
||||||
import type { ErrorWithCode, XmlMap, XmlValue } from './types.js'
|
import type { XmlMap, XmlValue } from './types.js'
|
||||||
|
|
||||||
const ENCODR = new TextEncoder()
|
export const sanitizeXmlETag = (etag: string): string =>
|
||||||
const chunkSize = 0x8000 // 32KB chunks
|
etag.replace(/^("|")+|("|")+$/g, '')
|
||||||
const HEXS = '0123456789abcdef'
|
|
||||||
|
|
||||||
export const getByteSize = (data: unknown): number => {
|
export const sanitizeETag = (etag: string | null): string | undefined =>
|
||||||
if (typeof data === 'string') {
|
etag?.replace(/^"+|"+$/g, '')
|
||||||
return ENCODR.encode(data).byteLength
|
|
||||||
}
|
/** Strips query string and hash from a URL to derive the object location. */
|
||||||
if (data instanceof ArrayBuffer) {
|
export function removeQueryString(urlString: string): string {
|
||||||
return data.byteLength
|
const urlObject = new URL(urlString)
|
||||||
}
|
urlObject.search = ''
|
||||||
if (ArrayBuffer.isView(data)) {
|
urlObject.hash = ''
|
||||||
return data.byteLength
|
return urlObject.href
|
||||||
}
|
|
||||||
if (data instanceof Blob) {
|
|
||||||
return data.size
|
|
||||||
}
|
|
||||||
throw new Error('Unsupported data type')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const textEncoder = new TextEncoder()
|
||||||
|
const HEXS = '0123456789abcdef'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Turn a raw ArrayBuffer into its hexadecimal representation.
|
* Turn a raw ArrayBuffer into its hexadecimal representation.
|
||||||
* @param {ArrayBuffer} buffer The raw bytes.
|
* @param {ArrayBuffer} buffer The raw bytes.
|
||||||
|
|
@ -34,28 +31,13 @@ export const hexFromBuffer = (buffer: ArrayBuffer): string => {
|
||||||
return hex
|
return hex
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Turn a raw ArrayBuffer into its base64 representation.
|
|
||||||
* @param {ArrayBuffer} buffer The raw bytes.
|
|
||||||
* @returns {string} Base64 string
|
|
||||||
*/
|
|
||||||
export const base64FromBuffer = (buffer: ArrayBuffer): string => {
|
|
||||||
const bytes = new Uint8Array(buffer)
|
|
||||||
let result = ''
|
|
||||||
for (let i = 0; i < bytes.length; i += chunkSize) {
|
|
||||||
const chunk = bytes.subarray(i, i + chunkSize)
|
|
||||||
result += btoa(String.fromCodePoint(...chunk))
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute SHA-256 hash of arbitrary string data.
|
* Compute SHA-256 hash of arbitrary string data.
|
||||||
* @param {string} content The content to be hashed.
|
* @param {string} content The content to be hashed.
|
||||||
* @returns {ArrayBuffer} The raw hash
|
* @returns {ArrayBuffer} The raw hash
|
||||||
*/
|
*/
|
||||||
export const sha256 = async (content: string): Promise<ArrayBuffer> => {
|
export const sha256 = async (content: string): Promise<ArrayBuffer> => {
|
||||||
const data = ENCODR.encode(content)
|
const data = textEncoder.encode(content)
|
||||||
|
|
||||||
return await globalThis.crypto.subtle.digest('SHA-256', data)
|
return await globalThis.crypto.subtle.digest('SHA-256', data)
|
||||||
}
|
}
|
||||||
|
|
@ -72,33 +54,16 @@ export const hmac = async (
|
||||||
): Promise<ArrayBuffer> => {
|
): Promise<ArrayBuffer> => {
|
||||||
const secret = await globalThis.crypto.subtle.importKey(
|
const secret = await globalThis.crypto.subtle.importKey(
|
||||||
'raw',
|
'raw',
|
||||||
typeof key === 'string' ? ENCODR.encode(key) : key,
|
typeof key === 'string' ? textEncoder.encode(key) : key,
|
||||||
{ name: 'HMAC', hash: 'SHA-256' },
|
{ name: 'HMAC', hash: 'SHA-256' },
|
||||||
false,
|
false,
|
||||||
['sign'],
|
['sign'],
|
||||||
)
|
)
|
||||||
const data = ENCODR.encode(content)
|
const data = textEncoder.encode(content)
|
||||||
|
|
||||||
return await globalThis.crypto.subtle.sign('HMAC', secret, data)
|
return await globalThis.crypto.subtle.sign('HMAC', secret, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize ETag value by removing quotes and XML entities
|
|
||||||
* @param etag ETag value to sanitize
|
|
||||||
* @returns Sanitized ETag
|
|
||||||
*/
|
|
||||||
export const sanitizeETag = (etag: string): string => {
|
|
||||||
const replaceChars: Record<string, string> = {
|
|
||||||
'"': '',
|
|
||||||
'"': '',
|
|
||||||
'"': '',
|
|
||||||
}
|
|
||||||
return etag.replaceAll(
|
|
||||||
/(^("|"|"))|(("|"|")$)/g,
|
|
||||||
(m) => replaceChars[m] || '',
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const entityMap = {
|
const entityMap = {
|
||||||
'"': '"',
|
'"': '"',
|
||||||
''': "'",
|
''': "'",
|
||||||
|
|
@ -107,20 +72,6 @@ const entityMap = {
|
||||||
'&': '&',
|
'&': '&',
|
||||||
} as const
|
} as const
|
||||||
|
|
||||||
/**
|
|
||||||
* Escape special characters for XML
|
|
||||||
* @param value String to escape
|
|
||||||
* @returns XML-escaped string
|
|
||||||
*/
|
|
||||||
export const escapeXml = (value: string): string => {
|
|
||||||
return value
|
|
||||||
.replaceAll('&', '&')
|
|
||||||
.replaceAll('<', '<')
|
|
||||||
.replaceAll('>', '>')
|
|
||||||
.replaceAll('"', '"')
|
|
||||||
.replaceAll("'", ''')
|
|
||||||
}
|
|
||||||
|
|
||||||
const unescapeXml = (value: string): string =>
|
const unescapeXml = (value: string): string =>
|
||||||
value.replaceAll(
|
value.replaceAll(
|
||||||
/&(quot|apos|lt|gt|amp);/g,
|
/&(quot|apos|lt|gt|amp);/g,
|
||||||
|
|
@ -182,7 +133,7 @@ const encodeAsHex = (c: string): string =>
|
||||||
* @param uriStr URI string to escape
|
* @param uriStr URI string to escape
|
||||||
* @returns Escaped URI string
|
* @returns Escaped URI string
|
||||||
*/
|
*/
|
||||||
export const uriEscape = (uriStr: string): string => {
|
const uriEscape = (uriStr: string): string => {
|
||||||
return encodeURIComponent(uriStr).replace(/[!'()*]/g, encodeAsHex)
|
return encodeURIComponent(uriStr).replace(/[!'()*]/g, encodeAsHex)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -195,17 +146,6 @@ export const uriResourceEscape = (string: string): string => {
|
||||||
return uriEscape(string).replaceAll('%2F', '/')
|
return uriEscape(string).replaceAll('%2F', '/')
|
||||||
}
|
}
|
||||||
|
|
||||||
export const extractErrCode = (e: unknown): string | undefined => {
|
|
||||||
if (typeof e !== 'object' || e === null) {
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
const err = e as ErrorWithCode
|
|
||||||
if (typeof err.code === 'string') {
|
|
||||||
return err.code
|
|
||||||
}
|
|
||||||
return typeof err.cause?.code === 'string' ? err.cause.code : undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
export class S3Error extends Error {
|
export class S3Error extends Error {
|
||||||
readonly code?: string
|
readonly code?: string
|
||||||
constructor(msg: string, code?: string, cause?: unknown) {
|
constructor(msg: string, code?: string, cause?: unknown) {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
import type { Body } from '@uppy/utils'
|
|
||||||
import { createAbortError } from '@uppy/utils'
|
import { createAbortError } from '@uppy/utils'
|
||||||
|
|
||||||
import type { AwsS3Part } from './index.js'
|
import type { AwsS3Part } from './index.js'
|
||||||
|
|
@ -22,9 +21,3 @@ export type UploadPartBytesResult = {
|
||||||
ETag: string
|
ETag: string
|
||||||
location?: string
|
location?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AwsBody extends Body {
|
|
||||||
location: string
|
|
||||||
key: string
|
|
||||||
bucket: string
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,14 @@
|
||||||
import { describe, expect, it, vi } from 'vitest'
|
import { HttpResponse, http } from 'msw'
|
||||||
|
import { setupServer } from 'msw/node'
|
||||||
|
import {
|
||||||
|
afterAll,
|
||||||
|
afterEach,
|
||||||
|
beforeAll,
|
||||||
|
describe,
|
||||||
|
expect,
|
||||||
|
it,
|
||||||
|
vi,
|
||||||
|
} from 'vitest'
|
||||||
|
|
||||||
import 'whatwg-fetch'
|
import 'whatwg-fetch'
|
||||||
import Core, { type Meta, type UppyFile } from '@uppy/core'
|
import Core, { type Meta, type UppyFile } from '@uppy/core'
|
||||||
|
|
@ -7,15 +17,145 @@ import AwsS3, { type AwsBody, type AwsS3Options } from '../src/index.js'
|
||||||
const KB = 1024
|
const KB = 1024
|
||||||
const MB = KB * KB
|
const MB = KB * KB
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Helpers for multipart upload tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Minimal XML responses that S3 returns for multipart operations */
|
||||||
|
const s3Responses = {
|
||||||
|
createMultipart: (uploadId: string, key: string) =>
|
||||||
|
`<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<InitiateMultipartUploadResult>
|
||||||
|
<UploadId>${uploadId}</UploadId>
|
||||||
|
<Key>${key}</Key>
|
||||||
|
</InitiateMultipartUploadResult>`,
|
||||||
|
|
||||||
|
uploadPart: (etag: string) => '',
|
||||||
|
|
||||||
|
listParts: (parts: { partNumber: number; etag: string }[]) =>
|
||||||
|
`<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<ListPartsResult>
|
||||||
|
${parts.map((p) => `<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${p.etag}</ETag></Part>`).join('')}
|
||||||
|
</ListPartsResult>`,
|
||||||
|
|
||||||
|
completeMultipart: (location: string, key: string) =>
|
||||||
|
`<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<CompleteMultipartUploadResult>
|
||||||
|
<Location>${location}</Location>
|
||||||
|
<Key>${key}</Key>
|
||||||
|
<Bucket>test-bucket</Bucket>
|
||||||
|
</CompleteMultipartUploadResult>`,
|
||||||
|
|
||||||
|
abortMultipart: () => '',
|
||||||
|
}
|
||||||
|
|
||||||
|
const server = setupServer()
|
||||||
|
const s3Url = 'https://test-bucket.s3.us-east-1.amazonaws.com/:key'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates signRequest + MSW handler state for multipart upload tests.
|
||||||
|
*/
|
||||||
|
function createMultipartMocks(opts: { uploadId?: string; key?: string } = {}) {
|
||||||
|
const uploadId = opts.uploadId ?? 'test-upload-id'
|
||||||
|
const key = opts.key ?? 'test-key'
|
||||||
|
|
||||||
|
// signRequest encodes operation details in the URL for MSW routing
|
||||||
|
const signRequest = vi.fn().mockImplementation(async (req: any) => {
|
||||||
|
const params = new URLSearchParams()
|
||||||
|
if (req.uploadId) params.set('uploadId', req.uploadId)
|
||||||
|
if (req.partNumber) params.set('partNumber', String(req.partNumber))
|
||||||
|
params.set('method', req.method)
|
||||||
|
return {
|
||||||
|
url: `https://test-bucket.s3.us-east-1.amazonaws.com/${req.key || key}?${params}`,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const operations: string[] = []
|
||||||
|
|
||||||
|
const registerHandlers = ({
|
||||||
|
hangNonCreate = false,
|
||||||
|
listParts = [] as { partNumber: number; etag: string }[],
|
||||||
|
} = {}) => {
|
||||||
|
const maybeHang = () =>
|
||||||
|
hangNonCreate ? (new Promise(() => {}) as Promise<any>) : null
|
||||||
|
|
||||||
|
server.use(
|
||||||
|
http.post(s3Url, ({ request }) => {
|
||||||
|
const hasUploadId = new URL(request.url).searchParams.has('uploadId')
|
||||||
|
if (!hasUploadId) {
|
||||||
|
operations.push('createMultipart')
|
||||||
|
return new HttpResponse(s3Responses.createMultipart(uploadId, key), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/xml' },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
operations.push('completeMultipart')
|
||||||
|
const hung = maybeHang()
|
||||||
|
if (hung) return hung
|
||||||
|
|
||||||
|
return new HttpResponse(
|
||||||
|
s3Responses.completeMultipart(
|
||||||
|
`https://test-bucket.s3.amazonaws.com/${key}`,
|
||||||
|
key,
|
||||||
|
),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'application/xml' } },
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
http.put(s3Url, () => {
|
||||||
|
operations.push('uploadPart')
|
||||||
|
const hung = maybeHang()
|
||||||
|
if (hung) return hung
|
||||||
|
return new HttpResponse('', {
|
||||||
|
status: 200,
|
||||||
|
headers: { ETag: '"etag-1"' },
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
http.get(s3Url, ({ request }) => {
|
||||||
|
const hasUploadId = new URL(request.url).searchParams.has('uploadId')
|
||||||
|
if (!hasUploadId) {
|
||||||
|
return new HttpResponse('Not Found', { status: 404 })
|
||||||
|
}
|
||||||
|
|
||||||
|
operations.push('listParts')
|
||||||
|
const hung = maybeHang()
|
||||||
|
if (hung) return hung
|
||||||
|
|
||||||
|
return new HttpResponse(s3Responses.listParts(listParts), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/xml' },
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
http.delete(s3Url, () => {
|
||||||
|
operations.push('abortMultipart')
|
||||||
|
return new HttpResponse('', { status: 204 })
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return { signRequest, operations, uploadId, key, registerHandlers }
|
||||||
|
}
|
||||||
|
|
||||||
describe('AwsS3', () => {
|
describe('AwsS3', () => {
|
||||||
|
beforeAll(() => {
|
||||||
|
server.listen({ onUnhandledRequest: 'error' })
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
server.resetHandlers()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
server.close()
|
||||||
|
})
|
||||||
|
|
||||||
it('Registers AwsS3 upload plugin', () => {
|
it('Registers AwsS3 upload plugin', () => {
|
||||||
const core = new Core().use(AwsS3, {
|
const core = new Core().use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
endpoint: 'https://companion.example.com',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
|
companionEndpoint: 'https://companion.example.com',
|
||||||
})
|
})
|
||||||
|
|
||||||
// @ts-expect-error private property
|
|
||||||
const pluginNames = core[Symbol.for('uppy test: getPlugins')](
|
const pluginNames = core[Symbol.for('uppy test: getPlugins')](
|
||||||
'uploader',
|
'uploader',
|
||||||
).map((plugin: AwsS3<Meta, AwsBody>) => plugin.constructor.name)
|
).map((plugin: AwsS3<Meta, AwsBody>) => plugin.constructor.name)
|
||||||
|
|
@ -23,34 +163,25 @@ describe('AwsS3', () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('configuration validation', () => {
|
describe('configuration validation', () => {
|
||||||
it('throws if bucket is not provided', () => {
|
|
||||||
expect(() => {
|
|
||||||
const core = new Core()
|
|
||||||
// @ts-expect-error - testing missing required option
|
|
||||||
core.use(AwsS3, {})
|
|
||||||
}).toThrow('`bucket` option is required')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('throws if region is not provided', () => {
|
|
||||||
expect(() => {
|
|
||||||
const core = new Core()
|
|
||||||
core.use(AwsS3, { bucket: 'test-bucket' })
|
|
||||||
}).toThrow('`region` option is required')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('throws if no signing method is provided', () => {
|
it('throws if no signing method is provided', () => {
|
||||||
expect(() => {
|
expect(() => {
|
||||||
const core = new Core()
|
const core = new Core()
|
||||||
core.use(AwsS3, { bucket: 'test-bucket', region: 'us-east-1' })
|
// @ts-expect-error - testing runtime validation, so omit required options
|
||||||
}).toThrow('`endpoint`, `signRequest`, or `getCredentials` is required')
|
core.use(AwsS3, {
|
||||||
|
s3Endpoint: 'https://companion.example.com',
|
||||||
|
region: 'us-east-1',
|
||||||
|
})
|
||||||
|
}).toThrow(
|
||||||
|
'One of options `companionEndpoint`, `signRequest`, or `getCredentials` is required',
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('accepts endpoint option', () => {
|
it('accepts endpoint option', () => {
|
||||||
const core = new Core()
|
const core = new Core()
|
||||||
core.use(AwsS3, {
|
core.use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
endpoint: 'https://companion.example.com',
|
companionEndpoint: 'https://companion.example.com',
|
||||||
})
|
})
|
||||||
expect(core.getPlugin('AwsS3')).toBeDefined()
|
expect(core.getPlugin('AwsS3')).toBeDefined()
|
||||||
})
|
})
|
||||||
|
|
@ -58,7 +189,7 @@ describe('AwsS3', () => {
|
||||||
it('accepts signRequest option', () => {
|
it('accepts signRequest option', () => {
|
||||||
const core = new Core()
|
const core = new Core()
|
||||||
core.use(AwsS3, {
|
core.use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
signRequest: vi.fn(),
|
signRequest: vi.fn(),
|
||||||
})
|
})
|
||||||
|
|
@ -68,7 +199,7 @@ describe('AwsS3', () => {
|
||||||
it('accepts getCredentials option', () => {
|
it('accepts getCredentials option', () => {
|
||||||
const core = new Core()
|
const core = new Core()
|
||||||
core.use(AwsS3, {
|
core.use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
getCredentials: vi.fn(),
|
getCredentials: vi.fn(),
|
||||||
})
|
})
|
||||||
|
|
@ -89,9 +220,9 @@ describe('AwsS3', () => {
|
||||||
|
|
||||||
it('defaults to multipart for files > 100MB', () => {
|
it('defaults to multipart for files > 100MB', () => {
|
||||||
const core = new Core<Meta, AwsBody>().use(AwsS3, {
|
const core = new Core<Meta, AwsBody>().use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
endpoint: 'https://companion.example.com',
|
companionEndpoint: 'https://companion.example.com',
|
||||||
})
|
})
|
||||||
const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody>
|
const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody>
|
||||||
const shouldUseMultipart = opts.shouldUseMultipart as (
|
const shouldUseMultipart = opts.shouldUseMultipart as (
|
||||||
|
|
@ -108,9 +239,9 @@ describe('AwsS3', () => {
|
||||||
|
|
||||||
it('handles very large files', () => {
|
it('handles very large files', () => {
|
||||||
const core = new Core<Meta, AwsBody>().use(AwsS3, {
|
const core = new Core<Meta, AwsBody>().use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
endpoint: 'https://companion.example.com',
|
companionEndpoint: 'https://companion.example.com',
|
||||||
})
|
})
|
||||||
const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody>
|
const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options<Meta, AwsBody>
|
||||||
const shouldUseMultipart = opts.shouldUseMultipart as (
|
const shouldUseMultipart = opts.shouldUseMultipart as (
|
||||||
|
|
@ -127,7 +258,7 @@ describe('AwsS3', () => {
|
||||||
const signRequest = vi.fn().mockRejectedValue(new Error('Test stop'))
|
const signRequest = vi.fn().mockRejectedValue(new Error('Test stop'))
|
||||||
|
|
||||||
const core = new Core().use(AwsS3, {
|
const core = new Core().use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
signRequest,
|
signRequest,
|
||||||
shouldUseMultipart: false,
|
shouldUseMultipart: false,
|
||||||
|
|
@ -156,7 +287,7 @@ describe('AwsS3', () => {
|
||||||
const signRequest = vi.fn().mockRejectedValue(new Error('Sign failed'))
|
const signRequest = vi.fn().mockRejectedValue(new Error('Sign failed'))
|
||||||
|
|
||||||
const core = new Core().use(AwsS3, {
|
const core = new Core().use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
signRequest,
|
signRequest,
|
||||||
shouldUseMultipart: false,
|
shouldUseMultipart: false,
|
||||||
|
|
@ -191,7 +322,7 @@ describe('AwsS3', () => {
|
||||||
)
|
)
|
||||||
|
|
||||||
const core = new Core().use(AwsS3, {
|
const core = new Core().use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
signRequest,
|
signRequest,
|
||||||
shouldUseMultipart: false,
|
shouldUseMultipart: false,
|
||||||
|
|
@ -222,7 +353,7 @@ describe('AwsS3', () => {
|
||||||
)
|
)
|
||||||
|
|
||||||
const core = new Core().use(AwsS3, {
|
const core = new Core().use(AwsS3, {
|
||||||
bucket: 'test-bucket',
|
s3Endpoint: 'https://companion.example.com',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
signRequest,
|
signRequest,
|
||||||
shouldUseMultipart: false,
|
shouldUseMultipart: false,
|
||||||
|
|
@ -244,4 +375,120 @@ describe('AwsS3', () => {
|
||||||
expect(result?.successful).toHaveLength(0)
|
expect(result?.successful).toHaveLength(0)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('Golden Retriever resume state (s3Multipart)', () => {
|
||||||
|
it('persists s3Multipart on file state after creating multipart upload', async () => {
|
||||||
|
const { signRequest, uploadId, registerHandlers } = createMultipartMocks()
|
||||||
|
// After createMultipart succeeds, hang on subsequent requests so we can inspect state
|
||||||
|
registerHandlers({ hangNonCreate: true })
|
||||||
|
|
||||||
|
const core = new Core().use(AwsS3, {
|
||||||
|
s3Endpoint: 'https://companion.example.com',
|
||||||
|
region: 'us-east-1',
|
||||||
|
signRequest,
|
||||||
|
shouldUseMultipart: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
const fileId = core.addFile({
|
||||||
|
source: 'test',
|
||||||
|
name: 'big.dat',
|
||||||
|
type: 'application/octet-stream',
|
||||||
|
data: new File([new Uint8Array(6 * MB)], 'big.dat'),
|
||||||
|
})
|
||||||
|
|
||||||
|
const uploadPromise = core.upload()
|
||||||
|
|
||||||
|
// Wait for createMultipart to complete and state to be persisted
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||||
|
|
||||||
|
const file = core.getFile(fileId)
|
||||||
|
expect(file.s3Multipart).toBeDefined()
|
||||||
|
expect(file.s3Multipart?.uploadId).toBe(uploadId)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
core.cancelAll()
|
||||||
|
await uploadPromise
|
||||||
|
})
|
||||||
|
|
||||||
|
it('clears s3Multipart when upload is aborted via cancelAll', async () => {
|
||||||
|
const { signRequest, registerHandlers } = createMultipartMocks({
|
||||||
|
uploadId: 'cancel-test-id',
|
||||||
|
key: 'cancel-key',
|
||||||
|
})
|
||||||
|
registerHandlers({ hangNonCreate: true })
|
||||||
|
|
||||||
|
const core = new Core().use(AwsS3, {
|
||||||
|
s3Endpoint: 'https://companion.example.com',
|
||||||
|
region: 'us-east-1',
|
||||||
|
signRequest,
|
||||||
|
shouldUseMultipart: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
const fileId = core.addFile({
|
||||||
|
source: 'test',
|
||||||
|
name: 'big.dat',
|
||||||
|
type: 'application/octet-stream',
|
||||||
|
data: new File([new Uint8Array(6 * MB)], 'big.dat'),
|
||||||
|
})
|
||||||
|
|
||||||
|
const uploadPromise = core.upload()
|
||||||
|
|
||||||
|
// Wait for createMultipart, then cancel
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||||
|
core.cancelAll()
|
||||||
|
|
||||||
|
await uploadPromise
|
||||||
|
|
||||||
|
const file = core.getFile(fileId)
|
||||||
|
// s3Multipart should be cleared so retries don't use a dead uploadId
|
||||||
|
expect(file?.s3Multipart).toBeUndefined()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('uses persisted s3Multipart key for resume (listParts, not createMultipart)', async () => {
|
||||||
|
const persistedKey = 'persisted-object-key'
|
||||||
|
const persistedUploadId = 'persisted-upload-id'
|
||||||
|
const { signRequest, operations, registerHandlers } =
|
||||||
|
createMultipartMocks({
|
||||||
|
uploadId: persistedUploadId,
|
||||||
|
key: persistedKey,
|
||||||
|
})
|
||||||
|
registerHandlers()
|
||||||
|
|
||||||
|
const core = new Core().use(AwsS3, {
|
||||||
|
s3Endpoint: 'https://companion.example.com',
|
||||||
|
region: 'us-east-1',
|
||||||
|
signRequest,
|
||||||
|
shouldUseMultipart: false, // Would normally be simple upload
|
||||||
|
})
|
||||||
|
|
||||||
|
const fileId = core.addFile({
|
||||||
|
source: 'test',
|
||||||
|
name: 'big.dat',
|
||||||
|
type: 'application/octet-stream',
|
||||||
|
data: new File([new Uint8Array(6 * MB)], 'big.dat'),
|
||||||
|
})
|
||||||
|
|
||||||
|
// Simulate Golden Retriever restoring s3Multipart state
|
||||||
|
core.setFileState(fileId, {
|
||||||
|
s3Multipart: { uploadId: persistedUploadId, key: persistedKey },
|
||||||
|
})
|
||||||
|
|
||||||
|
const uploadPromise = core.upload()
|
||||||
|
|
||||||
|
// Wait for the resume flow to call listParts (via fetch), then cancel
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||||
|
|
||||||
|
// Should have resumed (listParts) instead of creating a new multipart upload
|
||||||
|
expect(operations).toContain('listParts')
|
||||||
|
expect(operations).not.toContain('createMultipart')
|
||||||
|
|
||||||
|
// The signRequest calls should use the persisted key, not a generated one
|
||||||
|
const signedKeys = signRequest.mock.calls.map((call: any) => call[0].key)
|
||||||
|
expect(signedKeys.every((k: string) => k === persistedKey)).toBe(true)
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
core.cancelAll()
|
||||||
|
await uploadPromise
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -1,273 +0,0 @@
|
||||||
import { beforeAll, describe, expect, it, vi } from 'vitest'
|
|
||||||
import { S3mini } from '../../src/s3-client/index.js'
|
|
||||||
import { createSigV4Signer } from '../../src/s3-client/signer.js'
|
|
||||||
import { randomBytes } from '../test-utils/browser-crypto.js'
|
|
||||||
|
|
||||||
let _providerName
|
|
||||||
|
|
||||||
export const beforeRun = (raw, name, providerSpecific) => {
|
|
||||||
if (!raw) {
|
|
||||||
console.error(
|
|
||||||
'No credentials found. Please set the BUCKET_ENV_ environment variables.',
|
|
||||||
)
|
|
||||||
describe.skip(name, () => {
|
|
||||||
it('skipped', () => {
|
|
||||||
expect(true).toBe(true)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
console.log('Running tests for bucket:', name)
|
|
||||||
const credentials = {
|
|
||||||
provider: raw[0],
|
|
||||||
accessKeyId: raw[1],
|
|
||||||
secretAccessKey: raw[2],
|
|
||||||
endpoint: raw[3],
|
|
||||||
region: raw[4],
|
|
||||||
}
|
|
||||||
describe(`:::: ${credentials.provider} ::::`, () => {
|
|
||||||
expect(credentials.provider).toBe(name)
|
|
||||||
_providerName = credentials.provider
|
|
||||||
expect(credentials.accessKeyId).toBeDefined()
|
|
||||||
expect(credentials.secretAccessKey).toBeDefined()
|
|
||||||
expect(credentials.endpoint).toBeDefined()
|
|
||||||
expect(credentials.region).toBeDefined()
|
|
||||||
testRunner(credentials)
|
|
||||||
if (providerSpecific) {
|
|
||||||
providerSpecific(credentials)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const EIGHT_MB = 8 * 1024 * 1024
|
|
||||||
const key_bin = 'test-multipart.bin'
|
|
||||||
|
|
||||||
const large_buffer = randomBytes(EIGHT_MB * 3.2)
|
|
||||||
const content = 'some content'
|
|
||||||
const key = 'first-test-object.txt'
|
|
||||||
const key_list_parts = 'test-list-parts.bin'
|
|
||||||
const key_abort_multipart = 'test-abort-multipart.bin'
|
|
||||||
|
|
||||||
const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart]
|
|
||||||
|
|
||||||
export const cleanupTestBeforeAll = (s3client) => {
|
|
||||||
beforeAll(async () => {
|
|
||||||
for (const key of FILE_KEYS) {
|
|
||||||
try {
|
|
||||||
await s3client.deleteObject(key)
|
|
||||||
} catch {
|
|
||||||
// intentionally ignore the errors as the objects don't exists. still feels hacky tbh
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- 2 ■ A separate describe makes test output nicer -----------------------
|
|
||||||
export const testRunner = (bucket) => {
|
|
||||||
vi.setConfig({ testTimeout: 120_000 })
|
|
||||||
|
|
||||||
const presigner = createSigV4Signer({
|
|
||||||
accessKeyId: bucket.accessKeyId,
|
|
||||||
secretAccessKey: bucket.secretAccessKey,
|
|
||||||
region: bucket.region,
|
|
||||||
endpoint: bucket.endpoint,
|
|
||||||
})
|
|
||||||
|
|
||||||
const s3client = new S3mini({
|
|
||||||
endpoint: bucket.endpoint,
|
|
||||||
region: bucket.region,
|
|
||||||
signRequest: presigner,
|
|
||||||
})
|
|
||||||
|
|
||||||
cleanupTestBeforeAll(s3client)
|
|
||||||
|
|
||||||
it('instantiates s3client', () => {
|
|
||||||
expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation
|
|
||||||
})
|
|
||||||
|
|
||||||
// we don't need an explicit eTag method as we already get eTag in the putOject response
|
|
||||||
it('putObject uploads successfully and returns ETag', async () => {
|
|
||||||
const response = await s3client.putObject(key, content, 'text/plain')
|
|
||||||
expect(response.status).toBe(200)
|
|
||||||
expect(response.headers.get('etag')).toBeDefined()
|
|
||||||
await s3client.deleteObject(key)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('putObject handles binary data', async () => {
|
|
||||||
const binaryData = new Uint8Array(6).fill(0xff)
|
|
||||||
|
|
||||||
const response = await s3client.putObject(
|
|
||||||
key,
|
|
||||||
binaryData,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
|
|
||||||
expect(response).toBeDefined()
|
|
||||||
expect(response.status).toBe(200)
|
|
||||||
expect(response.headers.get('etag')).toBeDefined()
|
|
||||||
|
|
||||||
// cleanup
|
|
||||||
|
|
||||||
await s3client.deleteObject(key)
|
|
||||||
})
|
|
||||||
|
|
||||||
// test getMultipartUploadId
|
|
||||||
|
|
||||||
it('getMultipartUploadId returns a valid uploadId', async () => {
|
|
||||||
const uploadId = await s3client.getMultipartUploadId(
|
|
||||||
key_bin,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
expect(uploadId).toBeDefined()
|
|
||||||
expect(typeof uploadId).toBe('string')
|
|
||||||
expect(uploadId.length).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
// cleanup
|
|
||||||
await s3client.abortMultipartUpload(key, uploadId)
|
|
||||||
})
|
|
||||||
|
|
||||||
// test uploadPart
|
|
||||||
|
|
||||||
it('uploadPart returns partNumber and Etag', async () => {
|
|
||||||
const partData = randomBytes(EIGHT_MB)
|
|
||||||
|
|
||||||
const uploadId = await s3client.getMultipartUploadId(
|
|
||||||
key_bin,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
|
|
||||||
// upload part
|
|
||||||
const partResult = await s3client.uploadPart(key_bin, uploadId, partData, 1)
|
|
||||||
|
|
||||||
expect(partResult).toBeDefined()
|
|
||||||
expect(partResult.partNumber).toBe(1)
|
|
||||||
expect(partResult.etag).toBeDefined()
|
|
||||||
expect(typeof partResult.etag).toBe('string')
|
|
||||||
expect(partResult.etag.length).toBe(32)
|
|
||||||
|
|
||||||
// cleanup
|
|
||||||
await s3client.abortMultipartUpload(key, uploadId)
|
|
||||||
})
|
|
||||||
|
|
||||||
// end to end multipart flow
|
|
||||||
|
|
||||||
it('completeMultipartUpload assembles parts correctly', async () => {
|
|
||||||
// key - key_bin
|
|
||||||
const partSize = EIGHT_MB
|
|
||||||
const totalParts = Math.ceil(large_buffer.length / partSize)
|
|
||||||
|
|
||||||
const uploadId = await s3client.getMultipartUploadId(
|
|
||||||
key_bin,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
expect(uploadId).toBeDefined()
|
|
||||||
|
|
||||||
// upload all parts
|
|
||||||
const uploadPromises = []
|
|
||||||
for (let i = 0; i < totalParts; i++) {
|
|
||||||
const partBuffer = large_buffer.subarray(i * partSize, (i + 1) * partSize)
|
|
||||||
uploadPromises.push(
|
|
||||||
s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
const uploadResponses = await Promise.all(uploadPromises)
|
|
||||||
|
|
||||||
// verify all parts uploaded succesfully
|
|
||||||
|
|
||||||
expect(uploadResponses.length).toBe(totalParts)
|
|
||||||
|
|
||||||
uploadResponses.forEach((response, index) => {
|
|
||||||
expect(response.partNumber).toBe(index + 1)
|
|
||||||
expect(response.etag).toBeDefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
// create multipart upload
|
|
||||||
|
|
||||||
const parts = uploadResponses.map((response) => ({
|
|
||||||
partNumber: response.partNumber,
|
|
||||||
etag: response.etag,
|
|
||||||
}))
|
|
||||||
|
|
||||||
const completeResponse = await s3client.completeMultipartUpload(
|
|
||||||
key_bin,
|
|
||||||
uploadId,
|
|
||||||
parts,
|
|
||||||
)
|
|
||||||
|
|
||||||
expect(completeResponse).toBeDefined()
|
|
||||||
expect(typeof completeResponse).toBe('object')
|
|
||||||
expect(completeResponse.etag).toBeDefined()
|
|
||||||
expect(typeof completeResponse.etag).toBe('string')
|
|
||||||
expect(completeResponse.etag.length).toBe(32 + 2)
|
|
||||||
|
|
||||||
// cleanup
|
|
||||||
|
|
||||||
await s3client.deleteObject(key_bin)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('abortMultipartUpload cancels upload successfully', async () => {
|
|
||||||
// start upload
|
|
||||||
const uploadId = await s3client.getMultipartUploadId(
|
|
||||||
key_abort_multipart,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
|
|
||||||
expect(uploadId).toBeDefined()
|
|
||||||
|
|
||||||
const partData = randomBytes(EIGHT_MB)
|
|
||||||
await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1)
|
|
||||||
|
|
||||||
// abort
|
|
||||||
const abortResult = await s3client.abortMultipartUpload(
|
|
||||||
key_abort_multipart,
|
|
||||||
uploadId,
|
|
||||||
)
|
|
||||||
|
|
||||||
expect(abortResult).toBeDefined()
|
|
||||||
expect(abortResult.status).toBe('Aborted')
|
|
||||||
expect(abortResult.key).toBe(key_abort_multipart)
|
|
||||||
expect(abortResult.uploadId).toBe(uploadId)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('listParts returns uploaded parts correctly', async () => {
|
|
||||||
const partSize = EIGHT_MB
|
|
||||||
|
|
||||||
const uploadId = await s3client.getMultipartUploadId(
|
|
||||||
key_list_parts,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
expect(uploadId).toBeDefined()
|
|
||||||
|
|
||||||
const part1Data = randomBytes(partSize)
|
|
||||||
const part2Data = randomBytes(partSize)
|
|
||||||
|
|
||||||
const part1Result = await s3client.uploadPart(
|
|
||||||
key_list_parts,
|
|
||||||
uploadId,
|
|
||||||
part1Data,
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
const part2Result = await s3client.uploadPart(
|
|
||||||
key_list_parts,
|
|
||||||
uploadId,
|
|
||||||
part2Data,
|
|
||||||
2,
|
|
||||||
)
|
|
||||||
|
|
||||||
const parts = await s3client.listParts(uploadId, key_list_parts)
|
|
||||||
|
|
||||||
expect(parts).toBeInstanceOf(Array)
|
|
||||||
expect(parts.length).toBe(2)
|
|
||||||
|
|
||||||
// verify part 1
|
|
||||||
expect(parts[0].partNumber).toBe(1)
|
|
||||||
expect(parts[0].etag).toBe(part1Result.etag)
|
|
||||||
|
|
||||||
// verify part 2
|
|
||||||
expect(parts[1].partNumber).toBe(2)
|
|
||||||
expect(parts[1].etag).toBe(part2Result.etag)
|
|
||||||
|
|
||||||
// cleanup abort upload
|
|
||||||
await s3client.abortMultipartUpload(key, uploadId)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
14
packages/@uppy/aws-s3/tests/s3-client/config.ts
Normal file
14
packages/@uppy/aws-s3/tests/s3-client/config.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
export const accessKeyId = 'stsuser'
|
||||||
|
export const secretAccessKey = 'stspassword123'
|
||||||
|
|
||||||
|
export function getConfig(env: Record<string, string | undefined>) {
|
||||||
|
const config = env['VITE_MINIO_CONFIG']
|
||||||
|
if (!config) {
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
const [rootAccessKeyId, rootSecretAccessKey, endpoint, region] =
|
||||||
|
config.split(',')
|
||||||
|
// Use stsuser credentials for all tests (readwrite policy is sufficient)
|
||||||
|
// Root credentials are kept for Docker container startup
|
||||||
|
return { endpoint, region, rootAccessKeyId, rootSecretAccessKey }
|
||||||
|
}
|
||||||
|
|
@ -1,32 +1,23 @@
|
||||||
import { exec, spawn } from 'node:child_process'
|
import { exec, spawn } from 'node:child_process'
|
||||||
import { resolve } from 'node:path'
|
|
||||||
import { promisify } from 'node:util'
|
import { promisify } from 'node:util'
|
||||||
|
|
||||||
const CWD = resolve('.')
|
|
||||||
|
|
||||||
const execAsync = promisify(exec)
|
const execAsync = promisify(exec)
|
||||||
|
|
||||||
export async function getContainerName(serviceName) {
|
export async function getContainerName(serviceName: string) {
|
||||||
try {
|
// Find container by service name using docker ps
|
||||||
// Find container by service name using docker ps
|
const { stdout } = await execAsync(
|
||||||
const { stdout } = await execAsync(
|
`docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`,
|
||||||
`docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`,
|
)
|
||||||
)
|
const containerName = stdout.trim().split('\n')[0] // Get first matching container
|
||||||
const containerName = stdout.trim().split('\n')[0] // Get first matching container
|
if (!containerName) {
|
||||||
if (!containerName) {
|
throw new Error(`No running container found for service: ${serviceName}`)
|
||||||
throw new Error(`No running container found for service: ${serviceName}`)
|
|
||||||
}
|
|
||||||
return containerName
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(
|
|
||||||
`Failed to find container for service ${serviceName}: ${error.message}`,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
return containerName
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function execDockerCommand(
|
export async function execDockerCommand(
|
||||||
containerName,
|
containerName: string,
|
||||||
command,
|
command: string,
|
||||||
timeoutMs = 10000,
|
timeoutMs = 10000,
|
||||||
) {
|
) {
|
||||||
// If it's a service name (like 'garage'), find the actual container name
|
// If it's a service name (like 'garage'), find the actual container name
|
||||||
|
|
@ -34,9 +25,7 @@ export async function execDockerCommand(
|
||||||
if (['garage', 'minio', 'ceph'].includes(containerName)) {
|
if (['garage', 'minio', 'ceph'].includes(containerName)) {
|
||||||
try {
|
try {
|
||||||
actualContainerName = await getContainerName(containerName)
|
actualContainerName = await getContainerName(containerName)
|
||||||
console.log(
|
// console.log(`Found container: ${actualContainerName} for service: ${containerName}`,)
|
||||||
`Found container: ${actualContainerName} for service: ${containerName}`,
|
|
||||||
)
|
|
||||||
} catch (_error) {
|
} catch (_error) {
|
||||||
console.log(`Using container name as-is: ${containerName}`)
|
console.log(`Using container name as-is: ${containerName}`)
|
||||||
}
|
}
|
||||||
|
|
@ -74,27 +63,27 @@ export async function execDockerCommand(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function run(cmd, args) {
|
async function run(cmd: string, args: string[], env?: Record<string, string>) {
|
||||||
return new Promise((res, rej) => {
|
// console.log('running command:', cmd, args.join(' '))
|
||||||
const p = spawn(cmd, args, { cwd: CWD, stdio: 'inherit' })
|
return new Promise<void>((res, rej) => {
|
||||||
|
const p = spawn(cmd, args, {
|
||||||
|
stdio: 'inherit',
|
||||||
|
env: { ...process.env, ...env },
|
||||||
|
})
|
||||||
p.on('close', (code) =>
|
p.on('close', (code) =>
|
||||||
code === 0
|
code === 0
|
||||||
? res()
|
? res()
|
||||||
: rej(new Error(`${cmd} ${args.join(' ')} exited ${code}`)),
|
: rej(new Error(`${cmd} ${args.join(' ')} exited ${code}`)),
|
||||||
)
|
)
|
||||||
|
p.on('error', (err) => rej(err))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
export const composeUp = (file) =>
|
|
||||||
run('docker', ['compose', '-f', file, 'up', '-d', '--force-recreate'])
|
export const composeUpWait = (file: string, env?: Record<string, string>) =>
|
||||||
export const composeUpWait = (file) =>
|
run(
|
||||||
run('docker', [
|
'docker',
|
||||||
'compose',
|
['compose', '-f', file, 'up', '-d', '--force-recreate', '--wait'],
|
||||||
'-f',
|
env,
|
||||||
file,
|
)
|
||||||
'up',
|
export const composeDown = (file: string) =>
|
||||||
'-d',
|
|
||||||
'--force-recreate',
|
|
||||||
'--wait',
|
|
||||||
])
|
|
||||||
export const composeDown = (file) =>
|
|
||||||
run('docker', ['compose', '-f', file, 'down', '--remove-orphans', '-v'])
|
run('docker', ['compose', '-f', file, 'down', '--remove-orphans', '-v'])
|
||||||
|
|
@ -1,238 +0,0 @@
|
||||||
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts'
|
|
||||||
import { describe, expect, inject, it, vi } from 'vitest'
|
|
||||||
import { S3mini } from '../../src/s3-client/S3.js'
|
|
||||||
import { createSigV4Signer } from '../../src/s3-client/signer.js'
|
|
||||||
import { randomBytes } from '../test-utils/browser-crypto.js'
|
|
||||||
import { beforeRun, cleanupTestBeforeAll } from './_shared.test.js'
|
|
||||||
|
|
||||||
const name = 'minio'
|
|
||||||
|
|
||||||
// Get bucket configs from globalSetup via Vitest inject
|
|
||||||
const bucketConfigs = inject('bucketConfigs') || []
|
|
||||||
const raw = bucketConfigs.find((c) => c.provider === name)
|
|
||||||
? [
|
|
||||||
bucketConfigs.find((c) => c.provider === name).provider,
|
|
||||||
bucketConfigs.find((c) => c.provider === name).accessKeyId,
|
|
||||||
bucketConfigs.find((c) => c.provider === name).secretAccessKey,
|
|
||||||
bucketConfigs.find((c) => c.provider === name).endpoint,
|
|
||||||
bucketConfigs.find((c) => c.provider === name).region,
|
|
||||||
]
|
|
||||||
: null
|
|
||||||
|
|
||||||
/** Create STS client using @aws-sdk/client-sts */
|
|
||||||
function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) {
|
|
||||||
return new STSClient({
|
|
||||||
region,
|
|
||||||
endpoint,
|
|
||||||
credentials: { accessKeyId, secretAccessKey },
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Get temporary credentials via AssumeRole */
|
|
||||||
async function assumeRole(stsClient, { durationSeconds = 900 } = {}) {
|
|
||||||
const response = await stsClient.send(
|
|
||||||
new AssumeRoleCommand({
|
|
||||||
RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN
|
|
||||||
RoleSessionName: 'uppy-test',
|
|
||||||
DurationSeconds: durationSeconds,
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
const creds = response.Credentials
|
|
||||||
return {
|
|
||||||
AccessKeyId: creds.AccessKeyId,
|
|
||||||
SecretAccessKey: creds.SecretAccessKey,
|
|
||||||
SessionToken: creds.SessionToken,
|
|
||||||
Expiration: creds.Expiration?.toISOString() || creds.Expiration,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const minioSpecific = (bucket) => {
|
|
||||||
vi.setConfig({ testTimeout: 120_000 })
|
|
||||||
|
|
||||||
const presigner = createSigV4Signer({
|
|
||||||
accessKeyId: bucket.accessKeyId,
|
|
||||||
secretAccessKey: bucket.secretAccessKey,
|
|
||||||
region: bucket.region,
|
|
||||||
endpoint: bucket.endpoint,
|
|
||||||
})
|
|
||||||
|
|
||||||
const s3client = new S3mini({
|
|
||||||
endpoint: bucket.endpoint,
|
|
||||||
region: bucket.region,
|
|
||||||
signRequest: presigner,
|
|
||||||
})
|
|
||||||
|
|
||||||
cleanupTestBeforeAll(s3client)
|
|
||||||
|
|
||||||
// ===== signRequest tests =====
|
|
||||||
|
|
||||||
it('simple putObject upload', async () => {
|
|
||||||
const fileContents = new TextEncoder().encode(
|
|
||||||
'Hello from pre-signed URL test.',
|
|
||||||
)
|
|
||||||
|
|
||||||
const result = await s3client.putObject(
|
|
||||||
'presigned-test-file.txt',
|
|
||||||
fileContents,
|
|
||||||
'text/plain',
|
|
||||||
)
|
|
||||||
|
|
||||||
expect(result.ok).toBe(true)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('multipart upload with signRequest', async () => {
|
|
||||||
const key = `presigned-multipart-${Date.now()}.bin`
|
|
||||||
const partSize = 5 * 1024 * 1024 // 5MB
|
|
||||||
const part = randomBytes(partSize)
|
|
||||||
|
|
||||||
const uploadId = await s3client.getMultipartUploadId(
|
|
||||||
key,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
expect(uploadId).toBeDefined()
|
|
||||||
|
|
||||||
const uploaded = await s3client.uploadPart(key, uploadId, part, 1)
|
|
||||||
expect(uploaded.etag).toBeDefined()
|
|
||||||
|
|
||||||
const result = await s3client.completeMultipartUpload(key, uploadId, [
|
|
||||||
uploaded,
|
|
||||||
])
|
|
||||||
expect(result.etag).toBeDefined()
|
|
||||||
expect(result.location).toBeDefined()
|
|
||||||
expect(result.location).toContain(key)
|
|
||||||
expect(result.key).toBe(key)
|
|
||||||
|
|
||||||
await s3client.deleteObject(key)
|
|
||||||
})
|
|
||||||
|
|
||||||
// ===== getCredentials tests (STS) =====
|
|
||||||
|
|
||||||
describe('getCredentials (STS)', () => {
|
|
||||||
const stsEndpoint = new URL(bucket.endpoint).origin
|
|
||||||
|
|
||||||
it('should upload using getCredentials callback', async () => {
|
|
||||||
const testKey = `sts-getcreds-${Date.now()}.txt`
|
|
||||||
|
|
||||||
const s3 = new S3mini({
|
|
||||||
endpoint: bucket.endpoint,
|
|
||||||
region: bucket.region,
|
|
||||||
getCredentials: async () => {
|
|
||||||
const stsClient = createSTSClient({
|
|
||||||
endpoint: stsEndpoint,
|
|
||||||
accessKeyId: bucket.accessKeyId,
|
|
||||||
secretAccessKey: bucket.secretAccessKey,
|
|
||||||
region: bucket.region,
|
|
||||||
})
|
|
||||||
const creds = await assumeRole(stsClient)
|
|
||||||
return {
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: creds.AccessKeyId,
|
|
||||||
secretAccessKey: creds.SecretAccessKey,
|
|
||||||
sessionToken: creds.SessionToken,
|
|
||||||
expiration: creds.Expiration,
|
|
||||||
},
|
|
||||||
bucket: new URL(bucket.endpoint).pathname.slice(1),
|
|
||||||
region: bucket.region,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain')
|
|
||||||
expect(result.ok).toBe(true)
|
|
||||||
} finally {
|
|
||||||
await s3.deleteObject(testKey)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should cache credentials across multiple requests', async () => {
|
|
||||||
let fetchCount = 0
|
|
||||||
|
|
||||||
const s3 = new S3mini({
|
|
||||||
endpoint: bucket.endpoint,
|
|
||||||
region: bucket.region,
|
|
||||||
getCredentials: async () => {
|
|
||||||
fetchCount++
|
|
||||||
const stsClient = createSTSClient({
|
|
||||||
endpoint: stsEndpoint,
|
|
||||||
accessKeyId: bucket.accessKeyId,
|
|
||||||
secretAccessKey: bucket.secretAccessKey,
|
|
||||||
region: bucket.region,
|
|
||||||
})
|
|
||||||
const creds = await assumeRole(stsClient)
|
|
||||||
return {
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: creds.AccessKeyId,
|
|
||||||
secretAccessKey: creds.SecretAccessKey,
|
|
||||||
sessionToken: creds.SessionToken,
|
|
||||||
expiration: creds.Expiration,
|
|
||||||
},
|
|
||||||
bucket: new URL(bucket.endpoint).pathname.slice(1),
|
|
||||||
region: bucket.region,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
await s3.putObject(
|
|
||||||
`sts-cache-1-${Date.now()}.txt`,
|
|
||||||
'File 1',
|
|
||||||
'text/plain',
|
|
||||||
)
|
|
||||||
await s3.putObject(
|
|
||||||
`sts-cache-2-${Date.now()}.txt`,
|
|
||||||
'File 2',
|
|
||||||
'text/plain',
|
|
||||||
)
|
|
||||||
expect(fetchCount).toBe(1)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('should perform multipart upload with getCredentials', async () => {
|
|
||||||
const s3 = new S3mini({
|
|
||||||
endpoint: bucket.endpoint,
|
|
||||||
region: bucket.region,
|
|
||||||
getCredentials: async () => {
|
|
||||||
const stsClient = createSTSClient({
|
|
||||||
endpoint: stsEndpoint,
|
|
||||||
accessKeyId: bucket.accessKeyId,
|
|
||||||
secretAccessKey: bucket.secretAccessKey,
|
|
||||||
region: bucket.region,
|
|
||||||
})
|
|
||||||
const creds = await assumeRole(stsClient)
|
|
||||||
return {
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: creds.AccessKeyId,
|
|
||||||
secretAccessKey: creds.SecretAccessKey,
|
|
||||||
sessionToken: creds.SessionToken,
|
|
||||||
expiration: creds.Expiration,
|
|
||||||
},
|
|
||||||
bucket: new URL(bucket.endpoint).pathname.slice(1),
|
|
||||||
region: bucket.region,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
const key = `sts-multipart-${Date.now()}.bin`
|
|
||||||
const partSize = 5 * 1024 * 1024 // 5MB
|
|
||||||
const part = randomBytes(partSize)
|
|
||||||
|
|
||||||
const uploadId = await s3.getMultipartUploadId(
|
|
||||||
key,
|
|
||||||
'application/octet-stream',
|
|
||||||
)
|
|
||||||
expect(uploadId).toBeDefined()
|
|
||||||
|
|
||||||
const uploaded = await s3.uploadPart(key, uploadId, part, 1)
|
|
||||||
expect(uploaded.etag).toBeDefined()
|
|
||||||
|
|
||||||
const result = await s3.completeMultipartUpload(key, uploadId, [uploaded])
|
|
||||||
expect(result.etag).toBeDefined()
|
|
||||||
expect(result.location).toBeDefined()
|
|
||||||
expect(result.location).toContain(key)
|
|
||||||
expect(result.key).toBe(key)
|
|
||||||
|
|
||||||
await s3.deleteObject(key)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeRun(raw, name, minioSpecific)
|
|
||||||
456
packages/@uppy/aws-s3/tests/s3-client/minio.test.ts
Normal file
456
packages/@uppy/aws-s3/tests/s3-client/minio.test.ts
Normal file
|
|
@ -0,0 +1,456 @@
|
||||||
|
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts'
|
||||||
|
import { beforeAll, describe, expect, it } from 'vitest'
|
||||||
|
import { S3mini } from '../../src/s3-client/S3mini.js'
|
||||||
|
import { createSigV4Signer } from '../../src/s3-client/signer.js'
|
||||||
|
import { randomBytes } from '../test-utils/browser-crypto.js'
|
||||||
|
import { accessKeyId, getConfig, secretAccessKey } from './config.js'
|
||||||
|
|
||||||
|
// @ts-expect-error todo
|
||||||
|
const config = getConfig(import.meta.env)
|
||||||
|
|
||||||
|
const suiteName = 'MinIO S3 client tests'
|
||||||
|
|
||||||
|
if (config) {
|
||||||
|
const { endpoint, region } = config
|
||||||
|
const presigner = createSigV4Signer({
|
||||||
|
accessKeyId,
|
||||||
|
secretAccessKey,
|
||||||
|
region,
|
||||||
|
endpoint,
|
||||||
|
})
|
||||||
|
|
||||||
|
const s3client = new S3mini({
|
||||||
|
endpoint,
|
||||||
|
region,
|
||||||
|
signRequest: presigner,
|
||||||
|
})
|
||||||
|
|
||||||
|
const EIGHT_MB = 8 * 1024 * 1024
|
||||||
|
const key_bin = 'test-multipart.bin'
|
||||||
|
|
||||||
|
const large_buffer = randomBytes(EIGHT_MB * 3.2)
|
||||||
|
const content = 'some content'
|
||||||
|
const key = 'first-test-object.txt'
|
||||||
|
const key_list_parts = 'test-list-parts.bin'
|
||||||
|
const key_abort_multipart = 'test-abort-multipart.bin'
|
||||||
|
|
||||||
|
const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart]
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
for (const key of FILE_KEYS) {
|
||||||
|
try {
|
||||||
|
await s3client.deleteObject(key)
|
||||||
|
} catch {
|
||||||
|
// intentionally ignore the errors as the objects don't exists. still feels hacky tbh
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
/** Create STS client using @aws-sdk/client-sts */
|
||||||
|
function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) {
|
||||||
|
return new STSClient({
|
||||||
|
region,
|
||||||
|
endpoint,
|
||||||
|
credentials: { accessKeyId, secretAccessKey },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get temporary credentials via AssumeRole */
|
||||||
|
async function assumeRole(
|
||||||
|
stsClient: STSClient,
|
||||||
|
{ durationSeconds = 900 } = {},
|
||||||
|
) {
|
||||||
|
const response = await stsClient.send(
|
||||||
|
new AssumeRoleCommand({
|
||||||
|
RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN
|
||||||
|
RoleSessionName: 'uppy-test',
|
||||||
|
DurationSeconds: durationSeconds,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
const creds = response.Credentials!
|
||||||
|
return {
|
||||||
|
AccessKeyId: creds.AccessKeyId,
|
||||||
|
SecretAccessKey: creds.SecretAccessKey,
|
||||||
|
SessionToken: creds.SessionToken,
|
||||||
|
Expiration: creds.Expiration?.toISOString() || creds.Expiration,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== signRequest tests =====
|
||||||
|
|
||||||
|
describe(suiteName, () => {
|
||||||
|
it('simple putObject upload', async () => {
|
||||||
|
const key = 'presigned-test-file.txt'
|
||||||
|
const fileContents = new TextEncoder().encode(
|
||||||
|
'Hello from pre-signed URL test.',
|
||||||
|
)
|
||||||
|
|
||||||
|
const result = await s3client.putObject(key, fileContents, 'text/plain')
|
||||||
|
|
||||||
|
expect(result.location).toBeDefined()
|
||||||
|
expect(result.location).toContain(key)
|
||||||
|
// location should be a clean URL without query string (no signing params)
|
||||||
|
expect(result.location).not.toContain('X-Amz-Signature')
|
||||||
|
expect(result.location).not.toContain('?')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('multipart upload with signRequest', async () => {
|
||||||
|
const key = `presigned-multipart-${Date.now()}.bin`
|
||||||
|
const partSize = 5 * 1024 * 1024 // 5MB
|
||||||
|
const part = randomBytes(partSize)
|
||||||
|
|
||||||
|
const { uploadId } = await s3client.createMultipartUpload(
|
||||||
|
key,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
expect(uploadId).toBeDefined()
|
||||||
|
|
||||||
|
const partNumber = 1
|
||||||
|
const { etag } = await s3client.uploadPart(
|
||||||
|
key,
|
||||||
|
uploadId,
|
||||||
|
part,
|
||||||
|
partNumber,
|
||||||
|
)
|
||||||
|
expect(etag).toBeDefined()
|
||||||
|
|
||||||
|
const result = await s3client.completeMultipartUpload(key, uploadId, [
|
||||||
|
{ etag, partNumber },
|
||||||
|
])
|
||||||
|
expect(result.etag).toBeDefined()
|
||||||
|
expect(result.location).toBeDefined()
|
||||||
|
expect(result.location).toContain(key)
|
||||||
|
expect(result.key).toBe(key)
|
||||||
|
|
||||||
|
await s3client.deleteObject(key)
|
||||||
|
})
|
||||||
|
|
||||||
|
// ===== getCredentials tests (STS) =====
|
||||||
|
|
||||||
|
describe('getCredentials (STS)', () => {
|
||||||
|
const stsEndpoint = new URL(endpoint).origin
|
||||||
|
|
||||||
|
it('should upload using getCredentials callback', async () => {
|
||||||
|
const testKey = `sts-getcreds-${Date.now()}.txt`
|
||||||
|
|
||||||
|
const s3 = new S3mini({
|
||||||
|
endpoint,
|
||||||
|
region,
|
||||||
|
getCredentials: async () => {
|
||||||
|
const stsClient = createSTSClient({
|
||||||
|
endpoint: stsEndpoint,
|
||||||
|
accessKeyId,
|
||||||
|
secretAccessKey,
|
||||||
|
region,
|
||||||
|
})
|
||||||
|
const creds = await assumeRole(stsClient)
|
||||||
|
return {
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: creds.AccessKeyId!,
|
||||||
|
secretAccessKey: creds.SecretAccessKey!,
|
||||||
|
sessionToken: creds.SessionToken!,
|
||||||
|
expiration: creds.Expiration as string,
|
||||||
|
},
|
||||||
|
bucket: new URL(endpoint).pathname.slice(1),
|
||||||
|
region,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain')
|
||||||
|
expect(result.location).toBeDefined()
|
||||||
|
expect(result.location).toContain(testKey)
|
||||||
|
expect(result.location).not.toContain('X-Amz-Signature')
|
||||||
|
expect(result.location).not.toContain('?')
|
||||||
|
} finally {
|
||||||
|
await s3.deleteObject(testKey)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should cache credentials across multiple requests', async () => {
|
||||||
|
let fetchCount = 0
|
||||||
|
|
||||||
|
const s3 = new S3mini({
|
||||||
|
endpoint,
|
||||||
|
region,
|
||||||
|
getCredentials: async () => {
|
||||||
|
fetchCount++
|
||||||
|
const stsClient = createSTSClient({
|
||||||
|
endpoint: stsEndpoint,
|
||||||
|
accessKeyId,
|
||||||
|
secretAccessKey,
|
||||||
|
region,
|
||||||
|
})
|
||||||
|
const creds = await assumeRole(stsClient)
|
||||||
|
return {
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: creds.AccessKeyId!,
|
||||||
|
secretAccessKey: creds.SecretAccessKey!,
|
||||||
|
sessionToken: creds.SessionToken!,
|
||||||
|
expiration: creds.Expiration as string,
|
||||||
|
},
|
||||||
|
bucket: new URL(endpoint).pathname.slice(1),
|
||||||
|
region,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
await s3.putObject(
|
||||||
|
`sts-cache-1-${Date.now()}.txt`,
|
||||||
|
'File 1',
|
||||||
|
'text/plain',
|
||||||
|
)
|
||||||
|
await s3.putObject(
|
||||||
|
`sts-cache-2-${Date.now()}.txt`,
|
||||||
|
'File 2',
|
||||||
|
'text/plain',
|
||||||
|
)
|
||||||
|
expect(fetchCount).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should perform multipart upload with getCredentials', async () => {
|
||||||
|
const s3 = new S3mini({
|
||||||
|
endpoint,
|
||||||
|
region,
|
||||||
|
getCredentials: async () => {
|
||||||
|
const stsClient = createSTSClient({
|
||||||
|
endpoint: stsEndpoint,
|
||||||
|
accessKeyId,
|
||||||
|
secretAccessKey,
|
||||||
|
region,
|
||||||
|
})
|
||||||
|
const creds = await assumeRole(stsClient)
|
||||||
|
return {
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: creds.AccessKeyId!,
|
||||||
|
secretAccessKey: creds.SecretAccessKey!,
|
||||||
|
sessionToken: creds.SessionToken!,
|
||||||
|
expiration: creds.Expiration as string,
|
||||||
|
},
|
||||||
|
bucket: new URL(endpoint).pathname.slice(1),
|
||||||
|
region,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const key = `sts-multipart-${Date.now()}.bin`
|
||||||
|
const partSize = 5 * 1024 * 1024 // 5MB
|
||||||
|
const part = randomBytes(partSize)
|
||||||
|
|
||||||
|
const { uploadId } = await s3.createMultipartUpload(
|
||||||
|
key,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
expect(uploadId).toBeDefined()
|
||||||
|
|
||||||
|
const partNumber = 1
|
||||||
|
const { etag } = await s3.uploadPart(key, uploadId, part, partNumber)
|
||||||
|
expect(etag).toBeDefined()
|
||||||
|
|
||||||
|
const result = await s3.completeMultipartUpload(key, uploadId, [
|
||||||
|
{ etag, partNumber },
|
||||||
|
])
|
||||||
|
expect(result.etag).toBeDefined()
|
||||||
|
expect(result.location).toBeDefined()
|
||||||
|
expect(result.location).toContain(key)
|
||||||
|
expect(result.key).toBe(key)
|
||||||
|
|
||||||
|
await s3.deleteObject(key)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('instantiates s3client', () => {
|
||||||
|
expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation
|
||||||
|
})
|
||||||
|
|
||||||
|
// we don't need an explicit eTag method as we already get eTag in the putOject response
|
||||||
|
it('putObject uploads successfully and returns ETag', async () => {
|
||||||
|
const response = await s3client.putObject(key, content, 'text/plain')
|
||||||
|
expect(response.etag).toBeDefined()
|
||||||
|
await s3client.deleteObject(key)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('putObject handles binary data', async () => {
|
||||||
|
const binaryData = new Uint8Array(6).fill(0xff)
|
||||||
|
|
||||||
|
const response = await s3client.putObject(
|
||||||
|
key,
|
||||||
|
binaryData,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(response).toBeDefined()
|
||||||
|
expect(response.etag).toBeDefined()
|
||||||
|
|
||||||
|
// cleanup
|
||||||
|
|
||||||
|
await s3client.deleteObject(key)
|
||||||
|
})
|
||||||
|
|
||||||
|
// test createMultipartUpload
|
||||||
|
|
||||||
|
it('createMultipartUpload returns a valid uploadId', async () => {
|
||||||
|
const { uploadId } = await s3client.createMultipartUpload(
|
||||||
|
key_bin,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
expect(uploadId).toBeDefined()
|
||||||
|
expect(typeof uploadId).toBe('string')
|
||||||
|
expect(uploadId.length).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
// cleanup
|
||||||
|
await s3client.abortMultipartUpload(key, uploadId)
|
||||||
|
})
|
||||||
|
|
||||||
|
// test uploadPart
|
||||||
|
|
||||||
|
it('uploadPart returns partNumber and Etag', async () => {
|
||||||
|
const partData = randomBytes(EIGHT_MB)
|
||||||
|
|
||||||
|
const { uploadId } = await s3client.createMultipartUpload(
|
||||||
|
key_bin,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
|
||||||
|
// upload part
|
||||||
|
const partResult = await s3client.uploadPart(
|
||||||
|
key_bin,
|
||||||
|
uploadId,
|
||||||
|
partData,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(partResult).toBeDefined()
|
||||||
|
expect(partResult.etag).toBeDefined()
|
||||||
|
expect(typeof partResult.etag).toBe('string')
|
||||||
|
expect(partResult.etag.length).toBe(32)
|
||||||
|
|
||||||
|
// cleanup
|
||||||
|
await s3client.abortMultipartUpload(key, uploadId)
|
||||||
|
})
|
||||||
|
|
||||||
|
// end to end multipart flow
|
||||||
|
|
||||||
|
it('completeMultipartUpload assembles parts correctly', async () => {
|
||||||
|
// key - key_bin
|
||||||
|
const partSize = EIGHT_MB
|
||||||
|
const totalParts = Math.ceil(large_buffer.byteLength / partSize)
|
||||||
|
|
||||||
|
const { uploadId } = await s3client.createMultipartUpload(
|
||||||
|
key_bin,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
expect(uploadId).toBeDefined()
|
||||||
|
|
||||||
|
// upload all parts
|
||||||
|
const uploadPromises: Promise<{ etag: string }>[] = []
|
||||||
|
for (let i = 0; i < totalParts; i++) {
|
||||||
|
const partBuffer = large_buffer.subarray(
|
||||||
|
i * partSize,
|
||||||
|
(i + 1) * partSize,
|
||||||
|
)
|
||||||
|
uploadPromises.push(
|
||||||
|
s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const uploadResponses = await Promise.all(uploadPromises)
|
||||||
|
|
||||||
|
// verify all parts uploaded succesfully
|
||||||
|
|
||||||
|
expect(uploadResponses.length).toBe(totalParts)
|
||||||
|
|
||||||
|
uploadResponses.forEach((response) => {
|
||||||
|
expect(response.etag).toBeDefined()
|
||||||
|
})
|
||||||
|
|
||||||
|
// create multipart upload
|
||||||
|
|
||||||
|
const parts = uploadResponses.map((response, index) => ({
|
||||||
|
partNumber: index + 1,
|
||||||
|
etag: response.etag,
|
||||||
|
}))
|
||||||
|
|
||||||
|
const completeResponse = await s3client.completeMultipartUpload(
|
||||||
|
key_bin,
|
||||||
|
uploadId,
|
||||||
|
parts,
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(completeResponse).toBeDefined()
|
||||||
|
expect(typeof completeResponse).toBe('object')
|
||||||
|
expect(completeResponse.etag).toBeDefined()
|
||||||
|
expect(typeof completeResponse.etag).toBe('string')
|
||||||
|
expect(completeResponse.etag!.length).toBe(32 + 2)
|
||||||
|
|
||||||
|
// cleanup
|
||||||
|
|
||||||
|
await s3client.deleteObject(key_bin)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('abortMultipartUpload cancels upload successfully', async () => {
|
||||||
|
// start upload
|
||||||
|
const { uploadId } = await s3client.createMultipartUpload(
|
||||||
|
key_abort_multipart,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(uploadId).toBeDefined()
|
||||||
|
|
||||||
|
const partData = randomBytes(EIGHT_MB)
|
||||||
|
await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1)
|
||||||
|
|
||||||
|
// abort
|
||||||
|
await s3client.abortMultipartUpload(key_abort_multipart, uploadId)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('listParts returns uploaded parts correctly', async () => {
|
||||||
|
const partSize = EIGHT_MB
|
||||||
|
|
||||||
|
const { uploadId } = await s3client.createMultipartUpload(
|
||||||
|
key_list_parts,
|
||||||
|
'application/octet-stream',
|
||||||
|
)
|
||||||
|
expect(uploadId).toBeDefined()
|
||||||
|
|
||||||
|
const part1Data = randomBytes(partSize)
|
||||||
|
const part2Data = randomBytes(partSize)
|
||||||
|
|
||||||
|
const part1Result = await s3client.uploadPart(
|
||||||
|
key_list_parts,
|
||||||
|
uploadId,
|
||||||
|
part1Data,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
const part2Result = await s3client.uploadPart(
|
||||||
|
key_list_parts,
|
||||||
|
uploadId,
|
||||||
|
part2Data,
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
|
||||||
|
const parts = await s3client.listParts(uploadId, key_list_parts)
|
||||||
|
|
||||||
|
expect(parts).toBeInstanceOf(Array)
|
||||||
|
expect(parts.length).toBe(2)
|
||||||
|
|
||||||
|
// verify part 1
|
||||||
|
expect(parts[0].partNumber).toBe(1)
|
||||||
|
expect(parts[0].etag).toBe(part1Result.etag)
|
||||||
|
|
||||||
|
// verify part 2
|
||||||
|
expect(parts[1].partNumber).toBe(2)
|
||||||
|
expect(parts[1].etag).toBe(part2Result.etag)
|
||||||
|
|
||||||
|
// cleanup abort upload
|
||||||
|
await s3client.abortMultipartUpload(key, uploadId)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
console.warn(
|
||||||
|
'Skipping MinIO S3 client tests: missing env variable VITE_MINIO_CONFIG',
|
||||||
|
)
|
||||||
|
describe.skip(suiteName, () => {
|
||||||
|
it('noop', () => {})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -1,75 +0,0 @@
|
||||||
import * as dotenv from 'dotenv'
|
|
||||||
|
|
||||||
dotenv.config()
|
|
||||||
|
|
||||||
import { join } from 'node:path'
|
|
||||||
|
|
||||||
import { composeUp, composeUpWait, execDockerCommand } from './docker.js'
|
|
||||||
|
|
||||||
const composeFiles = {
|
|
||||||
minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'),
|
|
||||||
}
|
|
||||||
|
|
||||||
const bucketConfigs = Object.keys(process.env)
|
|
||||||
.filter((k) => k.startsWith('BUCKET_ENV_'))
|
|
||||||
.map((k) => {
|
|
||||||
const [provider, rootAccessKeyId, rootSecretAccessKey, endpoint, region] =
|
|
||||||
process.env[k].split(',')
|
|
||||||
// Use stsuser credentials for all tests (readwrite policy is sufficient)
|
|
||||||
// Root credentials are kept for Docker container startup
|
|
||||||
return {
|
|
||||||
provider,
|
|
||||||
accessKeyId: 'stsuser',
|
|
||||||
secretAccessKey: 'stspassword123',
|
|
||||||
rootAccessKeyId,
|
|
||||||
rootSecretAccessKey,
|
|
||||||
endpoint,
|
|
||||||
region,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
export default async function setup({ provide }) {
|
|
||||||
// Provide bucket configs to browser tests via Vitest's inject mechanism
|
|
||||||
provide('bucketConfigs', bucketConfigs)
|
|
||||||
|
|
||||||
for (const cfg of bucketConfigs) {
|
|
||||||
const composeFile = composeFiles[cfg.provider]
|
|
||||||
if (!composeFile) continue
|
|
||||||
console.log(`⏫ starting ${cfg.provider} image …`)
|
|
||||||
switch (cfg.provider) {
|
|
||||||
case 'minio':
|
|
||||||
process.env.MINIO_ROOT_USER = cfg.rootAccessKeyId
|
|
||||||
process.env.MINIO_ROOT_PASSWORD = cfg.rootSecretAccessKey
|
|
||||||
await composeUpWait(composeFile)
|
|
||||||
// biome-ignore lint/correctness/noSwitchDeclarations: no other switch blocks
|
|
||||||
const bucketName = new URL(cfg.endpoint).pathname.split('/')[1]
|
|
||||||
if (bucketName) {
|
|
||||||
await execDockerCommand(
|
|
||||||
'minio',
|
|
||||||
`mc mb local/${bucketName} --ignore-existing`,
|
|
||||||
30000,
|
|
||||||
)
|
|
||||||
// Create a user with readwrite policy for STS testing
|
|
||||||
// The root user cannot AssumeRole for itself - needs a regular user
|
|
||||||
try {
|
|
||||||
await execDockerCommand(
|
|
||||||
'minio',
|
|
||||||
`mc admin user add local stsuser stspassword123`,
|
|
||||||
15000,
|
|
||||||
)
|
|
||||||
await execDockerCommand(
|
|
||||||
'minio',
|
|
||||||
`mc admin policy attach local readwrite --user stsuser`,
|
|
||||||
15000,
|
|
||||||
)
|
|
||||||
} catch (err) {
|
|
||||||
// User may already exist, that's fine
|
|
||||||
console.log('STS user setup:', err.message || 'error occurred')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
default:
|
|
||||||
await composeUp(composeFile)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
48
packages/@uppy/aws-s3/tests/s3-client/setup.ts
Normal file
48
packages/@uppy/aws-s3/tests/s3-client/setup.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
import path from 'node:path'
|
||||||
|
import { getConfig } from './config'
|
||||||
|
import { composeDown, composeUpWait, execDockerCommand } from './docker'
|
||||||
|
|
||||||
|
const composeFile = path.join(__dirname, 'compose.minio.yaml')
|
||||||
|
|
||||||
|
const config = getConfig(process.env)
|
||||||
|
|
||||||
|
export async function setup() {
|
||||||
|
if (config == null) return
|
||||||
|
|
||||||
|
const { endpoint, rootAccessKeyId, rootSecretAccessKey } = config
|
||||||
|
|
||||||
|
console.log('⏫ starting minio image', composeFile)
|
||||||
|
process.env.MINIO_ROOT_USER = rootAccessKeyId
|
||||||
|
process.env.MINIO_ROOT_PASSWORD = rootSecretAccessKey
|
||||||
|
await composeUpWait(composeFile)
|
||||||
|
const bucketName = new URL(endpoint).pathname.split('/')[1]
|
||||||
|
await execDockerCommand(
|
||||||
|
'minio',
|
||||||
|
`mc mb local/${bucketName} --ignore-existing`,
|
||||||
|
30000,
|
||||||
|
)
|
||||||
|
// Create a user with readwrite policy for STS testing
|
||||||
|
// The root user cannot AssumeRole for itself - needs a regular user
|
||||||
|
try {
|
||||||
|
await execDockerCommand(
|
||||||
|
'minio',
|
||||||
|
`mc admin user add local stsuser stspassword123`,
|
||||||
|
)
|
||||||
|
await execDockerCommand(
|
||||||
|
'minio',
|
||||||
|
`mc admin policy attach local readwrite --user stsuser`,
|
||||||
|
)
|
||||||
|
} catch (err) {
|
||||||
|
// User may already exist, that's fine
|
||||||
|
console.log('STS user setup:', err.message || 'error occurred')
|
||||||
|
}
|
||||||
|
console.log(`✅ minio is ready`)
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function teardown() {
|
||||||
|
if (config == null) return
|
||||||
|
|
||||||
|
console.log(`⏬ stopping minio image …`)
|
||||||
|
await composeDown(composeFile)
|
||||||
|
console.log(`✅ minio stopped and cleaned up`)
|
||||||
|
}
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
import * as dotenv from 'dotenv'
|
|
||||||
|
|
||||||
dotenv.config()
|
|
||||||
|
|
||||||
import { join } from 'node:path'
|
|
||||||
import { composeDown } from './docker.js'
|
|
||||||
|
|
||||||
const composeFiles = {
|
|
||||||
minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'),
|
|
||||||
}
|
|
||||||
|
|
||||||
const bucketConfigs = Object.keys(process.env)
|
|
||||||
.filter((k) => k.startsWith('BUCKET_ENV_'))
|
|
||||||
.map((k) => {
|
|
||||||
const [provider, accessKeyId, secretAccessKey, endpoint, region] =
|
|
||||||
process.env[k].split(',')
|
|
||||||
return { provider, accessKeyId, secretAccessKey, endpoint, region }
|
|
||||||
})
|
|
||||||
|
|
||||||
export default async () => {
|
|
||||||
for (const cfg of bucketConfigs) {
|
|
||||||
const composeFile = composeFiles[cfg.provider]
|
|
||||||
if (!composeFile) continue // ignore unknown providers
|
|
||||||
|
|
||||||
console.log(`⏬ stopping ${cfg.provider} …`)
|
|
||||||
await composeDown(composeFile) // `docker compose -f … down`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
/**
|
|
||||||
* Generate random bytes using Web Crypto API.
|
|
||||||
* Web Crypto has a 65536 byte limit per getRandomValues call,
|
|
||||||
* so we chunk large requests.
|
|
||||||
* @param {number} size - Number of random bytes to generate
|
|
||||||
* @returns {Uint8Array} Random bytes
|
|
||||||
*/
|
|
||||||
export function randomBytes(size) {
|
|
||||||
const buffer = new Uint8Array(size)
|
|
||||||
const maxChunk = 65536 // Web Crypto API limit
|
|
||||||
|
|
||||||
for (let offset = 0; offset < size; offset += maxChunk) {
|
|
||||||
const chunkSize = Math.min(maxChunk, size - offset)
|
|
||||||
const chunk = buffer.subarray(offset, offset + chunkSize)
|
|
||||||
globalThis.crypto.getRandomValues(chunk)
|
|
||||||
}
|
|
||||||
|
|
||||||
return buffer
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute SHA-1 hash and return base64 string.
|
|
||||||
* @param {Uint8Array | ArrayBuffer} data - Data to hash
|
|
||||||
* @returns {Promise<string>} Base64-encoded SHA-1 hash
|
|
||||||
*/
|
|
||||||
export async function sha1Base64(data) {
|
|
||||||
const hash = await globalThis.crypto.subtle.digest('SHA-1', data)
|
|
||||||
const bytes = new Uint8Array(hash)
|
|
||||||
// Convert to base64 in chunks to avoid call stack issues with large data
|
|
||||||
const chunkSize = 0x8000
|
|
||||||
let result = ''
|
|
||||||
for (let i = 0; i < bytes.length; i += chunkSize) {
|
|
||||||
const chunk = bytes.subarray(i, i + chunkSize)
|
|
||||||
result += String.fromCharCode(...chunk)
|
|
||||||
}
|
|
||||||
return btoa(result)
|
|
||||||
}
|
|
||||||
17
packages/@uppy/aws-s3/tests/test-utils/browser-crypto.ts
Normal file
17
packages/@uppy/aws-s3/tests/test-utils/browser-crypto.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
/**
|
||||||
|
* Generate random bytes using Web Crypto API.
|
||||||
|
* Web Crypto has a 65536 byte limit per getRandomValues call,
|
||||||
|
* so we chunk large requests.
|
||||||
|
*/
|
||||||
|
export function randomBytes(size: number) {
|
||||||
|
const buffer = new Uint8Array(size)
|
||||||
|
const maxChunk = 65536 // Web Crypto API limit
|
||||||
|
|
||||||
|
for (let offset = 0; offset < size; offset += maxChunk) {
|
||||||
|
const chunkSize = Math.min(maxChunk, size - offset)
|
||||||
|
const chunk = buffer.subarray(offset, offset + chunkSize)
|
||||||
|
globalThis.crypto.getRandomValues(chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
return buffer
|
||||||
|
}
|
||||||
|
|
@ -3,21 +3,19 @@ import { defineConfig } from 'vitest/config'
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
test: {
|
test: {
|
||||||
testTimeout: 120_000,
|
testTimeout: 120_000,
|
||||||
globalSetup: ['./tests/s3-client/setup.js'],
|
globalSetup: ['tests/s3-client/setup.ts'],
|
||||||
setupFiles: ['./test-setup.mjs'],
|
|
||||||
projects: [
|
projects: [
|
||||||
{
|
{
|
||||||
test: {
|
test: {
|
||||||
name: 'aws-s3-node',
|
name: 's3-jsdom',
|
||||||
include: ['tests/index.test.ts', 'tests/createSignedURL.test.ts'],
|
include: ['**/*.test.ts'],
|
||||||
environment: 'jsdom',
|
environment: 'jsdom',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
test: {
|
test: {
|
||||||
name: 's3-client-browser',
|
name: 's3-browser',
|
||||||
include: ['tests/s3-client/*.test.{js,ts}'],
|
include: ['**/minio.test.ts'],
|
||||||
exclude: ['tests/s3-client/_shared.test.js'],
|
|
||||||
browser: {
|
browser: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
provider: 'playwright',
|
provider: 'playwright',
|
||||||
|
|
|
||||||
|
|
@ -662,11 +662,12 @@ export class Uppy<
|
||||||
...files[fileID].progress,
|
...files[fileID].progress,
|
||||||
...defaultProgress,
|
...defaultProgress,
|
||||||
},
|
},
|
||||||
// @ts-expect-error these typed are inserted
|
// @ts-expect-error these types are inserted
|
||||||
// into the namespace in their respective packages
|
// into the namespace in their respective packages
|
||||||
// but core isn't ware of those
|
// but core isn't aware of those
|
||||||
tus: undefined,
|
tus: undefined,
|
||||||
transloadit: undefined,
|
transloadit: undefined,
|
||||||
|
s3Multipart: undefined,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ export function fetcher(
|
||||||
options: FetcherOptions = {},
|
options: FetcherOptions = {},
|
||||||
): Promise<XMLHttpRequest> {
|
): Promise<XMLHttpRequest> {
|
||||||
const {
|
const {
|
||||||
body = null,
|
body,
|
||||||
headers = {},
|
headers = {},
|
||||||
method = 'GET',
|
method = 'GET',
|
||||||
onBeforeRequest = noop,
|
onBeforeRequest = noop,
|
||||||
|
|
|
||||||
|
|
@ -208,13 +208,13 @@ export default () => {
|
||||||
break
|
break
|
||||||
case 's3':
|
case 's3':
|
||||||
uppyDashboard.use(AwsS3, {
|
uppyDashboard.use(AwsS3, {
|
||||||
endpoint: COMPANION_URL,
|
companionEndpoint: COMPANION_URL,
|
||||||
shouldUseMultipart: false,
|
shouldUseMultipart: false,
|
||||||
})
|
})
|
||||||
break
|
break
|
||||||
case 's3-multipart':
|
case 's3-multipart':
|
||||||
uppyDashboard.use(AwsS3, {
|
uppyDashboard.use(AwsS3, {
|
||||||
endpoint: COMPANION_URL,
|
companionEndpoint: COMPANION_URL,
|
||||||
shouldUseMultipart: true,
|
shouldUseMultipart: true,
|
||||||
})
|
})
|
||||||
break
|
break
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@
|
||||||
"test": {
|
"test": {
|
||||||
"dependsOn": ["^test"],
|
"dependsOn": ["^test"],
|
||||||
"cache": false,
|
"cache": false,
|
||||||
"passThroughEnv": ["BUCKET_ENV_MINIO"]
|
"passThroughEnv": ["VITE_MINIO_CONFIG"]
|
||||||
},
|
},
|
||||||
"test:e2e": {
|
"test:e2e": {
|
||||||
"dependsOn": ["^test:e2e"],
|
"dependsOn": ["^test:e2e"],
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue