diff --git a/.github/workflows/bundlers.yml b/.github/workflows/bundlers.yml index 0f2035da8..23bbf645e 100644 --- a/.github/workflows/bundlers.yml +++ b/.github/workflows/bundlers.yml @@ -70,6 +70,12 @@ jobs: mkdir /tmp/artifacts && corepack yarn workspaces foreach --all --no-private pack --install-if-needed -o /tmp/artifacts/%s-${{ github.sha }}.tgz + - name: Debug packaged uppy dependencies + run: | + rm -rf /tmp/uppy-debug + mkdir -p /tmp/uppy-debug + tar -xzf /tmp/artifacts/uppy-${{ github.sha }}.tgz --strip-components 1 -C /tmp/uppy-debug + node -e 'const pkg=require("/tmp/uppy-debug/package.json");const fields=["dependencies","peerDependencies","optionalDependencies","devDependencies"];let found=false;for(const f of fields){if(!pkg[f])continue;for(const [k,v] of Object.entries(pkg[f])){if(String(v).startsWith("workspace:")){console.log(`${f}: ${k}=${v}`);found=true;}}}if(!found)console.log("No workspace:* entries in packaged uppy package.json");' - name: Upload artifact if: success() uses: actions/upload-artifact@v6 diff --git a/examples/aws-nodejs/index.js b/examples/aws-nodejs/index.js index 1010edf45..febb4f810 100644 --- a/examples/aws-nodejs/index.js +++ b/examples/aws-nodejs/index.js @@ -351,6 +351,80 @@ app.delete('/s3/multipart/:uploadId', (req, res, next) => { // === === +// === === +// This endpoint returns pre-signed URLs for S3 operations +// Used by the rewritten @uppy/aws-s3 plugin with signRequest option + +app.post('/s3/presign', async (req, res, next) => { + try { + const { method, key, uploadId, partNumber } = req.body + const client = getS3Client() + + if (!method || !key) { + return res.status(400).json({ error: 'method and key are required' }) + } + + const bucket = process.env.COMPANION_AWS_BUCKET + let command + + // Determine which command to use based on method and params + if (method === 'PUT' && uploadId && partNumber) { + // UploadPart + command = new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: parseInt(partNumber, 10), + }) + } else if (method === 'PUT') { + // PutObject (simple upload) + command = new PutObjectCommand({ + Bucket: bucket, + Key: key, + }) + } else if (method === 'POST' && !uploadId) { + // CreateMultipartUpload + command = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }) + } else if (method === 'POST' && uploadId) { + // CompleteMultipartUpload + command = new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + // Note: parts are sent in the request body, not in the presigned URL + }) + } else if (method === 'DELETE' && uploadId) { + // AbortMultipartUpload + command = new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }) + } else if (method === 'GET' && uploadId) { + // ListParts + command = new ListPartsCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }) + } else { + return res.status(400).json({ error: 'Unsupported operation' }) + } + + const url = await getSignedUrl(client, command, { expiresIn: 900 }) + + res.setHeader('Access-Control-Allow-Origin', accessControlAllowOrigin) + res.json({ url }) + } catch (err) { + next(err) + } +}) + +// === === + // === === app.get('/', (req, res) => { @@ -366,6 +440,21 @@ app.get('/withCustomEndpoints.html', (req, res) => { const htmlPath = path.join(__dirname, 'public', 'withCustomEndpoints.html') res.sendFile(htmlPath) }) +app.get('/rewrite-test.html', (req, res) => { + res.setHeader('Content-Type', 'text/html') + // Inject bucket config as JS variables + const config = `` + const htmlPath = path.join(__dirname, 'public', 'rewrite-test.html') + require('node:fs').readFile(htmlPath, 'utf8', (err, html) => { + if (err) return res.status(500).send('Error loading page') + // Inject config before + const modifiedHtml = html.replace('', `${config}`) + res.send(modifiedHtml) + }) +}) app.get('/uppy.min.mjs', (req, res) => { res.setHeader('Content-Type', 'text/javascript') @@ -410,4 +499,3 @@ app.listen(port, () => { console.log(`Example app listening on port ${port}.`) console.log(`Visit http://localhost:${port}/ on your browser to try it.`) }) -// === === diff --git a/examples/aws-nodejs/public/rewrite-test.html b/examples/aws-nodejs/public/rewrite-test.html new file mode 100644 index 000000000..3833fb9e2 --- /dev/null +++ b/examples/aws-nodejs/public/rewrite-test.html @@ -0,0 +1,127 @@ + + + + + Uppy – AWS S3 Plugin Rewrite Test + + + + +

AWS S3 Plugin Rewrite Test

+

Testing the rewritten @uppy/aws-s3 plugin using S3mini.

+ +
+ Test 1: Using getCredentials (STS) +
+

Uses the /s3/sts endpoint to get temporary credentials. S3mini handles signing internally.

+
+
+
+ +
+ Test 2: Using signRequest (Server Signing) +
+

Uses the /s3/presign endpoint to sign each request on the server.

+
+
+
+ + + + + + + diff --git a/packages/@uppy/aws-s3/src/MultipartUploader.ts b/packages/@uppy/aws-s3/src/MultipartUploader.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/@uppy/aws-s3/src/createSignedURL.ts b/packages/@uppy/aws-s3/src/createSignedURL.ts deleted file mode 100644 index a10fcb800..000000000 --- a/packages/@uppy/aws-s3/src/createSignedURL.ts +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Create a canonical request by concatenating the following strings, separated - * by newline characters. This helps ensure that the signature that you - * calculate and the signature that AWS calculates can match. - * - * @see https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request - * - * @param param0 - * @param param0.method – The HTTP method. - * @param param0.CanonicalUri – The URI-encoded version of the absolute - * path component URL (everything between the host and the question mark - * character (?) that starts the query string parameters). If the absolute path - * is empty, use a forward slash character (/). - * @param param0.CanonicalQueryString – The URL-encoded query string - * parameters, separated by ampersands (&). Percent-encode reserved characters, - * including the space character. Encode names and values separately. If there - * are empty parameters, append the equals sign to the parameter name before - * encoding. After encoding, sort the parameters alphabetically by key name. If - * there is no query string, use an empty string (""). - * @param param0.SignedHeaders – The request headers, - * that will be signed, and their values, separated by newline characters. - * For the values, trim any leading or trailing spaces, convert sequential - * spaces to a single space, and separate the values for a multi-value header - * using commas. You must include the host header (HTTP/1.1), and any x-amz-* - * headers in the signature. You can optionally include other standard headers - * in the signature, such as content-type. - * @param param0.HashedPayload – A string created using the payload in - * the body of the HTTP request as input to a hash function. This string uses - * lowercase hexadecimal characters. If the payload is empty, use an empty - * string as the input to the hash function. - */ -function createCanonicalRequest({ - method = 'PUT', - CanonicalUri = '/', - CanonicalQueryString = '', - SignedHeaders, - HashedPayload, -}: { - method?: string - CanonicalUri: string - CanonicalQueryString: string - SignedHeaders: Record - HashedPayload: string -}): string { - const headerKeys = Object.keys(SignedHeaders) - .map((k) => k.toLowerCase()) - .sort() - return [ - method, - CanonicalUri, - CanonicalQueryString, - ...headerKeys.map((k) => `${k}:${SignedHeaders[k]}`), - '', - headerKeys.join(';'), - HashedPayload, - ].join('\n') -} - -const ec = new TextEncoder() -const algorithm = { name: 'HMAC', hash: 'SHA-256' } - -async function digest(data: string): ReturnType { - const { subtle } = globalThis.crypto - return subtle.digest(algorithm.hash, ec.encode(data)) -} - -async function generateHmacKey(secret: string | ArrayBuffer) { - const { subtle } = globalThis.crypto - return subtle.importKey( - 'raw', - typeof secret === 'string' ? ec.encode(secret) : secret, - algorithm, - false, - ['sign'], - ) -} - -function arrayBufferToHexString(arrayBuffer: ArrayBuffer) { - const byteArray = new Uint8Array(arrayBuffer) - let hexString = '' - for (let i = 0; i < byteArray.length; i++) { - hexString += byteArray[i].toString(16).padStart(2, '0') - } - return hexString -} - -async function hash(key: Parameters[0], data: string) { - const { subtle } = globalThis.crypto - return subtle.sign(algorithm, await generateHmacKey(key), ec.encode(data)) -} - -/** - * @see https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html - */ -export default async function createSignedURL({ - accountKey, - accountSecret, - sessionToken, - bucketName, - Key, - Region, - expires, - uploadId, - partNumber, -}: { - accountKey: string - accountSecret: string - sessionToken: string - bucketName: string - Key: string - Region: string - expires: string | number - uploadId?: string - partNumber?: string | number -}): Promise { - const Service = 's3' - const host = `${Service}.${Region}.amazonaws.com` - /** - * List of char out of `encodeURI()` is taken from ECMAScript spec. - * Note that the `/` character is purposefully not included in list below. - * - * @see https://tc39.es/ecma262/#sec-encodeuri-uri - */ - const CanonicalUri = `/${bucketName}/${encodeURI(Key).replace(/[;?:@&=+$,#!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}` - const payload = 'UNSIGNED-PAYLOAD' - - const requestDateTime = new Date().toISOString().replace(/[-:]|\.\d+/g, '') // YYYYMMDDTHHMMSSZ - const date = requestDateTime.slice(0, 8) // YYYYMMDD - const scope = `${date}/${Region}/${Service}/aws4_request` - - const url = new URL(`https://${host}${CanonicalUri}`) - // N.B.: URL search params needs to be added in the ASCII order - url.searchParams.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256') - url.searchParams.set('X-Amz-Content-Sha256', payload) - url.searchParams.set('X-Amz-Credential', `${accountKey}/${scope}`) - url.searchParams.set('X-Amz-Date', requestDateTime) - url.searchParams.set('X-Amz-Expires', expires as string) - // We are signing on the client, so we expect there's going to be a session token: - url.searchParams.set('X-Amz-Security-Token', sessionToken) - url.searchParams.set('X-Amz-SignedHeaders', 'host') - // Those two are present only for Multipart Uploads: - if (partNumber) url.searchParams.set('partNumber', partNumber as string) - if (uploadId) url.searchParams.set('uploadId', uploadId) - url.searchParams.set( - 'x-id', - partNumber && uploadId ? 'UploadPart' : 'PutObject', - ) - - // Step 1: Create a canonical request - const canonical = createCanonicalRequest({ - CanonicalUri, - CanonicalQueryString: url.search.slice(1), - SignedHeaders: { - host, - }, - HashedPayload: payload, - }) - - // Step 2: Create a hash of the canonical request - const hashedCanonical = arrayBufferToHexString(await digest(canonical)) - - // Step 3: Create a string to sign - const stringToSign = [ - `AWS4-HMAC-SHA256`, // The algorithm used to create the hash of the canonical request. - requestDateTime, // The date and time used in the credential scope. - scope, // The credential scope. This restricts the resulting signature to the specified Region and service. - hashedCanonical, // The hash of the canonical request. - ].join('\n') - - // Step 4: Calculate the signature - const kDate = await hash(`AWS4${accountSecret}`, date) - const kRegion = await hash(kDate, Region) - const kService = await hash(kRegion, Service) - const kSigning = await hash(kService, 'aws4_request') - const signature = arrayBufferToHexString(await hash(kSigning, stringToSign)) - - // Step 5: Add the signature to the request - url.searchParams.set('X-Amz-Signature', signature) - - return url -} diff --git a/packages/@uppy/aws-s3/src/index.ts b/packages/@uppy/aws-s3/src/index.ts index 1b654278e..cf05b93b8 100644 --- a/packages/@uppy/aws-s3/src/index.ts +++ b/packages/@uppy/aws-s3/src/index.ts @@ -1,5 +1,707 @@ +import { + BasePlugin, + type DefinePluginOpts, + EventManager, + type PluginOpts, + type Uppy, +} from '@uppy/core' +import type { Body, Meta, UppyFile } from '@uppy/utils' +import { + AbortController, + filterFilesToEmitUploadStarted, + filterFilesToUpload, +} from '@uppy/utils' +import packageJson from '../package.json' with { type: 'json' } +import S3mini from './s3-client/S3.js' +import type * as IT from './s3-client/types.js' + +// ============================================================================ +// Types +// ============================================================================ + +/** Part information for multipart uploads */ export interface AwsS3Part { PartNumber?: number Size?: number ETag?: string } + +type PartUploadedCallback = ( + file: UppyFile, + part: { PartNumber: number; ETag: string }, +) => void + +declare module '@uppy/core' { + export interface UppyEventMap { + 's3-multipart:part-uploaded': PartUploadedCallback + } +} + +export interface AwsS3Options + extends PluginOpts { + bucket: string + region?: string + endpoint?: string + + /** + * Custom function to sign requests. + * Called with request details, should return signed headers. + * Alternative to using Companion endpoint. + */ + signRequest?: IT.signRequestFn + + /** + * Function to retrieve temporary credentials for client-side signing. + * When provided, S3mini handles signing internally using SigV4. + * Alternative to signRequest or endpoint. + */ + getCredentials?: IT.getCredentialsFn + + /** + * Whether to use multipart uploads. + * - `true`: Always use multipart + * - `false`: Always use simple PUT + * - `function`: Called with file, return true for multipart + * Default: Use multipart for files > 100MB + */ + shouldUseMultipart?: boolean | ((file: UppyFile) => boolean) + getChunkSize?: (file: { size: number }) => number + allowedMetaFields?: string[] | boolean + + /** + * Custom function to generate the S3 object key. + * Default: `{randomId}-{filename}` + */ + generateObjectKey?: (file: UppyFile) => string +} + +// ============================================================================ +// Constants +// ============================================================================ + +const MB = 1024 * 1024 + +/** Minimum chunk size required by S3 (5MB) */ +const MIN_CHUNK_SIZE = 5 * MB + +/** Maximum number of parts allowed by S3 */ +const MAX_PARTS = 10000 + +const defaultOptions = { + shouldUseMultipart: (file: UppyFile) => (file.size || 0) > 100 * MB, + allowedMetaFields: true, +} satisfies Partial> + +// ============================================================================ +// S3Uploader Types +// ============================================================================ + +interface S3UploaderOptions { + uppy: Uppy + s3Client: S3mini + file: UppyFile + key: string + shouldUseMultipart?: boolean + getChunkSize?: (file: { size: number }) => number + onProgress?: (bytesUploaded: number, bytesTotal: number) => void + onPartComplete?: (part: { PartNumber: number; ETag: string }) => void + onSuccess?: (result: UploadResult) => void + onError?: (err: Error) => void + onAbort?: () => void + log?: (message: string | Error, type?: 'error' | 'warning') => void +} + +interface UploadResult { + location: string + key: string + bucket?: string + uploadId?: string +} + +interface Chunk { + index: number + start: number + end: number + size: number +} + +interface ChunkState { + uploaded: number + etag?: string +} + +/** Reason for pausing (not a real error) */ +const pausingUploadReason = Symbol('pausing upload, not an actual error') + +// ============================================================================ +// S3Uploader Class +// ============================================================================ + +class S3Uploader { + readonly #s3Client: S3mini + readonly #file: UppyFile + readonly #data: Blob + readonly #key: string + readonly #options: S3UploaderOptions + readonly #eventManager: EventManager + + #chunks: Chunk[] = [] + #chunkState: ChunkState[] = [] + #shouldUseMultipart: boolean = false + #uploadId?: string + #uploadHasStarted: boolean = false + #abortController: AbortController = new AbortController() + + constructor(data: Blob, options: S3UploaderOptions) { + this.#s3Client = options.s3Client + this.#file = options.file + this.#data = data + this.#key = options.key + this.#options = options + this.#eventManager = new EventManager(options.uppy) + this.#initChunks() + this.#setupEvents() + } + + #setupEvents(): void { + const fileId = this.#file.id + + this.#eventManager.onFileRemove(fileId, () => { + this.abort() + this.#options.onAbort?.() + }) + + this.#eventManager.onCancelAll(fileId, () => { + this.abort() + this.#options.onAbort?.() + }) + + this.#eventManager.onFilePause(fileId, (isPaused) => { + if (isPaused) { + this.pause() + } else { + this.start() + } + }) + + this.#eventManager.onPauseAll(fileId, () => { + this.pause() + }) + + this.#eventManager.onResumeAll(fileId, () => { + this.start() + }) + + this.#eventManager.onRetry(fileId, () => { + this.start() + }) + + this.#eventManager.onRetryAll(fileId, () => { + this.start() + }) + } + + #initChunks(): void { + const fileSize = this.#data.size + + // Step 1: Determine if we should use multipart + // - If explicitly set to boolean, use that + // - Otherwise, use multipart for files larger than MIN_CHUNK_SIZE (5MB) + if (typeof this.#options.shouldUseMultipart === 'boolean') { + this.#shouldUseMultipart = this.#options.shouldUseMultipart + } else { + this.#shouldUseMultipart = fileSize > MIN_CHUNK_SIZE + } + + // Step 2: Force simple upload if file is too small for multipart + // (S3 requires minimum 5MB parts, except for the last part) + if (fileSize <= MIN_CHUNK_SIZE) { + this.#shouldUseMultipart = false + } + + // Step 3: Create chunks based on upload strategy + if (this.#shouldUseMultipart) { + // Calculate chunk size: at least MIN_CHUNK_SIZE, but may be larger for huge files + let chunkSize = this.#getChunkSize(fileSize) + chunkSize = Math.max(chunkSize, MIN_CHUNK_SIZE) + + // Ensure we don't exceed MAX_PARTS (S3 limit: 10,000 parts) + if (Math.ceil(fileSize / chunkSize) > MAX_PARTS) { + chunkSize = Math.ceil(fileSize / MAX_PARTS) + } + + // Create chunk definitions + for ( + let offset = 0, index = 0; + offset < fileSize; + offset += chunkSize, index++ + ) { + const end = Math.min(offset + chunkSize, fileSize) + this.#chunks.push({ index, start: offset, end, size: end - offset }) + } + } else { + // Simple upload: single chunk for the entire file + this.#chunks = [{ index: 0, start: 0, end: fileSize, size: fileSize }] + } + + this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 })) + } + + #getChunkSize(fileSize: number): number { + if (this.#options.getChunkSize) { + return this.#options.getChunkSize({ size: fileSize }) + } + return Math.max(MIN_CHUNK_SIZE, Math.ceil(fileSize / MAX_PARTS)) + } + + start(): void { + if (this.#uploadHasStarted) { + // Abort any pending operations (if not already aborted) + if (!this.#abortController.signal.aborted) { + this.#abortController.abort(pausingUploadReason) + } + // Always create a fresh AbortController for resume + this.#abortController = new AbortController() + this.#resumeUpload() + } else { + this.#createUpload() + } + } + + pause(): void { + this.#abortController.abort(pausingUploadReason) + this.#abortController = new AbortController() + } + + abort(opts?: { abortOnS3?: boolean }): void { + this.#abortController.abort() + // Clean up event listeners + this.#eventManager.remove() + if (opts?.abortOnS3 !== false && this.#uploadId) { + this.#s3Client + .abortMultipartUpload(this.#key, this.#uploadId) + .catch((abortErr) => { + this.#options.log?.(abortErr, 'warning') + }) + } + } + + async #createUpload(): Promise { + this.#uploadHasStarted = true + try { + if (this.#shouldUseMultipart) { + await this.#multipartUpload() + } else { + await this.#simpleUpload() + } + } catch (err) { + this.#onError(err as Error) + } + } + + async #resumeUpload(): Promise { + if (!this.#uploadId) { + await this.#createUpload() + return + } + try { + const existingParts = await this.#s3Client.listParts( + this.#uploadId, + this.#key, + ) + for (const part of existingParts) { + const chunkIndex = part.partNumber - 1 + if (chunkIndex >= 0 && chunkIndex < this.#chunkState.length) { + this.#chunkState[chunkIndex].uploaded = this.#chunks[chunkIndex].size + this.#chunkState[chunkIndex].etag = part.etag + } + } + await this.#uploadRemainingParts() + } catch (err) { + this.#onError(err as Error) + } + } + + async #simpleUpload(): Promise { + const signal = this.#abortController.signal + if (signal.aborted) { + throw new Error('Upload aborted', { cause: signal.reason }) + } + + await this.#s3Client.putObject( + this.#key, + this.#data, + this.#file.type || 'application/octet-stream', + ) + + this.#chunkState[0].uploaded = this.#data.size + this.#onProgress() + this.#onSuccess({ + location: `${this.#s3Client.endpoint}/${this.#key}`, + key: this.#key, + }) + } + + async #multipartUpload(): Promise { + const signal = this.#abortController.signal + if (signal.aborted) { + throw new Error('Upload aborted', { cause: signal.reason }) + } + + this.#uploadId = await this.#s3Client.getMultipartUploadId( + this.#key, + this.#file.type || 'application/octet-stream', + ) + await this.#uploadRemainingParts() + } + + async #uploadRemainingParts(): Promise { + const signal = this.#abortController.signal + // Collect already-uploaded parts (from resume) + const parts = this.#chunkState + .map((state, i) => + state.etag ? { partNumber: i + 1, etag: state.etag } : null, + ) + .filter((p): p is NonNullable => p !== null) + + for (let i = 0; i < this.#chunks.length; i++) { + if (signal.aborted) { + throw new Error('Upload aborted', { cause: signal.reason }) + } + if (this.#chunkState[i].etag) continue + + const chunk = this.#chunks[i] + const partNumber = i + 1 + const chunkData = this.#data.slice(chunk.start, chunk.end) + + const part = await this.#s3Client.uploadPart( + this.#key, + this.#uploadId!, + chunkData, + partNumber, + ) + + this.#chunkState[i].uploaded = chunk.size + this.#chunkState[i].etag = part.etag + parts.push({ partNumber: part.partNumber, etag: part.etag }) + this.#onProgress() + + if (this.#options.onPartComplete) { + this.#options.onPartComplete({ + PartNumber: part.partNumber, + ETag: part.etag, + }) + } + } + + if (signal.aborted) { + throw new Error('Upload aborted', { cause: signal.reason }) + } + + const result = await this.#s3Client.completeMultipartUpload( + this.#key, + this.#uploadId!, + parts, + ) + + this.#onSuccess({ + location: result.location, + key: result.key, + bucket: result.bucket, + uploadId: this.#uploadId, + }) + } + + #onProgress(): void { + if (!this.#options.onProgress) return + const bytesUploaded = this.#chunkState.reduce( + (sum, state) => sum + state.uploaded, + 0, + ) + this.#options.onProgress(bytesUploaded, this.#data.size) + } + + #onSuccess(result: UploadResult): void { + this.#options.onSuccess?.(result) + } + + #onError(err: Error): void { + if ((err as any).cause === pausingUploadReason) return + // Also ignore abort signals from intentional cancellation + if (err.name === 'AbortError') return + // If we intentionally aborted, don't report any subsequent errors + // (e.g., S3 returning 404 NoSuchUpload after we aborted the upload) + if (this.#abortController.signal.aborted) return + if (this.#uploadId) { + this.#s3Client + .abortMultipartUpload(this.#key, this.#uploadId) + .catch((abortErr) => { + this.#options.log?.(abortErr, 'warning') + }) + } + this.#options.onError?.(err) + } +} + +// ============================================================================ +// Plugin Class +// ============================================================================ + +export default class AwsS3 extends BasePlugin< + DefinePluginOpts, keyof typeof defaultOptions>, + M, + B +> { + static VERSION = packageJson.version + + #s3Client!: S3mini + #uploaders: Record | null> = {} + + constructor(uppy: Uppy, opts: AwsS3Options) { + super(uppy, { ...defaultOptions, ...opts }) + this.type = 'uploader' + this.id = this.opts.id || 'AwsS3' + } + + install(): void { + this.#setResumableUploadsCapability(true) + this.#initS3Client() + this.uppy.addUploader(this.#upload) + this.uppy.on('cancel-all', this.#resetResumableCapability) + } + + uninstall(): void { + this.#setResumableUploadsCapability(false) + this.uppy.removeUploader(this.#upload) + this.uppy.off('cancel-all', this.#resetResumableCapability) + // Abort and clean up any in-flight uploads + for (const fileId of Object.keys(this.#uploaders)) { + const uploader = this.#uploaders[fileId] + if (uploader) { + uploader.abort() + } + } + } + + // -------------------------------------------------------------------------- + // Resumable Uploads Capability + // -------------------------------------------------------------------------- + + #setResumableUploadsCapability = (value: boolean): void => { + const { capabilities } = this.uppy.getState() + this.uppy.setState({ + capabilities: { + ...capabilities, + resumableUploads: value, + }, + }) + } + + #resetResumableCapability = (): void => { + this.#setResumableUploadsCapability(true) + } + + // -------------------------------------------------------------------------- + // S3 Client Initialization + // -------------------------------------------------------------------------- + + #initS3Client(): void { + const { endpoint, signRequest, getCredentials, bucket, region } = this.opts + + if (typeof bucket !== 'string' || bucket.trim() === '') { + throw new Error( + 'AwsS3: `bucket` option is required and must be a non-empty string', + ) + } + + if (typeof region !== 'string' || region.trim() === '') { + throw new Error( + 'AwsS3: `region` option is required and must be a non-empty string', + ) + } + + const bucketName = bucket.trim() + + if (!signRequest && !getCredentials && !endpoint) { + throw new Error( + 'AwsS3: `endpoint`, `signRequest`, or `getCredentials` is required', + ) + } + + const s3Endpoint = `https://${bucketName}.s3.${region}.amazonaws.com` + + if (getCredentials) { + // Mode: Temporary credentials (client-side signing) + this.#s3Client = new S3mini({ + endpoint: s3Endpoint, + getCredentials, + region, + }) + } else if (signRequest) { + // Mode: Custom signing function + this.#s3Client = new S3mini({ + endpoint: s3Endpoint, + signRequest, + region, + }) + } else { + // Mode: Companion signing (endpoint is guaranteed to be set here) + this.#s3Client = new S3mini({ + endpoint: s3Endpoint, + signRequest: this.#createCompanionSigner(endpoint!), + region, + }) + } + } + + /** + * Creates a signing function that calls Companion's /s3/sign endpoint. + */ + #createCompanionSigner(companionUrl: string): IT.signRequestFn { + return async (request) => { + const response = await fetch(`${companionUrl}/s3/sign`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }) + if (!response.ok) { + throw new Error(`Failed to sign request: ${response.statusText}`) + } + return response.json() + } + } + + // -------------------------------------------------------------------------- + // Upload Entry Point + // -------------------------------------------------------------------------- + + #upload = async (fileIDs: string[]): Promise => { + if (fileIDs.length === 0) return + + const files = this.uppy.getFilesByIds(fileIDs) + const filesToUpload = filterFilesToUpload(files) + const filesToEmit = filterFilesToEmitUploadStarted(filesToUpload) + + this.uppy.emit('upload-start', filesToEmit) + + const promises = filesToUpload.map((file) => { + if (file.isRemote) { + // Remote files not yet supported in this minimal implementation + return Promise.reject( + new Error('Remote file uploads not yet supported'), + ) + } + return this.#uploadLocalFile(file) + }) + + await Promise.allSettled(promises) + } + + // -------------------------------------------------------------------------- + // Local File Upload + // -------------------------------------------------------------------------- + + async #uploadLocalFile(file: UppyFile): Promise { + return new Promise((resolve, reject) => { + const data = file.data as Blob + const key = this.#generateKey(file) + const shouldMultipart = this.#shouldUseMultipart(file) + + try { + // Create uploader (events are wired internally) + const uploader = new S3Uploader(data, { + uppy: this.uppy, + s3Client: this.#s3Client, + file, + key, + shouldUseMultipart: shouldMultipart, + getChunkSize: this.opts.getChunkSize, + log: (...args) => this.uppy.log(...args), + + onProgress: (bytesUploaded, bytesTotal) => { + this.uppy.emit('upload-progress', file, { + uploadStarted: file.progress.uploadStarted ?? Date.now(), + bytesUploaded, + bytesTotal, + }) + }, + + onPartComplete: (part) => { + this.uppy.emit('s3-multipart:part-uploaded', file, part) + }, + + onSuccess: (result: UploadResult) => { + this.uppy.emit('upload-success', file, { + status: 200, + body: { + location: result.location, + key: result.key, + bucket: result.bucket, + } as unknown as B, + uploadURL: result.location, + }) + this.#cleanup(file.id) + resolve() + }, + + onError: (err) => { + this.uppy.emit('upload-error', file, err) + this.#cleanup(file.id) + reject(err) + }, + + onAbort: () => { + this.#cleanup(file.id) + resolve() // Normal completion, not an error + }, + }) + + // Store uploader for external abort if needed + this.#uploaders[file.id] = uploader + + // Start the upload + uploader.start() + } catch (err) { + // Cleanup on synchronous failure during setup + this.#cleanup(file.id) + reject(err) + } + }) + } + + #shouldUseMultipart(file: UppyFile): boolean { + const { shouldUseMultipart } = this.opts + if (typeof shouldUseMultipart === 'function') { + return shouldUseMultipart(file) + } + if (typeof shouldUseMultipart === 'boolean') { + return shouldUseMultipart + } + // Default: multipart for files > 100MB + return (file.size || 0) > 100 * MB + } + + #generateKey(file: UppyFile): string { + if (this.opts.generateObjectKey) { + return this.opts.generateObjectKey(file) + } + // Default: {randomId}-{filename} + const randomId = crypto.randomUUID() + return `${randomId}-${file.name}` + } + + #cleanup(fileId: string): void { + if (this.#uploaders[fileId]) { + delete this.#uploaders[fileId] + } + } +} + +export type { AwsS3Options as AwsS3MultipartOptions } + +/** Body type for AWS S3 upload responses */ +export interface AwsBody extends Body { + location: string + key: string + bucket?: string +} diff --git a/packages/@uppy/aws-s3/src/s3-client/S3.ts b/packages/@uppy/aws-s3/src/s3-client/S3.ts index 61c1be2bb..cae467cf5 100644 --- a/packages/@uppy/aws-s3/src/s3-client/S3.ts +++ b/packages/@uppy/aws-s3/src/s3-client/S3.ts @@ -91,7 +91,7 @@ class S3mini { // Cache the promise so concurrent calls wait for the same fetch if (this.cachedCredentialsPromise == null) { - this.cachedCredentialsPromise = this.getCredentials!() + this.cachedCredentialsPromise = this.getCredentials!({}) .then((creds) => { this.cachedCredentials = creds return creds @@ -188,7 +188,8 @@ class S3mini { if ( typeof data === 'string' || data instanceof ArrayBuffer || - ArrayBuffer.isView(data) + ArrayBuffer.isView(data) || + data instanceof Blob ) { return data as BodyInit } diff --git a/packages/@uppy/aws-s3/src/s3-client/types.ts b/packages/@uppy/aws-s3/src/s3-client/types.ts index 36c84cad2..9b1c63d2c 100644 --- a/packages/@uppy/aws-s3/src/s3-client/types.ts +++ b/packages/@uppy/aws-s3/src/s3-client/types.ts @@ -129,6 +129,6 @@ export interface XmlMap { /** * Binary data types supported in browser environments. - * Use ArrayBuffer or Uint8Array - Buffer is not available in browsers. + * Use ArrayBuffer, Uint8Array, or Blob - Buffer is not available in browsers. */ -export type BinaryData = ArrayBuffer | Uint8Array +export type BinaryData = ArrayBuffer | Uint8Array | Blob diff --git a/packages/@uppy/aws-s3/tests/index.test.ts b/packages/@uppy/aws-s3/tests/index.test.ts new file mode 100644 index 000000000..d3c9490af --- /dev/null +++ b/packages/@uppy/aws-s3/tests/index.test.ts @@ -0,0 +1,247 @@ +import { describe, expect, it, vi } from 'vitest' + +import 'whatwg-fetch' +import Core, { type Meta, type UppyFile } from '@uppy/core' +import AwsS3, { type AwsBody, type AwsS3Options } from '../src/index.js' + +const KB = 1024 +const MB = KB * KB + +describe('AwsS3', () => { + it('Registers AwsS3 upload plugin', () => { + const core = new Core().use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + endpoint: 'https://companion.example.com', + }) + + // @ts-expect-error private property + const pluginNames = core[Symbol.for('uppy test: getPlugins')]( + 'uploader', + ).map((plugin: AwsS3) => plugin.constructor.name) + expect(pluginNames).toContain('AwsS3') + }) + + describe('configuration validation', () => { + it('throws if bucket is not provided', () => { + expect(() => { + const core = new Core() + // @ts-expect-error - testing missing required option + core.use(AwsS3, {}) + }).toThrow('`bucket` option is required') + }) + + it('throws if region is not provided', () => { + expect(() => { + const core = new Core() + core.use(AwsS3, { bucket: 'test-bucket' }) + }).toThrow('`region` option is required') + }) + + it('throws if no signing method is provided', () => { + expect(() => { + const core = new Core() + core.use(AwsS3, { bucket: 'test-bucket', region: 'us-east-1' }) + }).toThrow('`endpoint`, `signRequest`, or `getCredentials` is required') + }) + + it('accepts endpoint option', () => { + const core = new Core() + core.use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + endpoint: 'https://companion.example.com', + }) + expect(core.getPlugin('AwsS3')).toBeDefined() + }) + + it('accepts signRequest option', () => { + const core = new Core() + core.use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + signRequest: vi.fn(), + }) + expect(core.getPlugin('AwsS3')).toBeDefined() + }) + + it('accepts getCredentials option', () => { + const core = new Core() + core.use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + getCredentials: vi.fn(), + }) + expect(core.getPlugin('AwsS3')).toBeDefined() + }) + }) + + describe('shouldUseMultipart', () => { + const MULTIPART_THRESHOLD = 100 * MB + + // Helper that creates a mock file without allocating memory + const createFile = (size: number): UppyFile => + ({ + name: 'test.dat', + size, + data: { size } as Blob, + }) as unknown as UppyFile + + it('defaults to multipart for files > 100MB', () => { + const core = new Core().use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + endpoint: 'https://companion.example.com', + }) + const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options + const shouldUseMultipart = opts.shouldUseMultipart as ( + file: UppyFile, + ) => boolean + + expect(shouldUseMultipart(createFile(MULTIPART_THRESHOLD + 1))).toBe(true) + expect(shouldUseMultipart(createFile(MULTIPART_THRESHOLD))).toBe(false) + expect(shouldUseMultipart(createFile(MULTIPART_THRESHOLD - 1))).toBe( + false, + ) + expect(shouldUseMultipart(createFile(0))).toBe(false) + }) + + it('handles very large files', () => { + const core = new Core().use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + endpoint: 'https://companion.example.com', + }) + const opts = core.getPlugin('AwsS3')!.opts as AwsS3Options + const shouldUseMultipart = opts.shouldUseMultipart as ( + file: UppyFile, + ) => boolean + + expect(shouldUseMultipart(createFile(70 * 1024 * MB))).toBe(true) // 70GB + expect(shouldUseMultipart(createFile(400 * 1024 * MB))).toBe(true) // 400GB + }) + }) + + describe('upload events', () => { + it('emits upload-start when upload begins', async () => { + const signRequest = vi.fn().mockRejectedValue(new Error('Test stop')) + + const core = new Core().use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + signRequest, + shouldUseMultipart: false, + }) + + core.addFile({ + source: 'test', + name: 'test.txt', + type: 'text/plain', + data: new File([new Uint8Array(1024)], 'test.txt'), + }) + + const uploadStartHandler = vi.fn() + core.on('upload-start', uploadStartHandler) + + try { + await core.upload() + } catch { + // Expected + } + + expect(uploadStartHandler).toHaveBeenCalledTimes(1) + }) + + it('emits upload-error on failure', async () => { + const signRequest = vi.fn().mockRejectedValue(new Error('Sign failed')) + + const core = new Core().use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + signRequest, + shouldUseMultipart: false, + }) + + core.addFile({ + source: 'test', + name: 'test.txt', + type: 'text/plain', + data: new File([new Uint8Array(1024)], 'test.txt'), + }) + + const uploadErrorHandler = vi.fn() + core.on('upload-error', uploadErrorHandler) + + try { + await core.upload() + } catch { + // Expected + } + + expect(uploadErrorHandler).toHaveBeenCalledTimes(1) + }) + }) + + describe('abort', () => { + it('aborts when file is removed', async () => { + const signRequest = vi + .fn() + .mockImplementation( + () => new Promise((resolve) => setTimeout(resolve, 100)), + ) + + const core = new Core().use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + signRequest, + shouldUseMultipart: false, + }) + + core.addFile({ + source: 'test', + name: 'test.txt', + type: 'text/plain', + data: new File([new Uint8Array(1024)], 'test.txt'), + }) + + const fileId = Object.keys(core.getState().files)[0] + const uploadPromise = core.upload() + setTimeout(() => core.removeFile(fileId), 10) + + const result = await uploadPromise + // When a file is removed mid-upload, it should not appear in successful uploads + expect(result).toBeDefined() + expect(result?.successful).toHaveLength(0) + }) + + it('aborts when cancelAll is called', async () => { + const signRequest = vi + .fn() + .mockImplementation( + () => new Promise((resolve) => setTimeout(resolve, 100)), + ) + + const core = new Core().use(AwsS3, { + bucket: 'test-bucket', + region: 'us-east-1', + signRequest, + shouldUseMultipart: false, + }) + + core.addFile({ + source: 'test', + name: 'test.txt', + type: 'text/plain', + data: new File([new Uint8Array(1024)], 'test.txt'), + }) + + const uploadPromise = core.upload() + setTimeout(() => core.cancelAll(), 10) + + const result = await uploadPromise + // When cancelAll is called, no files should complete successfully + expect(result).toBeDefined() + expect(result?.successful).toHaveLength(0) + }) + }) +}) diff --git a/packages/@uppy/aws-s3/tests/s3-client/minio.test.js b/packages/@uppy/aws-s3/tests/s3-client/minio.test.js index 08b4de247..493a88dd9 100644 --- a/packages/@uppy/aws-s3/tests/s3-client/minio.test.js +++ b/packages/@uppy/aws-s3/tests/s3-client/minio.test.js @@ -2,6 +2,7 @@ import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts' import { describe, expect, inject, it, vi } from 'vitest' import { S3mini } from '../../src/s3-client/S3.js' import { createSigV4Signer } from '../../src/s3-client/signer.js' +import { randomBytes } from '../test-utils/browser-crypto.js' import { beforeRun, cleanupTestBeforeAll } from './_shared.test.js' const name = 'minio' @@ -82,13 +83,7 @@ const minioSpecific = (bucket) => { it('multipart upload with signRequest', async () => { const key = `presigned-multipart-${Date.now()}.bin` const partSize = 5 * 1024 * 1024 // 5MB - - const part = new Uint8Array(partSize) - for (let i = 0; i < part.length; i += 65536) { - crypto.getRandomValues( - new Uint8Array(part.buffer, i, Math.min(65536, part.length - i)), - ) - } + const part = randomBytes(partSize) const uploadId = await s3client.getMultipartUploadId( key, @@ -215,13 +210,7 @@ const minioSpecific = (bucket) => { const key = `sts-multipart-${Date.now()}.bin` const partSize = 5 * 1024 * 1024 // 5MB - - const part = new Uint8Array(partSize) - for (let i = 0; i < part.length; i += 65536) { - crypto.getRandomValues( - new Uint8Array(part.buffer, i, Math.min(65536, part.length - i)), - ) - } + const part = randomBytes(partSize) const uploadId = await s3.getMultipartUploadId( key, diff --git a/packages/uppy/src/bundle.ts b/packages/uppy/src/bundle.ts index d59bec5c6..5c03dce1d 100644 --- a/packages/uppy/src/bundle.ts +++ b/packages/uppy/src/bundle.ts @@ -19,6 +19,7 @@ export const views = { ProviderView } // Acquirers export { default as Audio } from '@uppy/audio' // Uploaders +export { default as AwsS3 } from '@uppy/aws-s3' export { default as Box } from '@uppy/box' // Miscellaneous export { default as Compressor } from '@uppy/compressor' diff --git a/packages/uppy/src/index.ts b/packages/uppy/src/index.ts index d0981107d..de33c8399 100644 --- a/packages/uppy/src/index.ts +++ b/packages/uppy/src/index.ts @@ -1,6 +1,7 @@ // Acquirers export type { AudioOptions } from '@uppy/audio' // Uploaders +export type { AwsS3Options } from '@uppy/aws-s3' export type { BoxOptions } from '@uppy/box' // Miscellaneous export type { CompressorOptions } from '@uppy/compressor'