From 0c7f02aef2abfebb32ee71910855fb1740ee88b8 Mon Sep 17 00:00:00 2001 From: prakash Date: Tue, 7 Jul 2026 18:40:40 +0530 Subject: [PATCH] update imports after merge remove stale files --- .../aws-s3/src/HTTPCommunicationQueue.ts | 444 ------------------ .../@uppy/aws-s3/src/MultipartUploader.ts | 265 ----------- packages/@uppy/aws-s3/src/S3Uploader.ts | 4 +- packages/@uppy/aws-s3/src/index.ts | 6 +- .../@uppy/aws-s3/src/s3-client/S3Client.ts | 2 +- packages/@uppy/aws-s3/src/s3-client/S3mini.ts | 2 +- 6 files changed, 7 insertions(+), 716 deletions(-) delete mode 100644 packages/@uppy/aws-s3/src/HTTPCommunicationQueue.ts delete mode 100644 packages/@uppy/aws-s3/src/MultipartUploader.ts diff --git a/packages/@uppy/aws-s3/src/HTTPCommunicationQueue.ts b/packages/@uppy/aws-s3/src/HTTPCommunicationQueue.ts deleted file mode 100644 index 3e1ae1117..000000000 --- a/packages/@uppy/aws-s3/src/HTTPCommunicationQueue.ts +++ /dev/null @@ -1,444 +0,0 @@ -import type { Body, Meta, UppyFile } from '@uppy/core' -import type { - RateLimitedQueue, - WrapPromiseFunctionType, -} from '@uppy/core/utils' -import type AwsS3Multipart from './index.js' -import type { - AwsS3MultipartOptions, - AwsS3UploadParameters, - uploadPartBytes, -} from './index.js' -import { type Chunk, pausingUploadReason } from './MultipartUploader.js' -import type { UploadPartBytesResult, UploadResult } from './utils.js' -import { throwIfAborted } from './utils.js' - -function removeMetadataFromURL(urlString: string) { - const urlObject = new URL(urlString) - urlObject.search = '' - urlObject.hash = '' - return urlObject.href -} - -export class HTTPCommunicationQueue { - #abortMultipartUpload!: WrapPromiseFunctionType< - AwsS3Multipart['abortMultipartUpload'] - > - - #cache = new WeakMap() - - #createMultipartUpload!: WrapPromiseFunctionType< - AwsS3Multipart['createMultipartUpload'] - > - - #fetchSignature!: WrapPromiseFunctionType['signPart']> - - #getUploadParameters!: WrapPromiseFunctionType< - AwsS3Multipart['getUploadParameters'] - > - - #listParts!: WrapPromiseFunctionType['listParts']> - - #previousRetryDelay!: number - - #requests - - #retryDelays!: { values: () => Iterator } - - #sendCompletionRequest!: WrapPromiseFunctionType< - AwsS3Multipart['completeMultipartUpload'] - > - - #setS3MultipartState - - #uploadPartBytes!: WrapPromiseFunctionType - - #getFile - - constructor( - requests: RateLimitedQueue, - options: AwsS3MultipartOptions, - setS3MultipartState: (file: UppyFile, result: UploadResult) => void, - getFile: (file: UppyFile) => UppyFile, - ) { - this.#requests = requests - this.#setS3MultipartState = setS3MultipartState - this.#getFile = getFile - this.setOptions(options) - } - - setOptions(options: Partial>): void { - const requests = this.#requests - - if ('abortMultipartUpload' in options) { - this.#abortMultipartUpload = requests.wrapPromiseFunction( - options.abortMultipartUpload as any, - { priority: 1 }, - ) - } - if ('createMultipartUpload' in options) { - this.#createMultipartUpload = requests.wrapPromiseFunction( - options.createMultipartUpload as any, - { priority: -1 }, - ) - } - if ('signPart' in options) { - this.#fetchSignature = requests.wrapPromiseFunction( - options.signPart as any, - ) - } - if ('listParts' in options) { - this.#listParts = requests.wrapPromiseFunction(options.listParts as any) - } - if ('completeMultipartUpload' in options) { - this.#sendCompletionRequest = requests.wrapPromiseFunction( - options.completeMultipartUpload as any, - { priority: 1 }, - ) - } - if ('retryDelays' in options) { - this.#retryDelays = options.retryDelays ?? [] - } - if ('uploadPartBytes' in options) { - this.#uploadPartBytes = requests.wrapPromiseFunction( - options.uploadPartBytes as any, - { priority: Infinity }, - ) - } - if ('getUploadParameters' in options) { - this.#getUploadParameters = requests.wrapPromiseFunction( - options.getUploadParameters as any, - ) - } - } - - async #shouldRetry(err: any, retryDelayIterator: Iterator) { - const requests = this.#requests - const status = err?.source?.status - - // TODO: this retry logic is taken out of Tus. We should have a centralized place for retrying, - // perhaps the rate limited queue, and dedupe all plugins with that. - if (status == null) { - return false - } - if (status === 403 && err.message === 'Request has expired') { - if (!requests.isPaused) { - // We don't want to exhaust the retryDelayIterator as long as there are - // more than one request in parallel, to give slower connection a chance - // to catch up with the expiry set in Companion. - if (requests.limit === 1 || this.#previousRetryDelay == null) { - const next = retryDelayIterator.next() - if (next == null || next.done) { - return false - } - // If there are more than 1 request done in parallel, the RLQ limit is - // decreased and the failed request is requeued after waiting for a bit. - // If there is only one request in parallel, the limit can't be - // decreased, so we iterate over `retryDelayIterator` as we do for - // other failures. - // `#previousRetryDelay` caches the value so we can re-use it next time. - this.#previousRetryDelay = next.value - } - // No need to stop the other requests, we just want to lower the limit. - requests.rateLimit(0) - await new Promise((resolve) => - setTimeout(resolve, this.#previousRetryDelay), - ) - } - } else if (status === 429) { - // HTTP 429 Too Many Requests => to avoid the whole download to fail, pause all requests. - if (!requests.isPaused) { - const next = retryDelayIterator.next() - if (next == null || next.done) { - return false - } - requests.rateLimit(next.value) - } - } else if (status > 400 && status < 500 && status !== 409) { - // HTTP 4xx, the server won't send anything, it's doesn't make sense to retry - return false - } else if (typeof navigator !== 'undefined' && navigator.onLine === false) { - // The navigator is offline, let's wait for it to come back online. - if (!requests.isPaused) { - requests.pause() - window.addEventListener( - 'online', - () => { - requests.resume() - }, - { once: true }, - ) - } - } else { - // Other error code means the request can be retried later. - const next = retryDelayIterator.next() - if (next == null || next.done) { - return false - } - await new Promise((resolve) => setTimeout(resolve, next.value)) - } - return true - } - - async getUploadId( - file: UppyFile, - signal: AbortSignal, - ): Promise { - let cachedResult: Promise | UploadResult | undefined - // As the cache is updated asynchronously, there could be a race condition - // where we just miss a new result so we loop here until we get nothing back, - // at which point it's out turn to create a new cache entry. - for (;;) { - if (file.data == null) throw new Error('File data is empty') - cachedResult = this.#cache.get(file.data) - if (cachedResult == null) break - try { - return await cachedResult - } catch { - // In case of failure, we want to ignore the cached error. - // At this point, either there's a new cached value, or we'll exit the loop a create a new one. - } - } - - const promise = this.#createMultipartUpload(this.#getFile(file), signal) - - const abortPromise = () => { - if (file.data == null) throw new Error('File data is empty') - promise.abort(signal.reason) - this.#cache.delete(file.data) - } - signal.addEventListener('abort', abortPromise, { once: true }) - this.#cache.set(file.data, promise) - promise.then( - async (result) => { - signal.removeEventListener('abort', abortPromise) - this.#setS3MultipartState(file, result) - if (file.data == null) throw new Error('File data is empty') - this.#cache.set(file.data, result) - }, - () => { - signal.removeEventListener('abort', abortPromise) - if (file.data == null) throw new Error('File data is empty') - this.#cache.delete(file.data) - }, - ) - - return promise - } - - async abortFileUpload(file: UppyFile): Promise { - if (file.data == null) throw new Error('File data is empty') - const result = this.#cache.get(file.data) - if (result == null) { - // If the createMultipartUpload request never was made, we don't - // need to send the abortMultipartUpload request. - return - } - // Remove the cache entry right away for follow-up requests do not try to - // use the soon-to-be aborted cached values. - this.#cache.delete(file.data) - this.#setS3MultipartState(file, Object.create(null)) - let awaitedResult: UploadResult - try { - awaitedResult = await result - } catch { - // If the cached result rejects, there's nothing to abort. - return - } - await this.#abortMultipartUpload(this.#getFile(file), awaitedResult) - } - - async #nonMultipartUpload( - file: UppyFile, - chunk: Chunk, - signal?: AbortSignal, - ) { - const { - method = 'POST', - url, - fields, - headers, - } = await this.#getUploadParameters(this.#getFile(file), { - signal, - }).abortOn(signal) - - let body: FormData | Blob - const data = chunk.getData() - if (method.toUpperCase() === 'POST') { - const formData = new FormData() - Object.entries(fields!).forEach(([key, value]) => { - formData.set(key, value) - }) - formData.set('file', data) - body = formData - } else { - body = data - } - - const { onProgress, onComplete } = chunk - - const result = (await this.#uploadPartBytes({ - signature: { url, headers, method } as any, - body, - size: data.size, - onProgress, - onComplete, - signal, - }).abortOn(signal)) as unknown as B // todo this doesn't make sense - - // Note: `fields.key` is not returned by old Companion versions. - // See https://github.com/transloadit/uppy/pull/5602 - const key = fields?.key - this.#setS3MultipartState(file, { key: key! }) - - return { - ...result, - location: - (result.location as string | undefined) ?? removeMetadataFromURL(url), - bucket: fields?.bucket, - key, - } - } - - async uploadFile( - file: UppyFile, - chunks: Chunk[], - signal: AbortSignal, - ): Promise> { - throwIfAborted(signal) - if (chunks.length === 1 && !chunks[0].shouldUseMultipart) { - return this.#nonMultipartUpload(file, chunks[0], signal) - } - const { uploadId, key } = await this.getUploadId(file, signal) - throwIfAborted(signal) - try { - const parts = await Promise.all( - chunks.map((chunk, i) => this.uploadChunk(file, i + 1, chunk, signal)), - ) - throwIfAborted(signal) - return await this.#sendCompletionRequest( - this.#getFile(file), - { key, uploadId, parts, signal }, - signal, - ).abortOn(signal) - } catch (err) { - if (err?.cause !== pausingUploadReason && err?.name !== 'AbortError') { - // We purposefully don't wait for the promise and ignore its status, - // because we want the error `err` to bubble up ASAP to report it to the - // user. A failure to abort is not that big of a deal anyway. - this.abortFileUpload(file) - } - throw err - } - } - - restoreUploadFile(file: UppyFile, uploadIdAndKey: UploadResult): void { - if (file.data == null) throw new Error('File data is empty') - this.#cache.set(file.data, uploadIdAndKey) - } - - async resumeUploadFile( - file: UppyFile, - chunks: Array, - signal: AbortSignal, - ): Promise { - throwIfAborted(signal) - if ( - chunks.length === 1 && - chunks[0] != null && - !chunks[0].shouldUseMultipart - ) { - return this.#nonMultipartUpload(file, chunks[0], signal) - } - const { uploadId, key } = await this.getUploadId(file, signal) - throwIfAborted(signal) - const alreadyUploadedParts = await this.#listParts( - this.#getFile(file), - { uploadId, key, signal }, - signal, - ).abortOn(signal) - throwIfAborted(signal) - const parts = await Promise.all( - chunks.map((chunk, i) => { - const partNumber = i + 1 - const alreadyUploadedInfo = alreadyUploadedParts.find( - ({ PartNumber }) => PartNumber === partNumber, - ) - if (alreadyUploadedInfo == null) { - return this.uploadChunk(file, partNumber, chunk!, signal) - } - // Already uploaded chunks are set to null. If we are restoring the upload, we need to mark it as already uploaded. - chunk?.setAsUploaded?.() - return { PartNumber: partNumber, ETag: alreadyUploadedInfo.ETag } - }), - ) - throwIfAborted(signal) - return this.#sendCompletionRequest( - this.#getFile(file), - { key, uploadId, parts, signal }, - signal, - ).abortOn(signal) - } - - async uploadChunk( - file: UppyFile, - partNumber: number, - chunk: Chunk, - signal: AbortSignal, - ): Promise { - throwIfAborted(signal) - const { uploadId, key } = await this.getUploadId(file, signal) - - const signatureRetryIterator = this.#retryDelays.values() - const chunkRetryIterator = this.#retryDelays.values() - const shouldRetrySignature = () => { - const next = signatureRetryIterator.next() - if (next == null || next.done) { - return null - } - return next.value - } - - for (;;) { - throwIfAborted(signal) - const chunkData = chunk.getData() - const { onProgress, onComplete } = chunk - let signature: AwsS3UploadParameters - - try { - signature = await this.#fetchSignature(this.#getFile(file), { - // Always defined for multipart uploads - uploadId: uploadId!, - key, - partNumber, - body: chunkData, - signal, - }).abortOn(signal) - } catch (err) { - const timeout = shouldRetrySignature() - if (timeout == null || signal.aborted) { - throw err - } - await new Promise((resolve) => setTimeout(resolve, timeout)) - continue - } - - throwIfAborted(signal) - try { - return { - PartNumber: partNumber, - ...(await this.#uploadPartBytes({ - signature, - body: chunkData, - size: chunkData.size, - onProgress, - onComplete, - signal, - }).abortOn(signal)), - } - } catch (err) { - if (!(await this.#shouldRetry(err, chunkRetryIterator))) throw err - } - } - } -} diff --git a/packages/@uppy/aws-s3/src/MultipartUploader.ts b/packages/@uppy/aws-s3/src/MultipartUploader.ts deleted file mode 100644 index afd050d79..000000000 --- a/packages/@uppy/aws-s3/src/MultipartUploader.ts +++ /dev/null @@ -1,265 +0,0 @@ -import type { Uppy } from '@uppy/core' -import type { Body, Meta, UppyFile } from '@uppy/core/utils' -import { AbortController } from '@uppy/core/utils' -import type { HTTPCommunicationQueue } from './HTTPCommunicationQueue.js' - -const MB = 1024 * 1024 - -interface MultipartUploaderOptions { - getChunkSize?: (file: { size: number }) => number - onProgress?: (bytesUploaded: number, bytesTotal: number) => void - onPartComplete?: (part: { PartNumber: number; ETag: string }) => void - shouldUseMultipart?: boolean | ((file: UppyFile) => boolean) - onSuccess?: (result: B) => void - onError?: (err: unknown) => void - companionComm: HTTPCommunicationQueue - file: UppyFile - log: Uppy['log'] - - uploadId?: string - key: string -} - -const defaultOptions = { - getChunkSize(file: { size: number }) { - return Math.ceil(file.size / 10000) - }, - onProgress() {}, - onPartComplete() {}, - onSuccess() {}, - onError(err: unknown) { - throw err - }, -} satisfies Partial> - -export interface Chunk { - getData: () => Blob - onProgress: (ev: ProgressEvent) => void - onComplete: (etag: string) => void - shouldUseMultipart: boolean - setAsUploaded?: () => void -} - -function ensureInt(value: T): T extends number | string ? number : never { - if (typeof value === 'string') { - // @ts-expect-error TS is not able to recognize it's fine. - return parseInt(value, 10) - } - if (typeof value === 'number') { - // @ts-expect-error TS is not able to recognize it's fine. - return value - } - throw new TypeError('Expected a number') -} - -export const pausingUploadReason = Symbol('pausing upload, not an actual error') - -/** - * A MultipartUploader instance is used per file upload to determine whether a - * upload should be done as multipart or as a regular S3 upload - * (based on the user-provided `shouldUseMultipart` option value) and to manage - * the chunk splitting. - */ -class MultipartUploader { - options: MultipartUploaderOptions & - Required, keyof typeof defaultOptions>> - - #abortController = new AbortController() - - #chunks: Array = [] - - #chunkState: { uploaded: number; etag?: string; done?: boolean }[] = [] - - /** - * The (un-chunked) data to upload. - */ - #data: Blob - - #file: UppyFile - - #uploadHasStarted = false - - #onError: (err: unknown) => void - - #onSuccess: (result: B) => void - - #shouldUseMultipart: MultipartUploaderOptions['shouldUseMultipart'] - - #isRestoring: boolean - - #onReject = (err: unknown) => - (err as any)?.cause === pausingUploadReason ? null : this.#onError(err) - - #maxMultipartParts = 10_000 - - #minPartSize = 5 * MB - - constructor(data: Blob, options: MultipartUploaderOptions) { - this.options = { - ...defaultOptions, - ...options, - } - // Use default `getChunkSize` if it was null or something - this.options.getChunkSize ??= defaultOptions.getChunkSize - - this.#data = data - this.#file = options.file - this.#onSuccess = this.options.onSuccess - this.#onError = this.options.onError - this.#shouldUseMultipart = this.options.shouldUseMultipart - - // When we are restoring an upload, we already have an UploadId and a Key. Otherwise - // we need to call `createMultipartUpload` to get an `uploadId` and a `key`. - // Non-multipart uploads are not restorable. - this.#isRestoring = (options.uploadId && options.key) as any as boolean - - this.#initChunks() - } - - // initChunks checks the user preference for using multipart uploads (opts.shouldUseMultipart) - // and calculates the optimal part size. When using multipart part uploads every part except for the last has - // to be at least 5 MB and there can be no more than 10K parts. - // This means we sometimes need to change the preferred part size from the user in order to meet these requirements. - #initChunks() { - const fileSize = this.#data.size - const shouldUseMultipart = - typeof this.#shouldUseMultipart === 'function' - ? this.#shouldUseMultipart(this.#file) - : Boolean(this.#shouldUseMultipart) - - if (shouldUseMultipart && fileSize > this.#minPartSize) { - // At least 5MB per request: - let chunkSize = Math.max( - this.options.getChunkSize(this.#data) as number, // Math.max can take undefined but TS does not think so - this.#minPartSize, - ) - let arraySize = Math.floor(fileSize / chunkSize) - - // At most 10k requests per file: - if (arraySize > this.#maxMultipartParts) { - arraySize = this.#maxMultipartParts - chunkSize = fileSize / this.#maxMultipartParts - } - this.#chunks = Array(arraySize) - - for (let offset = 0, j = 0; offset < fileSize; offset += chunkSize, j++) { - const end = Math.min(fileSize, offset + chunkSize) - - // Defer data fetching/slicing until we actually need the data, because it's slow if we have a lot of files - const getData = () => { - const i2 = offset - return this.#data.slice(i2, end) - } - - this.#chunks[j] = { - getData, - onProgress: this.#onPartProgress(j), - onComplete: this.#onPartComplete(j), - shouldUseMultipart, - } - if (this.#isRestoring) { - const size = - offset + chunkSize > fileSize ? fileSize - offset : chunkSize - // setAsUploaded is called by listPart, to keep up-to-date the - // quantity of data that is left to actually upload. - this.#chunks[j]!.setAsUploaded = () => { - this.#chunks[j] = null - this.#chunkState[j].uploaded = size - } - } - } - } else { - this.#chunks = [ - { - getData: () => this.#data, - onProgress: this.#onPartProgress(0), - onComplete: this.#onPartComplete(0), - shouldUseMultipart, - }, - ] - } - - this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 })) - } - - #createUpload() { - this.options.companionComm - .uploadFile( - this.#file, - this.#chunks as Chunk[], - this.#abortController.signal, - ) - .then(this.#onSuccess, this.#onReject) - this.#uploadHasStarted = true - } - - #resumeUpload() { - this.options.companionComm - .resumeUploadFile(this.#file, this.#chunks, this.#abortController.signal) - .then(this.#onSuccess, this.#onReject) - } - - #onPartProgress = (index: number) => (ev: ProgressEvent) => { - if (!ev.lengthComputable) return - - this.#chunkState[index].uploaded = ensureInt(ev.loaded) - - const totalUploaded = this.#chunkState.reduce((n, c) => n + c.uploaded, 0) - this.options.onProgress(totalUploaded, this.#data.size) - } - - #onPartComplete = (index: number) => (etag: string) => { - // This avoids the net::ERR_OUT_OF_MEMORY in Chromium Browsers. - this.#chunks[index] = null - this.#chunkState[index].etag = etag - this.#chunkState[index].done = true - - const part = { - PartNumber: index + 1, - ETag: etag, - } - this.options.onPartComplete(part) - } - - #abortUpload() { - this.#abortController.abort() - this.options.companionComm - .abortFileUpload(this.#file) - .catch((err: unknown) => this.options.log(err as Error)) - } - - start(): void { - if (this.#uploadHasStarted) { - if (!this.#abortController.signal.aborted) - this.#abortController.abort(pausingUploadReason) - this.#abortController = new AbortController() - this.#resumeUpload() - } else if (this.#isRestoring) { - this.options.companionComm.restoreUploadFile(this.#file, { - uploadId: this.options.uploadId, - key: this.options.key, - }) - this.#resumeUpload() - } else { - this.#createUpload() - } - } - - pause(): void { - this.#abortController.abort(pausingUploadReason) - // Swap it out for a new controller, because this instance may be resumed later. - this.#abortController = new AbortController() - } - - abort(opts?: { really?: boolean }): void { - if (opts?.really) this.#abortUpload() - else this.pause() - } - - // biome-ignore lint/correctness/noUnusedPrivateClassMembers: accessed via Symbol in tests - private [Symbol.for('uppy test: getChunkState')]() { - return this.#chunkState - } -} - -export default MultipartUploader diff --git a/packages/@uppy/aws-s3/src/S3Uploader.ts b/packages/@uppy/aws-s3/src/S3Uploader.ts index ce493d226..bc0818a83 100644 --- a/packages/@uppy/aws-s3/src/S3Uploader.ts +++ b/packages/@uppy/aws-s3/src/S3Uploader.ts @@ -1,5 +1,5 @@ import { EventManager, type Uppy } from '@uppy/core' -import type { Body, LocalUppyFile, Meta } from '@uppy/utils' +import type { Body, LocalUppyFile, Meta } from '@uppy/core/utils' import type S3Client from './s3-client/S3Client.js' /** Persisted S3 multipart state for Golden Retriever resume support */ @@ -8,7 +8,7 @@ interface S3MultipartState { key: string } -declare module '@uppy/utils' { +declare module '@uppy/core/utils' { export interface LocalUppyFile { s3Multipart?: S3MultipartState } diff --git a/packages/@uppy/aws-s3/src/index.ts b/packages/@uppy/aws-s3/src/index.ts index 205decbd7..11833ead6 100644 --- a/packages/@uppy/aws-s3/src/index.ts +++ b/packages/@uppy/aws-s3/src/index.ts @@ -1,4 +1,4 @@ -import type { RequestClient } from '@uppy/companion-client' +import type { RequestClient } from '@uppy/core/companion-client' import { BasePlugin, type DefinePluginOpts, @@ -11,13 +11,13 @@ import type { Meta, RemoteUppyFile, UppyFile, -} from '@uppy/utils' +} from '@uppy/core/utils' import { filterFilesToEmitUploadStarted, filterFilesToUpload, getAllowedMetaFields, TaskQueue, -} from '@uppy/utils' +} from '@uppy/core/utils' import packageJson from '../package.json' with { type: 'json' } import S3Uploader, { type UploadResult } from './S3Uploader.js' import S3Companion from './s3-client/CompanionS3.js' diff --git a/packages/@uppy/aws-s3/src/s3-client/S3Client.ts b/packages/@uppy/aws-s3/src/s3-client/S3Client.ts index eab0d9b55..52465861a 100644 --- a/packages/@uppy/aws-s3/src/s3-client/S3Client.ts +++ b/packages/@uppy/aws-s3/src/s3-client/S3Client.ts @@ -1,4 +1,4 @@ -import { fetcher } from '@uppy/utils' +import { fetcher } from '@uppy/core/utils' import * as C from './consts.js' import type * as IT from './types.js' diff --git a/packages/@uppy/aws-s3/src/s3-client/S3mini.ts b/packages/@uppy/aws-s3/src/s3-client/S3mini.ts index 26cf3a61d..6fe8b4a23 100644 --- a/packages/@uppy/aws-s3/src/s3-client/S3mini.ts +++ b/packages/@uppy/aws-s3/src/s3-client/S3mini.ts @@ -280,7 +280,7 @@ class S3mini extends S3Client { } /** - * Core XHR upload implementation using @uppy/utils/fetcher. + * Core XHR upload implementation using @uppy/core/utils fetcher. * * Features: * - Automatic retry with exponential backoff (3 attempts)