mirror of
https://github.com/transloadit/uppy.git
synced 2026-07-17 16:50:22 +00:00
update imports after merge remove stale files
This commit is contained in:
prakash
parent
e51c1ad99b
commit
0c7f02aef2
6 changed files with 7 additions and 716 deletions
|
|
@ -1,444 +0,0 @@
|
|||
import type { Body, Meta, UppyFile } from '@uppy/core'
|
||||
import type {
|
||||
RateLimitedQueue,
|
||||
WrapPromiseFunctionType,
|
||||
} from '@uppy/core/utils'
|
||||
import type AwsS3Multipart from './index.js'
|
||||
import type {
|
||||
AwsS3MultipartOptions,
|
||||
AwsS3UploadParameters,
|
||||
uploadPartBytes,
|
||||
} from './index.js'
|
||||
import { type Chunk, pausingUploadReason } from './MultipartUploader.js'
|
||||
import type { UploadPartBytesResult, UploadResult } from './utils.js'
|
||||
import { throwIfAborted } from './utils.js'
|
||||
|
||||
function removeMetadataFromURL(urlString: string) {
|
||||
const urlObject = new URL(urlString)
|
||||
urlObject.search = ''
|
||||
urlObject.hash = ''
|
||||
return urlObject.href
|
||||
}
|
||||
|
||||
export class HTTPCommunicationQueue<M extends Meta, B extends Body> {
|
||||
#abortMultipartUpload!: WrapPromiseFunctionType<
|
||||
AwsS3Multipart<M, B>['abortMultipartUpload']
|
||||
>
|
||||
|
||||
#cache = new WeakMap()
|
||||
|
||||
#createMultipartUpload!: WrapPromiseFunctionType<
|
||||
AwsS3Multipart<M, B>['createMultipartUpload']
|
||||
>
|
||||
|
||||
#fetchSignature!: WrapPromiseFunctionType<AwsS3Multipart<M, B>['signPart']>
|
||||
|
||||
#getUploadParameters!: WrapPromiseFunctionType<
|
||||
AwsS3Multipart<M, B>['getUploadParameters']
|
||||
>
|
||||
|
||||
#listParts!: WrapPromiseFunctionType<AwsS3Multipart<M, B>['listParts']>
|
||||
|
||||
#previousRetryDelay!: number
|
||||
|
||||
#requests
|
||||
|
||||
#retryDelays!: { values: () => Iterator<number> }
|
||||
|
||||
#sendCompletionRequest!: WrapPromiseFunctionType<
|
||||
AwsS3Multipart<M, B>['completeMultipartUpload']
|
||||
>
|
||||
|
||||
#setS3MultipartState
|
||||
|
||||
#uploadPartBytes!: WrapPromiseFunctionType<uploadPartBytes>
|
||||
|
||||
#getFile
|
||||
|
||||
constructor(
|
||||
requests: RateLimitedQueue,
|
||||
options: AwsS3MultipartOptions<M, B>,
|
||||
setS3MultipartState: (file: UppyFile<M, B>, result: UploadResult) => void,
|
||||
getFile: (file: UppyFile<M, B>) => UppyFile<M, B>,
|
||||
) {
|
||||
this.#requests = requests
|
||||
this.#setS3MultipartState = setS3MultipartState
|
||||
this.#getFile = getFile
|
||||
this.setOptions(options)
|
||||
}
|
||||
|
||||
setOptions(options: Partial<AwsS3MultipartOptions<M, B>>): void {
|
||||
const requests = this.#requests
|
||||
|
||||
if ('abortMultipartUpload' in options) {
|
||||
this.#abortMultipartUpload = requests.wrapPromiseFunction(
|
||||
options.abortMultipartUpload as any,
|
||||
{ priority: 1 },
|
||||
)
|
||||
}
|
||||
if ('createMultipartUpload' in options) {
|
||||
this.#createMultipartUpload = requests.wrapPromiseFunction(
|
||||
options.createMultipartUpload as any,
|
||||
{ priority: -1 },
|
||||
)
|
||||
}
|
||||
if ('signPart' in options) {
|
||||
this.#fetchSignature = requests.wrapPromiseFunction(
|
||||
options.signPart as any,
|
||||
)
|
||||
}
|
||||
if ('listParts' in options) {
|
||||
this.#listParts = requests.wrapPromiseFunction(options.listParts as any)
|
||||
}
|
||||
if ('completeMultipartUpload' in options) {
|
||||
this.#sendCompletionRequest = requests.wrapPromiseFunction(
|
||||
options.completeMultipartUpload as any,
|
||||
{ priority: 1 },
|
||||
)
|
||||
}
|
||||
if ('retryDelays' in options) {
|
||||
this.#retryDelays = options.retryDelays ?? []
|
||||
}
|
||||
if ('uploadPartBytes' in options) {
|
||||
this.#uploadPartBytes = requests.wrapPromiseFunction(
|
||||
options.uploadPartBytes as any,
|
||||
{ priority: Infinity },
|
||||
)
|
||||
}
|
||||
if ('getUploadParameters' in options) {
|
||||
this.#getUploadParameters = requests.wrapPromiseFunction(
|
||||
options.getUploadParameters as any,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async #shouldRetry(err: any, retryDelayIterator: Iterator<number>) {
|
||||
const requests = this.#requests
|
||||
const status = err?.source?.status
|
||||
|
||||
// TODO: this retry logic is taken out of Tus. We should have a centralized place for retrying,
|
||||
// perhaps the rate limited queue, and dedupe all plugins with that.
|
||||
if (status == null) {
|
||||
return false
|
||||
}
|
||||
if (status === 403 && err.message === 'Request has expired') {
|
||||
if (!requests.isPaused) {
|
||||
// We don't want to exhaust the retryDelayIterator as long as there are
|
||||
// more than one request in parallel, to give slower connection a chance
|
||||
// to catch up with the expiry set in Companion.
|
||||
if (requests.limit === 1 || this.#previousRetryDelay == null) {
|
||||
const next = retryDelayIterator.next()
|
||||
if (next == null || next.done) {
|
||||
return false
|
||||
}
|
||||
// If there are more than 1 request done in parallel, the RLQ limit is
|
||||
// decreased and the failed request is requeued after waiting for a bit.
|
||||
// If there is only one request in parallel, the limit can't be
|
||||
// decreased, so we iterate over `retryDelayIterator` as we do for
|
||||
// other failures.
|
||||
// `#previousRetryDelay` caches the value so we can re-use it next time.
|
||||
this.#previousRetryDelay = next.value
|
||||
}
|
||||
// No need to stop the other requests, we just want to lower the limit.
|
||||
requests.rateLimit(0)
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, this.#previousRetryDelay),
|
||||
)
|
||||
}
|
||||
} else if (status === 429) {
|
||||
// HTTP 429 Too Many Requests => to avoid the whole download to fail, pause all requests.
|
||||
if (!requests.isPaused) {
|
||||
const next = retryDelayIterator.next()
|
||||
if (next == null || next.done) {
|
||||
return false
|
||||
}
|
||||
requests.rateLimit(next.value)
|
||||
}
|
||||
} else if (status > 400 && status < 500 && status !== 409) {
|
||||
// HTTP 4xx, the server won't send anything, it's doesn't make sense to retry
|
||||
return false
|
||||
} else if (typeof navigator !== 'undefined' && navigator.onLine === false) {
|
||||
// The navigator is offline, let's wait for it to come back online.
|
||||
if (!requests.isPaused) {
|
||||
requests.pause()
|
||||
window.addEventListener(
|
||||
'online',
|
||||
() => {
|
||||
requests.resume()
|
||||
},
|
||||
{ once: true },
|
||||
)
|
||||
}
|
||||
} else {
|
||||
// Other error code means the request can be retried later.
|
||||
const next = retryDelayIterator.next()
|
||||
if (next == null || next.done) {
|
||||
return false
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, next.value))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
async getUploadId(
|
||||
file: UppyFile<M, B>,
|
||||
signal: AbortSignal,
|
||||
): Promise<UploadResult> {
|
||||
let cachedResult: Promise<UploadResult> | UploadResult | undefined
|
||||
// As the cache is updated asynchronously, there could be a race condition
|
||||
// where we just miss a new result so we loop here until we get nothing back,
|
||||
// at which point it's out turn to create a new cache entry.
|
||||
for (;;) {
|
||||
if (file.data == null) throw new Error('File data is empty')
|
||||
cachedResult = this.#cache.get(file.data)
|
||||
if (cachedResult == null) break
|
||||
try {
|
||||
return await cachedResult
|
||||
} catch {
|
||||
// In case of failure, we want to ignore the cached error.
|
||||
// At this point, either there's a new cached value, or we'll exit the loop a create a new one.
|
||||
}
|
||||
}
|
||||
|
||||
const promise = this.#createMultipartUpload(this.#getFile(file), signal)
|
||||
|
||||
const abortPromise = () => {
|
||||
if (file.data == null) throw new Error('File data is empty')
|
||||
promise.abort(signal.reason)
|
||||
this.#cache.delete(file.data)
|
||||
}
|
||||
signal.addEventListener('abort', abortPromise, { once: true })
|
||||
this.#cache.set(file.data, promise)
|
||||
promise.then(
|
||||
async (result) => {
|
||||
signal.removeEventListener('abort', abortPromise)
|
||||
this.#setS3MultipartState(file, result)
|
||||
if (file.data == null) throw new Error('File data is empty')
|
||||
this.#cache.set(file.data, result)
|
||||
},
|
||||
() => {
|
||||
signal.removeEventListener('abort', abortPromise)
|
||||
if (file.data == null) throw new Error('File data is empty')
|
||||
this.#cache.delete(file.data)
|
||||
},
|
||||
)
|
||||
|
||||
return promise
|
||||
}
|
||||
|
||||
async abortFileUpload(file: UppyFile<M, B>): Promise<void> {
|
||||
if (file.data == null) throw new Error('File data is empty')
|
||||
const result = this.#cache.get(file.data)
|
||||
if (result == null) {
|
||||
// If the createMultipartUpload request never was made, we don't
|
||||
// need to send the abortMultipartUpload request.
|
||||
return
|
||||
}
|
||||
// Remove the cache entry right away for follow-up requests do not try to
|
||||
// use the soon-to-be aborted cached values.
|
||||
this.#cache.delete(file.data)
|
||||
this.#setS3MultipartState(file, Object.create(null))
|
||||
let awaitedResult: UploadResult
|
||||
try {
|
||||
awaitedResult = await result
|
||||
} catch {
|
||||
// If the cached result rejects, there's nothing to abort.
|
||||
return
|
||||
}
|
||||
await this.#abortMultipartUpload(this.#getFile(file), awaitedResult)
|
||||
}
|
||||
|
||||
async #nonMultipartUpload(
|
||||
file: UppyFile<M, B>,
|
||||
chunk: Chunk,
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
const {
|
||||
method = 'POST',
|
||||
url,
|
||||
fields,
|
||||
headers,
|
||||
} = await this.#getUploadParameters(this.#getFile(file), {
|
||||
signal,
|
||||
}).abortOn(signal)
|
||||
|
||||
let body: FormData | Blob
|
||||
const data = chunk.getData()
|
||||
if (method.toUpperCase() === 'POST') {
|
||||
const formData = new FormData()
|
||||
Object.entries(fields!).forEach(([key, value]) => {
|
||||
formData.set(key, value)
|
||||
})
|
||||
formData.set('file', data)
|
||||
body = formData
|
||||
} else {
|
||||
body = data
|
||||
}
|
||||
|
||||
const { onProgress, onComplete } = chunk
|
||||
|
||||
const result = (await this.#uploadPartBytes({
|
||||
signature: { url, headers, method } as any,
|
||||
body,
|
||||
size: data.size,
|
||||
onProgress,
|
||||
onComplete,
|
||||
signal,
|
||||
}).abortOn(signal)) as unknown as B // todo this doesn't make sense
|
||||
|
||||
// Note: `fields.key` is not returned by old Companion versions.
|
||||
// See https://github.com/transloadit/uppy/pull/5602
|
||||
const key = fields?.key
|
||||
this.#setS3MultipartState(file, { key: key! })
|
||||
|
||||
return {
|
||||
...result,
|
||||
location:
|
||||
(result.location as string | undefined) ?? removeMetadataFromURL(url),
|
||||
bucket: fields?.bucket,
|
||||
key,
|
||||
}
|
||||
}
|
||||
|
||||
async uploadFile(
|
||||
file: UppyFile<M, B>,
|
||||
chunks: Chunk[],
|
||||
signal: AbortSignal,
|
||||
): Promise<B & Partial<UploadPartBytesResult>> {
|
||||
throwIfAborted(signal)
|
||||
if (chunks.length === 1 && !chunks[0].shouldUseMultipart) {
|
||||
return this.#nonMultipartUpload(file, chunks[0], signal)
|
||||
}
|
||||
const { uploadId, key } = await this.getUploadId(file, signal)
|
||||
throwIfAborted(signal)
|
||||
try {
|
||||
const parts = await Promise.all(
|
||||
chunks.map((chunk, i) => this.uploadChunk(file, i + 1, chunk, signal)),
|
||||
)
|
||||
throwIfAborted(signal)
|
||||
return await this.#sendCompletionRequest(
|
||||
this.#getFile(file),
|
||||
{ key, uploadId, parts, signal },
|
||||
signal,
|
||||
).abortOn(signal)
|
||||
} catch (err) {
|
||||
if (err?.cause !== pausingUploadReason && err?.name !== 'AbortError') {
|
||||
// We purposefully don't wait for the promise and ignore its status,
|
||||
// because we want the error `err` to bubble up ASAP to report it to the
|
||||
// user. A failure to abort is not that big of a deal anyway.
|
||||
this.abortFileUpload(file)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
restoreUploadFile(file: UppyFile<M, B>, uploadIdAndKey: UploadResult): void {
|
||||
if (file.data == null) throw new Error('File data is empty')
|
||||
this.#cache.set(file.data, uploadIdAndKey)
|
||||
}
|
||||
|
||||
async resumeUploadFile(
|
||||
file: UppyFile<M, B>,
|
||||
chunks: Array<Chunk | null>,
|
||||
signal: AbortSignal,
|
||||
): Promise<B> {
|
||||
throwIfAborted(signal)
|
||||
if (
|
||||
chunks.length === 1 &&
|
||||
chunks[0] != null &&
|
||||
!chunks[0].shouldUseMultipart
|
||||
) {
|
||||
return this.#nonMultipartUpload(file, chunks[0], signal)
|
||||
}
|
||||
const { uploadId, key } = await this.getUploadId(file, signal)
|
||||
throwIfAborted(signal)
|
||||
const alreadyUploadedParts = await this.#listParts(
|
||||
this.#getFile(file),
|
||||
{ uploadId, key, signal },
|
||||
signal,
|
||||
).abortOn(signal)
|
||||
throwIfAborted(signal)
|
||||
const parts = await Promise.all(
|
||||
chunks.map((chunk, i) => {
|
||||
const partNumber = i + 1
|
||||
const alreadyUploadedInfo = alreadyUploadedParts.find(
|
||||
({ PartNumber }) => PartNumber === partNumber,
|
||||
)
|
||||
if (alreadyUploadedInfo == null) {
|
||||
return this.uploadChunk(file, partNumber, chunk!, signal)
|
||||
}
|
||||
// Already uploaded chunks are set to null. If we are restoring the upload, we need to mark it as already uploaded.
|
||||
chunk?.setAsUploaded?.()
|
||||
return { PartNumber: partNumber, ETag: alreadyUploadedInfo.ETag }
|
||||
}),
|
||||
)
|
||||
throwIfAborted(signal)
|
||||
return this.#sendCompletionRequest(
|
||||
this.#getFile(file),
|
||||
{ key, uploadId, parts, signal },
|
||||
signal,
|
||||
).abortOn(signal)
|
||||
}
|
||||
|
||||
async uploadChunk(
|
||||
file: UppyFile<M, B>,
|
||||
partNumber: number,
|
||||
chunk: Chunk,
|
||||
signal: AbortSignal,
|
||||
): Promise<UploadPartBytesResult & { PartNumber: number }> {
|
||||
throwIfAborted(signal)
|
||||
const { uploadId, key } = await this.getUploadId(file, signal)
|
||||
|
||||
const signatureRetryIterator = this.#retryDelays.values()
|
||||
const chunkRetryIterator = this.#retryDelays.values()
|
||||
const shouldRetrySignature = () => {
|
||||
const next = signatureRetryIterator.next()
|
||||
if (next == null || next.done) {
|
||||
return null
|
||||
}
|
||||
return next.value
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
throwIfAborted(signal)
|
||||
const chunkData = chunk.getData()
|
||||
const { onProgress, onComplete } = chunk
|
||||
let signature: AwsS3UploadParameters
|
||||
|
||||
try {
|
||||
signature = await this.#fetchSignature(this.#getFile(file), {
|
||||
// Always defined for multipart uploads
|
||||
uploadId: uploadId!,
|
||||
key,
|
||||
partNumber,
|
||||
body: chunkData,
|
||||
signal,
|
||||
}).abortOn(signal)
|
||||
} catch (err) {
|
||||
const timeout = shouldRetrySignature()
|
||||
if (timeout == null || signal.aborted) {
|
||||
throw err
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, timeout))
|
||||
continue
|
||||
}
|
||||
|
||||
throwIfAborted(signal)
|
||||
try {
|
||||
return {
|
||||
PartNumber: partNumber,
|
||||
...(await this.#uploadPartBytes({
|
||||
signature,
|
||||
body: chunkData,
|
||||
size: chunkData.size,
|
||||
onProgress,
|
||||
onComplete,
|
||||
signal,
|
||||
}).abortOn(signal)),
|
||||
}
|
||||
} catch (err) {
|
||||
if (!(await this.#shouldRetry(err, chunkRetryIterator))) throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,265 +0,0 @@
|
|||
import type { Uppy } from '@uppy/core'
|
||||
import type { Body, Meta, UppyFile } from '@uppy/core/utils'
|
||||
import { AbortController } from '@uppy/core/utils'
|
||||
import type { HTTPCommunicationQueue } from './HTTPCommunicationQueue.js'
|
||||
|
||||
const MB = 1024 * 1024
|
||||
|
||||
interface MultipartUploaderOptions<M extends Meta, B extends Body> {
|
||||
getChunkSize?: (file: { size: number }) => number
|
||||
onProgress?: (bytesUploaded: number, bytesTotal: number) => void
|
||||
onPartComplete?: (part: { PartNumber: number; ETag: string }) => void
|
||||
shouldUseMultipart?: boolean | ((file: UppyFile<M, B>) => boolean)
|
||||
onSuccess?: (result: B) => void
|
||||
onError?: (err: unknown) => void
|
||||
companionComm: HTTPCommunicationQueue<M, B>
|
||||
file: UppyFile<M, B>
|
||||
log: Uppy<M, B>['log']
|
||||
|
||||
uploadId?: string
|
||||
key: string
|
||||
}
|
||||
|
||||
const defaultOptions = {
|
||||
getChunkSize(file: { size: number }) {
|
||||
return Math.ceil(file.size / 10000)
|
||||
},
|
||||
onProgress() {},
|
||||
onPartComplete() {},
|
||||
onSuccess() {},
|
||||
onError(err: unknown) {
|
||||
throw err
|
||||
},
|
||||
} satisfies Partial<MultipartUploaderOptions<any, any>>
|
||||
|
||||
export interface Chunk {
|
||||
getData: () => Blob
|
||||
onProgress: (ev: ProgressEvent) => void
|
||||
onComplete: (etag: string) => void
|
||||
shouldUseMultipart: boolean
|
||||
setAsUploaded?: () => void
|
||||
}
|
||||
|
||||
function ensureInt<T>(value: T): T extends number | string ? number : never {
|
||||
if (typeof value === 'string') {
|
||||
// @ts-expect-error TS is not able to recognize it's fine.
|
||||
return parseInt(value, 10)
|
||||
}
|
||||
if (typeof value === 'number') {
|
||||
// @ts-expect-error TS is not able to recognize it's fine.
|
||||
return value
|
||||
}
|
||||
throw new TypeError('Expected a number')
|
||||
}
|
||||
|
||||
export const pausingUploadReason = Symbol('pausing upload, not an actual error')
|
||||
|
||||
/**
|
||||
* A MultipartUploader instance is used per file upload to determine whether a
|
||||
* upload should be done as multipart or as a regular S3 upload
|
||||
* (based on the user-provided `shouldUseMultipart` option value) and to manage
|
||||
* the chunk splitting.
|
||||
*/
|
||||
class MultipartUploader<M extends Meta, B extends Body> {
|
||||
options: MultipartUploaderOptions<M, B> &
|
||||
Required<Pick<MultipartUploaderOptions<M, B>, keyof typeof defaultOptions>>
|
||||
|
||||
#abortController = new AbortController()
|
||||
|
||||
#chunks: Array<Chunk | null> = []
|
||||
|
||||
#chunkState: { uploaded: number; etag?: string; done?: boolean }[] = []
|
||||
|
||||
/**
|
||||
* The (un-chunked) data to upload.
|
||||
*/
|
||||
#data: Blob
|
||||
|
||||
#file: UppyFile<M, B>
|
||||
|
||||
#uploadHasStarted = false
|
||||
|
||||
#onError: (err: unknown) => void
|
||||
|
||||
#onSuccess: (result: B) => void
|
||||
|
||||
#shouldUseMultipart: MultipartUploaderOptions<M, B>['shouldUseMultipart']
|
||||
|
||||
#isRestoring: boolean
|
||||
|
||||
#onReject = (err: unknown) =>
|
||||
(err as any)?.cause === pausingUploadReason ? null : this.#onError(err)
|
||||
|
||||
#maxMultipartParts = 10_000
|
||||
|
||||
#minPartSize = 5 * MB
|
||||
|
||||
constructor(data: Blob, options: MultipartUploaderOptions<M, B>) {
|
||||
this.options = {
|
||||
...defaultOptions,
|
||||
...options,
|
||||
}
|
||||
// Use default `getChunkSize` if it was null or something
|
||||
this.options.getChunkSize ??= defaultOptions.getChunkSize
|
||||
|
||||
this.#data = data
|
||||
this.#file = options.file
|
||||
this.#onSuccess = this.options.onSuccess
|
||||
this.#onError = this.options.onError
|
||||
this.#shouldUseMultipart = this.options.shouldUseMultipart
|
||||
|
||||
// When we are restoring an upload, we already have an UploadId and a Key. Otherwise
|
||||
// we need to call `createMultipartUpload` to get an `uploadId` and a `key`.
|
||||
// Non-multipart uploads are not restorable.
|
||||
this.#isRestoring = (options.uploadId && options.key) as any as boolean
|
||||
|
||||
this.#initChunks()
|
||||
}
|
||||
|
||||
// initChunks checks the user preference for using multipart uploads (opts.shouldUseMultipart)
|
||||
// and calculates the optimal part size. When using multipart part uploads every part except for the last has
|
||||
// to be at least 5 MB and there can be no more than 10K parts.
|
||||
// This means we sometimes need to change the preferred part size from the user in order to meet these requirements.
|
||||
#initChunks() {
|
||||
const fileSize = this.#data.size
|
||||
const shouldUseMultipart =
|
||||
typeof this.#shouldUseMultipart === 'function'
|
||||
? this.#shouldUseMultipart(this.#file)
|
||||
: Boolean(this.#shouldUseMultipart)
|
||||
|
||||
if (shouldUseMultipart && fileSize > this.#minPartSize) {
|
||||
// At least 5MB per request:
|
||||
let chunkSize = Math.max(
|
||||
this.options.getChunkSize(this.#data) as number, // Math.max can take undefined but TS does not think so
|
||||
this.#minPartSize,
|
||||
)
|
||||
let arraySize = Math.floor(fileSize / chunkSize)
|
||||
|
||||
// At most 10k requests per file:
|
||||
if (arraySize > this.#maxMultipartParts) {
|
||||
arraySize = this.#maxMultipartParts
|
||||
chunkSize = fileSize / this.#maxMultipartParts
|
||||
}
|
||||
this.#chunks = Array(arraySize)
|
||||
|
||||
for (let offset = 0, j = 0; offset < fileSize; offset += chunkSize, j++) {
|
||||
const end = Math.min(fileSize, offset + chunkSize)
|
||||
|
||||
// Defer data fetching/slicing until we actually need the data, because it's slow if we have a lot of files
|
||||
const getData = () => {
|
||||
const i2 = offset
|
||||
return this.#data.slice(i2, end)
|
||||
}
|
||||
|
||||
this.#chunks[j] = {
|
||||
getData,
|
||||
onProgress: this.#onPartProgress(j),
|
||||
onComplete: this.#onPartComplete(j),
|
||||
shouldUseMultipart,
|
||||
}
|
||||
if (this.#isRestoring) {
|
||||
const size =
|
||||
offset + chunkSize > fileSize ? fileSize - offset : chunkSize
|
||||
// setAsUploaded is called by listPart, to keep up-to-date the
|
||||
// quantity of data that is left to actually upload.
|
||||
this.#chunks[j]!.setAsUploaded = () => {
|
||||
this.#chunks[j] = null
|
||||
this.#chunkState[j].uploaded = size
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.#chunks = [
|
||||
{
|
||||
getData: () => this.#data,
|
||||
onProgress: this.#onPartProgress(0),
|
||||
onComplete: this.#onPartComplete(0),
|
||||
shouldUseMultipart,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
|
||||
}
|
||||
|
||||
#createUpload() {
|
||||
this.options.companionComm
|
||||
.uploadFile(
|
||||
this.#file,
|
||||
this.#chunks as Chunk[],
|
||||
this.#abortController.signal,
|
||||
)
|
||||
.then(this.#onSuccess, this.#onReject)
|
||||
this.#uploadHasStarted = true
|
||||
}
|
||||
|
||||
#resumeUpload() {
|
||||
this.options.companionComm
|
||||
.resumeUploadFile(this.#file, this.#chunks, this.#abortController.signal)
|
||||
.then(this.#onSuccess, this.#onReject)
|
||||
}
|
||||
|
||||
#onPartProgress = (index: number) => (ev: ProgressEvent) => {
|
||||
if (!ev.lengthComputable) return
|
||||
|
||||
this.#chunkState[index].uploaded = ensureInt(ev.loaded)
|
||||
|
||||
const totalUploaded = this.#chunkState.reduce((n, c) => n + c.uploaded, 0)
|
||||
this.options.onProgress(totalUploaded, this.#data.size)
|
||||
}
|
||||
|
||||
#onPartComplete = (index: number) => (etag: string) => {
|
||||
// This avoids the net::ERR_OUT_OF_MEMORY in Chromium Browsers.
|
||||
this.#chunks[index] = null
|
||||
this.#chunkState[index].etag = etag
|
||||
this.#chunkState[index].done = true
|
||||
|
||||
const part = {
|
||||
PartNumber: index + 1,
|
||||
ETag: etag,
|
||||
}
|
||||
this.options.onPartComplete(part)
|
||||
}
|
||||
|
||||
#abortUpload() {
|
||||
this.#abortController.abort()
|
||||
this.options.companionComm
|
||||
.abortFileUpload(this.#file)
|
||||
.catch((err: unknown) => this.options.log(err as Error))
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.#uploadHasStarted) {
|
||||
if (!this.#abortController.signal.aborted)
|
||||
this.#abortController.abort(pausingUploadReason)
|
||||
this.#abortController = new AbortController()
|
||||
this.#resumeUpload()
|
||||
} else if (this.#isRestoring) {
|
||||
this.options.companionComm.restoreUploadFile(this.#file, {
|
||||
uploadId: this.options.uploadId,
|
||||
key: this.options.key,
|
||||
})
|
||||
this.#resumeUpload()
|
||||
} else {
|
||||
this.#createUpload()
|
||||
}
|
||||
}
|
||||
|
||||
pause(): void {
|
||||
this.#abortController.abort(pausingUploadReason)
|
||||
// Swap it out for a new controller, because this instance may be resumed later.
|
||||
this.#abortController = new AbortController()
|
||||
}
|
||||
|
||||
abort(opts?: { really?: boolean }): void {
|
||||
if (opts?.really) this.#abortUpload()
|
||||
else this.pause()
|
||||
}
|
||||
|
||||
// biome-ignore lint/correctness/noUnusedPrivateClassMembers: accessed via Symbol in tests
|
||||
private [Symbol.for('uppy test: getChunkState')]() {
|
||||
return this.#chunkState
|
||||
}
|
||||
}
|
||||
|
||||
export default MultipartUploader
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { EventManager, type Uppy } from '@uppy/core'
|
||||
import type { Body, LocalUppyFile, Meta } from '@uppy/utils'
|
||||
import type { Body, LocalUppyFile, Meta } from '@uppy/core/utils'
|
||||
import type S3Client from './s3-client/S3Client.js'
|
||||
|
||||
/** Persisted S3 multipart state for Golden Retriever resume support */
|
||||
|
|
@ -8,7 +8,7 @@ interface S3MultipartState {
|
|||
key: string
|
||||
}
|
||||
|
||||
declare module '@uppy/utils' {
|
||||
declare module '@uppy/core/utils' {
|
||||
export interface LocalUppyFile<M extends Meta, B extends Body> {
|
||||
s3Multipart?: S3MultipartState
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { RequestClient } from '@uppy/companion-client'
|
||||
import type { RequestClient } from '@uppy/core/companion-client'
|
||||
import {
|
||||
BasePlugin,
|
||||
type DefinePluginOpts,
|
||||
|
|
@ -11,13 +11,13 @@ import type {
|
|||
Meta,
|
||||
RemoteUppyFile,
|
||||
UppyFile,
|
||||
} from '@uppy/utils'
|
||||
} from '@uppy/core/utils'
|
||||
import {
|
||||
filterFilesToEmitUploadStarted,
|
||||
filterFilesToUpload,
|
||||
getAllowedMetaFields,
|
||||
TaskQueue,
|
||||
} from '@uppy/utils'
|
||||
} from '@uppy/core/utils'
|
||||
import packageJson from '../package.json' with { type: 'json' }
|
||||
import S3Uploader, { type UploadResult } from './S3Uploader.js'
|
||||
import S3Companion from './s3-client/CompanionS3.js'
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { fetcher } from '@uppy/utils'
|
||||
import { fetcher } from '@uppy/core/utils'
|
||||
import * as C from './consts.js'
|
||||
import type * as IT from './types.js'
|
||||
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ class S3mini extends S3Client {
|
|||
}
|
||||
|
||||
/**
|
||||
* Core XHR upload implementation using @uppy/utils/fetcher.
|
||||
* Core XHR upload implementation using @uppy/core/utils fetcher.
|
||||
*
|
||||
* Features:
|
||||
* - Automatic retry with exponential backoff (3 attempts)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue