mirror of
https://github.com/transloadit/uppy.git
synced 2026-07-24 02:38:58 +00:00
@uppy/aws-s3-multipart: add support for presigned URL batching (#3056)
* Add first basic test for aws-s3-multipart * Add another test for GH actions run * Add whatwg-fetch to aws-s3-multipart package.json This is to satisfy linter errors, and the other s3 plugin does this as well * Adding more tests * More test progress * Tests working with nock * Add another test, for part uploaded event * Remove spec and refactor to beforeEach * S3 multipart batch presign working...in theory * Working batching and tests * Min needed for presign batch functionality * Fix lint issues * Fix syntax error in companion * Add companion batch presign endpoint * Improve tests * Fix each -> forEach * Remove .prettierrc * Adding docs and allow for headers with batch prepare * Review fixes * Rename prepareUploadPart to prepareUploadParts and use breaking changes to batch presign URLs there. * Get rid of unnecessary batchPartPresign and minNeededForPresignBatch options. Use Math.ceil(limit / 2) for min needed instead. * Fix up tests. * Fix up type tests. * Update documentation. * Review fixes * Change _ private methods to use # syntax * Minor code and docs improvements * Change Promise.resolve() microtasks to async/await
This commit is contained in:
Martin Brennan
GitHub
parent
8fb6078256
commit
d613b849a6
9 changed files with 358 additions and 107 deletions
|
|
@ -26,6 +26,10 @@
|
|||
"@uppy/companion-client": "file:../companion-client",
|
||||
"@uppy/utils": "file:../utils"
|
||||
},
|
||||
"devDependencies": {
|
||||
"whatwg-fetch": "3.6.2",
|
||||
"nock": "^13.1.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@uppy/core": "^1.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ class MultipartUploader {
|
|||
// upload was created already. That also ensures that the sequencing is right
|
||||
// (so the `OP` definitely happens if the upload is created).
|
||||
//
|
||||
// This mostly exists to make `_abortUpload` work well: only sending the abort request if
|
||||
// This mostly exists to make `#abortUpload` work well: only sending the abort request if
|
||||
// the upload was already created, and if the createMultipartUpload request is still in flight,
|
||||
// aborting it immediately after it finishes.
|
||||
this.createdPromise = Promise.reject() // eslint-disable-line prefer-promise-reject-errors
|
||||
|
|
@ -58,8 +58,9 @@ class MultipartUploader {
|
|||
this.partsInProgress = 0
|
||||
this.chunks = null
|
||||
this.chunkState = null
|
||||
this.lockedCandidatesForBatch = []
|
||||
|
||||
this._initChunks()
|
||||
this.#initChunks()
|
||||
|
||||
this.createdPromise.catch(() => {}) // silence uncaught rejection warning
|
||||
}
|
||||
|
|
@ -71,11 +72,11 @@ class MultipartUploader {
|
|||
*
|
||||
* @returns {boolean}
|
||||
*/
|
||||
_aborted () {
|
||||
#aborted () {
|
||||
return this.abortController.signal.aborted
|
||||
}
|
||||
|
||||
_initChunks () {
|
||||
#initChunks () {
|
||||
const chunks = []
|
||||
const desiredChunkSize = this.options.getChunkSize(this.file)
|
||||
// at least 5MB per request, at most 10k requests
|
||||
|
|
@ -100,10 +101,10 @@ class MultipartUploader {
|
|||
}))
|
||||
}
|
||||
|
||||
_createUpload () {
|
||||
#createUpload () {
|
||||
this.createdPromise = Promise.resolve().then(() => this.options.createMultipartUpload())
|
||||
return this.createdPromise.then((result) => {
|
||||
if (this._aborted()) throw createAbortError()
|
||||
if (this.#aborted()) throw createAbortError()
|
||||
|
||||
const valid = typeof result === 'object' && result
|
||||
&& typeof result.uploadId === 'string'
|
||||
|
|
@ -116,18 +117,19 @@ class MultipartUploader {
|
|||
this.uploadId = result.uploadId
|
||||
|
||||
this.options.onStart(result)
|
||||
this._uploadParts()
|
||||
this.#uploadParts()
|
||||
}).catch((err) => {
|
||||
this._onError(err)
|
||||
this.#onError(err)
|
||||
})
|
||||
}
|
||||
|
||||
_resumeUpload () {
|
||||
return Promise.resolve().then(() => this.options.listParts({
|
||||
uploadId: this.uploadId,
|
||||
key: this.key,
|
||||
})).then((parts) => {
|
||||
if (this._aborted()) throw createAbortError()
|
||||
async #resumeUpload () {
|
||||
try {
|
||||
const parts = await this.options.listParts({
|
||||
uploadId: this.uploadId,
|
||||
key: this.key,
|
||||
})
|
||||
if (this.#aborted()) throw createAbortError()
|
||||
|
||||
parts.forEach((part) => {
|
||||
const i = part.PartNumber - 1
|
||||
|
|
@ -146,26 +148,40 @@ class MultipartUploader {
|
|||
})
|
||||
}
|
||||
})
|
||||
this._uploadParts()
|
||||
}).catch((err) => {
|
||||
this._onError(err)
|
||||
})
|
||||
this.#uploadParts()
|
||||
} catch (err) {
|
||||
this.#onError(err)
|
||||
}
|
||||
}
|
||||
|
||||
_uploadParts () {
|
||||
#uploadParts () {
|
||||
if (this.isPaused) return
|
||||
|
||||
const need = this.options.limit - this.partsInProgress
|
||||
if (need === 0) return
|
||||
|
||||
// All parts are uploaded.
|
||||
if (this.chunkState.every((state) => state.done)) {
|
||||
this._completeUpload()
|
||||
this.#completeUpload()
|
||||
return
|
||||
}
|
||||
|
||||
// For a 100MB file, with the default min chunk size of 5MB and a limit of 10:
|
||||
//
|
||||
// Total 20 parts
|
||||
// ---------
|
||||
// Need 1 is 10
|
||||
// Need 2 is 5
|
||||
// Need 3 is 5
|
||||
const need = this.options.limit - this.partsInProgress
|
||||
const completeChunks = this.chunkState.filter((state) => state.done).length
|
||||
const remainingChunks = this.chunks.length - completeChunks
|
||||
let minNeeded = Math.ceil(this.options.limit / 2)
|
||||
if (minNeeded > remainingChunks) {
|
||||
minNeeded = remainingChunks
|
||||
}
|
||||
if (need < minNeeded) return
|
||||
|
||||
const candidates = []
|
||||
for (let i = 0; i < this.chunkState.length; i++) {
|
||||
if (this.lockedCandidatesForBatch.includes(i)) continue
|
||||
const state = this.chunkState[i]
|
||||
if (state.done || state.busy) continue
|
||||
|
||||
|
|
@ -174,18 +190,22 @@ class MultipartUploader {
|
|||
break
|
||||
}
|
||||
}
|
||||
if (candidates.length === 0) return
|
||||
|
||||
candidates.forEach((index) => {
|
||||
this._uploadPartRetryable(index).then(() => {
|
||||
// Continue uploading parts
|
||||
this._uploadParts()
|
||||
}, (err) => {
|
||||
this._onError(err)
|
||||
this.#prepareUploadParts(candidates).then((result) => {
|
||||
candidates.forEach((index) => {
|
||||
const partNumber = index + 1
|
||||
const prePreparedPart = { url: result.presignedUrls[partNumber], headers: result.headers }
|
||||
this.#uploadPartRetryable(index, prePreparedPart).then(() => {
|
||||
this.#uploadParts()
|
||||
}, (err) => {
|
||||
this.#onError(err)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
_retryable ({ before, attempt, after }) {
|
||||
#retryable ({ before, attempt, after }) {
|
||||
const { retryDelays } = this.options
|
||||
const { signal } = this.abortController
|
||||
|
||||
|
|
@ -201,7 +221,7 @@ class MultipartUploader {
|
|||
}
|
||||
|
||||
const doAttempt = (retryAttempt) => attempt().catch((err) => {
|
||||
if (this._aborted()) throw createAbortError()
|
||||
if (this.#aborted()) throw createAbortError()
|
||||
|
||||
if (shouldRetry(err) && retryAttempt < retryDelays.length) {
|
||||
return delay(retryDelays[retryAttempt], { signal })
|
||||
|
|
@ -219,53 +239,62 @@ class MultipartUploader {
|
|||
})
|
||||
}
|
||||
|
||||
_uploadPartRetryable (index) {
|
||||
return this._retryable({
|
||||
async #prepareUploadParts (candidates) {
|
||||
this.lockedCandidatesForBatch.push(...candidates)
|
||||
|
||||
const result = await this.options.prepareUploadParts({
|
||||
key: this.key,
|
||||
uploadId: this.uploadId,
|
||||
partNumbers: candidates.map((index) => index + 1),
|
||||
})
|
||||
|
||||
const valid = typeof result?.presignedUrls === 'object'
|
||||
if (!valid) {
|
||||
throw new TypeError(
|
||||
'AwsS3/Multipart: Got incorrect result from `prepareUploadParts()`, expected an object `{ presignedUrls }`.'
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
#uploadPartRetryable (index, prePreparedPart) {
|
||||
return this.#retryable({
|
||||
before: () => {
|
||||
this.partsInProgress += 1
|
||||
},
|
||||
attempt: () => this._uploadPart(index),
|
||||
attempt: () => this.#uploadPart(index, prePreparedPart),
|
||||
after: () => {
|
||||
this.partsInProgress -= 1
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
_uploadPart (index) {
|
||||
#uploadPart (index, prePreparedPart) {
|
||||
const body = this.chunks[index]
|
||||
this.chunkState[index].busy = true
|
||||
|
||||
return Promise.resolve().then(() => this.options.prepareUploadPart({
|
||||
key: this.key,
|
||||
uploadId: this.uploadId,
|
||||
body,
|
||||
number: index + 1,
|
||||
})).then((result) => {
|
||||
const valid = typeof result === 'object' && result
|
||||
&& typeof result.url === 'string'
|
||||
if (!valid) {
|
||||
throw new TypeError('AwsS3/Multipart: Got incorrect result from `prepareUploadPart()`, expected an object `{ url }`.')
|
||||
}
|
||||
const valid = typeof prePreparedPart?.url === 'string'
|
||||
if (!valid) {
|
||||
throw new TypeError('AwsS3/Multipart: Got incorrect result for `prePreparedPart`, expected an object `{ url }`.')
|
||||
}
|
||||
|
||||
return result
|
||||
}).then(({ url, headers }) => {
|
||||
if (this._aborted()) {
|
||||
this.chunkState[index].busy = false
|
||||
throw createAbortError()
|
||||
}
|
||||
const { url, headers } = prePreparedPart
|
||||
if (this.#aborted()) {
|
||||
this.chunkState[index].busy = false
|
||||
throw createAbortError()
|
||||
}
|
||||
|
||||
return this._uploadPartBytes(index, url, headers)
|
||||
})
|
||||
return this.#uploadPartBytes(index, url, headers)
|
||||
}
|
||||
|
||||
_onPartProgress (index, sent, total) {
|
||||
#onPartProgress (index, sent, total) {
|
||||
this.chunkState[index].uploaded = ensureInt(sent)
|
||||
|
||||
const totalUploaded = this.chunkState.reduce((n, c) => n + c.uploaded, 0)
|
||||
this.options.onProgress(totalUploaded, this.file.size)
|
||||
}
|
||||
|
||||
_onPartComplete (index, etag) {
|
||||
#onPartComplete (index, etag) {
|
||||
this.chunkState[index].etag = etag
|
||||
this.chunkState[index].done = true
|
||||
|
||||
|
|
@ -278,7 +307,7 @@ class MultipartUploader {
|
|||
this.options.onPartComplete(part)
|
||||
}
|
||||
|
||||
_uploadPartBytes (index, url, headers) {
|
||||
#uploadPartBytes (index, url, headers) {
|
||||
const body = this.chunks[index]
|
||||
const { signal } = this.abortController
|
||||
|
||||
|
|
@ -307,7 +336,7 @@ class MultipartUploader {
|
|||
xhr.upload.addEventListener('progress', (ev) => {
|
||||
if (!ev.lengthComputable) return
|
||||
|
||||
this._onPartProgress(index, ev.loaded, ev.total)
|
||||
this.#onPartProgress(index, ev.loaded, ev.total)
|
||||
})
|
||||
|
||||
xhr.addEventListener('abort', (ev) => {
|
||||
|
|
@ -328,7 +357,7 @@ class MultipartUploader {
|
|||
return
|
||||
}
|
||||
|
||||
this._onPartProgress(index, body.size, body.size)
|
||||
this.#onPartProgress(index, body.size, body.size)
|
||||
|
||||
// NOTE This must be allowed by CORS.
|
||||
const etag = ev.target.getResponseHeader('ETag')
|
||||
|
|
@ -337,7 +366,7 @@ class MultipartUploader {
|
|||
return
|
||||
}
|
||||
|
||||
this._onPartComplete(index, etag)
|
||||
this.#onPartComplete(index, etag)
|
||||
defer.resolve()
|
||||
})
|
||||
|
||||
|
|
@ -355,22 +384,23 @@ class MultipartUploader {
|
|||
return promise
|
||||
}
|
||||
|
||||
_completeUpload () {
|
||||
async #completeUpload () {
|
||||
// Parts may not have completed uploading in sorted order, if limit > 1.
|
||||
this.parts.sort((a, b) => a.PartNumber - b.PartNumber)
|
||||
|
||||
return Promise.resolve().then(() => this.options.completeMultipartUpload({
|
||||
key: this.key,
|
||||
uploadId: this.uploadId,
|
||||
parts: this.parts,
|
||||
})).then((result) => {
|
||||
try {
|
||||
const result = await this.options.completeMultipartUpload({
|
||||
key: this.key,
|
||||
uploadId: this.uploadId,
|
||||
parts: this.parts,
|
||||
})
|
||||
this.options.onSuccess(result)
|
||||
}, (err) => {
|
||||
this._onError(err)
|
||||
})
|
||||
} catch (err) {
|
||||
this.#onError(err)
|
||||
}
|
||||
}
|
||||
|
||||
_abortUpload () {
|
||||
#abortUpload () {
|
||||
this.abortController.abort()
|
||||
|
||||
this.createdPromise.then(() => {
|
||||
|
|
@ -383,7 +413,7 @@ class MultipartUploader {
|
|||
})
|
||||
}
|
||||
|
||||
_onError (err) {
|
||||
#onError (err) {
|
||||
if (err && err.name === 'AbortError') {
|
||||
return
|
||||
}
|
||||
|
|
@ -394,9 +424,9 @@ class MultipartUploader {
|
|||
start () {
|
||||
this.isPaused = false
|
||||
if (this.uploadId) {
|
||||
this._resumeUpload()
|
||||
this.#resumeUpload()
|
||||
} else {
|
||||
this._createUpload()
|
||||
this.#createUpload()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -413,7 +443,7 @@ class MultipartUploader {
|
|||
|
||||
if (!really) return this.pause()
|
||||
|
||||
this._abortUpload()
|
||||
this.#abortUpload()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ module.exports = class AwsS3Multipart extends BasePlugin {
|
|||
retryDelays: [0, 1000, 3000, 5000],
|
||||
createMultipartUpload: this.createMultipartUpload.bind(this),
|
||||
listParts: this.listParts.bind(this),
|
||||
prepareUploadPart: this.prepareUploadPart.bind(this),
|
||||
prepareUploadParts: this.prepareUploadParts.bind(this),
|
||||
abortMultipartUpload: this.abortMultipartUpload.bind(this),
|
||||
completeMultipartUpload: this.completeMultipartUpload.bind(this),
|
||||
}
|
||||
|
|
@ -101,11 +101,11 @@ module.exports = class AwsS3Multipart extends BasePlugin {
|
|||
.then(assertServerError)
|
||||
}
|
||||
|
||||
prepareUploadPart (file, { key, uploadId, number }) {
|
||||
this.assertHost('prepareUploadPart')
|
||||
prepareUploadParts (file, { key, uploadId, partNumbers }) {
|
||||
this.assertHost('prepareUploadParts')
|
||||
|
||||
const filename = encodeURIComponent(key)
|
||||
return this.client.get(`s3/multipart/${uploadId}/${number}?key=${filename}`)
|
||||
return this.client.get(`s3/multipart/${uploadId}/batch?key=${filename}?partNumbers=${partNumbers.join(',')}`)
|
||||
.then(assertServerError)
|
||||
}
|
||||
|
||||
|
|
@ -191,7 +191,7 @@ module.exports = class AwsS3Multipart extends BasePlugin {
|
|||
// .bind to pass the file object to each handler.
|
||||
createMultipartUpload: this.opts.createMultipartUpload.bind(this, file),
|
||||
listParts: this.opts.listParts.bind(this, file),
|
||||
prepareUploadPart: this.opts.prepareUploadPart.bind(this, file),
|
||||
prepareUploadParts: this.opts.prepareUploadParts.bind(this, file),
|
||||
completeMultipartUpload: this.opts.completeMultipartUpload.bind(this, file),
|
||||
abortMultipartUpload: this.opts.abortMultipartUpload.bind(this, file),
|
||||
getChunkSize: this.opts.getChunkSize ? this.opts.getChunkSize.bind(this) : null,
|
||||
|
|
|
|||
164
packages/@uppy/aws-s3-multipart/src/index.test.js
Normal file
164
packages/@uppy/aws-s3-multipart/src/index.test.js
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
require('whatwg-fetch')
|
||||
const nock = require('nock')
|
||||
const Core = require('@uppy/core')
|
||||
const AwsS3Multipart = require('.')
|
||||
|
||||
const KB = 1024
|
||||
const MB = KB * KB
|
||||
|
||||
describe('AwsS3Multipart', () => {
|
||||
beforeEach(() => nock.disableNetConnect())
|
||||
|
||||
it('Registers AwsS3Multipart upload plugin', () => {
|
||||
const core = new Core()
|
||||
core.use(AwsS3Multipart)
|
||||
|
||||
const pluginNames = core[Symbol.for('uppy test: getPlugins')]('uploader').map((plugin) => plugin.constructor.name)
|
||||
expect(pluginNames).toContain('AwsS3Multipart')
|
||||
})
|
||||
|
||||
describe('companionUrl assertion', () => {
|
||||
it('Throws an error for main functions if configured without companionUrl', () => {
|
||||
const core = new Core()
|
||||
core.use(AwsS3Multipart)
|
||||
const awsS3Multipart = core.getPlugin('AwsS3Multipart')
|
||||
|
||||
const err = 'Expected a `companionUrl` option'
|
||||
const file = {}
|
||||
const opts = {}
|
||||
|
||||
expect(() => awsS3Multipart.opts.createMultipartUpload(file)).toThrow(
|
||||
err
|
||||
)
|
||||
expect(() => awsS3Multipart.opts.listParts(file, opts)).toThrow(err)
|
||||
expect(() => awsS3Multipart.opts.completeMultipartUpload(file, opts)).toThrow(err)
|
||||
expect(() => awsS3Multipart.opts.abortMultipartUpload(file, opts)).toThrow(err)
|
||||
expect(() => awsS3Multipart.opts.prepareUploadParts(file, opts)).toThrow(err)
|
||||
})
|
||||
})
|
||||
|
||||
describe('without companionUrl (custom main functions)', () => {
|
||||
let core
|
||||
let awsS3Multipart
|
||||
|
||||
beforeEach(() => {
|
||||
core = new Core()
|
||||
core.use(AwsS3Multipart, {
|
||||
createMultipartUpload: jest.fn(() => {
|
||||
return {
|
||||
uploadId: '6aeb1980f3fc7ce0b5454d25b71992',
|
||||
key: 'test/upload/multitest.dat',
|
||||
}
|
||||
}),
|
||||
completeMultipartUpload: jest.fn(() => Promise.resolve({ location: 'test' })),
|
||||
abortMultipartUpload: jest.fn(),
|
||||
prepareUploadParts: jest.fn(() => {
|
||||
const presignedUrls = {}
|
||||
const possiblePartNumbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
possiblePartNumbers.forEach((partNumber) => {
|
||||
presignedUrls[
|
||||
partNumber
|
||||
] = `https://bucket.s3.us-east-2.amazonaws.com/test/upload/multitest.dat?partNumber=${partNumber}&uploadId=6aeb1980f3fc7ce0b5454d25b71992&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIATEST%2F20210729%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20210729T014044Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host&X-Amz-Signature=test`
|
||||
})
|
||||
return { presignedUrls }
|
||||
}),
|
||||
})
|
||||
awsS3Multipart = core.getPlugin('AwsS3Multipart')
|
||||
})
|
||||
|
||||
it('Calls the prepareUploadParts function totalChunks / limit times', (done) => {
|
||||
const scope = nock(
|
||||
'https://bucket.s3.us-east-2.amazonaws.com'
|
||||
).defaultReplyHeaders({
|
||||
'access-control-allow-method': 'PUT',
|
||||
'access-control-allow-origin': '*',
|
||||
'access-control-expose-headers': 'ETag',
|
||||
})
|
||||
scope
|
||||
.options((uri) => uri.includes('test/upload/multitest.dat'))
|
||||
.reply(200, '')
|
||||
scope
|
||||
.options((uri) => uri.includes('test/upload/multitest.dat'))
|
||||
.reply(200, '')
|
||||
scope
|
||||
.put((uri) => uri.includes('test/upload/multitest.dat'))
|
||||
.reply(200, '', { ETag: 'test1' })
|
||||
scope
|
||||
.put((uri) => uri.includes('test/upload/multitest.dat'))
|
||||
.reply(200, '', { ETag: 'test2' })
|
||||
|
||||
// 6MB file will give us 2 chunks, so there will be 2 PUT and 2 OPTIONS
|
||||
// calls to the presigned URL from 1 prepareUploadParts calls
|
||||
const fileSize = 5 * MB + 1 * MB
|
||||
core.addFile({
|
||||
source: 'jest',
|
||||
name: 'multitest.dat',
|
||||
type: 'application/octet-stream',
|
||||
data: new File([Buffer.alloc(fileSize)], {
|
||||
type: 'application/octet-stream',
|
||||
}),
|
||||
})
|
||||
core.upload().then(() => {
|
||||
expect(
|
||||
awsS3Multipart.opts.prepareUploadParts.mock.calls.length
|
||||
).toEqual(1)
|
||||
scope.done()
|
||||
done()
|
||||
})
|
||||
})
|
||||
|
||||
it('Calls prepareUploadParts with a Math.ceil(limit / 2) minimum, instead of one at a time for the remaining chunks after the first limit batch', (done) => {
|
||||
const scope = nock(
|
||||
'https://bucket.s3.us-east-2.amazonaws.com'
|
||||
).defaultReplyHeaders({
|
||||
'access-control-allow-method': 'PUT',
|
||||
'access-control-allow-origin': '*',
|
||||
'access-control-expose-headers': 'ETag',
|
||||
})
|
||||
scope
|
||||
.options((uri) => uri.includes('test/upload/multitest.dat'))
|
||||
.reply(200, '')
|
||||
scope
|
||||
.put((uri) => uri.includes('test/upload/multitest.dat'))
|
||||
.reply(200, '', { ETag: 'test' })
|
||||
scope.persist()
|
||||
|
||||
// 50MB file will give us 10 chunks, so there will be 10 PUT and 10 OPTIONS
|
||||
// calls to the presigned URL from 3 prepareUploadParts calls
|
||||
//
|
||||
// The first prepareUploadParts call will be for 5 parts, the second
|
||||
// will be for 3 parts, the third will be for 2 parts.
|
||||
const fileSize = 50 * MB
|
||||
core.addFile({
|
||||
source: 'jest',
|
||||
name: 'multitest.dat',
|
||||
type: 'application/octet-stream',
|
||||
data: new File([Buffer.alloc(fileSize)], {
|
||||
type: 'application/octet-stream',
|
||||
}),
|
||||
})
|
||||
core.upload().then(() => {
|
||||
expect(
|
||||
awsS3Multipart.opts.prepareUploadParts.mock.calls.length
|
||||
).toEqual(3)
|
||||
expect(awsS3Multipart.opts.prepareUploadParts.mock.calls[0][1].partNumbers).toEqual([1, 2, 3, 4, 5])
|
||||
expect(awsS3Multipart.opts.prepareUploadParts.mock.calls[1][1].partNumbers).toEqual([6, 7, 8])
|
||||
expect(awsS3Multipart.opts.prepareUploadParts.mock.calls[2][1].partNumbers).toEqual([9, 10])
|
||||
const completeCall = awsS3Multipart.opts.completeMultipartUpload.mock.calls[0][1]
|
||||
expect(completeCall.parts).toEqual([
|
||||
{ ETag: 'test', PartNumber: 1 },
|
||||
{ ETag: 'test', PartNumber: 2 },
|
||||
{ ETag: 'test', PartNumber: 3 },
|
||||
{ ETag: 'test', PartNumber: 4 },
|
||||
{ ETag: 'test', PartNumber: 5 },
|
||||
{ ETag: 'test', PartNumber: 6 },
|
||||
{ ETag: 'test', PartNumber: 7 },
|
||||
{ ETag: 'test', PartNumber: 8 },
|
||||
{ ETag: 'test', PartNumber: 9 },
|
||||
{ ETag: 'test', PartNumber: 10 },
|
||||
])
|
||||
done()
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -19,10 +19,10 @@ interface AwsS3MultipartOptions extends PluginOptions {
|
|||
file: UppyFile,
|
||||
opts: { uploadId: string; key: string }
|
||||
) => MaybePromise<AwsS3Part[]>
|
||||
prepareUploadPart?: (
|
||||
prepareUploadParts?: (
|
||||
file: UppyFile,
|
||||
partData: { uploadId: string; key: string; body: Blob; number: number }
|
||||
) => MaybePromise<{ url: string, headers?: { [k: string]: string } }>
|
||||
partData: { uploadId: string; key: string; partNumbers: Array<number> }
|
||||
) => MaybePromise<{ presignedUrls: { [k: number]: string }, headers?: { [k: string]: string } }>
|
||||
abortMultipartUpload?: (
|
||||
file: UppyFile,
|
||||
opts: { uploadId: string; key: string }
|
||||
|
|
|
|||
|
|
@ -17,13 +17,12 @@ import type { AwsS3Part } from '..'
|
|||
expectType<string>(opts.key)
|
||||
return []
|
||||
},
|
||||
prepareUploadPart (file, part) {
|
||||
prepareUploadParts (file, partData) {
|
||||
expectType<UppyFile>(file)
|
||||
expectType<string>(part.uploadId)
|
||||
expectType<string>(part.key)
|
||||
expectType<Blob>(part.body)
|
||||
expectType<number>(part.number)
|
||||
return { url: '' }
|
||||
expectType<string>(partData.uploadId)
|
||||
expectType<string>(partData.key)
|
||||
expectType<Array<number>>(partData.partNumbers)
|
||||
return { presignedUrls: {} }
|
||||
},
|
||||
abortMultipartUpload (file, opts) {
|
||||
expectType<UppyFile>(file)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ describe('AwsS3', () => {
|
|||
expect(awsS3.opts.getUploadParameters).toThrow()
|
||||
})
|
||||
|
||||
it('Does not throw an error with campanionUrl configured', () => {
|
||||
it('Does not throw an error with companionUrl configured', () => {
|
||||
const core = new Core()
|
||||
core.use(AwsS3, { companionUrl: 'https://uppy-companion.myapp.com/' })
|
||||
const awsS3 = core.getPlugin('AwsS3')
|
||||
|
|
|
|||
|
|
@ -204,6 +204,61 @@ module.exports = function s3 (config) {
|
|||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parameters for uploading a batch of parts.
|
||||
*
|
||||
* Expected URL parameters:
|
||||
* - uploadId - The uploadId returned from `createMultipartUpload`.
|
||||
* Expected query parameters:
|
||||
* - key - The object key in the S3 bucket.
|
||||
* - partNumbers - A comma separated list of part numbers representing
|
||||
* indecies in the file (1-10000).
|
||||
* Response JSON:
|
||||
* - presignedUrls - The URLs to upload to, including signed query parameters,
|
||||
* in an object mapped to part numbers.
|
||||
*/
|
||||
function batchSignPartsUpload (req, res, next) {
|
||||
// @ts-ignore The `companion` property is added by middleware before reaching here.
|
||||
const client = req.companion.s3Client
|
||||
const { uploadId } = req.params
|
||||
const { key, partNumbers } = req.query
|
||||
|
||||
if (typeof key !== 'string') {
|
||||
return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' })
|
||||
}
|
||||
|
||||
if (typeof partNumbers !== 'string') {
|
||||
return res.status(400).json({ error: 's3: the part numbers must be passed as a comma separated query parameter. For example: "?partNumbers=4,6,7,21"' })
|
||||
}
|
||||
|
||||
const partNumbersArray = partNumbers.split(',')
|
||||
partNumbersArray.forEach((partNumber) => {
|
||||
if (!parseInt(partNumber, 10)) {
|
||||
return res.status(400).json({ error: 's3: the part numbers must be a number between 1 and 10000.' })
|
||||
}
|
||||
})
|
||||
|
||||
Promise.all(
|
||||
partNumbersArray.map((partNumber) => {
|
||||
return client.getSignedUrlPromise('uploadPart', {
|
||||
Bucket: config.bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: partNumber,
|
||||
Body: '',
|
||||
Expires: config.expires,
|
||||
})
|
||||
})
|
||||
).then((urls) => {
|
||||
const presignedUrls = Object.fromEntries(
|
||||
partNumbersArray.map((partNumber, index) => [partNumber, urls[index]])
|
||||
)
|
||||
res.json({ presignedUrls })
|
||||
}).catch((err) => {
|
||||
next(err)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Abort a multipart upload, deleting already uploaded parts.
|
||||
*
|
||||
|
|
@ -287,6 +342,7 @@ module.exports = function s3 (config) {
|
|||
.get('/multipart/:uploadId', getUploadedParts)
|
||||
.get('/multipart/:uploadId/:partNumber', signPartUpload)
|
||||
.post('/multipart/:uploadId/complete', completeMultipartUpload)
|
||||
.get(`/multipart/:uploadId/batch`, batchSignPartsUpload)
|
||||
.delete('/multipart/:uploadId', abortMultipartUpload)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,13 +39,15 @@ const { AwsS3Multipart } = Uppy
|
|||
|
||||
The `@uppy/aws-s3-multipart` plugin has the following configurable options:
|
||||
|
||||
### `limit: 0`
|
||||
### `limit: 5`
|
||||
|
||||
The maximum amount of chunks to upload simultaneously. Set to `0` to disable limiting.
|
||||
The maximum amount of chunks to upload simultaneously. This affects [`prepareUploadParts()`](#prepareUploadParts-file-partData) as well; after the initial batch of `limit` parts is presigned, a minimum of `limit / 2` rounded up will be presigned at a time. You should set the limit carefully. Setting it to a value too high could cause issues where the presigned URLs begin to expire before the chunks they are for start uploading. Too low and you will end up with a lot of extra round trips to your server (or Companion) than necessary to presign URLs. If the default chunk size of 5MB is used, a `limit` between 5 and 15 is recommended.
|
||||
|
||||
For example, with a 50MB file and a `limit` of 5 we end up with 10 chunks. 5 of these are presigned in one batch, then 3, then 2, for a total of 3 round trips to the server via [`prepareUploadParts()`](#prepareUploadParts-file-partData) and 10 requests sent to AWS via the presigned URLs generated.
|
||||
|
||||
### `retryDelays: [0, 1000, 3000, 5000]`
|
||||
|
||||
When uploading a chunk fails, automatically try again after the millisecond intervals specified in this array. By default, we first retry instantly; if that fails, we retry after 1 second; if that fails, we retry after 3 seconds, etc.
|
||||
When uploading a chunk to S3 using a presigned URL fails, automatically try again after the millisecond intervals specified in this array. By default, we first retry instantly; if that fails, we retry after 1 second; if that fails, we retry after 3 seconds, etc.
|
||||
|
||||
Set to `null` to disable automatic retries, and fail instantly if any chunk fails to upload.
|
||||
|
||||
|
|
@ -69,7 +71,7 @@ This option correlates to the [RequestCredentials value](https://developer.mozil
|
|||
|
||||
A function that returns the minimum chunk size to use when uploading the given file.
|
||||
|
||||
The S3 Multipart plugin uploads files in chunks. Each chunk requires a signing request ([`prepareUploadPart()`](#prepareUploadPart-file-partData)). To reduce the amount of requests for large files, you can choose a larger chunk size, at the cost of having to re-upload more data if one chunk fails to upload.
|
||||
The S3 Multipart plugin uploads files in chunks. Chunks are sent in batches to have presigned URLs generated via ([`prepareUploadParts()`](#prepareUploadParts-file-partData)). To reduce the amount of requests for large files, you can choose a larger chunk size, at the cost of having to re-upload more data if one chunk fails to upload.
|
||||
|
||||
S3 requires a minimum chunk size of 5MB, and supports at most 10,000 chunks per multipart upload. If `getChunkSize()` returns a size that's too small, Uppy will increase it to S3's minimum requirements.
|
||||
|
||||
|
|
@ -99,29 +101,25 @@ Return a Promise for an array of S3 Part objects, as returned by the S3 Multipar
|
|||
|
||||
The default implementation calls out to Companion's S3 signing endpoints.
|
||||
|
||||
### `prepareUploadPart(file, partData)`
|
||||
### `prepareUploadParts(file, partData)`
|
||||
|
||||
A function that generates a signed URL to upload a single part. Receives the `file` object from Uppy's state. The `partData` argument is an object with keys:
|
||||
A function that generates a batch of signed URLs for the specified part numbers. Receives the `file` object from Uppy's state. The `partData` argument is an object with keys:
|
||||
|
||||
- `uploadId` - The UploadID of this Multipart upload.
|
||||
- `key` - The object key in the S3 bucket.
|
||||
- `body` - A [`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob) of this part's contents.
|
||||
- `number` - The index of this part in the file (`PartNumber` in S3 terminology).
|
||||
- `partNumbers` - An array of indecies of this part in the file (`PartNumber` in S3 terminology). Note that part numbers are _not_ zero-based.
|
||||
|
||||
Return a Promise for an object with keys:
|
||||
|
||||
- `url` - The presigned URL to upload a part. This can be generated on the server using the S3 SDK like so:
|
||||
- `presignedUrls` - A JavaScript object with the part numbers as keys and the presigned URL for each part as the value. An example of what the return value should look like:
|
||||
|
||||
<!-- eslint-disable node/handle-callback-err -->
|
||||
```js
|
||||
sdkInstance.getSignedUrl('uploadPart', {
|
||||
Bucket: 'target',
|
||||
Key: partData.key,
|
||||
UploadId: partData.uploadId,
|
||||
PartNumber: partData.number,
|
||||
Body: '', // Empty, because it is uploaded later
|
||||
Expires: 5 * 60,
|
||||
}, (err, url) => { /* there's the url! */ })
|
||||
// for partNumbers [1, 2, 3]
|
||||
return {
|
||||
1: 'https://bucket.region.amazonaws.com/path/to/file.jpg?partNumber=1&...',
|
||||
2: 'https://bucket.region.amazonaws.com/path/to/file.jpg?partNumber=2&...',
|
||||
3: 'https://bucket.region.amazonaws.com/path/to/file.jpg?partNumber=3&...',
|
||||
}
|
||||
```
|
||||
- `headers` - **(Optional)** Custom headers that should be sent to the S3 presigned URL.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue