diff --git a/packages/@uppy/companion/package.json b/packages/@uppy/companion/package.json index 19a5a9717..b5121c197 100644 --- a/packages/@uppy/companion/package.json +++ b/packages/@uppy/companion/package.json @@ -54,10 +54,10 @@ "purest": "3.0.0", "redis": "2.8.0", "request": "2.85.0", - "serialize-error": "2.1.0", - "tus-js-client": "1.6.0", - "uuid": "3.3.2", - "validator": "9.4.1", + "serialize-error": "^2.1.0", + "tus-js-client": "github:ifedapoolarewaju/tus-js-client#6f6b88d554eef1ac26e3888f9bb6b675ea8e021c", + "uuid": "2.0.2", + "validator": "^9.0.0", "ws": "1.1.5" }, "devDependencies": { diff --git a/packages/@uppy/companion/src/server/Uploader.js b/packages/@uppy/companion/src/server/Uploader.js index 2c7da7b11..5e849e581 100644 --- a/packages/@uppy/companion/src/server/Uploader.js +++ b/packages/@uppy/companion/src/server/Uploader.js @@ -1,4 +1,5 @@ const fs = require('fs') +const stream = require('stream') const path = require('path') const tus = require('tus-js-client') const uuid = require('uuid') @@ -38,13 +39,23 @@ class Uploader { this.options = options this.token = uuid.v4() this.options.path = `${this.options.pathPrefix}/${Uploader.FILE_NAME_PREFIX}-${this.token}` - this.writer = fs.createWriteStream(this.options.path, { mode: 0o666 }) // no executable files + this.streamsEnded = false + this.duplexStream = new stream.PassThrough() + .on('error', (err) => logger.error(`${this.shortToken} ${err}`, 'uploader.duplex.error')) + this.writeStream = fs.createWriteStream(this.options.path, { mode: 0o666 }) // no executable files .on('error', (err) => logger.error(`${this.shortToken} ${err}`, 'uploader.write.error')) /** @type {number} */ this.emittedProgress = 0 this.storage = options.storage } + /** + * the number of bytes written into the streams + */ + get bytesWritten () { + return this.writeStream.bytesWritten + } + /** * Validate the options passed down to the uplaoder * @@ -105,28 +116,31 @@ class Uploader { * @param {Buffer | Buffer[]} chunk */ handleChunk (chunk) { - logger.debug(`${this.shortToken} ${this.writer.bytesWritten} bytes`, 'uploader.download.progress') + logger.debug(`${this.shortToken} ${this.bytesWritten} bytes`, 'uploader.download.progress') const protocol = this.options.protocol || 'multipart' // The download has completed; close the file and start an upload if necessary. if (chunk === null) { - if (this.options.endpoint && protocol === 'multipart') { - this.writer.on('finish', () => { + this.writeStream.on('finish', () => { + this.streamsEnded = true + if (this.options.endpoint && protocol === 'multipart') { this.uploadMultipart() - }) - } - return this.writer.end() + } + }) + + this.duplexStream.end() + return this.writeStream.end() } - this.writer.write(chunk, () => { + this.writeToStreams(chunk, () => { if (protocol === 's3-multipart' && !this.s3Upload) { return this.uploadS3Streaming() } if (!this.options.endpoint) return if (protocol === 'tus' && !this.tus) { - return this.uploadTus() + return this.uploadTus(true) } }) } @@ -136,11 +150,11 @@ class Uploader { * @param {object} resp */ handleResponse (resp) { - resp.pipe(this.writer) + resp.pipe(this.writeStream) const protocol = this.options.protocol || 'multipart' - this.writer.on('finish', () => { + this.writeStream.on('finish', () => { if (protocol === 's3-multipart') { this.uploadS3Full() } @@ -148,7 +162,7 @@ class Uploader { if (!this.options.endpoint) return if (protocol === 'tus') { - this.uploadTus() + this.uploadTus(false) } if (protocol === 'multipart') { this.uploadMultipart() @@ -156,6 +170,23 @@ class Uploader { }) } + /** + * @param {Buffer | Buffer[]} chunk + * @param {function} cb + */ + writeToStreams (chunk, cb) { + const done = [] + const onDone = () => { + done.push(true) + if (done.length >= 2) { + cb() + } + } + + this.duplexStream.write(chunk, onDone) + this.writeStream.write(chunk, onDone) + } + getResponse () { if (this._errRespMessage) { return { body: this._errRespMessage, status: 400 } @@ -179,6 +210,9 @@ class Uploader { */ emitProgress (bytesUploaded, bytesTotal) { bytesTotal = bytesTotal || this.options.size + if (this.tus.options.uploadLengthDeferred && this.streamsEnded) { + bytesTotal = this.bytesWritten + } const percentage = (bytesUploaded / bytesTotal * 100) const formatPercentage = percentage.toFixed(2) logger.debug( @@ -229,21 +263,27 @@ class Uploader { emitter().emit(this.token, dataToEmit) } - uploadTus () { + /** + * + * @param {boolean} deferLength + */ + uploadTus (deferLength) { const fname = path.basename(this.options.path) const ftype = this.options.metadata.type const metadata = Object.assign({ filename: fname, filetype: ftype }, this.options.metadata || {}) - const file = fs.createReadStream(this.options.path) + const file = deferLength ? this.duplexStream : fs.createReadStream(this.options.path) const uploader = this // @ts-ignore this.tus = new tus.Upload(file, { endpoint: this.options.endpoint, uploadUrl: this.options.uploadUrl, + // @ts-ignore + uploadLengthDeferred: deferLength, resume: true, - uploadSize: this.options.size || fs.statSync(this.options.path).size, + uploadSize: deferLength ? null : (this.options.size || fs.statSync(this.options.path).size), metadata, - chunkSize: this.writer.bytesWritten, + chunkSize: this.bytesWritten, /** * * @param {Error} error @@ -260,15 +300,6 @@ class Uploader { onProgress (bytesUploaded, bytesTotal) { uploader.emitProgress(bytesUploaded, bytesTotal) }, - /** - * - * @param {number} chunkSize - * @param {number} bytesUploaded - * @param {number} bytesTotal - */ - onChunkComplete (chunkSize, bytesUploaded, bytesTotal) { - uploader.tus.options.chunkSize = uploader.writer.bytesWritten - bytesUploaded - }, onSuccess () { uploader.emitSuccess(uploader.tus.url) uploader.cleanUp() @@ -339,7 +370,7 @@ class Uploader { tail: true }) - this.writer.on('finish', () => { + this.writeStream.on('finish', () => { file.close() })