From 98f54c0899b0b59d1e02bcc0ee41e04a6f85a97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9e=20Kooi?= Date: Mon, 20 Nov 2017 10:02:50 +0100 Subject: [PATCH 1/5] ignore bundled files from lint --- .eslintignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintignore b/.eslintignore index b3330d45a..e0bdfe0c8 100644 --- a/.eslintignore +++ b/.eslintignore @@ -10,3 +10,4 @@ website/public/** website/themes/uppy/source/js/smooth-scroll.min.js website/themes/uppy/source/js/uppy.js website/themes/uppy/source/uppy/** +bundle.js From 7a7dd752f8ac8f19c9c3ec76de9b1742ead80c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9e=20Kooi?= Date: Mon, 20 Nov 2017 10:04:44 +0100 Subject: [PATCH 2/5] xhrupload: Limit amount of simultaneous uploads --- src/core/Utils.js | 40 +++++++++++++++++++++++++++++++++++++++- src/core/Utils.test.js | 29 +++++++++++++++++++++++++++++ src/plugins/XHRUpload.js | 14 +++++++++----- 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/src/core/Utils.js b/src/core/Utils.js index eba412400..1fb2f8f00 100644 --- a/src/core/Utils.js +++ b/src/core/Utils.js @@ -526,6 +526,43 @@ function settle (promises) { }) } +/** + * Limit the amount of simultaneously pending Promises. + * Returns a function that, when passed a function `fn`, + * will make sure that at most `limit` calls to `fn` are pending. + * + * @param {number} limit + * @return {function()} + */ +function limitPromises (limit) { + let pending = 0 + const queue = [] + return (fn) => { + return (...args) => { + const call = () => { + pending++ + const promise = fn(...args) + promise.then(onfinish, onfinish) + return promise + } + + if (pending >= limit) { + return new Promise((resolve, reject) => { + queue.push(() => { + call().then(resolve, reject) + }) + }) + } + return call() + } + } + function onfinish () { + pending-- + const next = queue.shift() + if (next) next() + } +} + module.exports = { generateFileID, toArray, @@ -553,5 +590,6 @@ module.exports = { findAllDOMElements, getSocketHost, emitSocketProgress, - settle + settle, + limitPromises } diff --git a/src/core/Utils.test.js b/src/core/Utils.test.js index 3269baf30..5c293b2af 100644 --- a/src/core/Utils.test.js +++ b/src/core/Utils.test.js @@ -370,4 +370,33 @@ describe('core/utils', () => { }) }) }) + + describe('limitPromises', () => { + it('should run at most N promises at the same time', () => { + let pending = 0 + function fn () { + pending++ + return new Promise((resolve) => setTimeout(resolve, 10)) + .then(() => pending--) + } + + const limit = utils.limitPromises(4) + const fn2 = limit(fn) + + const result = Promise.all([ + fn2(), fn2(), fn2(), fn2(), + fn2(), fn2(), fn2(), fn2(), + fn2(), fn2() + ]) + + expect(pending).toBe(4) + setTimeout(() => { + expect(pending).toBe(4) + }, 10) + + return result.then(() => { + expect(pending).toBe(0) + }) + }) + }) }) diff --git a/src/plugins/XHRUpload.js b/src/plugins/XHRUpload.js index 453e6b1cc..b83643ad0 100644 --- a/src/plugins/XHRUpload.js +++ b/src/plugins/XHRUpload.js @@ -5,7 +5,8 @@ const UppySocket = require('../core/UppySocket') const { emitSocketProgress, getSocketHost, - settle + settle, + limitPromises } = require('../core/Utils') module.exports = class XHRUpload extends Plugin { @@ -248,19 +249,22 @@ module.exports = class XHRUpload extends Plugin { } uploadFiles (files) { - const promises = files.map((file, i) => { + const actions = files.map((file, i) => { const current = parseInt(i, 10) + 1 const total = files.length if (file.error) { - return Promise.reject(new Error(file.error)) + return () => Promise.reject(new Error(file.error)) } else if (file.isRemote) { - return this.uploadRemote(file, current, total) + return this.uploadRemote.bind(this, file, current, total) } else { - return this.upload(file, current, total) + return this.upload.bind(this, file, current, total) } }) + const promises = actions + .map(limitPromises(5)) + .map((action) => action()) return settle(promises) } From ea8ab5d78039f87541d29bc90644b7b76c242449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9e=20Kooi?= Date: Mon, 20 Nov 2017 10:14:52 +0100 Subject: [PATCH 3/5] xhrupload: Configurable limit --- src/core/Utils.test.js | 31 ++++++++++++++++++++++++------- src/plugins/XHRUpload.js | 11 ++++++++++- website/src/docs/xhrupload.md | 4 ++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/core/Utils.test.js b/src/core/Utils.test.js index 5c293b2af..275897ebc 100644 --- a/src/core/Utils.test.js +++ b/src/core/Utils.test.js @@ -372,14 +372,14 @@ describe('core/utils', () => { }) describe('limitPromises', () => { - it('should run at most N promises at the same time', () => { - let pending = 0 - function fn () { - pending++ - return new Promise((resolve) => setTimeout(resolve, 10)) - .then(() => pending--) - } + let pending = 0 + function fn () { + pending++ + return new Promise((resolve) => setTimeout(resolve, 10)) + .then(() => pending--) + } + it('should run at most N promises at the same time', () => { const limit = utils.limitPromises(4) const fn2 = limit(fn) @@ -398,5 +398,22 @@ describe('core/utils', () => { expect(pending).toBe(0) }) }) + + it('should accept Infinity as limit', () => { + const limit = utils.limitPromises(Infinity) + const fn2 = limit(fn) + + const result = Promise.all([ + fn2(), fn2(), fn2(), fn2(), + fn2(), fn2(), fn2(), fn2(), + fn2(), fn2() + ]) + + expect(pending).toBe(10) + + return result.then(() => { + expect(pending).toBe(0) + }) + }) }) }) diff --git a/src/plugins/XHRUpload.js b/src/plugins/XHRUpload.js index b83643ad0..d74fb6129 100644 --- a/src/plugins/XHRUpload.js +++ b/src/plugins/XHRUpload.js @@ -33,6 +33,7 @@ module.exports = class XHRUpload extends Plugin { headers: {}, locale: defaultLocale, timeout: 30 * 1000, + limit: 0, getResponseData (xhr) { return JSON.parse(xhr.response) }, @@ -51,6 +52,13 @@ module.exports = class XHRUpload extends Plugin { this.i18n = this.translator.translate.bind(this.translator) this.handleUpload = this.handleUpload.bind(this) + + // Simultaneous upload limiting is shared across all uploads with this plugin. + if (typeof this.opts.limit === 'number' && this.opts.limit !== 0) { + this.limitUploads = limitPromises(this.opts.limit) + } else { + this.limitUploads = (fn) => fn + } } getOptions (file) { @@ -263,8 +271,9 @@ module.exports = class XHRUpload extends Plugin { }) const promises = actions - .map(limitPromises(5)) + .map(this.limitUploads) .map((action) => action()) + return settle(promises) } diff --git a/website/src/docs/xhrupload.md b/website/src/docs/xhrupload.md index dc0640147..d67df5721 100644 --- a/website/src/docs/xhrupload.md +++ b/website/src/docs/xhrupload.md @@ -108,5 +108,9 @@ Set to `0` to disable this check. The default is 30 seconds. +### `limit: 0` + +Limit the amount of uploads going on at the same time. Passing `0` means no limit. + [FormData]: https://developer.mozilla.org/en-US/docs/Web/API/FormData [XHR.timeout]: https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/timeout From 2b9883d4bf166f89be7feb35e025cc4639ed4921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9e=20Kooi?= Date: Mon, 20 Nov 2017 12:20:32 +0100 Subject: [PATCH 4/5] changelog: tick simultaneous upload limiting --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3684c5e5..7f367f89e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,7 +95,7 @@ Theme: 🎄 Christmas edition - [ ] core: Redux PR (#216 / @arturi, @goto-bus-stop, @richardwillars) - [ ] core: css-in-js, while keeping non-random classnames (ideally prefixed) and useful preprocessor features. also see simple https://github.com/codemirror/CodeMirror/blob/master/lib/codemirror.css (@arturi, @goto-bus-stop) - [ ] core: improve on Redux PR #216 to allow using Redux (or any other solution) for all Uppy state management, instead of proxy-only (@goto-bus-stop, @arturi) -- [ ] core: limit amount of simultaneous uploads, queuing? #360 (@goto-bus-stop) +- [x] core: limit amount of simultaneous uploads, queuing? #360 (#427 / @goto-bus-stop) - [ ] core: research !important styles to be immune to any environment/page. Maybe use smth like `postcss-safe-important`, http://cleanslatecss.com/ Or increase specificity (with .uppy prefix) (@arturi) - [ ] dashboard: allow minimizing the Dashboard during upload (Uppy then becomes just a tiny progress indicator) (@arturi) - [ ] dashboard: cancel button for any kind of uploads? currently resume/pause only for tus, and cancel for XHR (@arturi, @goto-bus-stop) From 863e2f1e09a8e9d382e04f1bfc9014606075e882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9e=20Kooi?= Date: Mon, 27 Nov 2017 19:08:17 +0100 Subject: [PATCH 5/5] xhrupload: more procedural --- src/plugins/XHRUpload.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/plugins/XHRUpload.js b/src/plugins/XHRUpload.js index 3e3604374..ed7d7280e 100644 --- a/src/plugins/XHRUpload.js +++ b/src/plugins/XHRUpload.js @@ -270,9 +270,10 @@ module.exports = class XHRUpload extends Plugin { } }) - const promises = actions - .map(this.limitUploads) - .map((action) => action()) + const promises = actions.map((action) => { + const limitedAction = this.limitUploads(action) + return limitedAction() + }) return settle(promises) }