Merge pull request #427 from transloadit/feature/xhr-limit

Add XHRUpload option to limit simultaneous uploads

.eslintignore

@@ -10,3 +10,4 @@ website/public/**
 website/themes/uppy/source/js/smooth-scroll.min.js
 website/themes/uppy/source/js/uppy.js
 website/themes/uppy/source/uppy/**
+bundle.js

CHANGELOG.md

@@ -95,7 +95,7 @@ Theme: 🎄 Christmas edition
 - [ ] core: Redux PR (#216 / @arturi, @goto-bus-stop, @richardwillars)
 - [ ] core: css-in-js, while keeping non-random classnames (ideally prefixed) and useful preprocessor features. also see simple https://github.com/codemirror/CodeMirror/blob/master/lib/codemirror.css (@arturi, @goto-bus-stop)
 - [ ] core: improve on Redux PR #216 to allow using Redux (or any other solution) for all Uppy state management, instead of proxy-only (@goto-bus-stop, @arturi)
-- [ ] core: limit amount of simultaneous uploads, queuing? #360 (@goto-bus-stop)
+- [x] core: limit amount of simultaneous uploads, queuing? #360 (#427 / @goto-bus-stop)
 - [ ] core: research !important styles to be immune to any environment/page. Maybe use smth like `postcss-safe-important`, http://cleanslatecss.com/ Or increase specificity (with .uppy prefix) (@arturi)
 - [ ] dashboard: allow minimizing the Dashboard during upload (Uppy then becomes just a tiny progress indicator) (@arturi)
 - [ ] dashboard: cancel button for any kind of uploads? currently resume/pause only for tus, and cancel for XHR (@arturi, @goto-bus-stop)

src/core/Utils.js

@@ -526,6 +526,43 @@ function settle (promises) {
   })
 }
 
+/**
+ * Limit the amount of simultaneously pending Promises.
+ * Returns a function that, when passed a function `fn`,
+ * will make sure that at most `limit` calls to `fn` are pending.
+ *
+ * @param {number} limit
+ * @return {function()}
+ */
+function limitPromises (limit) {
+  let pending = 0
+  const queue = []
+  return (fn) => {
+    return (...args) => {
+      const call = () => {
+        pending++
+        const promise = fn(...args)
+        promise.then(onfinish, onfinish)
+        return promise
+      }
+
+      if (pending >= limit) {
+        return new Promise((resolve, reject) => {
+          queue.push(() => {
+            call().then(resolve, reject)
+          })
+        })
+      }
+      return call()
+    }
+  }
+  function onfinish () {
+    pending--
+    const next = queue.shift()
+    if (next) next()
+  }
+}
+
 module.exports = {
   generateFileID,
   toArray,
@@ -553,5 +590,6 @@ module.exports = {
   findAllDOMElements,
   getSocketHost,
   emitSocketProgress,
-  settle
+  settle,
+  limitPromises
 }

src/core/Utils.test.js

@@ -370,4 +370,50 @@ describe('core/utils', () => {
       })
     })
   })
+
+  describe('limitPromises', () => {
+    let pending = 0
+    function fn () {
+      pending++
+      return new Promise((resolve) => setTimeout(resolve, 10))
+        .then(() => pending--)
+    }
+
+    it('should run at most N promises at the same time', () => {
+      const limit = utils.limitPromises(4)
+      const fn2 = limit(fn)
+
+      const result = Promise.all([
+        fn2(), fn2(), fn2(), fn2(),
+        fn2(), fn2(), fn2(), fn2(),
+        fn2(), fn2()
+      ])
+
+      expect(pending).toBe(4)
+      setTimeout(() => {
+        expect(pending).toBe(4)
+      }, 10)
+
+      return result.then(() => {
+        expect(pending).toBe(0)
+      })
+    })
+
+    it('should accept Infinity as limit', () => {
+      const limit = utils.limitPromises(Infinity)
+      const fn2 = limit(fn)
+
+      const result = Promise.all([
+        fn2(), fn2(), fn2(), fn2(),
+        fn2(), fn2(), fn2(), fn2(),
+        fn2(), fn2()
+      ])
+
+      expect(pending).toBe(10)
+
+      return result.then(() => {
+        expect(pending).toBe(0)
+      })
+    })
+  })
 })

src/plugins/XHRUpload.js

@@ -5,7 +5,8 @@ const UppySocket = require('../core/UppySocket')
 const {
   emitSocketProgress,
   getSocketHost,
-  settle
+  settle,
+  limitPromises
 } = require('../core/Utils')
 
 module.exports = class XHRUpload extends Plugin {
@@ -32,6 +33,7 @@ module.exports = class XHRUpload extends Plugin {
       headers: {},
       locale: defaultLocale,
       timeout: 30 * 1000,
+      limit: 0,
       getResponseData (xhr) {
         return JSON.parse(xhr.response)
       },
@@ -50,6 +52,13 @@ module.exports = class XHRUpload extends Plugin {
     this.i18n = this.translator.translate.bind(this.translator)
 
     this.handleUpload = this.handleUpload.bind(this)
+
+    // Simultaneous upload limiting is shared across all uploads with this plugin.
+    if (typeof this.opts.limit === 'number' && this.opts.limit !== 0) {
+      this.limitUploads = limitPromises(this.opts.limit)
+    } else {
+      this.limitUploads = (fn) => fn
+    }
   }
 
   getOptions (file) {
@@ -248,19 +257,24 @@ module.exports = class XHRUpload extends Plugin {
   }
 
   uploadFiles (files) {
-    const promises = files.map((file, i) => {
+    const actions = files.map((file, i) => {
       const current = parseInt(i, 10) + 1
       const total = files.length
 
       if (file.error) {
-        return Promise.reject(new Error(file.error))
+        return () => Promise.reject(new Error(file.error))
       } else if (file.isRemote) {
-        return this.uploadRemote(file, current, total)
+        return this.uploadRemote.bind(this, file, current, total)
       } else {
-        return this.upload(file, current, total)
+        return this.upload.bind(this, file, current, total)
       }
     })
 
+    const promises = actions.map((action) => {
+      const limitedAction = this.limitUploads(action)
+      return limitedAction()
+    })
+
     return settle(promises)
   }
 

website/src/docs/xhrupload.md

@@ -108,5 +108,9 @@ Set to `0` to disable this check.
 
 The default is 30 seconds.
 
+### `limit: 0`
+
+Limit the amount of uploads going on at the same time. Passing `0` means no limit.
+
 [FormData]: https://developer.mozilla.org/en-US/docs/Web/API/FormData
 [XHR.timeout]: https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/timeout