Mirrors/uppy
1
0
Fork 0
mirror of https://github.com/transloadit/uppy.git synced 2026-07-17 16:50:22 +00:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into restore/pr-6284

Browse source
This commit is contained in:
prakash 2026-06-20 01:37:42 +05:30
commit 5585a796af
No known key found for this signature in database
406 changed files with 15280 additions and 11993 deletions
Download patch file Download diff file Expand all files Collapse all files

45
packages/@uppy/angular/.eslintrc.json
View file

@ -1,45 +0,0 @@
{
"ignorePatterns": ["projects/**/*"],
"overrides": [
{
"files": ["*.ts"],
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended",
"plugin:@angular-eslint/recommended",
"plugin:@angular-eslint/template/process-inline-templates"
],
"rules": {
// eslint-disable-line import/newline-after-import
"@angular-eslint/directive-selector": [
"error",
{
"type": "attribute",
"prefix": "app",
"style": "camelCase"
}
],
"@typescript-eslint/semi": ["error", "never"],
"import/no-unresolved": "off",
"import/prefer-default-export": "off",
"@angular-eslint/component-selector": [
"error",
{
"type": "element",
"prefix": "app",
"style": "kebab-case"
}
],
"semi": ["error", "never"]
}
},
{
"files": ["*.html"],
"extends": [
"plugin:@angular-eslint/template/recommended",
"plugin:@angular-eslint/template/accessibility"
],
"rules": {}
}
]
}

11
packages/@uppy/angular/package.json
View file

@ -17,22 +17,15 @@
"@angular/platform-browser": "^19.2.17",
"@angular/platform-browser-dynamic": "^19.2.17",
"@angular/router": "^19.2.17",
"rxjs": "~7.8.0",
"tslib": "^2.3.0",
"rxjs": "~7.8.2",
"tslib": "^2.8.1",
"zone.js": "~0.15.0"
},
"devDependencies": {
"@angular-devkit/build-angular": "^19.0.0",
"@angular-eslint/eslint-plugin": "^18.0.1",
"@angular-eslint/eslint-plugin-template": "^18.0.1",
"@angular/cli": "^19.0.0",
"@angular/compiler-cli": "^19.2.17",
"@types/jasmine": "~5.1.0",
"@typescript-eslint/eslint-plugin": "^7.2.0",
"@typescript-eslint/parser": "^7.2.0",
"@typescript-eslint/types": "^7.2.0",
"@typescript-eslint/utils": "^7.2.0",
"eslint": "^8.57.0",
"jasmine-core": "~5.1.0",
"karma": "~6.4.0",
"karma-chrome-launcher": "~3.2.0",

46
packages/@uppy/angular/projects/uppy/angular/.eslintrc.json
View file

@ -1,46 +0,0 @@
{
"extends": "../../../.eslintrc.json",
"ignorePatterns": ["!**/*"],
"overrides": [
{
"files": ["*.ts"],
"parserOptions": {
"project": [
"packages/@uppy/angular/projects/angular/tsconfig.lib.json",
"packages/@uppy/angular/projects/angular/tsconfig.spec.json"
],
"createDefaultProgram": true
},
"rules": {
"@angular-eslint/component-selector": [
"error",
{
"type": "element",
"prefix": "uppy",
"style": "kebab-case"
}
],
"@angular-eslint/directive-selector": [
"error",
{
"type": "attribute",
"prefix": "uppy",
"style": "camelCase"
}
],
"dot-notation": "error",
"indent": "error",
"no-empty-function": "off",
"no-shadow": "error",
"no-unused-expressions": "error",
"no-use-before-define": "off",
"quotes": "error",
"semi": "error"
}
},
{
"files": ["*.html"],
"rules": {}
}
]
}

2
packages/@uppy/angular/projects/uppy/angular/package.json
View file

@ -22,7 +22,7 @@
"prepublishOnly": "rm -fr * && cp -r ../../../dist/uppy/angular .."
},
"dependencies": {
"tslib": "^2.0.0"
"tslib": "^2.8.1"
},
"peerDependencies": {
"@angular/common": "^17.0.0 || ^18.0.0 || ^19.0.0 || ^20.0.0",

10
packages/@uppy/audio/package.json
View file

@ -41,16 +41,16 @@
},
"dependencies": {
"@uppy/utils": "workspace:^",
"preact": "^10.26.10"
"preact": "^10.29.2"
},
"devDependencies": {
"cssnano": "^7.0.7",
"jsdom": "^26.1.0",
"postcss": "^8.5.6",
"cssnano": "^8.0.1",
"jsdom": "^29.1.1",
"postcss": "^8.5.15",
"postcss-cli": "^11.0.1",
"sass": "^1.89.2",
"typescript": "^5.8.3",
"vitest": "^3.2.4"
"vitest": "^4.1.6"
},
"peerDependencies": {
"@uppy/core": "workspace:^"

4
packages/@uppy/audio/src/Audio.tsx
View file

@ -284,7 +284,9 @@ export default class Audio<M extends Meta, B extends Body> extends UIPlugin<
#stop = async () => {
if (this.#stream) {
const audioTracks = this.#stream.getAudioTracks()
audioTracks.forEach((track) => track.stop())
audioTracks.forEach((track) => {
track.stop()
})
}
if (this.#recorder) {

3
packages/@uppy/aws-s3/package.json
View file

@ -45,11 +45,12 @@
"@aws-sdk/client-s3": "^3.362.0",
"@aws-sdk/client-sts": "^3.362.0",
"@aws-sdk/s3-request-presigner": "^3.362.0",
"@types/node": "^20.19.0",
"@uppy/core": "workspace:^",
"jsdom": "^26.1.0",
"nock": "^13.1.0",
"typescript": "^5.8.3",
"vitest": "^3.2.4",
"vitest": "^4.1.6",
"whatwg-fetch": "3.6.2"
},
"peerDependencies": {

3
packages/@uppy/aws-s3/tsconfig.json
View file

@ -2,7 +2,8 @@
"extends": "../../../tsconfig.shared",
"compilerOptions": {
"emitDeclarationOnly": false,
"noEmit": true
"noEmit": true,
"types": ["preact", "node"]
},
"include": ["./package.json", "./src/**/*.*"],
"references": [

2
packages/@uppy/aws-s3/vitest.config.ts
View file

@ -12,7 +12,7 @@ export default defineConfig({
exclude: ['**/minio.test.ts'],
environment: 'jsdom',
},
}
},
{
test: {
name: 's3-browser',

2
packages/@uppy/box/package.json
View file

@ -37,7 +37,7 @@
"@uppy/companion-client": "workspace:^",
"@uppy/provider-views": "workspace:^",
"@uppy/utils": "workspace:^",
"preact": "^10.26.10"
"preact": "^10.29.2"
},
"peerDependencies": {
"@uppy/core": "workspace:^"

4
packages/@uppy/companion-client/package.json
View file

@ -41,9 +41,9 @@
"p-retry": "^6.1.0"
},
"devDependencies": {
"jsdom": "^26.1.0",
"jsdom": "^29.1.1",
"typescript": "^5.8.3",
"vitest": "^3.2.4"
"vitest": "^4.1.6"
},
"peerDependencies": {
"@uppy/core": "workspace:^"

137
packages/@uppy/companion-client/src/Provider.ts
View file

@ -5,8 +5,11 @@ import type {
UnknownProviderPlugin,
Uppy,
} from '@uppy/core'
import type { CompanionClientProvider, RequestOptions } from '@uppy/utils'
import { isOriginAllowed } from './getAllowedHosts.js'
import {
type CompanionClientProvider,
getSocketHost,
type RequestOptions,
} from '@uppy/utils'
import type { CompanionPluginOptions } from './index.js'
import RequestClient, { authErrorStatusCode } from './RequestClient.js'
@ -135,22 +138,27 @@ export default class Provider<M extends Meta, B extends Body>
authUrl({
authFormData,
query,
authCallbackToken,
}: {
authFormData: unknown
query: Record<string, string>
authCallbackToken?: string | undefined
}): string {
const params = new URLSearchParams({
const searchParams = new URLSearchParams({
...query,
// This is only used for Companion instances configured to accept multiple origins.
// `origin` is only used for Companion instances configured to accept multiple origins.
state: btoa(JSON.stringify({ origin: getOrigin() })),
...this.authQuery({ authFormData }),
})
if (this.preAuthToken) {
params.set('uppyPreAuthToken', this.preAuthToken)
searchParams.set('uppyPreAuthToken', this.preAuthToken)
}
if (authCallbackToken) {
searchParams.set('authCallbackToken', authCallbackToken)
}
return `${this.hostname}/${this.id}/connect?${params}`
return `${this.hostname}/${this.id}/connect?${searchParams}`
}
protected async loginSimpleAuth({
@ -181,67 +189,62 @@ export default class Provider<M extends Meta, B extends Body>
signal: AbortSignal
}): Promise<void> {
await this.ensurePreAuth()
signal.throwIfAborted()
const link = this.authUrl({ query: { uppyVersions }, authFormData })
// Important: We need to do this synchronously, or else browsers might block the popup.
// (We cannot wait until the websocket connection is established).
// This opens up a race condtition if the websocket is not connected before the user
// completes(or cancels) authentication, but that’s a small compromose we gotta make.
const authCallbackToken = crypto.randomUUID()
const link = this.authUrl({
query: { uppyVersions },
authFormData,
authCallbackToken,
})
const authWindow = window.open(link, '_blank')
let interval: number | undefined
let handleMessage: ((e: MessageEvent<any>) => void) | undefined
let webSocket: WebSocket | undefined
try {
return await new Promise((resolve, reject) => {
handleMessage = (e: MessageEvent<any>) => {
if (e.source !== authWindow) {
let jsonData = ''
try {
// TODO improve our uppy logger so that it can take an arbitrary number of arguments,
// each either objects, errors or strings,
// then we don’t have to manually do these things like json stringify when logging.
// the logger should never throw an error.
jsonData = JSON.stringify(e.data)
} catch (_err) {
// in case JSON.stringify fails (ignored)
const host = getSocketHost(this.opts.companionUrl)
// Note that this promise is not guaranteed to settle in all cases
const token = await new Promise<string>((resolve, reject) => {
webSocket = new WebSocket(
`${host}/api2/auth-callback/token/${authCallbackToken}`,
)
webSocket.addEventListener('close', () => {
reject(new Error('Socket closed'))
})
webSocket.addEventListener('error', (error) => {
this.uppy.log(
`Companion socket error ${JSON.stringify(error)}, closing socket`,
'warning',
)
webSocket?.close() // 'close' event will be emitted
})
webSocket.addEventListener('message', (e) => {
try {
const { token, error } = JSON.parse(e.data)
if (error) {
reject(new Error('Authentication reported error'))
} else if (!token) {
reject(new Error('Authentication did not return a token'))
} else {
resolve(token)
}
this.uppy.log(
`ignoring event from unknown source ${jsonData}`,
'warning',
)
return
} catch (err) {
reject(err)
}
})
const { companionAllowedHosts } = this.#getPlugin().opts
if (!isOriginAllowed(e.origin, companionAllowedHosts)) {
this.uppy.log(
`ignoring event from ${e.origin} vs allowed pattern ${companionAllowedHosts}`,
'warning',
)
// We cannot reject here because the page might send events from other origins
// before sending the "real" auth completed event.
// for example Box has a "Pendo" tool that sends events to the opener
// https://github.com/transloadit/uppy/pull/5719
return
}
// Check if it's a string before doing the JSON.parse to maintain support
// for older Companion versions that used object references
const data = typeof e.data === 'string' ? JSON.parse(e.data) : e.data
if (data.error) {
const { uppy } = this
const message = uppy.i18n('authAborted')
uppy.info({ message }, 'warning', 5000)
reject(new Error('auth aborted'))
return
}
if (!data.token) {
reject(new Error('did not receive token from auth window'))
return
}
resolve(this.setAuthToken(data.token))
}
signal.addEventListener('abort', () =>
reject(new Error('Authentication was aborted')),
)
// poll for user closure of the window, so we can reject when it happens
if (authWindow) {
@ -251,15 +254,21 @@ export default class Provider<M extends Meta, B extends Body>
}
}, 500)
}
signal.addEventListener('abort', () => reject(new Error('Aborted')))
window.addEventListener('message', handleMessage)
})
this.setAuthToken(token)
} catch (err) {
const message = this.uppy.i18n('authAborted')
this.uppy.info({ message }, 'warning', 5000)
this.uppy.log(`Authentication failed: ${err.message}`, 'warning')
throw err
} finally {
// cleanup:
authWindow?.close()
window.clearInterval(interval)
if (handleMessage) window.removeEventListener('message', handleMessage)
// if we don't setTimeout, the window doesn't really close (I don't know why).
setTimeout(() => authWindow?.close(), 1)
this.uppy.log(`Closing auth callback socket ${authCallbackToken}`)
webSocket?.close()
clearInterval(interval)
}
}

347
packages/@uppy/companion-client/src/RequestClient.ts
View file

@ -234,12 +234,13 @@ export default class RequestClient<M extends Meta, B extends Body> {
}
/**
* Remote uploading consists of two steps:
* 1. #requestSocketToken which starts the download/upload in companion and returns a unique token for the upload.
* Then companion will halt the upload until:
* 2. #awaitRemoteFileUpload is called, which will open/ensure a websocket connection towards companion, with the
* previously generated token provided. It returns a promise that will resolve/reject once the file has finished
* uploading or is otherwise done (failed, canceled)
* Remote uploading uses a single queue admission per attempt:
* 1. Acquire one queue slot.
* 2. Reuse an existing serverToken or request a new one from Companion.
* 3. Open/maintain the websocket and wait for Companion to finish.
*
* This prevents socket tokens from being created long before their websocket
* session is admitted by the queue.
*/
async uploadRemoteFile(
file: RemoteUppyFile<M, B>,
@ -248,72 +249,46 @@ export default class RequestClient<M extends Meta, B extends Body> {
): Promise<void> {
try {
const { signal, getQueue } = options || {}
const queue = getQueue()
return await pRetry(
async () => {
// if we already have a serverToken, assume that we are resuming the existing server upload id
const existingServerToken = this.uppy.getFile(file.id)?.serverToken
if (existingServerToken != null) {
this.uppy.log(
`Connecting to exiting websocket ${existingServerToken}`,
)
return this.#awaitRemoteFileUpload({
file,
queue: getQueue(),
signal,
})
}
const queueRemoteUploadAttempt = queue.wrapPromiseFunction(
async () => {
const currentFile = this.uppy.getFile(file.id) as
| RemoteUppyFile<M, B>
| undefined
if (currentFile == null) return undefined
const queueRequestSocketToken = getQueue().wrapPromiseFunction(
async (
...args: [
{
file: RemoteUppyFile<M, B>
postBody: Record<string, unknown>
signal: AbortSignal
},
]
) => {
try {
return await this.#requestSocketToken(...args)
} catch (outerErr) {
// throwing AbortError will cause p-retry to stop retrying
if (outerErr.isAuthError) throw new AbortError(outerErr)
let serverToken = currentFile.serverToken
if (outerErr.cause == null) throw outerErr
const err = outerErr.cause
if (serverToken != null) {
this.uppy.log(`Connecting to exiting websocket ${serverToken}`)
} else {
serverToken = await this.#requestSocketTokenWithRetryStrategy({
file: currentFile,
postBody: reqBody,
signal,
})
const isRetryableHttpError = () =>
[408, 409, 429, 418, 423].includes(err.statusCode) ||
(err.statusCode >= 500 &&
err.statusCode <= 599 &&
![501, 505].includes(err.statusCode))
if (err.name === 'HttpError' && !isRetryableHttpError())
throw new AbortError(err)
if (!this.uppy.getFile(file.id)) return undefined
// p-retry will retry most other errors,
// but it will not retry TypeError (except network error TypeErrors)
throw err
this.uppy.setFileState(file.id, { serverToken })
}
const latestFile = this.uppy.getFile(file.id) as
| RemoteUppyFile<M, B>
| undefined
if (latestFile == null) return undefined
return this.#awaitRemoteFileUpload({
file: latestFile,
signal,
})
},
{ priority: -1 },
)
const serverToken = await queueRequestSocketToken({
file,
postBody: reqBody,
signal,
}).abortOn(signal)
if (!this.uppy.getFile(file.id)) return undefined // has file since been removed?
this.uppy.setFileState(file.id, { serverToken })
return this.#awaitRemoteFileUpload({
file: this.uppy.getFile(file.id) as RemoteUppyFile<M, B>, // re-fetching file because it might have changed in the meantime
queue: getQueue(),
signal,
})
return queueRemoteUploadAttempt().abortOn(signal)
},
{
retries: retryCount,
@ -360,6 +335,39 @@ export default class RequestClient<M extends Meta, B extends Body> {
return res.token
}
#requestSocketTokenWithRetryStrategy = async ({
file,
postBody,
signal,
}: {
file: RemoteUppyFile<M, B>
postBody: Record<string, unknown>
signal: AbortSignal
}): Promise<string> => {
try {
return await this.#requestSocketToken({ file, postBody, signal })
} catch (outerErr) {
// throwing AbortError will cause p-retry to stop retrying
if (outerErr.isAuthError) throw new AbortError(outerErr)
if (outerErr.cause == null) throw outerErr
const err = outerErr.cause
const isRetryableHttpError = () =>
[408, 409, 429, 418, 423].includes(err.statusCode) ||
(err.statusCode >= 500 &&
err.statusCode <= 599 &&
![501, 505].includes(err.statusCode))
if (err.name === 'HttpError' && !isRetryableHttpError()) {
throw new AbortError(err)
}
// p-retry will retry most other errors,
// but it will not retry TypeError (except network error TypeErrors)
throw err
}
}
/**
* This method will ensure a websocket for the specified file and returns a promise that resolves
* when the file has finished downloading, or rejects if it fails.
@ -367,11 +375,9 @@ export default class RequestClient<M extends Meta, B extends Body> {
*/
async #awaitRemoteFileUpload({
file,
queue,
signal,
}: {
file: RemoteUppyFile<M, B>
queue: any
signal: AbortSignal
}): Promise<void> {
let removeEventHandlers: () => void
@ -430,126 +436,113 @@ export default class RequestClient<M extends Meta, B extends Body> {
function resetActivityTimeout() {
clearTimeout(activityTimeout)
if (isPaused) return
activityTimeout = setTimeout(
() =>
onFatalError(
new Error(
'Timeout waiting for message from Companion socket',
),
),
socketActivityTimeoutMs,
)
activityTimeout = setTimeout(() => {
onFatalError(
new Error('Timeout waiting for message from Companion socket'),
)
}, socketActivityTimeoutMs)
}
try {
await queue
.wrapPromiseFunction(async () => {
const reconnectWebsocket = async () =>
new Promise((_, rejectSocket) => {
socket = new WebSocket(`${host}/api/${token}`)
const reconnectWebsocket = async () =>
new Promise((_, rejectSocket) => {
socket = new WebSocket(`${host}/api/${token}`)
resetActivityTimeout()
resetActivityTimeout()
socket.addEventListener('close', () => {
socket = undefined
rejectSocket(new Error('Socket closed unexpectedly'))
})
socket.addEventListener('error', (error) => {
this.uppy.log(
`Companion socket error ${JSON.stringify(
error,
)}, closing socket`,
'warning',
)
socket?.close() // will 'close' event to be emitted
})
socket.addEventListener('open', () => {
sendState()
})
socket.addEventListener('message', (e) => {
resetActivityTimeout()
try {
const { action, payload } = JSON.parse(e.data)
switch (action) {
case 'progress': {
emitSocketProgress(
this,
payload,
this.uppy.getFile(file.id),
)
break
}
case 'success': {
// payload.response is sent from companion for xhr-upload (aka uploadMultipart in companion) and
// s3 multipart (aka uploadS3Multipart)
// but not for tus/transloadit (aka uploadTus)
// responseText is a string which may or may not be in JSON format
// this means that an upload destination of xhr or s3 multipart MUST respond with valid JSON
// to companion, or the JSON.parse will crash
const text = payload.response?.responseText
this.uppy.emit(
'upload-success',
this.uppy.getFile(file.id),
{
uploadURL: payload.url,
status: payload.response?.status ?? 200,
body: text
? (JSON.parse(text) as B)
: undefined,
},
)
socketAbortController?.abort?.()
resolve()
break
}
case 'error': {
const { message } = payload.error
throw Object.assign(new Error(message), {
cause: payload.error,
})
}
default:
this.uppy.log(
`Companion socket unknown action ${action}`,
'warning',
)
}
} catch (err) {
onFatalError(err)
}
})
const closeSocket = () => {
this.uppy.log(`Closing socket ${file.id}`)
clearTimeout(activityTimeout)
if (socket) socket.close()
socket = undefined
}
socketAbortController.signal.addEventListener(
'abort',
() => {
closeSocket()
},
)
})
await pRetry(reconnectWebsocket, {
retries: retryCount,
signal: socketAbortController.signal,
onFailedAttempt: () => {
if (socketAbortController.signal.aborted) return // don't log in this case
this.uppy.log(`Retrying websocket ${file.id}`)
},
socket.addEventListener('close', () => {
socket = undefined
rejectSocket(new Error('Socket closed unexpectedly'))
})
})()
.abortOn(socketAbortController.signal)
socket.addEventListener('error', (error) => {
this.uppy.log(
`Companion socket error ${JSON.stringify(
error,
)}, closing socket`,
'warning',
)
socket?.close() // will 'close' event to be emitted
})
socket.addEventListener('open', () => {
sendState()
})
socket.addEventListener('message', (e) => {
resetActivityTimeout()
try {
const { action, payload } = JSON.parse(e.data)
switch (action) {
case 'progress': {
emitSocketProgress(
this,
payload,
this.uppy.getFile(file.id),
)
break
}
case 'success': {
// payload.response is sent from companion for xhr-upload (aka uploadMultipart in companion) and
// s3 multipart (aka uploadS3Multipart)
// but not for tus/transloadit (aka uploadTus)
// responseText is a string which may or may not be in JSON format
// this means that an upload destination of xhr or s3 multipart MUST respond with valid JSON
// to companion, or the JSON.parse will crash
const text = payload.response?.responseText
this.uppy.emit(
'upload-success',
this.uppy.getFile(file.id),
{
uploadURL: payload.url,
status: payload.response?.status ?? 200,
body: text ? (JSON.parse(text) as B) : undefined,
},
)
socketAbortController?.abort?.()
resolve()
break
}
case 'error': {
const { message } = payload.error
throw Object.assign(new Error(message), {
cause: payload.error,
})
}
default:
this.uppy.log(
`Companion socket unknown action ${action}`,
'warning',
)
}
} catch (err) {
onFatalError(err)
}
})
const closeSocket = () => {
this.uppy.log(`Closing socket ${file.id}`)
clearTimeout(activityTimeout)
if (socket) socket.close()
socket = undefined
}
socketAbortController.signal.addEventListener('abort', () => {
closeSocket()
})
})
await pRetry(reconnectWebsocket, {
retries: retryCount,
signal: socketAbortController.signal,
onFailedAttempt: () => {
if (socketAbortController.signal.aborted) return // don't log in this case
this.uppy.log(`Retrying websocket ${file.id}`)
},
})
} catch (err) {
if (socketAbortController.signal.aborted) return
onFatalError(err)

19
packages/@uppy/companion-client/src/getAllowedHosts.test.ts
View file

@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
import getAllowedHosts, { isOriginAllowed } from './getAllowedHosts.js'
import getAllowedHosts from './getAllowedHosts.js'
describe('getAllowedHosts', () => {
it('can convert companionAllowedHosts', () => {
@ -30,20 +30,3 @@ describe('getAllowedHosts', () => {
)
})
})
describe('isOriginAllowed', () => {
it('should check origin', () => {
expect(isOriginAllowed('a', [/^.+$/])).toBeTruthy()
expect(isOriginAllowed('a', ['^.+$'])).toBeTruthy()
expect(
isOriginAllowed('www.transloadit.com', ['^www\\.transloadit\\.com$']),
).toBeTruthy()
expect(
isOriginAllowed('www.transloadit.com', ['^transloadit\\.com$']),
).toBeFalsy()
expect(isOriginAllowed('match', ['fail', 'match'])).toBeTruthy()
expect(
isOriginAllowed('www.transloadit.com', ['\\.transloadit\\.com$']),
).toBeTruthy()
})
})

12
packages/@uppy/companion-client/src/getAllowedHosts.ts
View file

@ -47,15 +47,3 @@ export default function getAllowedHosts(
ret = escapeRegex(ret)
return ret
}
export function isOriginAllowed(
origin: string,
allowedOrigin: string | RegExp | Array<string | RegExp> | undefined,
): boolean {
const patterns = Array.isArray(allowedOrigin)
? allowedOrigin.map(wrapInRegex)
: [wrapInRegex(allowedOrigin)]
return patterns.some(
(pattern) => pattern?.test(origin) || pattern?.test(`${origin}/`),
) // allowing for trailing '/'
}

2
packages/@uppy/companion/.gitignore vendored
View file

@ -34,6 +34,6 @@ test/output/*
.DS_Store
# Transpiled
./lib/
./dist/
infra/kube/companion/uppy-env.yaml
scripts/.tl-deploy-hosts-danger.txt

7
packages/@uppy/companion/CHANGELOG.md
View file

@ -1,5 +1,12 @@
# @uppy/companion
## 6.2.2
### Patch Changes
- 49db42d: Fix bug with 429 not returning JSON response with message
- 4652dc6: upgrade @aws-sdk/ deps in @uppy/companion
## 6.2.1
### Patch Changes

2
packages/@uppy/companion/KUBERNETES.md
View file

@ -38,8 +38,6 @@ data:
COMPANION_BOX_SECRET: 'YOUR BOX SECRET'
COMPANION_GOOGLE_KEY: 'YOUR GOOGLE KEY'
COMPANION_GOOGLE_SECRET: 'YOUR GOOGLE SECRET'
COMPANION_INSTAGRAM_KEY: 'YOUR INSTAGRAM KEY'
COMPANION_INSTAGRAM_SECRET: 'YOUR INSTAGRAM SECRET'
COMPANION_AWS_KEY: 'YOUR AWS KEY'
COMPANION_AWS_SECRET: 'YOUR AWS SECRET'
COMPANION_AWS_BUCKET: 'YOUR AWS S3 BUCKET'

7
packages/@uppy/companion/README.md
View file

@ -8,7 +8,7 @@ Companion is a server integration for
[Uppy](https://github.com/transloadit/uppy) file uploader.
It handles the server-to-server communication between your server and file
storage providers such as Google Drive, Dropbox, Instagram, etc. **Companion is
storage providers such as Google Drive, Dropbox, etc. **Companion is
not a target to upload files to**. For this, use a <https://tus.io> server (if
you want resumable) or your existing Apache/Nginx server (if you don’t).
[See here for full documentation](https://uppy.io/docs/companion/)
@ -68,7 +68,10 @@ on, you call the `socket` method like so.
// ...
const server = app.listen(PORT)
companion.socket(server)
// Pass the same `options` object you passed to `companion.app(options)` —
// `companion.socket` needs `options.server` to compute the external base path
// for incoming WS URLs (important behind reverse proxies).
companion.socket(server, options)
```
### Run as standalone server

9
packages/@uppy/companion/__mocks__/express-prom-bundle.js → packages/@uppy/companion/__mocks__/express-prom-bundle.ts
View file

@ -3,7 +3,14 @@ class Gauge {
}
export default function () {
const middleware = (req, res, next) => {
type Req = { url?: string }
type Res = {
setHeader: (key: string, value: string) => void
end: (s?: string) => void
}
type Next = () => void
const middleware = (req: Req, res: Res, next: Next) => {
// simulate prometheus metrics endpoint:
if (req.url === '/metrics') {
res.setHeader('Content-Type', 'text/plain')

21
packages/@uppy/companion/__mocks__/tus-js-client.js
View file

@ -1,21 +0,0 @@
export class Upload {
constructor(file, options) {
this.url = 'https://tus.endpoint/files/foo-bar'
this.options = options
}
_triggerProgressThenSuccess() {
this.options.onProgress(this.options.uploadSize, this.options.uploadSize)
setTimeout(() => this.options.onSuccess(), 100)
}
start() {
setTimeout(this._triggerProgressThenSuccess.bind(this), 100)
}
abort() {
// noop
}
}
export default { Upload }

42
packages/@uppy/companion/__mocks__/tus-js-client.ts Normal file
View file

@ -0,0 +1,42 @@
type UploadOptions = {
uploadSize: number
onProgress: (bytesUploaded: number, bytesTotal: number) => void
onSuccess: () => void
} & Record<string, unknown>
let lastUploadFile: unknown
export function __getLastUploadFile(): unknown {
return lastUploadFile
}
export function __resetTusMockState(): void {
lastUploadFile = undefined
}
export class Upload {
url: string
options: UploadOptions
constructor(file: unknown, options: UploadOptions) {
lastUploadFile = file
this.url = 'https://tus.endpoint/files/foo-bar'
this.options = options
}
_triggerProgressThenSuccess() {
this.options.onProgress(this.options.uploadSize, this.options.uploadSize)
setTimeout(() => this.options.onSuccess(), 100)
}
start() {
setTimeout(this._triggerProgressThenSuccess.bind(this), 100)
}
abort() {
// noop
}
}
export default { Upload }

3
packages/@uppy/companion/bin/companion
View file

@ -1,3 +0,0 @@
#!/usr/bin/env node
import '../lib/standalone/start-server.js'

4
packages/@uppy/companion/env_example
View file

@ -26,10 +26,6 @@ COMPANION_GOOGLE_KEY=
COMPANION_GOOGLE_SECRET=
COMPANION_GOOGLE_SECRET_FILE=
COMPANION_INSTAGRAM_KEY=
COMPANION_INSTAGRAM_SECRET=
COMPANION_INSTAGRAM_SECRET_FILE=
COMPANION_FACEBOOK_KEY=
COMPANION_FACEBOOK_SECRET=
COMPANION_FACEBOOK_SECRET_FILE=

2
packages/@uppy/companion/nodemon.json
View file

@ -9,5 +9,5 @@
"debug": true,
"watch": ["/app/", "/src/"],
"ext": "js dust html ejs css scss rb json htpasswd",
"exec": "node /app/lib/standalone/start-server.js"
"exec": "node /app/dist/standalone/start-server.js"
}

124
packages/@uppy/companion/package.json
View file

@ -1,8 +1,8 @@
{
"name": "@uppy/companion",
"version": "6.2.1",
"version": "6.2.2",
"description": "OAuth helper and remote fetcher for Uppy's (https://uppy.io) extensible file upload widget with support for drag&drop, resumable uploads, previews, restrictions, file processing/encoding, remote providers like Dropbox and Google Drive, S3 and more :dog:",
"types": "lib/companion.d.ts",
"types": "dist/companion.d.ts",
"author": "Transloadit.com",
"license": "MIT",
"sideEffects": false,
@ -13,8 +13,8 @@
"url": "git+https://github.com/transloadit/uppy.git"
},
"exports": {
".": "./lib/companion.js",
"./standalone": "./lib/standalone/index.js",
".": "./dist/companion.js",
"./standalone": "./dist/standalone/index.js",
"./package.json": "./package.json"
},
"keywords": [
@ -32,84 +32,92 @@
"express",
"realtime"
],
"bin": "./bin/companion",
"bin": "./dist/bin/companion.js",
"dependencies": {
"@aws-sdk/client-s3": "^3.338.0",
"@aws-sdk/client-sts": "^3.338.0",
"@aws-sdk/lib-storage": "^3.338.0",
"@aws-sdk/s3-presigned-post": "^3.338.0",
"@aws-sdk/s3-request-presigner": "^3.338.0",
"body-parser": "1.20.4",
"@aws-sdk/client-s3": "3.1047.0",
"@aws-sdk/client-sts": "3.1047.0",
"@aws-sdk/lib-storage": "3.1047.0",
"@aws-sdk/s3-presigned-post": "3.1047.0",
"@aws-sdk/s3-request-presigner": "3.1047.0",
"body-parser": "2.2.2",
"common-tags": "1.8.2",
"connect-redis": "7.1.1",
"content-disposition": "^0.5.4",
"content-disposition": "2.0.0",
"cookie-parser": "1.4.7",
"cors": "^2.8.5",
"escape-goat": "3.0.0",
"escape-string-regexp": "4.0.0",
"express": "4.21.2",
"cors": "2.8.6",
"escape-goat": "4.0.0",
"escape-string-regexp": "5.0.0",
"express": "^5.2.1",
"express-interceptor": "1.2.0",
"express-prom-bundle": "7",
"express-session": "1.18.1",
"express-prom-bundle": "^8.0.0",
"express-session": "1.19.0",
"fast-safe-stringify": "^2.1.1",
"formdata-node": "^6.0.3",
"got": "^13.0.0",
"grant": "^5.4.24",
"helmet": "^7.1.0",
"ioredis": "^5.3.2",
"ipaddr.js": "^2.0.1",
"jsonwebtoken": "9.0.2",
"lodash": "^4.17.23",
"mime-types": "2.1.35",
"moment": "^2.29.2",
"moment-timezone": "^0.5.31",
"morgan": "1.10.0",
"helmet": "8.1.0",
"ioredis": "5.10.1",
"ipaddr.js": "2.4.0",
"jsonwebtoken": "9.0.3",
"lodash": "4.18.1",
"mime-types": "3.0.2",
"moment": "2.30.1",
"moment-timezone": "0.6.2",
"morgan": "1.10.1",
"ms": "2.1.3",
"node-schedule": "2.1.1",
"p-map": "^7.0.3",
"prom-client": "15.1.2",
"serialize-error": "^11.0.0",
"serialize-javascript": "^6.0.0",
"supports-color": "8.x",
"tus-js-client": "^4.1.0",
"validator": "^13.15.22",
"webdav": "^5.8.0",
"ws": "8.17.1"
"p-map": "7.0.4",
"prom-client": "15.1.3",
"serialize-error": "13.0.1",
"serialize-javascript": "7.0.5",
"supports-color": "10.2.2",
"tus-js-client": "4.3.1",
"validator": "13.15.35",
"webdav": "5.10.0",
"ws": "8.20.1",
"zod": "4.4.3"
},
"devDependencies": {
"@types/compression": "1.7.0",
"@types/cookie-parser": "1.4.2",
"@types/common-tags": "1.8.4",
"@types/compression": "1.8.1",
"@types/content-disposition": "0.5.9",
"@types/cookie-parser": "1.4.10",
"@types/cors": "2.8.6",
"@types/eslint": "^8.2.0",
"@types/express-session": "1.17.3",
"@types/http-proxy": "^1",
"@types/jsonwebtoken": "8.3.7",
"@types/lodash": "4.14.191",
"@types/eslint": "9.6.1",
"@types/express-session": "1.19.0",
"@types/http-proxy": "1.17.17",
"@types/jsonwebtoken": "9.0.10",
"@types/lodash": "4.17.24",
"@types/mime-types": "3.0.1",
"@types/morgan": "1.7.37",
"@types/ms": "0.7.31",
"@types/node": "^20.19.0",
"@types/request": "2.48.8",
"@types/webpack": "^5.28.0",
"@types/ws": "8.5.3",
"execa": "^9.6.0",
"http-proxy": "^1.18.1",
"nock": "^13.1.3",
"supertest": "6.2.4",
"typescript": "^5.8.3",
"vitest": "^3.2.4"
"@types/ms": "2.1.0",
"@types/node": "20.19.41",
"@types/node-schedule": "2.1.8",
"@types/request": "2.48.13",
"@types/serialize-javascript": "5.0.4",
"@types/supertest": "7.2.0",
"@types/validator": "13.15.10",
"@types/webpack": "5.28.5",
"@types/ws": "8.18.1",
"execa": "9.6.1",
"http-proxy": "1.18.1",
"nock": "^14.0.15",
"supertest": "7.2.2",
"typescript": "6.0.3",
"vitest": "4.1.6"
},
"files": [
"bin/",
"lib/",
"dist/",
"src/",
"CHANGELOG.md"
],
"scripts": {
"build": "tsc --build tsconfig.build.json",
"deploy": "kubectl apply -f infra/kube/companion-kube.yml",
"start": "node ./lib/standalone/start-server.js",
"start": "node ./dist/standalone/start-server.js",
"start:dev": "bash start-dev",
"typecheck": "tsc --build",
"typecheck": "tsc",
"check": "yarn typecheck && yarn test",
"test": "vitest run --silent='passed-only'"
},
"engines": {

3
packages/@uppy/companion/src/bin/companion.ts Normal file
View file

@ -0,0 +1,3 @@
#!/usr/bin/env node
import '../standalone/start-server.js'

76
packages/@uppy/companion/src/companion.js → packages/@uppy/companion/src/companion.ts
View file

@ -11,6 +11,10 @@ import {
validateConfig,
} from './config/companion.js'
import grantConfigFn from './config/grant.js'
import type {
CompanionInitOptions,
CredentialsFetchResponse,
} from './schemas/index.js'
import googlePicker from './server/controllers/googlePicker.js'
import * as controllers from './server/controllers/index.js'
import s3 from './server/controllers/s3.js'
@ -28,17 +32,22 @@ import {
ProviderUserError,
} from './server/provider/error.js'
import * as providerManager from './server/provider/index.js'
import type Provider from './server/provider/Provider.js'
import { isOAuthProvider } from './server/provider/Provider.js'
import * as redis from './server/redis.js'
import socket from './server/socket.js'
import type { CompanionRuntimeOptions } from './types/companion-options.js'
export { socket }
const grantConfig = grantConfigFn()
export function setLoggerProcessName({ loggerProcessName }) {
if (loggerProcessName != null) logger.setProcessName(loggerProcessName)
export function setLoggerProcessName({
loggerProcessName,
}: Pick<CompanionRuntimeOptions, 'loggerProcessName'>) {
if (loggerProcessName != null) {
logger.setProcessName(loggerProcessName)
}
}
// intercepts grantJS' default response error when something goes
@ -83,11 +92,8 @@ export const errors = {
/**
* Entry point into initializing the Companion app.
*
* @param {object} optionsArg
* @returns {{ app: import('express').Express, emitter: any }}}
*/
export function app(optionsArg = {}) {
export function app(optionsArg: CompanionInitOptions) {
setLoggerProcessName(optionsArg)
validateConfig(optionsArg)
@ -96,13 +102,17 @@ export function app(optionsArg = {}) {
const providers = providerManager.getDefaultProviders()
const { customProviders } = options
const customProviders = options.customProviders
if (customProviders) {
providerManager.addCustomProviders(customProviders, providers, grantConfig)
providerManager.addCustomProviders(
customProviders,
providers as Record<string, typeof Provider>,
grantConfig,
)
}
const getOauthProvider = (providerName) =>
providers[providerName]?.oauthProvider
const getOauthProvider = (providerName: string) =>
providers[providerName as keyof typeof providers]?.oauthProvider
providerManager.addProviderOptions(options, grantConfig, getOauthProvider)
@ -115,21 +125,29 @@ export function app(optionsArg = {}) {
const app = express()
// Needed for e.g. s3 `/params` endpoint metadata, like `metadata[key]=value` to be parsed into a metadata object
app.set('query parser', 'extended')
if (options.metrics) {
app.use(middlewares.metrics({ path: options.server.path }))
}
app.get('/health', (_req, res) => res.end())
app.use(cookieParser()) // server tokens are added to cookies
app.use(interceptGrantErrorResponse)
// override provider credentials at request time
// Making `POST` request to the `/connect/:provider/:override?` route requires a form body parser middleware:
// Making `POST` request to the `/connect/:provider{/:override}` route requires a form body parser middleware:
// See https://github.com/simov/grant#dynamic-http
app.use(
'/connect/:oauthProvider/:override?',
'/connect/:oauthProvider{/:override}',
express.urlencoded({ extended: false }),
getCredentialsOverrideMiddleware(providers, options),
getCredentialsOverrideMiddleware(
providers as Record<string, typeof Provider>,
options,
),
)
app.use(grant.default.express(grantConfig))
@ -144,7 +162,7 @@ export function app(optionsArg = {}) {
app.use(middlewares.cors(options))
// add uppy options to the request object so it can be accessed by subsequent handlers.
app.use('*', middlewares.getCompanionMiddleware(options))
app.use(middlewares.getCompanionMiddleware(options))
app.use('/s3', s3(options.s3))
if (options.enableUrlEndpoint) app.use('/url', url())
if (options.enableGooglePickerEndpoint)
@ -217,7 +235,7 @@ export function app(optionsArg = {}) {
)
app.get(
'/:providerName/list/:id?',
'/:providerName/list{/:id}',
middlewares.hasSessionAndProvider,
middlewares.verifyToken,
controllers.list,
@ -263,19 +281,28 @@ export function app(optionsArg = {}) {
// Used for testing dynamic credentials only, normally this would run on a separate server.
if (options.testDynamicOauthCredentials) {
logger.info('Dynamic credentials test endpoint enabled')
app.post('/:providerName/test-dynamic-oauth-credentials', (req, res) => {
if (req.query.secret !== options.testDynamicOauthCredentialsSecret)
const providedSecret = req.query['secret']
console.log('Received request for dynamic credentials test endpoint', {
providedSecret,
})
if (
typeof providedSecret !== 'string' ||
providedSecret !== options.testDynamicOauthCredentialsSecret
) {
throw new Error('Invalid secret')
}
const { providerName } = req.params
// for simplicity, we just return the normal credentials for the provider, but in a real-world scenario,
// we would query based on parameters
const { key, secret } = options.providerOptions[providerName] ?? {
__proto__: null,
}
const { key, secret } = options.providerOptions[providerName]!
function getTransloaditGateway() {
const oauthProvider = getOauthProvider(providerName)
if (!isOAuthProvider(oauthProvider)) return undefined
if (!isOAuthProvider(oauthProvider))
throw new Error('Not an OAuth provider')
return getURLBuilder(options)('', true)
}
@ -285,7 +312,7 @@ export function app(optionsArg = {}) {
secret,
transloadit_gateway: getTransloaditGateway(),
origins: ['http://localhost:5173'],
},
} satisfies CredentialsFetchResponse,
}
logger.info(
@ -299,7 +326,10 @@ export function app(optionsArg = {}) {
app.param(
'providerName',
providerManager.getProviderMiddleware(providers, grantConfig),
providerManager.getProviderMiddleware(
providers as Record<string, typeof Provider>,
grantConfig,
),
)
if (app.get('env') !== 'test') {

164
packages/@uppy/companion/src/config/companion.js
View file

@ -1,164 +0,0 @@
import fs from 'node:fs'
import validator from 'validator'
import { defaultGetKey } from '../server/helpers/utils.js'
import logger from '../server/logger.js'
export const defaultOptions = {
server: {
protocol: 'http',
path: '',
},
providerOptions: {},
s3: {
endpoint: 'https://{service}.{region}.amazonaws.com',
conditions: [],
useAccelerateEndpoint: false,
getKey: defaultGetKey,
expires: 800, // seconds
},
enableUrlEndpoint: false,
enableGooglePickerEndpoint: false,
allowLocalUrls: false,
periodicPingUrls: [],
streamingUpload: true,
clientSocketConnectTimeout: 60000,
metrics: true,
}
/**
* @param {object} companionOptions
*/
export function getMaskableSecrets(companionOptions) {
const secrets = []
const { providerOptions, customProviders, s3 } = companionOptions
Object.keys(providerOptions).forEach((provider) => {
if (providerOptions[provider].secret) {
secrets.push(providerOptions[provider].secret)
}
})
if (customProviders) {
Object.keys(customProviders).forEach((provider) => {
if (customProviders[provider].config?.secret) {
secrets.push(customProviders[provider].config.secret)
}
})
}
if (s3?.secret) {
secrets.push(s3.secret)
}
return secrets
}
/**
* validates that the mandatory companion options are set.
* If it is invalid, it will console an error of unset options and exits the process.
* If it is valid, nothing happens.
*
* @param {object} companionOptions
*/
export const validateConfig = (companionOptions) => {
const mandatoryOptions = ['secret', 'filePath', 'server.host']
/** @type {string[]} */
const unspecified = []
mandatoryOptions.forEach((i) => {
const value = i
.split('.')
.reduce((prev, curr) => (prev ? prev[curr] : undefined), companionOptions)
if (!value) unspecified.push(`"${i}"`)
})
// vaidate that all required config is specified
if (unspecified.length) {
const messagePrefix =
'Please specify the following options to use companion:'
throw new Error(`${messagePrefix}\n${unspecified.join(',\n')}`)
}
// validate that specified filePath is writeable/readable.
try {
// @ts-ignore
fs.accessSync(
`${companionOptions.filePath}`,
fs.constants.R_OK | fs.constants.W_OK,
)
} catch (_err) {
throw new Error(
`No access to "${companionOptions.filePath}". Please ensure the directory exists and with read/write permissions.`,
)
}
const { providerOptions, periodicPingUrls, server } = companionOptions
if (server?.path) {
// see https://github.com/transloadit/uppy/issues/4271
// todo fix the code so we can allow `/`
if (server.path === '/')
throw new Error(
"If you want to use '/' as server.path, leave the 'path' variable unset",
)
}
if (providerOptions) {
const deprecatedOptions = {
microsoft: 'providerOptions.onedrive',
google: 'providerOptions.drive',
s3: 's3',
}
Object.keys(deprecatedOptions).forEach((deprecated) => {
if (Object.hasOwn(providerOptions, deprecated)) {
throw new Error(
`The Provider option "providerOptions.${deprecated}" is no longer supported. Please use the option "${deprecatedOptions[deprecated]}" instead.`,
)
}
})
}
if (
companionOptions.uploadUrls == null ||
companionOptions.uploadUrls.length === 0
) {
if (process.env.NODE_ENV === 'production')
throw new Error('uploadUrls is required')
logger.error(
'Running without uploadUrls is a security risk and Companion will refuse to start up when running in production (NODE_ENV=production)',
'startup.uploadUrls',
)
}
if (companionOptions.corsOrigins == null) {
throw new TypeError(
'Option corsOrigins is required. To disable security, pass true',
)
}
if (companionOptions.corsOrigins === '*') {
throw new TypeError(
'Option corsOrigins cannot be "*". To disable security, pass true',
)
}
if (
periodicPingUrls != null &&
(!Array.isArray(periodicPingUrls) ||
periodicPingUrls.some(
(url2) =>
!validator.isURL(url2, {
protocols: ['http', 'https'],
require_protocol: true,
require_tld: false,
}),
))
) {
throw new TypeError('Invalid periodicPingUrls')
}
if (companionOptions.maxFilenameLength <= 0) {
throw new TypeError('Option maxFilenameLength must be greater than 0')
}
}

152
packages/@uppy/companion/src/config/companion.ts Normal file
View file

@ -0,0 +1,152 @@
import fs from 'node:fs'
import type { PresignedPostOptions } from '@aws-sdk/s3-presigned-post'
import validator from 'validator'
import z from 'zod'
import type { CompanionInitOptions } from '../schemas/companion.js'
import { defaultGetKey } from '../server/helpers/utils.js'
import logger from '../server/logger.js'
const defaultS3Conditions: PresignedPostOptions['Conditions'] = []
const defaultPeriodicPingUrls: string[] = []
export const defaultOptions = {
server: {
protocol: 'http',
path: '',
},
providerOptions: {},
s3: {
endpoint: 'https://{service}.{region}.amazonaws.com',
conditions: defaultS3Conditions,
useAccelerateEndpoint: false,
getKey: defaultGetKey,
expires: 800, // seconds
},
enableUrlEndpoint: false,
enableGooglePickerEndpoint: false,
allowLocalUrls: false,
periodicPingUrls: defaultPeriodicPingUrls,
streamingUpload: true,
clientSocketConnectTimeout: 60000,
metrics: true,
}
/**
* Returns secrets that should be masked in log messages.
*/
export function getMaskableSecrets(
companionOptions: CompanionInitOptions,
): string[] {
const secrets: string[] = []
const { customProviders, providerOptions = {}, s3 } = companionOptions ?? {}
Object.keys(providerOptions).forEach((provider) => {
const secret = providerOptions[provider]?.secret
if (secret != null) secrets.push(secret)
})
if (customProviders) {
Object.keys(customProviders).forEach((provider) => {
const secret = customProviders[provider]?.config?.secret
if (secret != null) secrets.push(secret)
})
}
const s3Secret = s3?.['secret']
if (s3Secret != null) {
secrets.push(s3Secret)
}
return secrets
}
const validateConfigSchema = z.object({
filePath: z.string().nonempty(),
secret: z.string().nonempty(),
server: z.object({
host: z.string().nonempty(),
}),
periodicPingUrls: z
.string()
.refine(
(url) =>
validator.isURL(url, {
protocols: ['http', 'https'],
require_protocol: true,
require_tld: false,
}),
{
message: 'periodicPingUrls',
},
)
.array()
.optional(),
maxFilenameLength: z.number().positive().optional(),
})
/**
* Validates that the mandatory Companion options are set.
*
* If invalid, throws with an error explaining what needs to be fixed.
*/
export function validateConfig(companionOptions: CompanionInitOptions): void {
const parsedConfig = validateConfigSchema.parse(companionOptions)
const { filePath } = parsedConfig
// validate that specified filePath is writeable/readable.
try {
fs.accessSync(filePath, fs.constants.R_OK | fs.constants.W_OK)
} catch {
throw new Error(
`No access to "${filePath}". Please ensure the directory exists and with read/write permissions.`,
)
}
const { providerOptions, server, uploadUrls } = companionOptions
// see https://github.com/transloadit/uppy/issues/4271
// todo fix the code so we can allow `/`
if (server.path === '/') {
throw new Error(
"If you want to use '/' as server.path, leave the 'path' variable unset",
)
}
if (providerOptions) {
const deprecatedOptions: Record<string, string> = {
microsoft: 'providerOptions.onedrive',
google: 'providerOptions.drive',
s3: 's3',
}
Object.keys(deprecatedOptions).forEach((deprecated) => {
if (Object.hasOwn(providerOptions, deprecated)) {
throw new Error(
`The Provider option "providerOptions.${deprecated}" is no longer supported. Please use the option "${deprecatedOptions[deprecated]}" instead.`,
)
}
})
}
if (uploadUrls == null || uploadUrls.length === 0) {
if (process.env['NODE_ENV'] === 'production') {
throw new Error('uploadUrls is required')
}
logger.error(
'Running without uploadUrls is a security risk and Companion will refuse to start up when running in production (NODE_ENV=production)',
'startup.uploadUrls',
)
}
const { corsOrigins } = companionOptions
if (corsOrigins == null) {
throw new TypeError(
'Option corsOrigins is required. To disable security, pass true',
)
}
if (corsOrigins === '*') {
throw new TypeError(
'Option corsOrigins cannot be "*". To disable security, pass true',
)
}
}

27
packages/@uppy/companion/src/config/grant.js → packages/@uppy/companion/src/config/grant.ts
View file

@ -1,10 +1,27 @@
import type { GrantProvider } from 'grant'
export type GrantProviderStaticConfig = Pick<
GrantProvider,
| 'state'
| 'authorize_url'
| 'access_url'
| 'oauth'
| 'scope_delimiter'
| 'callback'
| 'scope'
| 'custom_params'
> & {
transport?: 'session' | 'state' | undefined // more specific types
custom_params?: Record<string, string> | undefined // we want more specific types than grant's `any`
}
export type GrantStaticConfig = Record<string, GrantProviderStaticConfig>
const defaults = {
transport: 'session',
state: true, // Enable CSRF check
}
} satisfies GrantProviderStaticConfig
// oauth configuration for provider services that are used.
export default () => {
export default function grantConfig(): GrantStaticConfig {
return {
// we need separate auth providers because scopes are different,
// and because it would be a too big rewrite to allow reuse of the same provider.
@ -39,10 +56,6 @@ export default () => {
access_url: 'https://api.box.com/oauth2/token',
callback: '/box/callback',
},
instagram: {
...defaults,
callback: '/instagram/callback',
},
facebook: {
...defaults,
scope: ['email', 'user_photos'],

113
packages/@uppy/companion/src/schemas/companion.ts Normal file
View file

@ -0,0 +1,113 @@
import type { ObjectCannedACL, S3ClientConfig } from '@aws-sdk/client-s3'
import type { PresignedPostOptions } from '@aws-sdk/s3-presigned-post'
import type { CorsOptions } from 'cors'
import type { Request } from 'express'
import type { RedisOptions } from 'ioredis'
import type Provider from '../server/provider/Provider.js'
// todo implement zod schema validation and remove manual typeof validation around in the code, also in providers/adapters
// see `validateConfig`
export interface ProviderOptions {
key?: string | undefined
secret?: string | undefined
credentialsURL?: string | undefined
verificationToken?: string | undefined
}
type ProviderConstructor = typeof Provider
export interface CustomProvider {
module: ProviderConstructor
config: ProviderOptions
}
export type CredentialsFetchResponse = Pick<
ProviderOptions,
'key' | 'secret' | 'verificationToken'
> & {
transloadit_gateway?: string
origins?: string[]
}
export interface CompanionInitOptions {
// required:
secret: string
filePath: string
server: {
host: string
protocol?: string | undefined
path?: string | undefined
implicitPath?: string | undefined
oauthDomain?: string | undefined
validHosts?: string[] | undefined
}
// optional:
preAuthSecret?: string | Buffer | undefined
loggerProcessName?: string | undefined
providerOptions?: Record<string, ProviderOptions> | undefined
customProviders?: Record<string, CustomProvider> | undefined
redisUrl?: string | undefined
redisOptions?: RedisOptions | undefined
redisPubSubScope?: string | undefined
sendSelfEndpoint?: string | undefined
enableUrlEndpoint?: boolean | undefined
enableGooglePickerEndpoint?: boolean | undefined
metrics?: boolean | undefined
periodicPingUrls?: string[] | undefined
periodicPingInterval?: number | undefined
periodicPingCount?: number | undefined
testDynamicOauthCredentials?: boolean | undefined
testDynamicOauthCredentialsSecret?: string | undefined
allowLocalUrls?: boolean | undefined
clientSocketConnectTimeout?: number | undefined
corsOrigins?: CorsOptions['origin'] | undefined
periodicPingStaticPayload?: unknown
s3?: {
/** @deprecated */
accessKeyId?: unknown
/** @deprecated */
secretAccessKey?: unknown
region?: string | undefined
endpoint?: string | undefined
bucket?: string | GetBucketFn | undefined
key?: string | undefined
getKey?: GetKeyFn | undefined
secret?: string | undefined
sessionToken?: string | undefined
conditions?: PresignedPostOptions['Conditions'] | undefined
forcePathStyle?: boolean
acl?: ObjectCannedACL | undefined
useAccelerateEndpoint?: boolean
expires: number
awsClientOptions?: S3ClientConfig & {
/** @deprecated */
accessKeyId?: unknown
/** @deprecated */
secretAccessKey?: unknown
}
}
maxFilenameLength?: number | undefined
uploadUrls?: RegExp[] | string[] | undefined | null
cookieDomain?: string | undefined
streamingUpload?: boolean | undefined
tusDeferredUploadLength?: boolean | undefined
maxFileSize?: number | undefined
chunkSize?: number | undefined
uploadHeaders?: Record<string, string> | undefined
}
export type GetKeyFn = (args: {
req: Request
filename: string
metadata: Record<string, unknown>
}) => string
export type GetBucketFn = (args: {
req: Request
filename?: string | undefined
metadata: Record<string, unknown>
}) => string

1
packages/@uppy/companion/src/schemas/index.ts Normal file
View file

@ -0,0 +1 @@
export * from './companion.js'

42
packages/@uppy/companion/test/with-load-balancer.mjs → packages/@uppy/companion/src/scripts/with-load-balancer.ts Executable file → Normal file
View file

@ -2,28 +2,32 @@
import http from 'node:http'
import process from 'node:process'
import { execaNode } from 'execa'
import { execa, execaNode } from 'execa'
import httpProxy from 'http-proxy'
const numInstances = 3
const lbPort = 3020
const companionStartPort = 3021
// Note: this file is compiled to `dist/scripts/*` and executed with plain `node`.
const repoRoot = new URL('../../../../../', import.meta.url)
// simple load balancer that will direct requests round robin between companion instances
function createLoadBalancer(baseUrls) {
function createLoadBalancer(baseUrls: string[]): http.Server {
const proxy = httpProxy.createProxyServer({ ws: true })
let i = 0
function getTarget() {
return baseUrls[i % baseUrls.length]
function getTarget(): string {
return baseUrls[i % baseUrls.length] ?? baseUrls[0] ?? 'http://localhost'
}
const server = http.createServer((req, res) => {
const target = getTarget()
// console.log('req', req.method, target, req.url)
proxy.web(req, res, { target }, (err) => {
console.error('Load balancer failed to proxy request', err.message)
const error = err instanceof Error ? err : new Error(String(err))
console.error('Load balancer failed to proxy request', error.message)
res.statusCode = 500
res.end()
})
@ -34,8 +38,9 @@ function createLoadBalancer(baseUrls) {
const target = getTarget()
// console.log('upgrade', target, req.url)
proxy.ws(req, socket, head, { target }, (err) => {
console.error('Load balancer failed to proxy websocket', err.message)
console.error(err)
const error = err instanceof Error ? err : new Error(String(err))
console.error('Load balancer failed to proxy websocket', error.message)
console.error(error)
socket.destroy()
})
i++
@ -49,22 +54,29 @@ function createLoadBalancer(baseUrls) {
const isWindows = process.platform === 'win32'
const isOSX = process.platform === 'darwin'
const startCompanion = ({ name, port }) =>
execaNode('packages/@uppy/companion/src/standalone/start-server.js', {
// Ensure we run the compiled JS output. This script spawns plain `node`, so it
// cannot rely on TypeScript loaders.
await execa('yarn', ['workspace', '@uppy/companion', 'build'], {
cwd: repoRoot,
stdio: 'inherit',
})
const startCompanion = ({ name, port }: { name: string; port: number }) =>
execaNode('packages/@uppy/companion/dist/standalone/start-server.js', {
nodeOptions: [
'-r',
'dotenv/config',
// Watch mode support is limited to Windows and macOS at the time of writing.
...(isWindows || isOSX
? ['--watch-path', 'packages/@uppy/companion/src', '--watch']
? ['--watch-path', 'packages/@uppy/companion/dist', '--watch']
: []),
],
cwd: new URL('../../../../', import.meta.url),
cwd: repoRoot,
stdio: 'inherit',
env: {
// Note: these env variables will override anything set in .env
...process.env,
COMPANION_PORT: port,
COMPANION_PORT: `${port}`,
COMPANION_SECRET: 'development', // multi instance will not work without secret set
COMPANION_PREAUTH_SECRET: 'development', // multi instance will not work without secret set
COMPANION_ALLOW_LOCAL_URLS: 'true',
@ -90,7 +102,7 @@ const companions = hosts.map(({ index, port }) =>
startCompanion({ name: `companion${index}`, port }),
)
let loadBalancer
let loadBalancer: http.Server | undefined
try {
loadBalancer = createLoadBalancer(
hosts.map(({ port }) => `http://localhost:${port}`),
@ -98,5 +110,7 @@ try {
await Promise.all(companions)
} finally {
loadBalancer?.close()
companions.forEach((companion) => companion.kill())
companions.forEach((companion) => {
companion.kill()
})
}

557
packages/@uppy/companion/src/server/Uploader.js → packages/@uppy/companion/src/server/Uploader.ts
View file

@ -3,16 +3,24 @@ import { once } from 'node:events'
import { createReadStream, createWriteStream, ReadStream } from 'node:fs'
import { stat, unlink } from 'node:fs/promises'
import { join } from 'node:path'
import type { EventEmitter, Readable as NodeReadableStream } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import type { PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
import { Upload } from '@aws-sdk/lib-storage'
import type { Request } from 'express'
import type { FormDataLike } from 'form-data-encoder'
import { FormData } from 'formdata-node'
import type { OptionsOfTextResponseBody, Response } from 'got'
import got from 'got'
import type { Redis } from 'ioredis'
import throttle from 'lodash/throttle.js'
import { serializeError } from 'serialize-error'
import tus from 'tus-js-client'
import validator from 'validator'
import type { CompanionRuntimeOptions } from '../types/companion-options.js'
import emitter from './emitter/index.js'
import headerSanitize from './header-blacklist.js'
import { isRecord, toError } from './helpers/type-guards.js'
import {
getBucket,
hasMatch,
@ -32,21 +40,73 @@ const PROTOCOLS = Object.freeze({
tus: 'tus',
})
function exceedsMaxFileSize(maxFileSize, size) {
return maxFileSize && size && size > maxFileSize
type UploadProtocol = (typeof PROTOCOLS)[keyof typeof PROTOCOLS]
type Metadata = Record<string, unknown> & { name?: string; type?: string }
type UploaderOptions = {
endpoint?: string
uploadUrl?: string
protocol?: UploadProtocol
size?: number
fieldname?: string
pathPrefix: string
s3?: { client: S3Client; options: CompanionRuntimeOptions['s3'] } | null
metadata: Metadata
companionOptions: Pick<
CompanionRuntimeOptions,
| 'uploadUrls'
| 'uploadHeaders'
| 'maxFileSize'
| 'maxFilenameLength'
| 'tusDeferredUploadLength'
> & {
streamingUpload?: CompanionRuntimeOptions['streamingUpload'] | undefined
}
storage?: Redis | null
headers?: Record<string, unknown>
httpMethod?: string
useFormData?: boolean
chunkSize?: number
providerName?: string
}
export interface ProgressPayload {
progress: string
bytesUploaded: number
bytesTotal: number
}
export interface UploadExtraDataResponse {
status?: number | undefined
statusText?: string | undefined
responseText?: string
headers?: Record<string, unknown>
}
export interface UploadExtraData {
response: UploadExtraDataResponse
bytesUploaded?: number
}
export interface UploadResult {
url: string | null
extraData?: UploadExtraData | undefined
}
function exceedsMaxFileSize(
maxFileSize: number | undefined,
size: number | undefined,
): boolean {
return maxFileSize !== undefined && size !== undefined && size > maxFileSize
}
export class ValidationError extends Error {
name = 'ValidationError'
override name = 'ValidationError'
}
/**
* Validate the options passed down to the uplaoder
*
* @param {UploaderOptions} options
*/
function validateOptions(options) {
// validate HTTP Method
function validateOptions(options: UploaderOptions): void {
// validate HTTP Method (optional)
if (options.httpMethod) {
if (typeof options.httpMethod !== 'string') {
throw new ValidationError('unsupported HTTP METHOD specified')
@ -62,25 +122,25 @@ function validateOptions(options) {
throw new ValidationError('maxFileSize exceeded')
}
// validate fieldname
// validate fieldname (optional)
if (options.fieldname != null && typeof options.fieldname !== 'string') {
throw new ValidationError('fieldname must be a string')
}
// validate metadata
// validate metadata (optional)
if (options.metadata != null && typeof options.metadata !== 'object') {
throw new ValidationError('metadata must be an object')
}
// validate headers
// validate headers (optional)
if (options.headers != null && typeof options.headers !== 'object') {
throw new ValidationError('headers must be an object')
}
// validate protocol
// validate protocol (optional)
if (
options.protocol &&
!Object.keys(PROTOCOLS).some((key) => PROTOCOLS[key] === options.protocol)
!Object.values(PROTOCOLS).includes(options.protocol)
) {
throw new ValidationError('unsupported protocol specified')
}
@ -93,14 +153,15 @@ function validateOptions(options) {
throw new ValidationError('no destination specified')
}
const validateUrl = (url) => {
const validateUrl = (url: string | undefined): void => {
if (url == null) return
const validatorOpts = { require_protocol: true, require_tld: false }
if (url && !validator.isURL(url, validatorOpts)) {
if (!validator.isURL(url, validatorOpts)) {
throw new ValidationError('invalid destination url')
}
const allowedUrls = options.companionOptions.uploadUrls
if (allowedUrls && url && !hasMatch(url, allowedUrls)) {
if (allowedUrls && !hasMatch(url, allowedUrls)) {
throw new ValidationError(
'upload destination does not match any allowed destinations',
)
@ -123,34 +184,49 @@ const states = {
}
export default class Uploader {
/** @type {import('ioredis').Redis} */
storage
static FILE_NAME_PREFIX = 'uppy-file'
static STORAGE_PREFIX = 'companion'
storage: Redis | null | undefined
providerName: string | undefined
options: UploaderOptions
token: string
fileName: string
size: number | undefined
uploadFileName: string
downloadedBytes: number
readStream: NodeReadableStream | null
tmpPath: string | null
tus: tus.Upload | null
fieldname: string
metadata: Metadata
throttledEmitProgress: (dataToEmit: {
action: string
payload: ProgressPayload
}) => void
/**
* Uploads file to destination based on the supplied protocol (tus, s3-multipart, multipart)
* For tus uploads, the deferredLength option is enabled, because file size value can be unreliable
* for some providers (Instagram particularly)
* for some providers.
*
* @typedef {object} UploaderOptions
* @property {string} endpoint
* @property {string} [uploadUrl]
* @property {string} protocol
* @property {number} [size]
* @property {string} [fieldname]
* @property {string} pathPrefix
* @property {any} [s3]
* @property {any} metadata
* @property {any} companionOptions
* @property {any} [storage]
* @property {any} [headers]
* @property {string} [httpMethod]
* @property {boolean} [useFormData]
* @property {number} [chunkSize]
* @property {string} [providerName]
*
* @param {UploaderOptions} optionsIn
* @param optionsIn
*/
constructor(optionsIn) {
constructor(optionsIn: UploaderOptions) {
validateOptions(optionsIn)
const options = {
@ -159,23 +235,24 @@ export default class Uploader {
...optionsIn.headers,
...optionsIn.companionOptions.uploadHeaders,
},
}
} satisfies UploaderOptions
this.providerName = options.providerName
this.options = options
this.token = randomUUID()
this.fileName = `${Uploader.FILE_NAME_PREFIX}-${this.token}`
this.options.metadata = {
const metadata = {
...(this.providerName != null && { provider: this.providerName }),
...(this.options.metadata || {}), // allow user to override provider
...(options.metadata || {}), // allow user to override provider
}
this.options.fieldname = this.options.fieldname || DEFAULT_FIELD_NAME
this.metadata = metadata
this.fieldname = options.fieldname || DEFAULT_FIELD_NAME
this.size = options.size
const { maxFilenameLength } = this.options.companionOptions
const { maxFilenameLength } = options.companionOptions
// Define upload file name
this.uploadFileName = truncateFilename(
this.options.metadata.name || this.fileName,
metadata.name || this.fileName,
maxFilenameLength,
)
@ -184,6 +261,8 @@ export default class Uploader {
this.downloadedBytes = 0
this.readStream = null
this.tmpPath = null
this.tus = null
if (this.options.protocol === PROTOCOLS.tus) {
emitter().on(`pause:${this.token}`, () => {
@ -218,44 +297,66 @@ export default class Uploader {
this.#canceled = true
this.abortReadStream(new Error('Canceled'))
})
this.throttledEmitProgress = throttle(
(dataToEmit: { action: string; payload: ProgressPayload }) => {
const {
payload: { bytesUploaded, bytesTotal, progress },
} = dataToEmit
logger.debug(
`${bytesUploaded} ${bytesTotal} ${progress}%`,
'uploader.total.progress',
this.shortToken,
)
this.saveState(dataToEmit)
emitter().emit(this.token, dataToEmit)
},
1000,
{ trailing: false },
)
}
#uploadState = states.idle
#canceled = false
abortReadStream(err) {
abortReadStream(err: Error): void {
this.#uploadState = states.done
if (this.readStream) this.readStream.destroy(err)
}
_getUploadProtocol() {
_getUploadProtocol(): UploadProtocol {
return this.options.protocol || PROTOCOLS.multipart
}
async _uploadByProtocol(req) {
async _uploadByProtocol(
req: Request,
stream: NodeReadableStream,
): Promise<UploadResult> {
const protocol = this._getUploadProtocol()
switch (protocol) {
case PROTOCOLS.multipart:
return this.#uploadMultipart(this.readStream)
return this.#uploadMultipart(stream)
case PROTOCOLS.s3Multipart:
return this.#uploadS3Multipart(this.readStream, req)
return this.#uploadS3Multipart(stream, req)
case PROTOCOLS.tus:
return this.#uploadTus(this.readStream)
return this.#uploadTus(stream)
default:
throw new Error('Invalid protocol')
}
}
async _downloadStreamAsFile(stream) {
async _downloadStreamAsFile(stream: NodeReadableStream): Promise<void> {
this.tmpPath = join(this.options.pathPrefix, this.fileName)
logger.debug('fully downloading file', 'uploader.download', this.shortToken)
const writeStream = createWriteStream(this.tmpPath)
const onData = (chunk) => {
this.downloadedBytes += chunk.length
const onData = (chunk: Buffer | string) => {
const len =
typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
this.downloadedBytes += len
if (
exceedsMaxFileSize(
this.options.companionOptions.maxFileSize,
@ -284,16 +385,14 @@ export default class Uploader {
this.readStream = fileStream
}
_canStream() {
return this.options.companionOptions.streamingUpload
_canStream(): boolean {
return !!this.options.companionOptions.streamingUpload
}
/**
*
* @param {import('stream').Readable} stream
* @param {import('express').Request} req
*/
async uploadStream(stream, req) {
async uploadStream(
stream: NodeReadableStream,
req: Request,
): Promise<UploadResult | undefined> {
try {
if (this.#uploadState !== states.idle)
throw new Error('Can only start an upload in the idle state')
@ -316,10 +415,15 @@ export default class Uploader {
}
if (this.#uploadState !== states.uploading) return undefined
const activeStream = this.readStream
if (!activeStream) throw new Error('No readable stream available')
const { url, extraData } = await Promise.race([
this._uploadByProtocol(req),
this._uploadByProtocol(req, activeStream),
// If we don't handle stream errors, we get unhandled error in node.
new Promise((resolve, reject) => this.readStream.on('error', reject)),
new Promise<never>((_resolve, reject) =>
activeStream.on('error', reject),
),
])
return { url, extraData }
} finally {
@ -331,16 +435,17 @@ export default class Uploader {
}
}
tryDeleteTmpPath() {
if (this.tmpPath) unlink(this.tmpPath).catch(() => {})
async tryDeleteTmpPath(): Promise<void> {
if (!this.tmpPath) return
try {
await unlink(this.tmpPath)
} catch {}
}
/**
*
* @param {import('stream').Readable} stream
* @param {import('express').Request} req
*/
async tryUploadStream(stream, req) {
async tryUploadStream(
stream: NodeReadableStream,
req: Request,
): Promise<void> {
try {
emitter().emit('upload-start', { token: this.token })
@ -366,43 +471,100 @@ export default class Uploader {
* returns a substring of the token. Used as traceId for logging
* we avoid using the entire token because this is meant to be a short term
* access token between uppy client and companion websocket
*
* @param {string} token the token to Shorten
* @returns {string}
*/
static shortenToken(token) {
static shortenToken(token: string): string {
return token.substring(0, 8)
}
static reqToOptions(req, size) {
const useFormDataIsSet = Object.hasOwn(req.body, 'useFormData')
const useFormData = useFormDataIsSet ? req.body.useFormData : true
static reqToOptions(req: Request, size: number | undefined): UploaderOptions {
const body: Record<string, unknown> = isRecord(req.body) ? req.body : {}
const useFormDataValue = body['useFormData']
const useFormData =
typeof useFormDataValue === 'boolean' ? useFormDataValue : true
const headers = body['headers']
if (headers != null && !isRecord(headers)) {
throw new ValidationError('headers must be an object')
}
const metadataValue = body['metadata']
if (metadataValue != null && !isRecord(metadataValue)) {
throw new ValidationError('metadata must be an object')
}
const metadata = metadataValue ?? {}
const httpMethod = body['httpMethod']
if (httpMethod != null && typeof httpMethod !== 'string') {
throw new ValidationError('unsupported HTTP METHOD specified')
}
const protocolValue = body['protocol']
let protocol: UploadProtocol | undefined
if (protocolValue != null) {
if (typeof protocolValue !== 'string')
throw new ValidationError('unsupported protocol specified')
if (
protocolValue === PROTOCOLS.multipart ||
protocolValue === PROTOCOLS.s3Multipart ||
protocolValue === PROTOCOLS.tus
) {
protocol = protocolValue
} else {
throw new ValidationError('unsupported protocol specified')
}
}
const endpoint = body['endpoint']
if (endpoint != null && typeof endpoint !== 'string') {
throw new ValidationError('invalid destination url')
}
const uploadUrl = body['uploadUrl']
if (uploadUrl != null && typeof uploadUrl !== 'string') {
throw new ValidationError('invalid destination url')
}
const fieldname = body['fieldname']
if (fieldname != null && typeof fieldname !== 'string') {
throw new ValidationError('fieldname must be a string')
}
const companionOptions = req.companion.options
const { filePath } = companionOptions
const chunkSizeValue = companionOptions['chunkSize']
const chunkSize =
typeof chunkSizeValue === 'number' ? chunkSizeValue : undefined
const storage = redis.client()
return {
// Client provided info (must be validated and not blindly trusted):
headers: req.body.headers,
httpMethod: req.body.httpMethod,
protocol: req.body.protocol,
endpoint: req.body.endpoint,
uploadUrl: req.body.uploadUrl,
metadata: req.body.metadata,
fieldname: req.body.fieldname,
...(headers != null && { headers }),
...(httpMethod != null && { httpMethod }),
...(protocol != null && { protocol }),
...(endpoint != null && { endpoint }),
...(uploadUrl != null && { uploadUrl }),
...(fieldname != null && { fieldname }),
useFormData,
metadata,
providerName: req.companion.providerName,
...(req.companion.providerName != null && {
providerName: req.companion.providerName,
}),
// Info coming from companion server configuration:
size,
companionOptions: req.companion.options,
pathPrefix: `${req.companion.options.filePath}`,
storage: redis.client(),
...(size != null && { size }),
companionOptions,
pathPrefix: filePath,
...(storage != null && { storage }),
s3: req.companion.s3Client
? {
client: req.companion.s3Client,
options: req.companion.options.s3,
options: companionOptions.s3,
}
: null,
chunkSize: req.companion.options.chunkSize,
...(chunkSize != null && { chunkSize }),
}
}
@ -415,7 +577,7 @@ export default class Uploader {
return Uploader.shortenToken(this.token)
}
async awaitReady(timeout) {
async awaitReady(timeout: number): Promise<void> {
logger.debug(
'waiting for socket connection',
'uploader.socket.wait',
@ -423,11 +585,9 @@ export default class Uploader {
)
const eventName = `connection:${this.token}`
await once(
emitter(),
eventName,
timeout && { signal: AbortSignal.timeout(timeout) },
)
await once(emitter() as EventEmitter, eventName, {
signal: AbortSignal.timeout(timeout),
})
logger.debug(
'socket connection received',
@ -437,10 +597,9 @@ export default class Uploader {
}
/**
* @typedef {{action: string, payload: object}} State
* @param {State} state
* Persist the latest upload state to Redis so a reconnecting client can resume.
*/
saveState(state) {
saveState(state: { action: string; payload: unknown }): void {
if (!this.storage) return
// make sure the keys get cleaned up.
// https://github.com/transloadit/uppy/issues/3748
@ -449,27 +608,10 @@ export default class Uploader {
this.storage.set(redisKey, jsonStringify(state), 'EX', keyExpirySec)
}
throttledEmitProgress = throttle(
(dataToEmit) => {
const { bytesUploaded, bytesTotal, progress } = dataToEmit.payload
logger.debug(
`${bytesUploaded} ${bytesTotal} ${progress}%`,
'uploader.total.progress',
this.shortToken,
)
this.saveState(dataToEmit)
emitter().emit(this.token, dataToEmit)
},
1000,
{ trailing: false },
)
/**
*
* @param {number} [bytesUploaded]
* @param {number | null} [bytesTotalIn]
*/
onProgress(bytesUploaded = 0, bytesTotalIn = 0) {
onProgress(
bytesUploaded = 0,
bytesTotalIn: number | null | undefined = 0,
): void {
const bytesTotal = bytesTotalIn || this.size || 0
// If fully downloading before uploading, combine downloaded and uploaded bytes
@ -510,12 +652,10 @@ export default class Uploader {
this.throttledEmitProgress(dataToEmit)
}
/**
*
* @param {string} url
* @param {object} extraData
*/
#emitSuccess(url, extraData) {
#emitSuccess(
url: string | null,
extraData: UploadExtraData | undefined,
): void {
const emitData = {
action: 'success',
payload: { ...extraData, complete: true, url },
@ -524,14 +664,11 @@ export default class Uploader {
emitter().emit(this.token, emitData)
}
/**
*
* @param {Error} err
*/
async #emitError(err) {
async #emitError(err: unknown): Promise<void> {
const error = toError(err)
// delete stack to avoid sending server info to client
// see PR discussion https://github.com/transloadit/uppy/pull/3832
const { stack, ...serializedErr } = serializeError(err)
const { stack, ...serializedErr } = serializeError(error)
const dataToEmit = {
action: 'error',
payload: { error: serializedErr },
@ -542,10 +679,8 @@ export default class Uploader {
/**
* start the tus upload
*
* @param {any} stream
*/
async #uploadTus(stream) {
async #uploadTus(stream: NodeReadableStream): Promise<UploadResult> {
const uploader = this
const isFileStream = stream instanceof ReadStream
@ -553,10 +688,16 @@ export default class Uploader {
// https://github.com/tus/tus-js-client/blob/4479b78032937ac14da9b0542e489ac6fe7e0bc7/lib/node/fileReader.js#L50
const chunkSize = this.options.chunkSize || (isFileStream ? Infinity : 50e6)
const tusRet = await new Promise((resolve, reject) => {
const tusOptions = {
endpoint: this.options.endpoint,
uploadUrl: this.options.uploadUrl,
type TusUploadOptions = ConstructorParameters<typeof tus.Upload>[1]
const tusRet = await new Promise<UploadResult>((resolve, reject) => {
const tusOptions: TusUploadOptions = {
...(this.options.endpoint != null && {
endpoint: this.options.endpoint,
}),
...(this.options.uploadUrl != null && {
uploadUrl: this.options.uploadUrl,
}),
retryDelays: [0, 1000, 3000, 5000],
chunkSize,
headers: headerSanitize(this.options.headers),
@ -565,35 +706,28 @@ export default class Uploader {
// file name and type as required by the tusd tus server
// https://github.com/tus/tusd/blob/5b376141903c1fd64480c06dde3dfe61d191e53d/unrouted_handler.go#L614-L646
filename: this.uploadFileName,
filetype: this.options.metadata.type,
...this.options.metadata,
...(this.metadata.type != null && {
filetype: this.metadata.type,
}),
...Object.fromEntries(
Object.entries(this.metadata).map(([k, v]) => [k, String(v)]),
),
},
/**
*
* @param {Error} error
*/
onError(error) {
logger.error(error, 'uploader.tus.error')
// deleting tus originalRequest field because it uses the same http-agent
// as companion, and this agent may contain sensitive request details (e.g headers)
// previously made to providers. Deleting the field would prevent it from getting leaked
// to the frontend etc.
// @ts-ignore
delete error.originalRequest
// @ts-ignore
delete error.originalResponse
Reflect.deleteProperty(error, 'originalRequest')
Reflect.deleteProperty(error, 'originalResponse')
reject(error)
},
/**
*
* @param {number} [bytesUploaded]
* @param {number} [bytesTotal]
*/
onProgress(bytesUploaded, bytesTotal) {
uploader.onProgress(bytesUploaded, bytesTotal)
},
onSuccess() {
resolve({ url: uploader.tus.url })
resolve({ url: uploader.tus?.url ?? null })
},
}
@ -609,6 +743,7 @@ export default class Uploader {
'tusDeferredUploadLength needs to be enabled if no file size is provided by the provider',
),
)
return
}
tusOptions.uploadLengthDeferred = false
tusOptions.uploadSize = this.size
@ -619,37 +754,37 @@ export default class Uploader {
this.tus.start()
})
// @ts-ignore
if (this.size != null && this.tus._size !== this.size) {
// @ts-ignore
logger.warn(
// @ts-expect-error _size is not typed
`Tus uploaded size ${this.tus._size} different from reported URL size ${this.size}`,
'upload.tus.mismatch.error',
)
if (this.tus != null && this.size != null) {
const tusSize: unknown = Reflect.get(this.tus, '_size')
if (typeof tusSize === 'number' && tusSize !== this.size) {
logger.warn(
`Tus uploaded size ${tusSize} different from reported URL size ${this.size}`,
'upload.tus.mismatch.error',
)
}
}
return tusRet
}
async #uploadMultipart(stream) {
async #uploadMultipart(stream: NodeReadableStream): Promise<UploadResult> {
if (!this.options.endpoint) {
throw new Error('No multipart endpoint set')
}
function getRespObj(response) {
function getRespObj(response: Response<string>): UploadExtraDataResponse {
// remove browser forbidden headers
const {
'set-cookie': deleted,
'set-cookie2': deleted2,
...responseHeaders
...headers
} = response.headers
return {
responseText: response.body,
status: response.statusCode,
statusText: response.statusMessage,
headers: responseHeaders,
headers,
}
}
@ -661,19 +796,23 @@ export default class Uploader {
})
const url = this.options.endpoint
const reqOptions = {
const reqOptions: OptionsOfTextResponseBody = {
headers: headerSanitize(this.options.headers),
isStream: false,
resolveBodyOnly: false,
responseType: 'text',
}
if (this.options.useFormData) {
const formData = new FormData()
Object.entries(this.options.metadata).forEach(([key, value]) =>
formData.append(key, value),
)
Object.entries(this.metadata).forEach(([key, value]) => {
formData.append(key, value)
})
// see https://github.com/octet-stream/form-data/blob/73a5a24e635938026538673f94cbae1249a3f5cc/readme.md?plain=1#L232
formData.set(this.options.fieldname, {
formData.set(this.fieldname, {
type: this.metadata.type || 'application/octet-stream',
name: this.uploadFileName,
[Symbol.toStringTag]: 'File',
stream() {
@ -681,18 +820,21 @@ export default class Uploader {
},
})
reqOptions.body = formData
reqOptions.body = formData as FormDataLike
} else {
reqOptions.headers['content-length'] = this.size
if (this.size != null) {
reqOptions.headers = {
...reqOptions.headers,
'content-length': `${this.size}`,
}
}
reqOptions.body = stream
}
try {
const httpMethod =
(this.options.httpMethod || '').toUpperCase() === 'PUT' ? 'put' : 'post'
const runRequest = got[httpMethod]
const response = await runRequest(url, reqOptions)
(this.options.httpMethod ?? '').toUpperCase() === 'PUT' ? 'put' : 'post'
const response = await got[httpMethod](url, reqOptions)
if (this.size != null && bytesUploaded !== this.size) {
const errMsg = `uploaded only ${bytesUploaded} of ${this.size} with status: ${response.statusCode}`
@ -714,12 +856,24 @@ export default class Uploader {
}
} catch (err) {
logger.error(err, 'upload.multipart.error')
const statusCode = err.response?.statusCode
const errObj = isRecord(err) ? err : null
const response =
errObj && isRecord(errObj['response']) ? errObj['response'] : null
const statusCode =
response && typeof response['statusCode'] === 'number'
? response['statusCode']
: undefined
if (statusCode != null) {
throw Object.assign(new Error(err.statusMessage), {
extraData: getRespObj(err.response),
const statusMessage =
typeof errObj?.['statusMessage'] === 'string'
? errObj['statusMessage']
: 'Request failed'
throw Object.assign(new Error(statusMessage), {
extraData: getRespObj(response as unknown as Response<string>),
})
}
throw new Error('Unknown multipart upload error', { cause: err })
}
}
@ -727,7 +881,10 @@ export default class Uploader {
/**
* Upload the file to S3 using a Multipart upload.
*/
async #uploadS3Multipart(stream, req) {
async #uploadS3Multipart(
stream: NodeReadableStream,
req: Request,
): Promise<UploadResult> {
if (!this.options.s3) {
throw new Error(
'The S3 client is not configured on this companion instance.',
@ -735,28 +892,33 @@ export default class Uploader {
}
const filename = this.uploadFileName
/**
* @type {{client: import('@aws-sdk/client-s3').S3Client, options: Record<string, any>}}
*/
const s3Options = this.options.s3
const { metadata } = this.options
const { metadata } = this
const { client, options } = s3Options
const params = {
Bucket: getBucket({ bucketOrFn: options.bucket, req, metadata }),
const params: PutObjectCommandInput = {
Bucket: getBucket({
bucketOrFn: options.bucket,
req,
metadata,
filename,
}),
Key: options.getKey({ req, filename, metadata }),
ContentType: metadata.type,
Metadata: rfc2047EncodeMetadata(metadata),
Body: stream,
...(options['acl'] != null && {
ACL: options['acl'],
}),
}
if (options.acl != null) params.ACL = options.acl
const upload = new Upload({
client,
params,
// using chunkSize as partSize too, see https://github.com/transloadit/uppy/pull/3511
partSize: this.options.chunkSize,
...(this.options.chunkSize != null && {
partSize: this.options.chunkSize,
}),
leavePartsOnError: true, // https://github.com/aws/aws-sdk-js-v3/issues/2311
})
@ -769,6 +931,8 @@ export default class Uploader {
url: data?.Location || null,
extraData: {
response: {
status: data.$metadata.httpStatusCode,
responseText: JSON.stringify(data),
headers: {
'content-type': 'application/json',
@ -778,6 +942,3 @@ export default class Uploader {
}
}
}
Uploader.FILE_NAME_PREFIX = 'uppy-file'
Uploader.STORAGE_PREFIX = 'companion'

87
packages/@uppy/companion/src/server/controllers/callback.js
View file

@ -1,87 +0,0 @@
/**
* oAuth callback. Encrypts the access token and sends the new token with the response,
*/
import serialize from 'serialize-javascript'
import * as tokenService from '../helpers/jwt.js'
import * as oAuthState from '../helpers/oauth-state.js'
import logger from '../logger.js'
const closePageHtml = (origin) => `
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<script>
// if window.opener is nullish, we want the following line to throw to avoid
// the window closing without informing the user.
window.opener.postMessage(${serialize({ error: true })}, ${serialize(origin)})
window.close()
</script>
</head>
<body>Authentication failed.</body>
</html>`
/**
*
* @param {object} req
* @param {object} res
* @param {Function} next
*/
export default function callback(req, res, next) {
const { providerName } = req.params
const grant = req.session.grant || {}
const grantDynamic = oAuthState.getGrantDynamicFromRequest(req)
const origin =
grantDynamic.state &&
oAuthState.getFromState(
grantDynamic.state,
'origin',
req.companion.options.secret,
)
if (!grant.response?.access_token) {
logger.debug(
`Did not receive access token for provider ${providerName}`,
null,
req.id,
)
logger.debug(grant.response, 'callback.oauth.resp', req.id)
return res.status(400).send(closePageHtml(origin))
}
const { access_token: accessToken, refresh_token: refreshToken } =
grant.response
req.companion.providerUserSession = {
accessToken,
refreshToken, // might be undefined for some providers
...req.companion.providerClass.grantDynamicToUserSession({ grantDynamic }),
}
logger.debug(
`Generating auth token for provider ${providerName}. refreshToken: ${refreshToken ? 'yes' : 'no'}`,
null,
req.id,
)
const uppyAuthToken = tokenService.generateEncryptedAuthToken(
{ [providerName]: req.companion.providerUserSession },
req.companion.options.secret,
req.companion.providerClass.authStateExpiry,
)
tokenService.addToCookiesIfNeeded(
req,
res,
uppyAuthToken,
req.companion.providerClass.authStateExpiry,
)
return res.redirect(
req.companion.buildURL(
`/${providerName}/send-token?uppyAuthToken=${uppyAuthToken}`,
true,
),
)
}

107
packages/@uppy/companion/src/server/controllers/callback.ts Normal file
View file

@ -0,0 +1,107 @@
/**
* oAuth callback. Encrypts the access token and sends the new token with the response,
*/
import type { NextFunction, Request, Response } from 'express'
import emitter from '../emitter/index.js'
import {
authCallbackErrorHtml,
legacyAuthCallbackHtml,
} from '../helpers/html.js'
import * as tokenService from '../helpers/jwt.js'
import * as oAuthState from '../helpers/oauth-state.js'
import logger from '../logger.js'
export default function callback(
req: Request,
res: Response,
next: NextFunction,
): void {
const providerName = req.params['providerName']
const { companion } = req
if (typeof providerName !== 'string' || providerName.length === 0) {
res.sendStatus(400)
return
}
const secret = req.companion.options.secret
const grantDynamic = oAuthState.getGrantDynamicFromRequest(req)
const origin =
grantDynamic.state &&
oAuthState.getFromState(grantDynamic.state, 'origin', secret)
const originString = typeof origin === 'string' ? origin : undefined
const accessToken = req.session?.grant?.response?.access_token
const refreshToken = req.session?.grant?.response?.refresh_token
if (!accessToken) {
logger.debug(
`Did not receive access token for provider ${providerName}`,
undefined,
req.id,
)
logger.debug(req.session?.grant?.response, 'callback.oauth.resp', req.id)
const authCallbackToken =
grantDynamic.state &&
oAuthState.getFromState(
grantDynamic.state,
'authCallbackToken',
companion.options.secret,
)
// only new Uppy clients will set an authCallbackToken in the state
// in that case, we send the token through the emitter.
if (authCallbackToken) {
emitter().emit(authCallbackToken, { error: true })
res.status(400).send(authCallbackErrorHtml())
} else {
// This is backwards compatible with old Uppy clients:
res
.status(400)
.send(legacyAuthCallbackHtml({ error: true }, originString))
}
return
}
const { providerClass } = req.companion
if (!providerClass) {
res.sendStatus(400)
return
}
req.companion.providerUserSession = {
accessToken,
refreshToken, // might be undefined for some providers
...providerClass.grantDynamicToUserSession({ grantDynamic }),
}
logger.debug(
`Generating auth token for provider ${providerName}. refreshToken: ${refreshToken ? 'yes' : 'no'}`,
undefined,
req.id,
)
const uppyAuthToken = tokenService.generateEncryptedAuthToken(
{ [providerName]: req.companion.providerUserSession },
secret,
providerClass.authStateExpiry,
)
tokenService.addToCookiesIfNeeded(
req,
res,
uppyAuthToken,
providerClass.authStateExpiry,
)
if (!req.companion.buildURL) {
res.sendStatus(500)
return
}
res.redirect(
req.companion.buildURL(
`/${providerName}/send-token?uppyAuthToken=${uppyAuthToken}`,
true,
),
)
}

88
packages/@uppy/companion/src/server/controllers/connect.js → packages/@uppy/companion/src/server/controllers/connect.ts
View file

@ -1,13 +1,16 @@
import type { CorsOptions } from 'cors'
import type { NextFunction, Request, Response } from 'express'
import * as oAuthState from '../helpers/oauth-state.js'
import { isRecord } from '../helpers/type-guards.js'
/**
* Derived from `cors` npm package.
* @see https://github.com/expressjs/cors/blob/791983ebc0407115bc8ae8e64830d440da995938/lib/index.js#L19-L34
* @param {string} origin
* @param {*} allowedOrigins
* @returns {boolean}
*/
function isOriginAllowed(origin, allowedOrigins) {
function isOriginAllowed(
origin: string,
allowedOrigins: CorsOptions['origin'],
): boolean {
if (Array.isArray(allowedOrigins)) {
return allowedOrigins.some((allowedOrigin) =>
isOriginAllowed(origin, allowedOrigin),
@ -16,33 +19,55 @@ function isOriginAllowed(origin, allowedOrigins) {
if (typeof allowedOrigins === 'string') {
return origin === allowedOrigins
}
return allowedOrigins.test?.(origin) ?? !!allowedOrigins
if (allowedOrigins instanceof RegExp) {
return allowedOrigins.test(origin)
}
return !!allowedOrigins
}
const queryString = (params, prefix = '?') => {
const queryString = (params: Record<string, string>, prefix = '?'): string => {
const str = new URLSearchParams(params).toString()
return str ? `${prefix}${str}` : ''
}
function encodeStateAndRedirect(req, res, stateObj) {
function encodeStateAndRedirect(
req: Request,
res: Response,
stateObj: oAuthState.OAuthState,
): void {
const { secret } = req.companion.options
const state = oAuthState.encodeState(stateObj, secret)
const { providerClass, providerGrantConfig } = req.companion
if (!providerClass || !req.companion.buildURL) {
res.sendStatus(400)
return
}
// pass along grant's dynamic config (if specified for the provider in its grant config `dynamic` section)
// this is needed for things like custom oauth domain (e.g. webdav)
const dynamicKeys = providerGrantConfig?.dynamic ?? []
const grantDynamicConfig = Object.fromEntries(
providerGrantConfig.dynamic?.flatMap((dynamicKey) => {
dynamicKeys.flatMap((dynamicKey) => {
const queryValue = req.query[dynamicKey]
const queryValueString =
typeof queryValue === 'string'
? queryValue
: Array.isArray(queryValue) && typeof queryValue[0] === 'string'
? queryValue[0]
: undefined
// note: when using credentialsURL (dynamic oauth credentials), dynamic has ['key', 'secret', 'redirect_uri']
// but in that case, query string is empty, so we need to only fetch these parameters from QS if they exist.
if (!queryValue) return []
return [[dynamicKey, queryValue]]
}) || [],
if (!queryValueString) return []
return [[dynamicKey, queryValueString]]
}),
)
const { oauthProvider } = providerClass
if (oauthProvider == null || oauthProvider.length === 0) {
res.sendStatus(400)
return
}
const qs = queryString({
...grantDynamicConfig,
state,
@ -52,10 +77,13 @@ function encodeStateAndRedirect(req, res, stateObj) {
res.redirect(req.companion.buildURL(`/connect/${oauthProvider}${qs}`, true))
}
function getClientOrigin(base64EncodedState) {
function getClientOrigin(base64EncodedState: unknown): string | undefined {
if (typeof base64EncodedState !== 'string') return undefined
try {
const { origin } = JSON.parse(atob(base64EncodedState))
return origin
const parsed: unknown = JSON.parse(atob(base64EncodedState))
if (!isRecord(parsed)) return undefined
const origin = parsed['origin']
return typeof origin === 'string' ? origin : undefined
} catch {
return undefined
}
@ -78,39 +106,49 @@ function getClientOrigin(base64EncodedState) {
* That's why we use the client-provided base64-encoded parameter, check if it
* matches origin(s) allowed in `corsOrigins` Companion option, and use that as
* our `targetOrigin` for the `window.postMessage()` call (see `send-token.js`).
*
* @param {object} req
* @param {object} res
*/
export default function connect(req, res, next) {
export default function connect(
req: Request,
res: Response,
next: NextFunction,
): void {
const stateObj = oAuthState.generateState()
if (req.companion.options.server.oauthDomain) {
if (req.companion.options.server.oauthDomain && req.companion.buildURL) {
stateObj.companionInstance = req.companion.buildURL('', true)
}
if (req.query.uppyPreAuthToken) {
stateObj.preAuthToken = req.query.uppyPreAuthToken
const preAuthTokenValue = req.query['uppyPreAuthToken']
if (typeof preAuthTokenValue === 'string') {
stateObj.preAuthToken = preAuthTokenValue
}
const authCallbackToken = req.query['authCallbackToken']
if (typeof authCallbackToken === 'string') {
stateObj.authCallbackToken = authCallbackToken
}
// Get the computed header generated by `cors` in a previous middleware.
stateObj.origin = res.getHeader('Access-Control-Allow-Origin')
let clientOrigin
let clientOrigin: string | undefined
if (!stateObj.origin) {
clientOrigin = getClientOrigin(req.query.state)
clientOrigin = getClientOrigin(req.query['state'])
}
if (!stateObj.origin && clientOrigin) {
const { corsOrigins } = req.companion.options
if (typeof corsOrigins === 'function') {
corsOrigins(clientOrigin, (err, finalOrigin) => {
if (err) next(err)
if (err) {
next(err)
return
}
stateObj.origin = finalOrigin
encodeStateAndRedirect(req, res, stateObj)
})
return
}
if (isOriginAllowed(clientOrigin, req.companion.options.corsOrigins)) {
if (isOriginAllowed(clientOrigin, req.companion.options['corsOrigins'])) {
stateObj.origin = clientOrigin
}
}

19
packages/@uppy/companion/src/server/controllers/deauth-callback.js → packages/@uppy/companion/src/server/controllers/deauth-callback.ts
View file

@ -1,21 +1,28 @@
import type { NextFunction, Request, Response } from 'express'
import { respondWithError } from '../provider/error.js'
export default async function deauthCallback(
{ body, companion, headers },
res,
next,
) {
req: Request,
res: Response,
next: NextFunction,
): Promise<void> {
const { body, companion, headers } = req
const { provider } = companion
if (!provider) {
res.sendStatus(400)
return
}
// we need the provider instance to decide status codes because
// this endpoint does not cater to a uniform client.
// It doesn't respond to Uppy client like other endpoints.
// Instead it responds to the providers themselves.
try {
const { data, status } = await companion.provider.deauthorizationCallback({
const { data, status } = await provider.deauthorizationCallback({
companion,
body,
headers,
})
res.status(status || 200).json(data)
res.status(status ?? 200).json(data)
} catch (err) {
if (respondWithError(err, res)) return
next(err)

24
packages/@uppy/companion/src/server/controllers/get.js
View file

@ -1,24 +0,0 @@
import { startDownUpload } from '../helpers/upload.js'
import logger from '../logger.js'
import { respondWithError } from '../provider/error.js'
export default async function get(req, res) {
const { id } = req.params
const { providerUserSession } = req.companion
const { provider } = req.companion
async function getSize() {
return provider.size({ id, providerUserSession, query: req.query })
}
const download = () =>
provider.download({ id, providerUserSession, query: req.query })
try {
await startDownUpload({ req, res, getSize, download })
} catch (err) {
logger.error(err, 'controller.get.error', req.id)
if (respondWithError(err, res)) return
res.status(500).json({ message: 'Failed to download file' })
}
}

37
packages/@uppy/companion/src/server/controllers/get.ts Normal file
View file

@ -0,0 +1,37 @@
import type { Request, Response } from 'express'
import { startDownUpload } from '../helpers/upload.js'
import logger from '../logger.js'
import { respondWithError } from '../provider/error.js'
export default async function get(req: Request, res: Response): Promise<void> {
const id = req.params['id']
if (typeof id !== 'string' || id.length === 0) {
res.sendStatus(400)
return
}
const { providerUserSession } = req.companion
const { provider } = req.companion
if (!provider) {
res.sendStatus(400)
return
}
const getSize = async () =>
provider.size({ id, providerUserSession, query: req.query })
const download = () =>
provider.download({
id,
providerUserSession,
query: req.query,
companion: req.companion,
})
try {
await startDownUpload({ req, res, getSize, download })
} catch (err) {
logger.error(err, 'controller.get.error', req.id)
if (respondWithError(err, res)) return
res.status(500).json({ message: 'Failed to download file' })
}
}

49
packages/@uppy/companion/src/server/controllers/googlePicker.js
View file

@ -1,49 +0,0 @@
import assert from 'node:assert'
import express from 'express'
import { downloadURL } from '../download.js'
import { validateURL } from '../helpers/request.js'
import { startDownUpload } from '../helpers/upload.js'
import logger from '../logger.js'
import { respondWithError } from '../provider/error.js'
import { streamGoogleFile } from '../provider/google/drive/index.js'
const getAuthHeader = (token) => ({ authorization: `Bearer ${token}` })
/**
*
* @param {object} req expressJS request object
* @param {object} res expressJS response object
*/
const get = async (req, res) => {
try {
logger.debug('Google Picker file import handler running', null, req.id)
const allowLocalUrls = false
const { accessToken, platform, fileId } = req.body
assert(platform === 'drive' || platform === 'photos')
if (platform === 'photos' && !validateURL(req.body.url, allowLocalUrls)) {
res.status(400).json({ error: 'Invalid URL' })
return
}
const download = () => {
if (platform === 'drive') {
return streamGoogleFile({ token: accessToken, id: fileId })
}
return downloadURL(req.body.url, allowLocalUrls, req.id, {
headers: getAuthHeader(accessToken),
})
}
await startDownUpload({ req, res, download, getSize: undefined })
} catch (err) {
logger.error(err, 'controller.googlePicker.error', req.id)
if (respondWithError(err, res)) return
res.status(500).json({ message: 'failed to fetch Google Picker URL' })
}
}
export default () => express.Router().post('/get', express.json(), get)

69
packages/@uppy/companion/src/server/controllers/googlePicker.ts Normal file
View file

@ -0,0 +1,69 @@
import type { Request, Response } from 'express'
import express from 'express'
import { z } from 'zod'
import { downloadURL } from '../download.js'
import { validateURL } from '../helpers/request.js'
import { startDownUpload } from '../helpers/upload.js'
import logger from '../logger.js'
import { respondWithError } from '../provider/error.js'
import { streamGoogleFile } from '../provider/google/drive/index.js'
const getAuthHeader = (token: string): { authorization: string } => ({
authorization: `Bearer ${token}`,
})
const googlePickerBodySchema = z.discriminatedUnion('platform', [
z.object({
platform: z.literal('drive'),
accessToken: z.string().min(1),
fileId: z.string().min(1),
}),
z.object({
platform: z.literal('photos'),
accessToken: z.string().min(1),
url: z.string().min(1),
}),
])
const get = async (req: Request, res: Response): Promise<void> => {
try {
logger.debug('Google Picker file import handler running', undefined, req.id)
const allowLocalUrls = false
const parsedBody = googlePickerBodySchema.safeParse(req.body)
if (!parsedBody.success) {
res.status(400).json({ error: 'Invalid request body' })
return
}
const { accessToken, platform } = parsedBody.data
if (
platform === 'photos' &&
!validateURL(parsedBody.data.url, allowLocalUrls)
) {
res.status(400).json({ error: 'Invalid URL' })
return
}
const download = () => {
if (platform === 'drive') {
return streamGoogleFile({
token: accessToken,
id: parsedBody.data.fileId,
})
}
return downloadURL(parsedBody.data.url, allowLocalUrls, req.id, {
headers: getAuthHeader(accessToken),
})
}
await startDownUpload({ req, res, download, getSize: undefined })
} catch (err) {
logger.error(err, 'controller.googlePicker.error', req.id)
if (respondWithError(err, res)) return
res.status(500).json({ message: 'failed to fetch Google Picker URL' })
}
}
export default () => express.Router().post('/get', express.json(), get)

0
packages/@uppy/companion/src/server/controllers/index.js → packages/@uppy/companion/src/server/controllers/index.ts
View file

18
packages/@uppy/companion/src/server/controllers/list.js
View file

@ -1,18 +0,0 @@
import { respondWithError } from '../provider/error.js'
export default async function list({ query, params, companion }, res, next) {
const { providerUserSession } = companion
try {
const data = await companion.provider.list({
companion,
providerUserSession,
directory: params.id,
query,
})
res.json(data)
} catch (err) {
if (respondWithError(err, res)) return
next(err)
}
}

29
packages/@uppy/companion/src/server/controllers/list.ts Normal file
View file

@ -0,0 +1,29 @@
import type { NextFunction, Request, Response } from 'express'
import { respondWithError } from '../provider/error.js'
export default async function list(
req: Request,
res: Response,
next: NextFunction,
): Promise<void> {
const { query, params, companion } = req
const id = params['id']
const { providerUserSession, provider } = companion
if (!provider || (typeof id !== 'string' && id != null)) {
res.sendStatus(400)
return
}
try {
const data = await provider.list({
companion,
providerUserSession,
directory: id,
query,
})
res.json(data)
} catch (err) {
if (respondWithError(err, res)) return
next(err)
}
}

42
packages/@uppy/companion/src/server/controllers/logout.js
View file

@ -1,42 +0,0 @@
import * as tokenService from '../helpers/jwt.js'
import { respondWithError } from '../provider/error.js'
/**
*
* @param {object} req
* @param {object} res
*/
export default async function logout(req, res, next) {
const cleanSession = () => {
if (req.session.grant) {
req.session.grant.state = null
req.session.grant.dynamic = null
}
}
const { companion } = req
const { providerUserSession } = companion
if (!providerUserSession) {
cleanSession()
res.json({ ok: true, revoked: false })
return
}
try {
const data = await companion.provider.logout({
providerUserSession,
companion,
})
delete companion.providerUserSession
tokenService.removeFromCookies(
res,
companion.options,
companion.providerClass.oauthProvider,
)
cleanSession()
res.json({ ok: true, ...data })
} catch (err) {
if (respondWithError(err, res)) return
next(err)
}
}

47
packages/@uppy/companion/src/server/controllers/logout.ts Normal file
View file

@ -0,0 +1,47 @@
import type { NextFunction, Request, Response } from 'express'
import * as tokenService from '../helpers/jwt.js'
import { respondWithError } from '../provider/error.js'
export default async function logout(
req: Request,
res: Response,
next: NextFunction,
): Promise<void> {
const cleanSession = (): void => {
const grant = req.session?.grant
if (grant) {
grant['state'] = null
grant['dynamic'] = null
}
}
const { companion } = req
const { providerUserSession, provider, providerClass } = companion
if (!providerUserSession) {
cleanSession()
res.json({ ok: true, revoked: false })
return
}
if (!provider) {
cleanSession()
res.sendStatus(400)
return
}
try {
const out = await provider.logout({
providerUserSession,
companion,
})
delete companion.providerUserSession
const oauthProvider = providerClass?.oauthProvider
if (oauthProvider != null) {
tokenService.removeFromCookies(res, companion.options, oauthProvider)
}
cleanSession()
res.json({ ok: true, ...out })
} catch (err) {
if (respondWithError(err, res)) return
next(err)
}
}

43
packages/@uppy/companion/src/server/controllers/oauth-redirect.js
View file

@ -1,43 +0,0 @@
import qs from 'node:querystring'
import { URL } from 'node:url'
import * as oAuthState from '../helpers/oauth-state.js'
import { hasMatch } from '../helpers/utils.js'
/**
*
* @param {object} req
* @param {object} res
*/
export default function oauthRedirect(req, res) {
const params = qs.stringify(req.query)
const { oauthProvider } = req.companion.providerClass
if (!req.companion.options.server.oauthDomain) {
res.redirect(
req.companion.buildURL(
`/connect/${oauthProvider}/callback?${params}`,
true,
),
)
return
}
const { state } = oAuthState.getGrantDynamicFromRequest(req)
if (!state) {
res.status(400).send('Cannot find state in session')
return
}
const handler = oAuthState.getFromState(
state,
'companionInstance',
req.companion.options.secret,
)
const handlerHostName = new URL(handler).host
if (hasMatch(handlerHostName, req.companion.options.server.validHosts)) {
const url = `${handler}/connect/${oauthProvider}/callback?${params}`
res.redirect(url)
return
}
res.status(400).send('Invalid Host in state')
}

58
packages/@uppy/companion/src/server/controllers/oauth-redirect.ts Normal file
View file

@ -0,0 +1,58 @@
import qs from 'node:querystring'
import { URL } from 'node:url'
import type { Request, Response } from 'express'
import * as oAuthState from '../helpers/oauth-state.js'
import { hasMatch } from '../helpers/utils.js'
export default function oauthRedirect(req: Request, res: Response): void {
const secret = req.companion.options.secret
const queryParams: Record<string, string | readonly string[]> = {}
for (const [key, value] of Object.entries(req.query)) {
if (typeof value === 'string') {
queryParams[key] = value
continue
}
if (Array.isArray(value) && value.every((i) => typeof i === 'string')) {
queryParams[key] = value
}
}
const params = qs.stringify(queryParams)
const { providerClass, buildURL } = req.companion
const oauthProvider = providerClass?.oauthProvider
if (!buildURL) {
res.sendStatus(500)
return
}
if (!req.companion.options.server.oauthDomain) {
res.redirect(buildURL(`/connect/${oauthProvider}/callback?${params}`, true))
return
}
const { state } = oAuthState.getGrantDynamicFromRequest(req)
if (!state) {
res.status(400).send('Cannot find state in session')
return
}
const handler = oAuthState.getFromState(state, 'companionInstance', secret)
if (typeof handler !== 'string' || handler.length === 0) {
res.status(400).send('Invalid Host in state')
return
}
let handlerHostName: string
try {
handlerHostName = new URL(handler).host
} catch {
res.status(400).send('Invalid Host in state')
return
}
if (
hasMatch(handlerHostName, req.companion.options.server.validHosts ?? [])
) {
const url = `${handler}/connect/${oauthProvider}/callback?${params}`
res.redirect(url)
return
}
res.status(400).send('Invalid Host in state')
}

21
packages/@uppy/companion/src/server/controllers/preauth.js
View file

@ -1,21 +0,0 @@
import * as tokenService from '../helpers/jwt.js'
import logger from '../logger.js'
export default function preauth(req, res) {
if (!req.body || !req.body.params) {
logger.info('invalid request data received', 'preauth.bad')
return res.sendStatus(400)
}
const providerConfig =
req.companion.options.providerOptions[req.params.providerName]
if (!providerConfig?.credentialsURL) {
return res.sendStatus(501)
}
const preAuthToken = tokenService.generateEncryptedToken(
req.body.params,
req.companion.options.preAuthSecret,
)
return res.json({ token: preAuthToken })
}

39
packages/@uppy/companion/src/server/controllers/preauth.ts Normal file
View file

@ -0,0 +1,39 @@
import type { Request, Response } from 'express'
import * as tokenService from '../helpers/jwt.js'
import { isRecord } from '../helpers/type-guards.js'
import logger from '../logger.js'
export default function preauth(req: Request, res: Response): void {
const body = isRecord(req.body) ? req.body : undefined
const params =
body && Object.hasOwn(body, 'params') ? body['params'] : undefined
if (!params) {
logger.info('invalid request data received', 'preauth.bad')
res.sendStatus(400)
return
}
const providerName = req.params['providerName']
if (typeof providerName !== 'string' || providerName.length === 0) {
res.sendStatus(400)
return
}
const credentialsURL =
req.companion.options.providerOptions[providerName]?.credentialsURL
if (!credentialsURL) {
res.sendStatus(501)
return
}
const { preAuthSecret } = req.companion.options
if (preAuthSecret == null) {
res.sendStatus(500)
return
}
const preAuthToken = tokenService.generateEncryptedToken(
params,
preAuthSecret,
)
res.json({ token: preAuthToken })
}

43
packages/@uppy/companion/src/server/controllers/refresh-token.js → packages/@uppy/companion/src/server/controllers/refresh-token.ts
View file

@ -1,3 +1,4 @@
import type { NextFunction, Request, Response } from 'express'
import * as tokenService from '../helpers/jwt.js'
import logger from '../logger.js'
import { respondWithError } from '../provider/error.js'
@ -5,27 +6,39 @@ import { respondWithError } from '../provider/error.js'
// https://www.dropboxforum.com/t5/Dropbox-API-Support-Feedback/Get-refresh-token-from-access-token/td-p/596739
// https://developers.dropbox.com/oauth-guide
// https://github.com/simov/grant/issues/149
export default async function refreshToken(req, res, next) {
const { providerName } = req.params
export default async function refreshToken(
req: Request,
res: Response,
next: NextFunction,
): Promise<void> {
const providerName = req.params['providerName']
if (typeof providerName !== 'string' || providerName.length === 0) {
res.sendStatus(400)
return
}
const { secret } = req.companion.options
const { key: clientId, secret: clientSecret } = req.companion.options
.providerOptions[providerName] ?? { __proto__: null }
const { redirect_uri: redirectUri } = req.companion.providerGrantConfig
const providerConfig = req.companion.options.providerOptions?.[providerName]
const clientId = providerConfig?.key
const clientSecret = providerConfig?.secret
const redirectUri = req.companion.providerGrantConfig?.redirect_uri
const { provider, providerClass } = req.companion
const { providerUserSession } = req.companion
// not all providers have refresh tokens
if (
providerUserSession.refreshToken == null ||
providerUserSession.refreshToken === ''
) {
if (!providerUserSession?.refreshToken) {
logger.warn('Tried to refresh token without having a token')
res.sendStatus(401)
return
}
if (!provider || !providerClass) {
res.sendStatus(400)
return
}
try {
const data = await req.companion.provider.refreshToken({
const { accessToken } = await provider.refreshToken({
redirectUri,
clientId,
clientSecret,
@ -34,25 +47,25 @@ export default async function refreshToken(req, res, next) {
req.companion.providerUserSession = {
...providerUserSession,
accessToken: data.accessToken,
accessToken,
}
logger.debug(
`Generating refreshed auth token for provider ${providerName}`,
null,
undefined,
req.id,
)
const uppyAuthToken = tokenService.generateEncryptedAuthToken(
{ [providerName]: req.companion.providerUserSession },
req.companion.options.secret,
req.companion.providerClass.authStateExpiry,
secret,
providerClass.authStateExpiry,
)
tokenService.addToCookiesIfNeeded(
req,
res,
uppyAuthToken,
req.companion.providerClass.authStateExpiry,
providerClass.authStateExpiry,
)
res.send({ uppyAuthToken })

180
packages/@uppy/companion/src/server/controllers/s3.js → packages/@uppy/companion/src/server/controllers/s3.ts
View file

@ -1,3 +1,5 @@
import assert from 'node:assert'
import type { Part, S3Client } from '@aws-sdk/client-s3'
import {
AbortMultipartUploadCommand,
CompleteMultipartUploadCommand,
@ -8,14 +10,29 @@ import {
import { GetFederationTokenCommand, STSClient } from '@aws-sdk/client-sts'
import { createPresignedPost } from '@aws-sdk/s3-presigned-post'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
import type { NextFunction, Request, Response, Router } from 'express'
import express from 'express'
import type { CompanionRuntimeOptions } from '../../types/companion-options.js'
import { isRecord } from '../helpers/type-guards.js'
import {
getBucket,
rfc2047EncodeMetadata,
truncateFilename,
} from '../helpers/utils.js'
export default function s3(config) {
export default function s3(
config: Pick<
CompanionRuntimeOptions['s3'],
| 'acl'
| 'getKey'
| 'expires'
| 'conditions'
| 'bucket'
| 'region'
| 'key'
| 'secret'
>,
): Router {
if (typeof config.acl !== 'string' && config.acl != null) {
throw new TypeError('s3: The `acl` option must be a string or null')
}
@ -23,10 +40,11 @@ export default function s3(config) {
throw new TypeError('s3: The `getKey` option must be a function')
}
function getS3Client(req, res, createPresignedPostMode = false) {
/**
* @type {import('@aws-sdk/client-s3').S3Client}
*/
function getS3Client(
req: Request,
res: Response,
createPresignedPostMode = false,
): S3Client | undefined {
const client = createPresignedPostMode
? req.companion.s3ClientCreatePresignedPost
: req.companion.s3Client
@ -52,11 +70,18 @@ export default function s3(config) {
* - url - The URL to upload to.
* - fields - Form fields to send along.
*/
function getUploadParameters(req, res, next) {
const client = getS3Client(req, res)
function getUploadParameters(
req: Request,
res: Response,
next: NextFunction,
) {
const client = getS3Client(req, res, true)
if (!client) return
const { metadata = {}, filename } = req.query
const filename = req.query['filename']
const metadata = isRecord(req.query['metadata'])
? req.query['metadata']
: {}
// Validate filename is provided and non-empty
if (typeof filename !== 'string' || filename === '') {
@ -88,17 +113,23 @@ export default function s3(config) {
return
}
const fields = {
const fields: Record<string, string> = {
success_action_status: '201',
'content-type': req.query.type,
...(typeof req.query['type'] === 'string' && {
'content-type': req.query['type'],
}),
}
if (config.acl != null) fields.acl = config.acl
if (config.acl != null) fields['acl'] = config.acl
Object.keys(metadata).forEach((metadataKey) => {
fields[`x-amz-meta-${metadataKey}`] = metadata[metadataKey]
Object.entries(metadata).forEach(([metadataKey, value]) => {
if (typeof value !== 'string') return
fields[`x-amz-meta-${metadataKey}`] = value
})
// `createPresignedPost` sometimes pulls in a nested copy of `@aws-sdk/client-s3`,
// which makes the `S3Client` type nominally incompatible. The instance is still
// compatible at runtime.
createPresignedPost(client, {
Bucket: bucket,
Expires: config.expires,
@ -129,11 +160,21 @@ export default function s3(config) {
* - key - The object key in the S3 bucket.
* - uploadId - The ID of this multipart upload, to be used in later requests.
*/
function createMultipartUpload(req, res, next) {
function createMultipartUpload(
req: Request,
res: Response,
next: NextFunction,
) {
const client = getS3Client(req, res)
if (!client) return
const { type, metadata = {}, filename } = req.body
const {
type,
filename,
metadata: maybeMetadata,
}: { type: unknown; filename: unknown; metadata: unknown } = req.body
const metadata = isRecord(maybeMetadata) ? maybeMetadata : {}
// Validate filename is provided and non-empty
if (typeof filename !== 'string' || filename === '') {
@ -149,8 +190,6 @@ export default function s3(config) {
req.companion.options.maxFilenameLength,
)
const key = config.getKey({ req, filename: truncatedFilename, metadata })
const bucket = getBucket({
bucketOrFn: config.bucket,
req,
@ -158,6 +197,8 @@ export default function s3(config) {
metadata,
})
const key = config.getKey({ req, filename: truncatedFilename, metadata })
if (typeof key !== 'string') {
res.status(500).json({
error: 's3: filename returned from `getKey` must be a string',
@ -174,10 +215,9 @@ export default function s3(config) {
Key: key,
ContentType: type,
Metadata: rfc2047EncodeMetadata(metadata),
...(config.acl != null && { ACL: config.acl }),
}
if (config.acl != null) params.ACL = config.acl
client.send(new CreateMultipartUploadCommand(params)).then((data) => {
res.json({
key: data.Key,
@ -200,13 +240,19 @@ export default function s3(config) {
* - ETag - a hash of this part's contents, used to refer to it.
* - Size - size of this part.
*/
function getUploadedParts(req, res, next) {
function getUploadedParts(req: Request, res: Response, next: NextFunction) {
const client = getS3Client(req, res)
if (!client) return
const s3Client = client
const { uploadId } = req.params
const { key } = req.query
assert(
typeof uploadId === 'string' && uploadId.length > 0,
's3: uploadId must be provided.',
)
if (typeof key !== 'string') {
res.status(400).json({
error:
@ -214,17 +260,18 @@ export default function s3(config) {
})
return
}
const keyStr = key
const bucket = getBucket({ bucketOrFn: config.bucket, req })
const parts = []
const parts: Part[] = []
function listPartsPage(startAt) {
client
const listPartsPage = (startAt?: string) => {
s3Client
.send(
new ListPartsCommand({
Bucket: bucket,
Key: key,
Key: keyStr,
UploadId: uploadId,
PartNumberMarker: startAt,
}),
@ -254,13 +301,18 @@ export default function s3(config) {
* Response JSON:
* - url - The URL to upload to, including signed query parameters.
*/
function signPartUpload(req, res, next) {
function signPartUpload(req: Request, res: Response, next: NextFunction) {
const client = getS3Client(req, res)
if (!client) return
const { uploadId, partNumber } = req.params
const { key } = req.query
const uploadId = req.params['uploadId']
const partNumber = req.params['partNumber']
const key = req.query['key']
if (typeof uploadId !== 'string' || uploadId.length === 0) {
res.status(400).json({ error: 's3: uploadId must be provided.' })
return
}
if (typeof key !== 'string') {
res.status(400).json({
error:
@ -268,7 +320,7 @@ export default function s3(config) {
})
return
}
if (!parseInt(partNumber, 10)) {
if (typeof partNumber !== 'string' || !parseInt(partNumber, 10)) {
res.status(400).json({
error: 's3: the part number must be a number between 1 and 10000.',
})
@ -283,7 +335,7 @@ export default function s3(config) {
Bucket: bucket,
Key: key,
UploadId: uploadId,
PartNumber: partNumber,
PartNumber: Number(partNumber),
Body: '',
}),
{ expiresIn: config.expires },
@ -305,12 +357,22 @@ export default function s3(config) {
* - presignedUrls - The URLs to upload to, including signed query parameters,
* in an object mapped to part numbers.
*/
function batchSignPartsUpload(req, res, next) {
function batchSignPartsUpload(
req: Request,
res: Response,
next: NextFunction,
) {
const client = getS3Client(req, res)
if (!client) return
const { uploadId } = req.params
const { key, partNumbers } = req.query
const uploadId = req.params['uploadId']
const key = req.query['key']
const partNumbers = req.query['partNumbers']
if (typeof uploadId !== 'string' || uploadId.length === 0) {
res.status(400).json({ error: 's3: uploadId must be provided.' })
return
}
if (typeof key !== 'string') {
res.status(400).json({
@ -354,9 +416,12 @@ export default function s3(config) {
}),
)
.then((urls) => {
const presignedUrls = Object.create(null)
const presignedUrls: Record<string, string> = Object.create(null)
for (let index = 0; index < partNumbersArray.length; index++) {
presignedUrls[partNumbersArray[index]] = urls[index]
const partNumber = partNumbersArray[index]
const url = urls[index]
if (!partNumber || !url) continue
presignedUrls[partNumber] = url
}
res.json({ presignedUrls })
})
@ -373,13 +438,22 @@ export default function s3(config) {
* Response JSON:
* Empty.
*/
function abortMultipartUpload(req, res, next) {
function abortMultipartUpload(
req: Request,
res: Response,
next: NextFunction,
) {
const client = getS3Client(req, res)
if (!client) return
const { uploadId } = req.params
const { key } = req.query
assert(
typeof uploadId === 'string' && uploadId.length > 0,
's3: uploadId must be provided.',
)
if (typeof key !== 'string') {
res.status(400).json({
error:
@ -413,13 +487,22 @@ export default function s3(config) {
* Response JSON:
* - location - The full URL to the object in the S3 bucket.
*/
function completeMultipartUpload(req, res, next) {
function completeMultipartUpload(
req: Request,
res: Response,
next: NextFunction,
) {
const client = getS3Client(req, res)
if (!client) return
const { uploadId } = req.params
const { key } = req.query
const { parts } = req.body
const { parts }: { parts: unknown } = req.body
assert(
typeof uploadId === 'string' && uploadId.length > 0,
's3: uploadId must be provided.',
)
if (typeof key !== 'string') {
res.status(400).json({
@ -479,8 +562,11 @@ export default function s3(config) {
],
}
let stsClient
function getSTSClient() {
let stsClient: STSClient | undefined
function getSTSClient(): STSClient | null {
if (config.region == null || config.key == null || config.secret == null) {
return null
}
if (stsClient == null) {
stsClient = new STSClient({
region: config.region,
@ -506,8 +592,20 @@ export default function s3(config) {
* - bucket: the S3 bucket name.
* - region: the region where that bucket is stored.
*/
function getTemporarySecurityCredentials(req, res, next) {
getSTSClient()
function getTemporarySecurityCredentials(
req: Request,
res: Response,
next: NextFunction,
) {
const sts = getSTSClient()
if (!sts || typeof config.bucket !== 'string') {
res.status(400).json({
error: 'This Companion server does not support STS credentials',
})
return
}
sts
.send(
new GetFederationTokenCommand({
// Name of the federated user. The name is used as an identifier for the

17
packages/@uppy/companion/src/server/controllers/search.js
View file

@ -1,17 +0,0 @@
import { respondWithError } from '../provider/error.js'
export default async function search({ query, companion }, res, next) {
const { providerUserSession } = companion
try {
const data = await companion.provider.search({
companion,
providerUserSession,
query,
})
res.json(data)
} catch (err) {
if (respondWithError(err, res)) return
next(err)
}
}

34
packages/@uppy/companion/src/server/controllers/search.ts Normal file
View file

@ -0,0 +1,34 @@
import type { NextFunction, Request, Response } from 'express'
import { respondWithError } from '../provider/error.js'
export default async function search(
req: Request,
res: Response,
next: NextFunction,
): Promise<void> {
const { query, companion } = req
const { providerUserSession, provider } = companion
if (!provider) {
res.sendStatus(400)
return
}
const buildURL = companion.buildURL
const q = query['q']
if (buildURL == null || typeof q !== 'string') {
res.sendStatus(500)
return
}
try {
const data = await provider.search({
companion: { buildURL },
providerUserSession,
query: { ...query, q },
})
res.json(data)
} catch (err) {
if (respondWithError(err, res)) return
next(err)
}
}

58
packages/@uppy/companion/src/server/controllers/send-token.ts Normal file
View file

@ -0,0 +1,58 @@
import type { NextFunction, Request, Response } from 'express'
import emitter from '../emitter/index.js'
import {
authCallbackSuccessHtml,
legacyAuthCallbackHtml,
} from '../helpers/html.js'
import * as oAuthState from '../helpers/oauth-state.js'
import { isOriginAllowed } from './connect.js'
export default function sendToken(
req: Request,
res: Response,
next: NextFunction,
): void {
const companion = req.companion
const uppyAuthToken = companion.authToken
const { secret } = companion.options
const { state } = oAuthState.getGrantDynamicFromRequest(req)
if (!state) {
next()
return
}
const clientOrigin = oAuthState.getFromState(state, 'origin', secret)
if (typeof clientOrigin !== 'string' || clientOrigin.length === 0) {
next()
return
}
const customerDefinedAllowedOrigins = oAuthState.getFromState(
state,
'customerDefinedAllowedOrigins',
secret,
)
if (
customerDefinedAllowedOrigins &&
!isOriginAllowed(clientOrigin, customerDefinedAllowedOrigins)
) {
next()
return
}
const authCallbackToken = oAuthState.getFromState(
state,
'authCallbackToken',
companion.options.secret,
)
// only new Uppy clients will set an authCallbackToken in the state
// in that case, we send the token through the emitter.
if (authCallbackToken) {
emitter().emit(authCallbackToken, { token: uppyAuthToken })
res.send(authCallbackSuccessHtml())
} else {
// This is backwards compatible with old Uppy clients:
res.send(legacyAuthCallbackHtml({ token: uppyAuthToken }, clientOrigin))
}
}

29
packages/@uppy/companion/src/server/controllers/simple-auth.js → packages/@uppy/companion/src/server/controllers/simple-auth.ts
View file

@ -1,12 +1,27 @@
import type { NextFunction, Request, Response } from 'express'
import * as tokenService from '../helpers/jwt.js'
import logger from '../logger.js'
import { respondWithError } from '../provider/error.js'
export default async function simpleAuth(req, res, next) {
const { providerName } = req.params
export default async function simpleAuth(
req: Request,
res: Response,
next: NextFunction,
): Promise<void> {
const providerName = req.params['providerName']
if (typeof providerName !== 'string' || providerName.length === 0) {
res.sendStatus(400)
return
}
const { secret } = req.companion.options
const { provider, providerClass } = req.companion
if (!provider || !providerClass) {
res.sendStatus(400)
return
}
try {
const simpleAuthResponse = await req.companion.provider.simpleAuth({
const simpleAuthResponse = await provider.simpleAuth({
requestBody: req.body,
})
@ -17,20 +32,20 @@ export default async function simpleAuth(req, res, next) {
logger.debug(
`Generating simple auth token for provider ${providerName}`,
null,
undefined,
req.id,
)
const uppyAuthToken = tokenService.generateEncryptedAuthToken(
{ [providerName]: req.companion.providerUserSession },
req.companion.options.secret,
req.companion.providerClass.authStateExpiry,
secret,
providerClass.authStateExpiry,
)
tokenService.addToCookiesIfNeeded(
req,
res,
uppyAuthToken,
req.companion.providerClass.authStateExpiry,
providerClass.authStateExpiry,
)
res.send({ uppyAuthToken })

22
packages/@uppy/companion/src/server/controllers/thumbnail.js → packages/@uppy/companion/src/server/controllers/thumbnail.ts
View file

@ -1,13 +1,21 @@
import type { NextFunction, Request, Response } from 'express'
import { respondWithError } from '../provider/error.js'
/**
*
* @param {object} req
* @param {object} res
*/
async function thumbnail(req, res, next) {
const { id } = req.params
async function thumbnail(
req: Request,
res: Response,
next: NextFunction,
): Promise<void> {
const id = req.params['id']
if (typeof id !== 'string' || id.length === 0) {
res.sendStatus(400)
return
}
const { provider, providerUserSession } = req.companion
if (!provider) {
res.sendStatus(400)
return
}
try {
const { stream, contentType } = await provider.thumbnail({

30
packages/@uppy/companion/src/server/controllers/url.js → packages/@uppy/companion/src/server/controllers/url.ts
View file

@ -1,3 +1,4 @@
import type { Request, Response } from 'express'
import express from 'express'
import { downloadURL } from '../download.js'
import { getURLMeta, validateURL } from '../helpers/request.js'
@ -6,25 +7,16 @@ import logger from '../logger.js'
import { respondWithError } from '../provider/error.js'
/**
* @callback downloadCallback
* @param {Error} err
* @param {string | Buffer | Buffer[]} chunk
* Fetch the size and content type of a URL.
*/
/**
* Fetches the size and content type of a URL
*
* @param {object} req expressJS request object
* @param {object} res expressJS response object
*/
const meta = async (req, res) => {
const meta = async (req: Request, res: Response): Promise<void> => {
try {
logger.debug('URL file import handler running', null, req.id)
logger.debug('URL file import handler running', undefined, req.id)
const { allowLocalUrls } = req.companion.options
if (!validateURL(req.body.url, allowLocalUrls)) {
logger.debug(
'Invalid request body detected. Exiting url meta handler.',
null,
undefined,
req.id,
)
res.status(400).json({ error: 'Invalid request body' })
@ -41,19 +33,15 @@ const meta = async (req, res) => {
}
/**
* Handles the reques of import a file from a remote URL, and then
* subsequently uploading it to the specified destination.
*
* @param {object} req expressJS request object
* @param {object} res expressJS response object
* Import a file from a remote URL, then upload it to the specified destination.
*/
const get = async (req, res) => {
logger.debug('URL file import handler running', null, req.id)
const get = async (req: Request, res: Response): Promise<void> => {
logger.debug('URL file import handler running', undefined, req.id)
const { allowLocalUrls } = req.companion.options
if (!validateURL(req.body.url, allowLocalUrls)) {
logger.debug(
'Invalid request body detected. Exiting url import handler.',
null,
undefined,
req.id,
)
res.status(400).json({ error: 'Invalid request body' })

18
packages/@uppy/companion/src/server/download.js → packages/@uppy/companion/src/server/download.ts
View file

@ -1,17 +1,21 @@
import type { Readable } from 'node:stream'
import { getProtectedGot } from './helpers/request.js'
import { prepareStream } from './helpers/utils.js'
import logger from './logger.js'
/**
* Downloads the content in the specified url, and passes the data
* to the callback chunk by chunk.
* Downloads the content at the given URL.
*
* @param {string} url
* @param {boolean} allowLocalIPs
* @param {string} traceId
* @returns {Promise}
* @param url - URL to download.
* @param allowLocalIPs - Whether to allow local/private IPs (disables SSRF protection).
* @param traceId - Request trace id for logging.
*/
export const downloadURL = async (url, allowLocalIPs, traceId, options) => {
export const downloadURL = async (
url: string,
allowLocalIPs: boolean,
traceId?: string,
options: Record<string, unknown> = {},
): Promise<{ stream: Readable; size: number | undefined }> => {
try {
const protectedGot = getProtectedGot({ allowLocalIPs })
const stream = protectedGot.stream.get(url, {

5
packages/@uppy/companion/src/server/emitter/default-emitter.js
View file

@ -1,5 +0,0 @@
import { EventEmitter } from 'node:events'
export default function defaultEmitter() {
return new EventEmitter()
}

6
packages/@uppy/companion/src/server/emitter/default-emitter.ts Normal file
View file

@ -0,0 +1,6 @@
import { EventEmitter } from 'node:events'
import type { EmitterLike } from './index.js'
export default function defaultEmitter(): EmitterLike {
return new EventEmitter()
}

19
packages/@uppy/companion/src/server/emitter/index.js
View file

@ -1,19 +0,0 @@
import nodeEmitter from './default-emitter.js'
import redisEmitter from './redis-emitter.js'
let emitter
/**
* Singleton event emitter that is shared between modules throughout the lifetime of the server.
* Used to transmit events (such as progress, upload completion) from controllers,
* such as the Google Drive 'get' controller, along to the client.
*/
export default function getEmitter(redisClient, redisPubSubScope) {
if (!emitter) {
emitter = redisClient
? redisEmitter(redisClient, redisPubSubScope)
: nodeEmitter()
}
return emitter
}

34
packages/@uppy/companion/src/server/emitter/index.ts Normal file
View file

@ -0,0 +1,34 @@
import type { Redis } from 'ioredis'
import nodeEmitter from './default-emitter.js'
import redisEmitter from './redis-emitter.js'
type Listener = (...args: unknown[]) => void
export interface EmitterLike {
on: (eventName: string, handler: Listener) => unknown
once: (eventName: string, handler: Listener) => unknown
off?: (eventName: string, handler: Listener) => unknown
emit: (eventName: string, ...args: unknown[]) => unknown
removeListener: (eventName: string, handler: Listener) => unknown
removeAllListeners: (eventName: string) => unknown
}
let emitter: EmitterLike | undefined
/**
* Singleton event emitter that is shared between modules throughout the lifetime of the server.
* Used to transmit events (such as progress, upload completion) from controllers,
* such as the Google Drive 'get' controller, along to the client.
*/
export default function getEmitter(
redisClient?: Redis | undefined,
redisPubSubScope?: string,
): EmitterLike {
if (!emitter) {
emitter = redisClient
? redisEmitter(redisClient, redisPubSubScope)
: nodeEmitter()
}
return emitter
}

89
packages/@uppy/companion/src/server/emitter/redis-emitter.js → packages/@uppy/companion/src/server/emitter/redis-emitter.ts
View file

@ -1,8 +1,10 @@
import { EventEmitter } from 'node:events'
import safeStringify from 'fast-safe-stringify'
import type { Redis } from 'ioredis'
import * as logger from '../logger.js'
import type { EmitterLike } from './index.js'
function replacer(key, value) {
function replacer(key: string, value: unknown): unknown {
// Remove the circular structure and internal ones
return key[0] === '_' || value === '[Circular]' ? undefined : value
}
@ -12,16 +14,18 @@ function replacer(key, value) {
* This is useful for when companion is running on multiple instances and events need
* to be distributed across.
*
* @param {import('ioredis').Redis} redisClient
* @param {string} redisPubSubScope
* @returns
* @param redisClient
* @param redisPubSubScope
*/
export default function redisEmitter(redisClient, redisPubSubScope) {
export default function redisEmitter(
redisClient: Redis,
redisPubSubScope?: string,
): EmitterLike {
const prefix = redisPubSubScope ? `${redisPubSubScope}:` : ''
const getPrefixedEventName = (eventName) => `${prefix}${eventName}`
const getPrefixedEventName = (eventName: string) => `${prefix}${eventName}`
const errorEmitter = new EventEmitter()
const handleError = (err) => errorEmitter.emit('error', err)
const handleError = (err: unknown) => errorEmitter.emit('error', err)
async function makeRedis() {
const publisher = redisClient.duplicate({ lazyConnect: true })
@ -42,9 +46,11 @@ export default function redisEmitter(redisClient, redisPubSubScope) {
/**
*
* @param {(a: Awaited<typeof redisPromise>) => void} fn
* @param fn
*/
async function runWhenConnected(fn) {
async function runWhenConnected(
fn: (clients: { subscriber: Redis; publisher: Redis }) => unknown,
): Promise<void> {
try {
await fn(await redisPromise)
} catch (err) {
@ -53,16 +59,23 @@ export default function redisEmitter(redisClient, redisPubSubScope) {
}
// because each event can have multiple listeners, we need to keep track of them
/** @type {Map<string, Map<() => unknown, () => unknown>>} */
const handlersByEventName = new Map()
const handlersByEventName: Map<
string,
Map<
(...args: unknown[]) => unknown,
(pattern: string, _channel: string, message: string) => unknown
>
> = new Map()
/**
* Remove an event listener
*
* @param {string} eventName name of the event
* @param {any} handler the handler of the event to remove
* @param eventName name of the event
* @param handler the handler of the event to remove
*/
async function removeListener(eventName, handler) {
type Handler = (...args: unknown[]) => unknown
async function removeListener(eventName: string, handler: Handler) {
if (eventName === 'error') {
errorEmitter.removeListener('error', handler)
return
@ -91,26 +104,34 @@ export default function redisEmitter(redisClient, redisPubSubScope) {
/**
*
* @param {string} eventName
* @param {*} handler
* @param {*} _once
* @param eventName
* @param handler
* @param _once
*/
async function addListener(eventName, handler, _once = false) {
async function addListener(
eventName: string,
handler: Handler,
_once = false,
) {
if (eventName === 'error') {
if (_once) errorEmitter.once('error', handler)
else errorEmitter.addListener('error', handler)
return
}
function actualHandler(pattern, channel, message) {
function actualHandler(pattern: string, _channel: string, message: string) {
if (pattern !== getPrefixedEventName(eventName)) {
return
}
if (_once) removeListener(eventName, handler)
let args
let args: unknown[]
try {
args = JSON.parse(message)
const parsed: unknown = JSON.parse(message)
if (!Array.isArray(parsed)) {
throw new Error('Expected JSON array')
}
args = parsed
} catch (_ex) {
handleError(
new Error(
@ -139,39 +160,39 @@ export default function redisEmitter(redisClient, redisPubSubScope) {
/**
* Add an event listener
*
* @param {string} eventName name of the event
* @param {any} handler the handler of the event
* @param eventName name of the event
* @param handler the handler of the event
*/
async function on(eventName, handler) {
async function on(eventName: string, handler: Handler) {
await addListener(eventName, handler)
}
/**
* Remove an event listener
*
* @param {string} eventName name of the event
* @param {any} handler the handler of the event
* @param eventName name of the event
* @param handler the handler of the event
*/
async function off(eventName, handler) {
async function off(eventName: string, handler: Handler) {
await removeListener(eventName, handler)
}
/**
* Add an event listener (will be triggered at most once)
*
* @param {string} eventName name of the event
* @param {any} handler the handler of the event
* @param eventName name of the event
* @param handler the handler of the event
*/
async function once(eventName, handler) {
async function once(eventName: string, handler: Handler) {
await addListener(eventName, handler, true)
}
/**
* Announce the occurrence of an event
*
* @param {string} eventName name of the event
* @param eventName name of the event
*/
async function emit(eventName, ...args) {
async function emit(eventName: string, ...args: unknown[]) {
await runWhenConnected(async ({ publisher }) =>
publisher.publish(
getPrefixedEventName(eventName),
@ -183,9 +204,9 @@ export default function redisEmitter(redisClient, redisPubSubScope) {
/**
* Remove all listeners of an event
*
* @param {string} eventName name of the event
* @param eventName name of the event
*/
async function removeAllListeners(eventName) {
async function removeAllListeners(eventName: string) {
if (eventName === 'error') {
errorEmitter.removeAllListeners(eventName)
return

22
packages/@uppy/companion/src/server/header-blacklist.js → packages/@uppy/companion/src/server/header-blacklist.ts
View file

@ -34,10 +34,10 @@ const forbiddenRegex = [/^proxy-.*$/, /^sec-.*$/]
/**
* Check if the header in parameter is a forbidden header.
*
* @param {string} header Header to check
* @param header Header to check
* @returns True if header is forbidden, false otherwise.
*/
const isForbiddenHeader = (header) => {
const isForbiddenHeader = (header: string): boolean => {
const headerLower = header.toLowerCase()
const forbidden =
forbiddenNames.indexOf(headerLower) >= 0 ||
@ -49,7 +49,9 @@ const isForbiddenHeader = (header) => {
return forbidden
}
export default function headerBlacklist(headers) {
export default function headerBlacklist(
headers: Record<string, unknown> | undefined,
): Record<string, string> {
if (
headers == null ||
typeof headers !== 'object' ||
@ -58,11 +60,11 @@ export default function headerBlacklist(headers) {
return {}
}
const headersCloned = { ...headers }
Object.keys(headersCloned).forEach((header) => {
if (isForbiddenHeader(header)) {
delete headersCloned[header]
}
})
return headersCloned
const out: Record<string, string> = {}
for (const [header, value] of Object.entries(headers)) {
if (isForbiddenHeader(header)) continue
if (typeof value !== 'string') continue
out[header] = value
}
return out
}

103
packages/@uppy/companion/src/server/controllers/send-token.js → packages/@uppy/companion/src/server/helpers/html.ts
View file

@ -1,13 +1,6 @@
import serialize from 'serialize-javascript'
import * as oAuthState from '../helpers/oauth-state.js'
import { isOriginAllowed } from './connect.js'
/**
*
* @param {string} token uppy auth token
* @param {string} origin url string
*/
const htmlContent = (token, origin) => {
export const authCallbackSuccessHtml = () => {
return `
<!DOCTYPE html>
<html>
@ -17,7 +10,61 @@ const htmlContent = (token, origin) => {
(function() {
'use strict';
var data = ${serialize({ token })};
document.addEventListener('DOMContentLoaded', function () {
window.close();
});
})();
</script>
</head>
<body>
<center>Authentication successful. You may now close this page.</center>
</body>
</html>`
}
export const authCallbackErrorHtml = () => {
return `
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<script>
(function() {
'use strict';
document.addEventListener('DOMContentLoaded', function () {
window.close();
});
})();
</script>
</head>
<body>
<center>Authentication failed. You may now close this page.</center>
</body>
</html>`
}
/**
* Generate an HTML page that will post a message to the opener window and then close itself.
* This is only used by old Uppy clients
*
* @param {unknown} data data payload
* @param {string} origin url string
*/
export const legacyAuthCallbackHtml = (
data: unknown,
origin: string | undefined,
) => {
return `
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<script>
(function() {
'use strict';
var data = ${serialize(data)};
var origin = ${serialize(origin)};
if (window.opener != null) {
@ -43,41 +90,3 @@ const htmlContent = (token, origin) => {
</body>
</html>`
}
/**
*
* @param {import('express').Request} req
* @param {import('express').Response} res
* @param {import('express').NextFunction} next
*/
export default function sendToken(req, res, next) {
// @ts-expect-error untyped
const { companion } = req
const uppyAuthToken = companion.authToken
const { state } = oAuthState.getGrantDynamicFromRequest(req)
if (!state) {
return next()
}
const clientOrigin = oAuthState.getFromState(
state,
'origin',
companion.options.secret,
)
const customerDefinedAllowedOrigins = oAuthState.getFromState(
state,
'customerDefinedAllowedOrigins',
companion.options.secret,
)
if (
customerDefinedAllowedOrigins &&
!isOriginAllowed(clientOrigin, customerDefinedAllowedOrigins)
) {
return next()
}
return res.send(htmlContent(uppyAuthToken, clientOrigin))
}

143
packages/@uppy/companion/src/server/helpers/jwt.js → packages/@uppy/companion/src/server/helpers/jwt.ts
View file

@ -1,4 +1,7 @@
import type { Request, Response } from 'express'
import jwt from 'jsonwebtoken'
import type { CompanionRuntimeOptions } from '../../types/companion-options.js'
import { isRecord } from './type-guards.js'
import { decrypt, encrypt } from './utils.js'
// The Uppy auth token is an encrypted JWT & JSON encoded container.
@ -19,74 +22,72 @@ import { decrypt, encrypt } from './utils.js'
// With 400 days, there's still a theoretical possibility but very low.
export const MAX_AGE_REFRESH_TOKEN = 60 * 60 * 24 * 400
export const MAX_AGE_24H = 60 * 60 * 24
/**
*
* @param {*} data
* @param {string} secret
* @param {number} maxAge
*/
const generateToken = (data, secret, maxAge) => {
type EncryptionSecret = string | Buffer
const generateToken = (
data: unknown,
secret: EncryptionSecret,
maxAge: number,
): string => {
return jwt.sign({ data }, secret, { expiresIn: maxAge })
}
/**
*
* @param {string} token
* @param {string} secret
*/
const verifyToken = (token, secret) => {
// @ts-ignore
return jwt.verify(token, secret, {}).data
const verifyJwtToken = (token: string, secret: EncryptionSecret) => {
const decoded = jwt.verify(token, secret, {})
if (!decoded || typeof decoded !== 'object' || !('data' in decoded)) {
throw new Error('Invalid token payload')
}
return decoded['data']
}
/**
*
* @param {*} payload
* @param {string} secret
*/
export const generateEncryptedToken = (
payload,
secret,
payload: unknown,
secret: EncryptionSecret,
maxAge = MAX_AGE_24H,
) => {
): string => {
// return payload // for easier debugging
return encrypt(generateToken(payload, secret, maxAge), secret)
}
/**
* @param {*} payload
* @param {string} secret
*/
export const generateEncryptedAuthToken = (payload, secret, maxAge) => {
export const generateEncryptedAuthToken = (
payload: unknown,
secret: EncryptionSecret,
maxAge?: number,
): string => {
return generateEncryptedToken(JSON.stringify(payload), secret, maxAge)
}
/**
*
* @param {string} token
* @param {string} secret
*/
export const verifyEncryptedToken = (token, secret) => {
const ret = verifyToken(decrypt(token, secret), secret)
export const verifyEncryptedToken = (
token: string,
secret: EncryptionSecret,
) => {
const ret = verifyJwtToken(decrypt(token, secret), secret)
if (!ret) throw new Error('No payload')
return ret
}
/**
*
* @param {string} token
* @param {string} secret
*/
export const verifyEncryptedAuthToken = (token, secret, providerName) => {
export const verifyEncryptedAuthToken = <T extends Record<string, unknown>>(
token: string,
secret: EncryptionSecret,
providerName: string,
): T => {
const json = verifyEncryptedToken(token, secret)
const tokens = JSON.parse(json)
if (!tokens[providerName])
if (typeof json !== 'string') {
throw new Error('Invalid token payload: expected string')
}
const tokens: T = JSON.parse(json)
if (!isRecord(tokens) || !Object.hasOwn(tokens, providerName))
throw new Error(`Missing token payload for provider ${providerName}`)
return tokens
}
function getCommonCookieOptions({ companionOptions }) {
const cookieOptions = {
function getCommonCookieOptions({
companionOptions,
}: {
companionOptions: CompanionRuntimeOptions
}): Record<string, unknown> {
const cookieOptions: Record<string, unknown> = {
httpOnly: true,
}
@ -95,18 +96,20 @@ function getCommonCookieOptions({ companionOptions }) {
// Note that sameSite cookies also require secure (which needs https), so thumbnails don't work from localhost
// to test locally, you can manually find the URL of the image and open it in a separate browser tab
if (companionOptions.server && companionOptions.server.protocol === 'https') {
cookieOptions.sameSite = 'none'
cookieOptions.secure = true
cookieOptions['sameSite'] = 'none'
cookieOptions['secure'] = true
}
if (companionOptions.cookieDomain) {
cookieOptions.domain = companionOptions.cookieDomain
const cookieDomain = companionOptions['cookieDomain']
if (cookieDomain != null) {
cookieOptions['domain'] = cookieDomain
}
return cookieOptions
}
const getCookieName = (oauthProvider) => `uppyAuthToken--${oauthProvider}`
const getCookieName = (oauthProvider: string): string =>
`uppyAuthToken--${oauthProvider}`
const addToCookies = ({
res,
@ -114,7 +117,13 @@ const addToCookies = ({
companionOptions,
oauthProvider,
maxAge = MAX_AGE_24H,
}) => {
}: {
res: Response
token: string
companionOptions: CompanionRuntimeOptions
oauthProvider: string
maxAge?: number
}): void => {
const cookieOptions = {
...getCommonCookieOptions({ companionOptions }),
maxAge: maxAge * 1000,
@ -124,26 +133,32 @@ const addToCookies = ({
res.cookie(getCookieName(oauthProvider), token, cookieOptions)
}
export const addToCookiesIfNeeded = (req, res, uppyAuthToken, maxAge) => {
export const addToCookiesIfNeeded = (
req: Request,
res: Response,
uppyAuthToken: string,
maxAge?: number,
): void => {
// some providers need the token in cookies for thumbnail/image requests
if (req.companion.provider.needsCookieAuth) {
addToCookies({
if (req.companion.provider?.needsCookieAuth) {
const oauthProvider = req.companion.providerClass?.oauthProvider
if (oauthProvider == null) return
const args = {
res,
token: uppyAuthToken,
companionOptions: req.companion.options,
oauthProvider: req.companion.providerClass.oauthProvider,
maxAge,
})
oauthProvider,
...(maxAge != null && { maxAge }),
}
addToCookies(args)
}
}
/**
*
* @param {object} res
* @param {object} companionOptions
* @param {string} oauthProvider
*/
export const removeFromCookies = (res, companionOptions, oauthProvider) => {
export const removeFromCookies = (
res: Response,
companionOptions: CompanionRuntimeOptions,
oauthProvider: string,
): void => {
// options must be identical to those given to res.cookie(), excluding expires and maxAge.
// https://expressjs.com/en/api.html#res.clearCookie
const cookieOptions = getCommonCookieOptions({ companionOptions })

26
packages/@uppy/companion/src/server/helpers/oauth-state.js
View file

@ -1,26 +0,0 @@
import crypto from 'node:crypto'
import { decrypt, encrypt } from './utils.js'
export const encodeState = (state, secret) => {
const encodedState = Buffer.from(JSON.stringify(state)).toString('base64')
return encrypt(encodedState, secret)
}
export const decodeState = (state, secret) => {
const encodedState = decrypt(state, secret)
return JSON.parse(atob(encodedState))
}
export const generateState = () => {
return {
id: crypto.randomBytes(10).toString('hex'),
}
}
export const getFromState = (state, name, secret) => {
return decodeState(state, secret)[name]
}
export const getGrantDynamicFromRequest = (req) => {
return req.session.grant?.dynamic ?? {}
}

57
packages/@uppy/companion/src/server/helpers/oauth-state.ts Normal file
View file

@ -0,0 +1,57 @@
import crypto from 'node:crypto'
import type { CompanionSession } from '../../types/express.js'
import { isRecord } from './type-guards.js'
import { decrypt, encrypt } from './utils.js'
export type OAuthState = {
id: string
origin?: string | string[] | number | boolean | undefined // weird type because this is what express's res.getHeader and cors callback combined can possibly return
preAuthToken?: string
companionInstance?: string
customerDefinedAllowedOrigins?: string[]
authCallbackToken?: string
}
export const encodeState = (
state: OAuthState,
secret: string | Buffer,
): string => {
const encodedState = Buffer.from(JSON.stringify(state)).toString('base64')
return encrypt(encodedState, secret)
}
export const decodeState = (
state: string,
secret: string | Buffer,
): OAuthState => {
const encodedState = decrypt(state, secret)
const parsed: unknown = JSON.parse(atob(encodedState))
if (!isOAuthState(parsed)) {
throw new Error('Invalid OAuth state payload')
}
return parsed
}
export const generateState = (): OAuthState => {
return {
id: crypto.randomBytes(10).toString('hex'),
}
}
function isOAuthState(value: unknown): value is OAuthState {
return isRecord(value) && typeof value['id'] === 'string'
}
export const getFromState = <T extends keyof OAuthState>(
state: string,
name: T,
secret: string | Buffer,
): OAuthState[T] | undefined => {
return decodeState(state, secret)[name]
}
export const getGrantDynamicFromRequest = (req: {
session?: CompanionSession
}) => {
return req.session?.grant?.dynamic ?? {}
}

146
packages/@uppy/companion/src/server/helpers/request.js → packages/@uppy/companion/src/server/helpers/request.ts
View file

@ -1,9 +1,11 @@
import dns from 'node:dns'
import http from 'node:http'
import https from 'node:https'
import type { LookupFunction } from 'node:net'
import path from 'node:path'
import type { Duplex } from 'node:stream'
import contentDisposition from 'content-disposition'
import got from 'got'
import got, { type Response } from 'got'
import ipaddr from 'ipaddr.js'
import validator from 'validator'
@ -12,16 +14,19 @@ export const FORBIDDEN_IP_ADDRESS = 'Forbidden IP address'
// Example scary IPs that should return false (ipv6-to-ipv4 mapped):
// ::FFFF:127.0.0.1
// ::ffff:7f00:1
const isDisallowedIP = (ipAddress) =>
const isDisallowedIP = (ipAddress: string): boolean =>
ipaddr.parse(ipAddress).range() !== 'unicast'
/**
* Validates that the download URL is secure
* Validates that the download URL is secure.
*
* @param {string} url the url to validate
* @param {boolean} allowLocalUrls whether to allow local addresses
* @param url - The URL to validate.
* @param allowLocalUrls - Whether to allow local addresses.
*/
const validateURL = (url, allowLocalUrls) => {
const validateURL = (
url: string | null | undefined,
allowLocalUrls: boolean,
): boolean => {
if (!url) {
return false
}
@ -43,8 +48,35 @@ export { validateURL }
/**
* Returns http Agent that will prevent requests to private IPs (to prevent SSRF)
*/
const getProtectedHttpAgent = ({ protocol, allowLocalIPs }) => {
function dnsLookup(hostname, options, callback) {
function getProtectedHttpAgent({
protocol,
allowLocalIPs,
}: {
protocol: 'http'
allowLocalIPs: boolean
}): typeof http.Agent
function getProtectedHttpAgent({
protocol,
allowLocalIPs,
}: {
protocol: 'https'
allowLocalIPs: boolean
}): typeof https.Agent
function getProtectedHttpAgent({
protocol,
allowLocalIPs,
}: {
protocol: string
allowLocalIPs: boolean
}): typeof http.Agent | typeof https.Agent
function getProtectedHttpAgent({
protocol,
allowLocalIPs,
}: {
protocol: string
allowLocalIPs: boolean
}): typeof http.Agent | typeof https.Agent {
const dnsLookup: LookupFunction = (hostname, options, callback) => {
dns.lookup(hostname, options, (err, addresses, maybeFamily) => {
if (err) {
callback(err, addresses, maybeFamily)
@ -53,7 +85,8 @@ const getProtectedHttpAgent = ({ protocol, allowLocalIPs }) => {
const toValidate = Array.isArray(addresses)
? addresses
: [{ address: addresses }]
: [{ address: addresses, family: maybeFamily }]
// because dns.lookup seems to be called with option `all: true`, if we are on an ipv6 system,
// `addresses` could contain a list of ipv4 addresses as well as ipv6 mapped addresses (rfc6052) which we cannot allow
// however we should still allow any valid ipv4 addresses, so we filter out the invalid addresses
@ -61,8 +94,9 @@ const getProtectedHttpAgent = ({ protocol, allowLocalIPs }) => {
? toValidate
: toValidate.filter(({ address }) => !isDisallowedIP(address))
const [firstValidAddress] = validAddresses
// and check if there's anything left after we filtered:
if (validAddresses.length === 0) {
if (firstValidAddress == null) {
callback(
new Error(
`Forbidden resolved IP address ${hostname} -> ${toValidate.map(({ address }) => address).join(', ')}`,
@ -75,23 +109,43 @@ const getProtectedHttpAgent = ({ protocol, allowLocalIPs }) => {
const ret = Array.isArray(addresses)
? validAddresses
: validAddresses[0].address
: firstValidAddress.address
callback(err, ret, maybeFamily)
})
}
return class HttpAgent extends (protocol.startsWith('https') ? https : http)
.Agent {
createConnection(options, callback) {
if (
ipaddr.isValid(options.host) &&
!allowLocalIPs &&
isDisallowedIP(options.host)
) {
callback(new Error(FORBIDDEN_IP_ADDRESS))
return undefined
const shouldBlockHost = (host: string | null | undefined): host is string =>
host != null && ipaddr.isValid(host) && isDisallowedIP(host)
if (protocol.startsWith('https')) {
return class HttpsAgent extends https.Agent {
override createConnection(
options: http.ClientRequestArgs,
callback?: (err: Error | null, stream: Duplex) => void,
): Duplex | null | undefined {
if (!allowLocalIPs && shouldBlockHost(options.host)) {
const socket = undefined as unknown as Duplex // not sure about this but it's how it always worked
callback?.(new Error(FORBIDDEN_IP_ADDRESS), socket)
return socket
}
return super.createConnection(
{ ...options, lookup: dnsLookup },
callback,
)
}
}
}
return class HttpAgent extends http.Agent {
override createConnection(
options: http.ClientRequestArgs,
callback?: (err: Error | null, stream: Duplex) => void,
): Duplex | null | undefined {
if (!allowLocalIPs && shouldBlockHost(options.host)) {
const socket = undefined as unknown as Duplex // not sure about this but it's how it always worked
callback?.(new Error(FORBIDDEN_IP_ADDRESS), socket)
return socket
}
// @ts-ignore
return super.createConnection({ ...options, lookup: dnsLookup }, callback)
}
}
@ -99,7 +153,7 @@ const getProtectedHttpAgent = ({ protocol, allowLocalIPs }) => {
export { getProtectedHttpAgent }
function getProtectedGot({ allowLocalIPs }) {
function getProtectedGot({ allowLocalIPs }: { allowLocalIPs: boolean }) {
const HttpAgent = getProtectedHttpAgent({ protocol: 'http', allowLocalIPs })
const HttpsAgent = getProtectedHttpAgent({
protocol: 'https',
@ -108,7 +162,6 @@ function getProtectedGot({ allowLocalIPs }) {
const httpAgent = new HttpAgent()
const httpsAgent = new HttpsAgent()
// @ts-ignore
return got.extend({ agent: { http: httpAgent, https: httpsAgent } })
}
@ -117,16 +170,23 @@ export { getProtectedGot }
/**
* Gets the size and content type of a url's content
*
* @param {string} url
* @param {boolean} allowLocalIPs
* @returns {Promise<{name: string, type: string, size: number}>}
* @param url - The URL to inspect.
* @param allowLocalIPs - Whether to allow local addresses (disables SSRF protection).
* @param options - Extra request options passed to got.
*/
export async function getURLMeta(
url,
url: string,
allowLocalIPs = false,
options = undefined,
options: Record<string, unknown> | undefined = undefined,
) {
async function requestWithMethod(method) {
type UrlMetaWithStatus = {
name: string
type: string | undefined
size: number | null
statusCode: number
}
async function requestWithMethod(method: 'HEAD' | 'GET') {
const protectedGot = getProtectedGot({ allowLocalIPs })
const stream = protectedGot.stream(url, {
method,
@ -134,18 +194,30 @@ export async function getURLMeta(
...options,
})
return new Promise((resolve, reject) =>
return new Promise<UrlMetaWithStatus>((resolve, reject) =>
stream
.on('response', (response) => {
.on('response', (response: Response) => {
// Can be undefined for unknown length URLs, e.g. transfer-encoding: chunked
const contentLength = parseInt(response.headers['content-length'], 10)
const contentLengthHeader = response.headers['content-length']
const contentLength =
contentLengthHeader != null
? parseInt(contentLengthHeader, 10)
: NaN
// If Content-Disposition with file name is missing, fallback to the URL path for the name,
// but if multiple files are served via query params like foo.com?file=file-1, foo.com?file=file-2,
// we add random string to avoid duplicate files
const filename = response.headers['content-disposition']
? contentDisposition.parse(response.headers['content-disposition'])
.parameters.filename
: path.basename(`${response.request.requestUrl}`)
const contentDispositionHeader =
response.headers['content-disposition']
let filename: string | undefined
if (contentDispositionHeader != null) {
const parsed = contentDisposition.parse(contentDispositionHeader)
const maybeFilename = parsed.parameters['filename']
if (maybeFilename != null) {
filename = maybeFilename
}
}
if (!filename)
filename = path.basename(`${response.request.requestUrl}`)
// No need to get the rest of the response, as we only want header (not really relevant for HEAD, but why not)
stream.destroy()

9
packages/@uppy/companion/src/server/helpers/type-guards.ts Normal file
View file

@ -0,0 +1,9 @@
export function isRecord(value: unknown): value is Record<string, unknown> {
return value != null && typeof value === 'object' && !Array.isArray(value)
}
export function toError(err: unknown): Error {
if (err instanceof Error) return err
if (typeof err === 'string') return new Error(err)
return new Error('Unknown error')
}

37
packages/@uppy/companion/src/server/helpers/upload.js → packages/@uppy/companion/src/server/helpers/upload.ts
View file

@ -1,19 +1,28 @@
import type { Readable } from 'node:stream'
import type { Request, Response } from 'express'
import logger from '../logger.js'
import Uploader from '../Uploader.js'
export async function startDownUpload({ req, res, getSize, download }) {
logger.debug('Starting download stream.', null, req.id)
export async function startDownUpload({
req,
res,
getSize,
download,
}: {
req: Request
res: Response
getSize?: (() => Promise<number | null | undefined>) | undefined
download: () => Promise<{ stream: Readable; size: number | undefined }>
}): Promise<void> {
logger.debug('Starting download stream.', undefined, req.id)
const { stream, size: maybeSize } = await download()
let size
// if we already know the size from the GET response content-length header, we can use that
if (
typeof maybeSize === 'number' &&
!Number.isNaN(maybeSize) &&
maybeSize > 0
) {
size = maybeSize
}
let size: number | null | undefined =
maybeSize != null && !Number.isNaN(maybeSize) && maybeSize > 0
? maybeSize
: undefined
// if not, we may need to explicitly get the size
// note that getSize might also return undefined/null, which is usually fine, it just means that
// the size is unknown and we cannot send the size to the Uploader
@ -22,8 +31,8 @@ export async function startDownUpload({ req, res, getSize, download }) {
}
const { clientSocketConnectTimeout } = req.companion.options
logger.debug('Instantiating uploader.', null, req.id)
const uploader = new Uploader(Uploader.reqToOptions(req, size))
logger.debug('Instantiating uploader.', undefined, req.id)
const uploader = new Uploader(Uploader.reqToOptions(req, size ?? undefined))
// "Forking" off the upload operation to background, so we can return the http request:
;(async () => {
@ -31,13 +40,13 @@ export async function startDownUpload({ req, res, getSize, download }) {
// the download, so that the client can receive all download/upload progress.
logger.debug(
'Waiting for socket connection before beginning remote download/upload.',
null,
undefined,
req.id,
)
await uploader.awaitReady(clientSocketConnectTimeout)
logger.debug(
'Socket connection received. Starting remote download/upload.',
null,
undefined,
req.id,
)

285
packages/@uppy/companion/src/server/helpers/utils.js
View file

@ -1,285 +0,0 @@
import crypto from 'node:crypto'
const authTagLength = 16
const nonceLength = 16
const encryptionKeyLength = 32
const ivLength = 12
/**
*
* @param {string} value
* @param {string[]} criteria
* @returns {boolean}
*/
export const hasMatch = (value, criteria) => {
return criteria.some((i) => {
return value === i || new RegExp(i).test(value)
})
}
/**
*
* @param {object} data
* @returns {string}
*/
export const jsonStringify = (data) => {
const cache = []
return JSON.stringify(data, (key, value) => {
if (typeof value === 'object' && value !== null) {
if (cache.indexOf(value) !== -1) {
// Circular reference found, discard key
return undefined
}
cache.push(value)
}
return value
})
}
// all paths are assumed to be '/' prepended
/**
* Returns a url builder
*
* @param {object} options companion options
*/
export function getURLBuilder(options) {
/**
* Builds companion targeted url
*
* @param {string} subPath the tail path of the url
* @param {boolean} isExternal if the url is for the external world
* @param {boolean} [excludeHost] if the server domain and protocol should be included
*/
const buildURL = (subPath, isExternal, excludeHost) => {
let path = ''
if (isExternal && options.server.implicitPath) {
path += options.server.implicitPath
}
if (options.server.path) {
path += options.server.path
}
path += subPath
if (excludeHost) {
return path
}
return `${options.server.protocol}://${options.server.host}${path}`
}
return buildURL
}
export const getRedirectPath = (providerName) => `/${providerName}/redirect`
/**
* Create an AES-CCM encryption key and initialization vector from the provided secret
* and a random nonce.
*
* @param {string|Buffer} secret
* @param {Buffer|undefined} nonce
*/
function createSecrets(secret, nonce) {
const key = crypto.hkdfSync(
'sha256',
secret,
new Uint8Array(32),
nonce,
encryptionKeyLength + ivLength,
)
const buf = Buffer.from(key)
return {
key: buf.subarray(0, encryptionKeyLength),
iv: buf.subarray(encryptionKeyLength, encryptionKeyLength + ivLength),
}
}
/**
* Encrypt a buffer or string with AES256 and a random iv.
*
* @param {string} input
* @param {string|Buffer} secret
* @returns {string} Ciphertext as a hex string, prefixed with 32 hex characters containing the iv.
*/
export const encrypt = (input, secret) => {
const nonce = crypto.randomBytes(nonceLength)
const { key, iv } = createSecrets(secret, nonce)
const cipher = crypto.createCipheriv('aes-256-ccm', key, iv, {
authTagLength,
})
const encrypted = Buffer.concat([
cipher.update(input, 'utf8'),
cipher.final(),
cipher.getAuthTag(),
])
// add nonce to encrypted string to use for decryption
return `${nonce.toString('hex')}${encrypted.toString('base64url')}`
}
/**
* Decrypt an iv-prefixed or string with AES256. The iv should be in the first 32 hex characters.
*
* @param {string} encrypted hex encoded string of encrypted data
* @param {string|Buffer} secret
* @returns {string} Decrypted value.
*/
export const decrypt = (encrypted, secret) => {
const nonceHexLength = nonceLength * 2 // because hex encoding uses 2 bytes per byte
// NOTE: The first 32 characters are the nonce, in hex format.
const nonce = Buffer.from(encrypted.slice(0, nonceHexLength), 'hex')
// The rest is the encrypted string, in base64url format.
const encryptionWithoutNonce = Buffer.from(
encrypted.slice(nonceHexLength),
'base64url',
)
// The last 16 bytes of the rest is the authentication tag
const authTag = encryptionWithoutNonce.subarray(-authTagLength)
// and the rest (from beginning) is the encrypted data
const encryptionWithoutNonceAndTag = encryptionWithoutNonce.subarray(
0,
-authTagLength,
)
if (nonce.length < nonceLength) {
throw new Error(
'Invalid encrypted value. Maybe it was generated with an old Companion version?',
)
}
const { key, iv } = createSecrets(secret, nonce)
const decipher = crypto.createDecipheriv('aes-256-ccm', key, iv, {
authTagLength,
})
decipher.setAuthTag(authTag)
const decrypted = Buffer.concat([
decipher.update(encryptionWithoutNonceAndTag),
decipher.final(),
])
return decrypted.toString('utf8')
}
export const defaultGetKey = ({ filename }) => {
return `${crypto.randomUUID()}-${filename}`
}
/**
* Our own HttpError in cases where we can't use `got`'s `HTTPError`
*/
export class HttpError extends Error {
statusCode
responseJson
constructor({ statusCode, responseJson }) {
super(`Request failed with status ${statusCode}`)
this.statusCode = statusCode
this.responseJson = responseJson
this.name = 'HttpError'
}
}
export const prepareStream = async (stream) =>
new Promise((resolve, reject) => {
stream
.on('response', (response) => {
const contentLengthStr = response.headers['content-length']
const contentLength = parseInt(contentLengthStr, 10)
const size =
!Number.isNaN(contentLength) && contentLength >= 0
? contentLength
: undefined
// Don't allow any more data to flow yet.
// https://github.com/request/request/issues/1990#issuecomment-184712275
stream.pause()
resolve({ size })
})
.on('error', (err) => {
// In this case the error object is not a normal GOT HTTPError where json is already parsed,
// we use our own HttpError error for this scenario.
if (
typeof err.response?.body === 'string' &&
typeof err.response?.statusCode === 'number'
) {
let responseJson
try {
responseJson = JSON.parse(err.response.body)
} catch (_err2) {
reject(err)
return
}
reject(
new HttpError({
statusCode: err.response.statusCode,
responseJson,
}),
)
return
}
reject(err)
})
})
export const getBasicAuthHeader = (key, secret) => {
const base64 = Buffer.from(`${key}:${secret}`, 'binary').toString('base64')
return `Basic ${base64}`
}
const rfc2047Encode = (dataIn) => {
const data = `${dataIn}`
// biome-ignore lint/suspicious/noControlCharactersInRegex: leave it for now
if (/^[\x00-\x7F]*$/.test(data)) return data // we return ASCII as is
return `=?UTF-8?B?${Buffer.from(data).toString('base64')}?=` // We encode non-ASCII strings
}
export const rfc2047EncodeMetadata = (metadata) =>
Object.fromEntries(
Object.entries(metadata).map((entry) => entry.map(rfc2047Encode)),
)
/**
*
* @param {{
* bucketOrFn: string | ((a: {
* req: import('express').Request,
* metadata: Record<string, string>,
* filename: string | undefined,
* }) => string),
* req: import('express').Request,
* metadata?: Record<string, string>,
* filename?: string,
* }} param0
* @returns
*/
export const getBucket = ({ bucketOrFn, req, metadata, filename }) => {
const bucket =
typeof bucketOrFn === 'function'
? bucketOrFn({ req, metadata, filename })
: bucketOrFn
if (typeof bucket !== 'string' || bucket === '') {
// This means a misconfiguration or bug
throw new TypeError(
's3: bucket key must be a string or a function resolving the bucket string',
)
}
return bucket
}
/**
* Truncate filename to a maximum length.
*
* @param {string} filename
* @param {number} maxFilenameLength
* @returns {string}
*/
export const truncateFilename = (filename, maxFilenameLength) => {
return filename.slice(maxFilenameLength * -1)
}

300
packages/@uppy/companion/src/server/helpers/utils.ts Normal file
View file

@ -0,0 +1,300 @@
import crypto from 'node:crypto'
import type { Request } from 'express'
import type { GetBucketFn } from '../../schemas/companion.js'
const authTagLength = 16
const nonceLength = 16
const encryptionKeyLength = 32
const ivLength = 12
export const hasMatch = (
value: string,
criteria: ReadonlyArray<string | RegExp>,
): boolean => {
return criteria.some((i) => {
if (i instanceof RegExp) return i.test(value)
return value === i || new RegExp(i).test(value)
})
}
export const jsonStringify = (data: unknown): string => {
const cache: unknown[] = []
return JSON.stringify(data, (_key, value) => {
if (typeof value === 'object' && value !== null) {
if (cache.indexOf(value) !== -1) {
// Circular reference found, discard key
return undefined
}
cache.push(value)
}
return value
})
}
// all paths are assumed to be '/' prepended
/**
* Returns a URL builder.
*
* The returned function builds Companion-targeted URLs, optionally including the
* server protocol/host for external use.
*/
export function getURLBuilder(options: {
server?: {
protocol?: string | undefined
host?: string | undefined
path?: string | undefined
implicitPath?: string | undefined
}
}) {
return (
subPath: string,
isExternal: boolean,
excludeHost?: boolean,
): string => {
const server = options.server ?? {}
let path = ''
if (isExternal && server.implicitPath) path += server.implicitPath
if (server.path) path += server.path
path += subPath
if (excludeHost) return path
return `${server.protocol}://${server.host}${path}`
}
}
export const getRedirectPath = (providerName: string): string =>
`/${providerName}/redirect`
/**
* Create an AES-CCM encryption key and initialization vector from the provided
* secret and a random nonce.
*/
function createSecrets(
secret: string | Buffer,
nonce: Buffer | undefined,
): { key: Buffer; iv: Buffer } {
const key = crypto.hkdfSync(
'sha256',
secret,
new Uint8Array(32),
nonce ?? new Uint8Array(0),
encryptionKeyLength + ivLength,
)
const buf = Buffer.from(key)
return {
key: buf.subarray(0, encryptionKeyLength),
iv: buf.subarray(encryptionKeyLength, encryptionKeyLength + ivLength),
}
}
/**
* Encrypt a string with AES-256-CCM and a random nonce.
* Ciphertext as a hex string, prefixed with 32 hex characters containing the iv.
*
* The returned ciphertext is prefixed with the nonce (hex), followed by the
* encrypted data (base64url).
*/
export const encrypt = (input: string, secret: string | Buffer): string => {
const nonce = crypto.randomBytes(nonceLength)
const { key, iv } = createSecrets(secret, nonce)
const cipher = crypto.createCipheriv('aes-256-ccm', key, iv, {
authTagLength,
})
const encrypted = Buffer.concat([
cipher.update(input, 'utf8'),
cipher.final(),
cipher.getAuthTag(),
])
return `${nonce.toString('hex')}${encrypted.toString('base64url')}`
}
/**
* Decrypt a nonce-prefixed ciphertext produced by {@link encrypt}.
* The iv should be in the first 32 hex characters.
*/
export const decrypt = (encrypted: string, secret: string | Buffer): string => {
const nonceHexLength = nonceLength * 2 // because hex encoding uses 2 bytes per byte
// NOTE: The first 32 characters are the nonce, in hex format.
const nonce = Buffer.from(encrypted.slice(0, nonceHexLength), 'hex')
// The rest is the encrypted string, in base64url format.
const encryptionWithoutNonce = Buffer.from(
encrypted.slice(nonceHexLength),
'base64url',
)
// The last 16 bytes of the rest is the authentication tag
const authTag = encryptionWithoutNonce.subarray(-authTagLength)
// and the rest (from beginning) is the encrypted data
const encryptionWithoutNonceAndTag = encryptionWithoutNonce.subarray(
0,
-authTagLength,
)
if (nonce.length < nonceLength) {
throw new Error(
'Invalid encrypted value. Maybe it was generated with an old Companion version?',
)
}
const { key, iv } = createSecrets(secret, nonce)
const decipher = crypto.createDecipheriv('aes-256-ccm', key, iv, {
authTagLength,
})
decipher.setAuthTag(authTag)
const decrypted = Buffer.concat([
decipher.update(encryptionWithoutNonceAndTag),
decipher.final(),
])
return decrypted.toString('utf8')
}
export const defaultGetKey = ({ filename }: { filename: string }): string => {
return `${crypto.randomUUID()}-${filename}`
}
/**
* Our own HttpError in cases where we can't use `got`'s `HTTPError`.
*/
export class HttpError extends Error {
statusCode: number
responseJson: unknown
constructor({
statusCode,
responseJson,
}: { statusCode: number; responseJson: unknown }) {
super(`Request failed with status ${statusCode}`)
this.statusCode = statusCode
this.responseJson = responseJson
this.name = 'HttpError'
}
}
type ResponseLike = { headers: Record<string, string | string[] | undefined> }
type StreamLike = NodeJS.ReadableStream & {
pause: () => void
on: {
(event: 'response', handler: (response: ResponseLike) => void): StreamLike
(event: 'error', handler: (err: unknown) => void): StreamLike
}
}
export const prepareStream = async (
stream: StreamLike,
): Promise<{ size: number | undefined }> =>
new Promise((resolve, reject) => {
stream
.on('response', (response) => {
const contentLengthStr = response.headers['content-length']
const contentLength =
typeof contentLengthStr === 'string'
? parseInt(contentLengthStr, 10)
: NaN
const size =
!Number.isNaN(contentLength) && contentLength >= 0
? contentLength
: undefined
// Don't allow any more data to flow yet.
stream.pause()
resolve({ size })
})
.on('error', (err) => {
if (!err || typeof err !== 'object' || !('response' in err)) {
reject(err)
return
}
const response = err.response
if (!response || typeof response !== 'object') {
reject(err)
return
}
const body = (response as { body?: unknown }).body
const statusCode = (response as { statusCode?: unknown }).statusCode
if (typeof body === 'string' && typeof statusCode === 'number') {
// In this case the error object is not a normal GOT HTTPError where json is already parsed,
// we use our own HttpError error for this scenario.
try {
const responseJson: unknown = JSON.parse(body)
reject(new HttpError({ statusCode, responseJson }))
return
} catch {
reject(err)
return
}
}
reject(err)
})
})
export const getBasicAuthHeader = (
key: string | undefined,
secret: string | undefined,
): string => {
const base64 = Buffer.from(`${key}:${secret}`, 'binary').toString('base64')
return `Basic ${base64}`
}
const rfc2047Encode = (dataIn: unknown): string => {
const data = String(dataIn)
// biome-ignore lint/suspicious/noControlCharactersInRegex: leave it for now
if (/^[\x00-\x7F]*$/.test(data)) return data // we return ASCII as is
return `=?UTF-8?B?${Buffer.from(data).toString('base64')}?=` // We encode non-ASCII strings
}
export const rfc2047EncodeMetadata = (
metadata: Record<string, unknown>,
): Record<string, string> =>
Object.fromEntries(
Object.entries(metadata).map(([key, value]) => [
rfc2047Encode(key),
rfc2047Encode(value),
]),
)
export const getBucket = ({
bucketOrFn,
req,
metadata,
filename,
}: {
bucketOrFn: string | GetBucketFn | undefined
req: Request
metadata?: Record<string, unknown>
filename?: string
}): string => {
const bucket =
typeof bucketOrFn === 'function'
? bucketOrFn({ req, metadata: metadata ?? {}, filename })
: bucketOrFn
if (typeof bucket !== 'string' || bucket === '') {
// This means a misconfiguration or bug
throw new TypeError(
's3: bucket key must be a string or a function resolving the bucket string',
)
}
return bucket
}
export const truncateFilename = (
filename: string,
maxFilenameLength?: number,
): string => {
if (
maxFilenameLength == null ||
!Number.isFinite(maxFilenameLength) ||
maxFilenameLength <= 0
) {
// Historically, passing `undefined` resulted in no truncation (slice(0)).
return filename
}
return filename.slice(maxFilenameLength * -1)
}

32
packages/@uppy/companion/src/server/jobs.js → packages/@uppy/companion/src/server/jobs.ts
View file

@ -3,10 +3,11 @@ import path from 'node:path'
import { setTimeout as sleep } from 'node:timers/promises'
import got from 'got'
import schedule from 'node-schedule'
import { isRecord } from './helpers/type-guards.js'
import * as logger from './logger.js'
import Uploader from './Uploader.js'
const cleanUpFinishedUploads = (dirPath) => {
const cleanUpFinishedUploads = (dirPath: string): void => {
logger.info(
`running clean up job for path: ${dirPath}`,
'jobs.cleanup.progress.read',
@ -34,8 +35,7 @@ const cleanUpFinishedUploads = (dirPath) => {
// we still delete the file if we can't get the stats
// but we also log the error
logger.error(err2, 'jobs.cleanup.stat.error')
// @ts-ignore
} else if (Date.now() - stats.mtime < twelveHoursAgo) {
} else if (Date.now() - stats.mtime.getTime() < twelveHoursAgo) {
logger.info(`skipping file ${file}`, 'jobs.cleanup.skip')
return
}
@ -52,15 +52,23 @@ const cleanUpFinishedUploads = (dirPath) => {
/**
* Runs a function every 24 hours, to clean up stale, upload related files.
*
* @param {string} dirPath path to the directory which you want to clean
* @param dirPath path to the directory which you want to clean
*/
export function startCleanUpJob(dirPath) {
export function startCleanUpJob(dirPath: string): void {
logger.info('starting clean up job', 'jobs.cleanup.start')
// run once a day
schedule.scheduleJob('0 23 * * *', () => cleanUpFinishedUploads(dirPath))
}
async function runPeriodicPing({ urls, payload, requestTimeout }) {
async function runPeriodicPing({
urls,
payload,
requestTimeout,
}: {
urls: string[]
payload: Record<string, unknown>
requestTimeout: number
}): Promise<void> {
// Run requests in parallel
await Promise.all(
urls.map(async (url) => {
@ -85,7 +93,14 @@ export async function startPeriodicPingJob({
staticPayload = {},
version,
processId,
}) {
}: {
urls: string[]
interval?: number | undefined
count?: number | undefined
staticPayload?: unknown | undefined
version: string
processId: string
}): Promise<void> {
if (urls.length === 0) return
logger.info('Starting periodic ping job', 'jobs.periodic.ping.start')
@ -122,11 +137,12 @@ export async function startPeriodicPingJob({
try {
requesting = true
const safeStaticPayload = isRecord(staticPayload) ? staticPayload : {}
const payload = {
version,
processId,
service: 'companion',
...staticPayload,
...safeStaticPayload,
}
await runPeriodicPing({ urls, payload, requestTimeout })
} finally {

132
packages/@uppy/companion/src/server/logger.js
View file

@ -1,132 +0,0 @@
import util from 'node:util'
import escapeStringRegexp from 'escape-string-regexp'
import supportsColors from 'supports-color'
let valuesToMask = []
/**
* Adds a list of strings that should be masked by the logger.
* This function can only be called once through out the life of the server.
*
* @param {Array} maskables a list of strings to be masked
*/
export function setMaskables(maskables) {
valuesToMask = maskables.map((i) => escapeStringRegexp(i))
}
/**
* Mask the secret content of a message
*
* @param {string} msg the message whose content should be masked
* @returns {string}
*/
function maskMessage(msg) {
let out = msg
for (const toBeMasked of valuesToMask) {
const toBeReplaced = new RegExp(toBeMasked, 'gi')
out = out.replace(toBeReplaced, '******')
}
return out
}
let processName = 'companion'
export function setProcessName(newProcessName) {
processName = newProcessName
}
const styleText =
typeof util.styleText === 'function' && supportsColors.stderr
? util.styleText
: (style, text) => text
/**
* message log
*
* @typedef {import('node:util').styleText} styleText
* @typedef {Parameters<styleText>[0]} Colors
*
* @param {object} params
* @param {string | Error} params.arg the message or error to log
* @param {string} params.tag a unique tag to easily search for this message
* @param {string} params.level error | info | debug
* @param {string} [params.traceId] a unique id to easily trace logs tied to a request
* @param {Colors} [params.color] Format(s) that can be passed to `util.styleText`.
*/
const log = ({ arg, tag = '', level, traceId = '', color = [] }) => {
const time = new Date().toISOString()
const whitespace = tag && traceId ? ' ' : ''
function msgToString() {
// We don't need to log stack trace on special errors that we ourselves have produced
// (to reduce log noise)
// @ts-ignore
if (
arg instanceof Error &&
arg.name === 'ProviderApiError' &&
typeof arg.message === 'string'
) {
return arg.message
}
if (typeof arg === 'string') return arg
return util.inspect(arg)
}
const msgString = msgToString()
const masked = maskMessage(msgString)
console.log(
styleText(
color,
`${processName}: ${time} [${level}] ${traceId}${whitespace}${tag}`,
),
styleText(color, masked),
)
}
/**
* INFO level log
*
* @param {string} msg the message to log
* @param {string} [tag] a unique tag to easily search for this message
* @param {string} [traceId] a unique id to easily trace logs tied to a request
*/
export function info(msg, tag, traceId) {
log({ arg: msg, tag, level: 'info', traceId })
}
/**
* WARN level log
*
* @param {string} msg the message to log
* @param {string} [tag] a unique tag to easily search for this message
* @param {string} [traceId] a unique id to easily trace logs tied to a request
*/
export function warn(msg, tag, traceId) {
log({ arg: msg, tag, level: 'warn', traceId, color: ['bold', 'yellow'] })
}
/**
* ERROR level log
*
* @param {string | Error} msg the message to log
* @param {string} [tag] a unique tag to easily search for this message
* @param {string} [traceId] a unique id to easily trace logs tied to a request
*/
export function error(msg, tag, traceId) {
log({ arg: msg, tag, level: 'error', traceId, color: ['bold', 'red'] })
}
/**
* DEBUG level log
*
* @param {string} msg the message to log
* @param {string} [tag] a unique tag to easily search for this message
* @param {string} [traceId] a unique id to easily trace logs tied to a request
*/
export function debug(msg, tag, traceId) {
if (process.env.NODE_ENV !== 'production') {
log({ arg: msg, tag, level: 'debug', traceId, color: ['bold', 'blue'] })
}
}
const logger = { setMaskables, setProcessName, info, warn, error, debug }
export default logger

148
packages/@uppy/companion/src/server/logger.ts Normal file
View file

@ -0,0 +1,148 @@
import util from 'node:util'
import escapeStringRegexp from 'escape-string-regexp'
import supportsColors from 'supports-color'
type StyleTextFn = (
style: Parameters<NonNullable<typeof util.styleText>>[0],
text: string,
) => string
type Colors = Parameters<StyleTextFn>[0]
type LogLevel = 'error' | 'info' | 'warn' | 'debug'
let valuesToMask: string[] = []
/**
* Adds a list of strings that should be masked by the logger.
* This is expected to be set during startup and not updated continuously.
*/
export function setMaskables(maskables: readonly string[]): void {
valuesToMask = maskables.map((i) => escapeStringRegexp(i))
}
/**
* Mask secret values in a log message.
*/
function maskMessage(msg: string): string {
let out = msg
for (const toBeMasked of valuesToMask) {
const toBeReplaced = new RegExp(toBeMasked, 'gi')
out = out.replace(toBeReplaced, '******')
}
return out
}
let processName = 'companion'
export function setProcessName(newProcessName: string): void {
processName = newProcessName
}
const styleText: StyleTextFn =
typeof util.styleText === 'function' && supportsColors.stderr
? (style, text) => util.styleText(style, text)
: (_style, text) => text
/**
* Logs a message.
*
* @param params.arg - The message or error to log.
* @param params.tag - A tag to easily search for this message.
* @param params.level - Log level.
* @param params.traceId - A unique id to correlate logs tied to a request.
* @param params.color - Format(s) that can be passed to `util.styleText`.
*/
function log(params: {
arg: unknown
tag?: string
level: LogLevel
traceId?: string
color?: Colors
}): void {
const { arg, level } = params
const tag = params.tag ?? ''
const traceId = params.traceId ?? ''
const color = params.color ?? []
const time = new Date().toISOString()
const whitespace = tag && traceId ? ' ' : ''
function msgToString(): string {
// We don't need to log stack trace on special errors that we ourselves have produced
// (to reduce log noise).
if (
arg instanceof Error &&
arg.name === 'ProviderApiError' &&
typeof arg.message === 'string'
) {
return arg.message
}
if (typeof arg === 'string') return arg
return util.inspect(arg)
}
const msgString = msgToString()
const masked = maskMessage(msgString)
console.log(
styleText(
color,
`${processName}: ${time} [${level}] ${traceId}${whitespace}${tag}`,
),
styleText(color, masked),
)
}
/**
* INFO level log.
*/
export function info(msg: unknown, tag?: string, traceId?: string): void {
log({
arg: msg,
level: 'info',
...(tag != null && { tag }),
...(traceId != null && { traceId }),
})
}
/**
* WARN level log.
*/
export function warn(msg: unknown, tag?: string, traceId?: string): void {
log({
arg: msg,
level: 'warn',
color: ['bold', 'yellow'],
...(tag != null && { tag }),
...(traceId != null && { traceId }),
})
}
/**
* ERROR level log.
*/
export function error(msg: unknown, tag?: string, traceId?: string): void {
log({
arg: msg,
level: 'error',
color: ['bold', 'red'],
...(tag != null && { tag }),
...(traceId != null && { traceId }),
})
}
/**
* DEBUG level log.
*/
export function debug(msg: unknown, tag?: string, traceId?: string): void {
if (process.env['NODE_ENV'] !== 'production') {
log({
arg: msg,
level: 'debug',
color: ['bold', 'blue'],
...(tag != null && { tag }),
...(traceId != null && { traceId }),
})
}
}
const logger = { setMaskables, setProcessName, info, warn, error, debug }
export default logger

156
packages/@uppy/companion/src/server/middlewares.js → packages/@uppy/companion/src/server/middlewares.ts
View file

@ -1,18 +1,21 @@
import corsImport from 'cors'
import type { NextFunction, Request, RequestHandler, Response } from 'express'
import promBundle from 'express-prom-bundle'
import packageJson from '../../package.json' with { type: 'json' }
import type { CompanionRuntimeOptions } from '../types/companion-options.js'
import type { ProviderUserSession } from '../types/express.js'
import * as tokenService from './helpers/jwt.js'
import { getURLBuilder } from './helpers/utils.js'
import * as logger from './logger.js'
import { isOAuthProvider } from './provider/Provider.js'
import getS3Client from './s3-client.js'
export const hasSessionAndProvider = (req, res, next) => {
export const hasSessionAndProvider: RequestHandler = (req, res, next) => {
if (!req.session) {
logger.debug(
'No session attached to req object. Exiting dispatcher.',
null,
undefined,
req.id,
)
return res.sendStatus(400)
@ -21,7 +24,7 @@ export const hasSessionAndProvider = (req, res, next) => {
if (!req.companion.provider) {
logger.debug(
'No provider/provider-handler found. Exiting dispatcher.',
null,
undefined,
req.id,
)
return res.sendStatus(400)
@ -30,38 +33,38 @@ export const hasSessionAndProvider = (req, res, next) => {
return next()
}
const isOAuthProviderReq = (req) =>
isOAuthProvider(req.companion.providerClass.oauthProvider)
const isSimpleAuthProviderReq = (req) =>
!!req.companion.providerClass.hasSimpleAuth
const isOAuthProviderReq = (req: Request) =>
isOAuthProvider(req.companion.providerClass?.oauthProvider)
const isSimpleAuthProviderReq = (req: Request) =>
!!req.companion.providerClass?.hasSimpleAuth
/**
* Middleware can be used to verify that the current request is to an OAuth provider
* This is because not all requests are supported by non-oauth providers (formerly known as SearchProviders)
*/
export const hasOAuthProvider = (req, res, next) => {
export const hasOAuthProvider: RequestHandler = (req, res, next) => {
if (!isOAuthProviderReq(req)) {
logger.debug('Provider does not support OAuth.', null, req.id)
logger.debug('Provider does not support OAuth.', undefined, req.id)
return res.sendStatus(400)
}
return next()
}
export const hasSimpleAuthProvider = (req, res, next) => {
export const hasSimpleAuthProvider: RequestHandler = (req, res, next) => {
if (!isSimpleAuthProviderReq(req)) {
logger.debug('Provider does not support simple auth.', null, req.id)
logger.debug('Provider does not support simple auth.', undefined, req.id)
return res.sendStatus(400)
}
return next()
}
export const hasBody = (req, res, next) => {
export const hasBody: RequestHandler = (req, res, next) => {
if (!req.body) {
logger.debug(
'No body attached to req object. Exiting dispatcher.',
null,
undefined,
req.id,
)
return res.sendStatus(400)
@ -70,8 +73,8 @@ export const hasBody = (req, res, next) => {
return next()
}
export const hasSearchQuery = (req, res, next) => {
if (typeof req.query.q !== 'string') {
export const hasSearchQuery: RequestHandler = (req, res, next) => {
if (typeof req.query['q'] !== 'string') {
logger.debug(
'search request has no search query',
'search.query.check',
@ -83,7 +86,7 @@ export const hasSearchQuery = (req, res, next) => {
return next()
}
export const verifyToken = (req, res, next) => {
export const verifyToken: RequestHandler = (req, res, next) => {
if (isOAuthProviderReq(req) || isSimpleAuthProviderReq(req)) {
// For OAuth / simple auth provider, we find the encrypted auth token from the header:
const token = req.companion.authToken
@ -92,16 +95,23 @@ export const verifyToken = (req, res, next) => {
res.sendStatus(401)
return
}
const { providerName } = req.params
const { secret } = req.companion.options
const providerName = req.params['providerName']
if (typeof providerName !== 'string' || providerName.length === 0) {
res.sendStatus(400)
return
}
try {
const payload = tokenService.verifyEncryptedAuthToken(
token,
req.companion.options.secret,
providerName,
)
const payload = tokenService.verifyEncryptedAuthToken<
Record<string, ProviderUserSession>
>(token, secret, providerName)
req.companion.providerUserSession = payload[providerName]
} catch (err) {
logger.error(err.message, 'token.verify.error', req.id)
logger.error(
err instanceof Error ? err.message : String(err),
'token.verify.error',
req.id,
)
res.sendStatus(401)
return
}
@ -112,8 +122,12 @@ export const verifyToken = (req, res, next) => {
// for non auth providers, we just load the static key from options
if (!isOAuthProviderReq(req)) {
const { providerOptions } = req.companion.options
const { providerName } = req.params
const key = providerOptions[providerName]?.key
const providerName = req.params['providerName']
const providerOption =
typeof providerName === 'string'
? providerOptions?.[providerName]
: undefined
const key = providerOption?.key
if (!key) {
logger.info(
`unconfigured credentials for ${providerName}`,
@ -132,31 +146,44 @@ export const verifyToken = (req, res, next) => {
}
// does not fail if token is invalid
export const gentleVerifyToken = (req, res, next) => {
const { providerName } = req.params
if (req.companion.authToken) {
export const gentleVerifyToken: RequestHandler = (req, res, next) => {
const providerName = req.params['providerName']
if (typeof providerName !== 'string' || providerName.length === 0) {
next()
return
}
const { secret } = req.companion.options
if (req.companion.authToken != null) {
try {
const payload = tokenService.verifyEncryptedAuthToken(
req.companion.authToken,
req.companion.options.secret,
providerName,
)
const payload = tokenService.verifyEncryptedAuthToken<
Record<string, ProviderUserSession>
>(req.companion.authToken, secret, providerName)
req.companion.providerUserSession = payload[providerName]
} catch (err) {
logger.error(err.message, 'token.gentle.verify.error', req.id)
logger.error(
err instanceof Error ? err.message : String(err),
'token.gentle.verify.error',
req.id,
)
}
}
next()
}
export const cookieAuthToken = (req, res, next) => {
req.companion.authToken =
req.cookies[`uppyAuthToken--${req.companion.providerClass.oauthProvider}`]
export const cookieAuthToken: RequestHandler = (req, res, next) => {
const oauthProvider = req.companion.providerClass?.oauthProvider
if (oauthProvider == null || oauthProvider.length === 0) {
return next()
}
req.companion.authToken = req.cookies[`uppyAuthToken--${oauthProvider}`]
return next()
}
export const cors =
(options = {}) =>
(options: {
corsOrigins?: CompanionRuntimeOptions['corsOrigins']
sendSelfEndpoint?: CompanionRuntimeOptions['sendSelfEndpoint']
}): RequestHandler =>
(req, res, next) => {
// HTTP headers are not case sensitive, and express always handles them in lower case, so that's why we lower case them.
// I believe that HTTP verbs are case sensitive, and should be uppercase.
@ -168,7 +195,10 @@ export const cors =
?.map((method) => method.trim().toLowerCase()),
)
if (options.sendSelfEndpoint) exposeHeadersSet.add('i-am')
const sendSelfEndpoint = options['sendSelfEndpoint']
if (sendSelfEndpoint != null) {
exposeHeadersSet.add('i-am')
}
// Needed for basic operation: https://github.com/transloadit/uppy/issues/3021
const allowedHeaders = [
@ -208,19 +238,33 @@ export const cors =
// Because we need to merge with existing headers, we need to call cors inside our own middleware
return corsImport({
credentials: true,
origin,
methods: Array.from(allowMethodsSet),
allowedHeaders: Array.from(allowHeadersSet).join(','),
exposedHeaders: Array.from(exposeHeadersSet).join(','),
...(origin !== undefined && { origin }),
})(req, res, next)
}
export const metrics = ({ path = undefined } = {}) => {
function hasPromClient(mw: unknown): mw is RequestHandler & {
promClient: {
collectDefaultMetrics: (opts: unknown) => void
Gauge: new (opts: unknown) => { set: (n: number) => void }
register: unknown
}
} {
return !!mw && typeof mw === 'function' && 'promClient' in mw
}
export const metrics = ({
path = undefined,
}: {
path?: string
} = {}): RequestHandler => {
const metricsMiddleware = promBundle({
includeMethod: true,
metricsPath: path ? `${path}/metrics` : undefined,
...(path && { metricsPath: `${path}/metrics` }),
})
// @ts-ignore Not in the typings, but it does exist
if (!hasPromClient(metricsMiddleware)) return metricsMiddleware
const { promClient } = metricsMiddleware
const { collectDefaultMetrics } = promClient
collectDefaultMetrics({ register: promClient.register })
@ -235,23 +279,21 @@ export const metrics = ({ path = undefined } = {}) => {
return metricsMiddleware
}
/**
*
* @param {object} options
*/
export const getCompanionMiddleware = (options) => {
/**
* @param {object} req
* @param {object} res
* @param {Function} next
*/
const middleware = (req, res, next) => {
export const getCompanionMiddleware = (
options: CompanionRuntimeOptions,
): RequestHandler => {
const middleware = (req: Request, _res: Response, next: NextFunction) => {
const s3Client = getS3Client(options, false)
const s3ClientCreatePresignedPost = getS3Client(options, true)
const authToken =
req.header('uppy-auth-token') || req.query['uppyAuthToken']
req.companion = {
options,
s3Client: getS3Client(options, false),
s3ClientCreatePresignedPost: getS3Client(options, true),
authToken: req.header('uppy-auth-token') || req.query.uppyAuthToken,
buildURL: getURLBuilder(options),
...(s3Client && { s3Client }),
...(s3ClientCreatePresignedPost && { s3ClientCreatePresignedPost }),
...(typeof authToken === 'string' && { authToken }),
}
next()
}

129
packages/@uppy/companion/src/server/provider/Provider.js
View file

@ -1,129 +0,0 @@
import { MAX_AGE_24H } from '../helpers/jwt.js'
/**
* Provider interface defines the specifications of any provider implementation
*/
export default class Provider {
/**
*
* @param {{providerName: string, allowLocalUrls: boolean, providerGrantConfig?: object, secret: string}} options
*/
constructor({ allowLocalUrls, providerGrantConfig, secret }) {
// Some providers might need cookie auth for the thumbnails fetched via companion
this.needsCookieAuth = false
this.allowLocalUrls = allowLocalUrls
this.providerGrantConfig = providerGrantConfig
this.secret = secret
// biome-ignore lint/correctness/noConstructorReturn: ...
return this
}
/**
* config to extend the grant config
*/
static getExtraGrantConfig() {
return {}
}
/**
* list the files and folders in the provider account
*
* @param {object} options
* @returns {Promise}
*/
async list(options) {
throw new Error('method not implemented')
}
/**
* search for files/folders in the provider account
*
* @param {object} options
* @returns {Promise}
*/
async search(options) {
throw new Error('method not implemented')
}
/**
* download a certain file from the provider account
*
* @param {object} options
* @returns {Promise}
*/
async download(options) {
throw new Error('method not implemented')
}
/**
* return a thumbnail for a provider file
*
* @param {object} options
* @returns {Promise}
*/
async thumbnail(options) {
throw new Error('method not implemented')
}
/**
* first Companion will try to get the size from the content-length response header,
* if that fails, it will call this method to get the size.
* So if your provider has a different method for getting the size, you can return the size here
*
* @param {object} options
* @returns {Promise}
*/
async size(options) {
return undefined
}
/**
* handle deauthorization notification from oauth providers
*
* @param {object} options
* @returns {Promise}
*/
async deauthorizationCallback(options) {
throw new Error('method not implemented')
}
/**
* Generate a new access token based on the refresh token
*/
async refreshToken(options) {
throw new Error('method not implemented')
}
/**
* @param {any} param0
* @returns {Promise<any>}
*/
async simpleAuth({ requestBody }) {
throw new Error('method not implemented')
}
/**
* Name of the OAuth provider (passed to Grant). Return empty string if no OAuth provider is needed.
*
* @returns {string}
*/
static get oauthProvider() {
return undefined
}
static grantDynamicToUserSession({ grantDynamic }) {
return {}
}
static get hasSimpleAuth() {
return false
}
static get authStateExpiry() {
return MAX_AGE_24H
}
}
// OAuth providers are those that have an `oauthProvider` set. It means they require OAuth authentication to work
export const isOAuthProvider = (oauthProvider) =>
typeof oauthProvider === 'string' && oauthProvider.length > 0

210
packages/@uppy/companion/src/server/provider/Provider.ts Normal file
View file

@ -0,0 +1,210 @@
import type { Readable } from 'node:stream'
import type {
BuildUrl,
CompanionContext,
GrantDynamic,
ProviderGrantConfig,
} from '../../types/express.js'
import { MAX_AGE_24H } from '../helpers/jwt.js'
// from express:
export interface Query {
[key: string]: undefined | string | string[] | Query | Query[]
}
export type CompanionLike = Pick<
CompanionContext,
'getProviderCredentials' | 'options'
>
export interface ProviderListItem {
isFolder: boolean
icon: string | null | undefined
id: string
name?: string | undefined
requestPath: string
modifiedDate?: string | undefined
mimeType?: string | null | undefined
size?: number | null | undefined
thumbnail?: string | null | undefined
}
// todo use these types in the Uppy client
export interface ProviderListResponse {
items: ProviderListItem[]
nextPagePath?: string | null | undefined
username?: string | null | undefined
}
export type ProviderSearchResponse = ProviderListResponse
/**
* Provider interface defines the specifications of any provider implementation
*/
export default class Provider<US = unknown> {
needsCookieAuth: boolean
allowLocalUrls: boolean
providerGrantConfig: ProviderGrantConfig | undefined
constructor({
allowLocalUrls,
providerGrantConfig,
}: {
allowLocalUrls: boolean
providerGrantConfig?: ProviderGrantConfig
}) {
// Some providers might need cookie auth for the thumbnails fetched via companion
this.needsCookieAuth = false
this.allowLocalUrls = allowLocalUrls
this.providerGrantConfig = providerGrantConfig
// biome-ignore lint/correctness/noConstructorReturn: ...
return this
}
/**
* config to extend the grant config
*/
static getExtraGrantConfig(): Record<string, unknown> {
return {}
}
/**
* List the files and folders in the provider account.
*
* This method should be overridden by provider implementations.
*/
async list(options: {
companion: CompanionContext
directory?: string | undefined
providerUserSession: US
query?: Query
}): Promise<ProviderListResponse> {
throw new Error('method not implemented')
}
/**
* Search for files and folders in the provider account.
*
* This method should be overridden by provider implementations.
*/
async search(options: {
providerUserSession: US
query: { q: string; path?: string; [k: string]: unknown }
companion: { buildURL: BuildUrl }
}): Promise<ProviderSearchResponse> {
throw new Error('method not implemented')
}
/**
* Download a certain file from the provider account.
*
* This method should be overridden by provider implementations.
*/
async download(options: {
companion: CompanionLike
id: string
providerUserSession: US
query: Query
}): Promise<{ stream: Readable; size: number | undefined }> {
throw new Error('method not implemented')
}
/**
* Return a thumbnail for a provider file.
*
* This method should be overridden by provider implementations.
*/
async thumbnail(options: {
id: string
providerUserSession: US
}): Promise<{ stream: Readable; contentType?: string }> {
throw new Error('method not implemented')
}
/**
* first Companion will try to get the size from the content-length response header,
* if that fails, it will call this method to get the size.
* So if your provider has a different method for getting the size, you can return the size here
*/
async size(options: {
id: string
providerUserSession: US
query: unknown
}): Promise<number | undefined> {
return undefined
}
/**
* Handle deauthorization notification from OAuth providers.
*
* This method should be overridden by provider implementations.
*/
async deauthorizationCallback(options: {
companion: CompanionLike
body: unknown
headers: Record<string, (string | string[]) | undefined>
}): Promise<{ data?: unknown; status?: number }> {
throw new Error('method not implemented')
}
/**
* Generate a new access token based on the refresh token
*/
async refreshToken(options: {
redirectUri: string | undefined
clientId: string | undefined
clientSecret: string | undefined
refreshToken: string
}): Promise<{ accessToken: string }> {
throw new Error('method not implemented')
}
/**
* Revoke/logout for a provider session (if supported).
*
* This method should be overridden by provider implementations.
*/
async logout(options: {
providerUserSession: US
companion: CompanionLike
}): Promise<{ revoked: boolean; manual_revoke_url?: string }> {
throw new Error('method not implemented')
}
async simpleAuth({ requestBody }: { requestBody: unknown }): Promise<object> {
throw new Error('method not implemented')
}
/**
* Name of the OAuth provider (passed to Grant). Return empty string if no OAuth provider is needed.
*/
static get oauthProvider(): string | undefined {
return undefined
}
static grantDynamicToUserSession({
grantDynamic,
}: {
grantDynamic: GrantDynamic
}): Record<string, unknown> {
return {}
}
static get hasSimpleAuth(): boolean {
return false
}
static get authStateExpiry(): number {
return MAX_AGE_24H
}
}
export type ProviderCtor = typeof Provider
// OAuth providers are those that have an `oauthProvider` set. It means they require OAuth authentication to work
export const isOAuthProvider = (
oauthProvider: string | undefined,
): oauthProvider is string =>
typeof oauthProvider === 'string' && oauthProvider.length > 0

77
packages/@uppy/companion/src/server/provider/box/adapter.js
View file

@ -1,77 +0,0 @@
import querystring from 'node:querystring'
import mime from 'mime-types'
const isFolder = (item) => {
return item.type === 'folder'
}
const getItemSize = (item) => {
return item.size
}
const getItemIcon = (item) => {
return item.type
}
const getItemSubList = (item) => {
return item.entries
}
const getItemName = (item) => {
return item.name || ''
}
const getMimeType = (item) => {
return mime.lookup(getItemName(item)) || null
}
const getItemId = (item) => {
return item.id
}
const getItemRequestPath = (item) => {
return item.id
}
const getItemModifiedDate = (item) => {
return item.modified_at
}
const getItemThumbnailUrl = (item) => {
return `/box/thumbnail/${getItemRequestPath(item)}`
}
const getNextPagePath = (data) => {
if (
data.total_count < data.limit ||
data.offset + data.limit > data.total_count
) {
return null
}
const query = { cursor: data.offset + data.limit }
return `?${querystring.stringify(query)}`
}
const adaptData = function adaptData(res, username, companion) {
const data = { username, items: [] }
const items = getItemSubList(res)
items.forEach((item) => {
data.items.push({
isFolder: isFolder(item),
icon: getItemIcon(item),
name: getItemName(item),
mimeType: getMimeType(item),
id: getItemId(item),
thumbnail: companion.buildURL(getItemThumbnailUrl(item), true),
requestPath: getItemRequestPath(item),
modifiedDate: getItemModifiedDate(item),
size: getItemSize(item),
})
})
data.nextPagePath = getNextPagePath(res)
return data
}
export default adaptData

99
packages/@uppy/companion/src/server/provider/box/adapter.ts Normal file
View file

@ -0,0 +1,99 @@
import querystring from 'node:querystring'
import mime from 'mime-types'
import type { BuildUrl } from '../../../types/express.js'
type BoxItem = {
type?: string
size?: number
entries?: BoxItem[]
name?: string
id: string
modified_at?: string
}
type BoxListResponse = {
entries: BoxItem[]
total_count: number
limit: number
offset: number
}
type CompanionLike = {
buildURL?: BuildUrl
}
const isFolder = (item: BoxItem): boolean => {
return item.type === 'folder'
}
const getItemSize = (item: BoxItem): number | undefined => {
return item.size
}
const getItemIcon = (item: BoxItem): string | undefined => {
return item.type
}
const getItemSubList = (item: BoxListResponse): BoxItem[] => {
return item.entries
}
const getItemName = (item: BoxItem): string => {
return item.name || ''
}
const getMimeType = (item: BoxItem): string | null => {
const mt = mime.lookup(getItemName(item))
return typeof mt === 'string' ? mt : null
}
const getItemId = (item: BoxItem): string => {
return item.id
}
const getItemRequestPath = (item: BoxItem): string => {
return item.id
}
const getItemModifiedDate = (item: BoxItem): string | undefined => {
return item.modified_at
}
const getItemThumbnailUrl = (item: BoxItem): string => {
return `/box/thumbnail/${getItemRequestPath(item)}`
}
const getNextPagePath = (data: BoxListResponse): string | null => {
if (
data.total_count < data.limit ||
data.offset + data.limit > data.total_count
) {
return null
}
const query = { cursor: data.offset + data.limit }
return `?${querystring.stringify(query)}`
}
const adaptData = function adaptData(
res: BoxListResponse,
username: string | undefined,
companion: CompanionLike,
) {
const items = getItemSubList(res).map((item) => ({
isFolder: isFolder(item),
icon: getItemIcon(item),
name: getItemName(item),
mimeType: getMimeType(item),
id: getItemId(item),
thumbnail: companion.buildURL?.(getItemThumbnailUrl(item), true),
requestPath: getItemRequestPath(item),
modifiedDate: getItemModifiedDate(item),
size: getItemSize(item),
}))
const nextPagePath = getNextPagePath(res)
return { username, items, nextPagePath }
}
export default adaptData

122
packages/@uppy/companion/src/server/provider/box/index.js → packages/@uppy/companion/src/server/provider/box/index.ts
View file

@ -1,13 +1,25 @@
import type { Readable } from 'node:stream'
import got from 'got'
import type { ProviderOptions } from '../../../schemas/companion.js'
import type { BuildUrl } from '../../../types/express.js'
import { isRecord } from '../../helpers/type-guards.js'
import { prepareStream } from '../../helpers/utils.js'
import Provider from '../Provider.js'
import Provider, { type ProviderListResponse, type Query } from '../Provider.js'
import { withProviderErrorHandling } from '../providerErrors.js'
import adaptData from './adapter.js'
const BOX_FILES_FIELDS = 'id,modified_at,name,permissions,size,type'
const BOX_THUMBNAIL_SIZE = 256
const getClient = ({ token }) =>
type BoxUserSession = { accessToken: string }
type CompanionLike = {
buildURL?: BuildUrl
options: {
providerOptions: Record<string, Pick<ProviderOptions, 'key' | 'secret'>>
}
}
const getClient = ({ token }: { token: string }) =>
got.extend({
prefixUrl: 'https://api.box.com/2.0',
headers: {
@ -15,53 +27,74 @@ const getClient = ({ token }) =>
},
})
async function getUserInfo({ token }) {
return getClient({ token }).get('users/me', { responseType: 'json' }).json()
async function getUserInfo({
token,
}: {
token: string
}): Promise<{ login?: string }> {
return getClient({ token })
.get('users/me', { responseType: 'json' })
.json<{ login?: string }>()
}
async function list({ directory, query, token }) {
async function list({
directory,
query,
token,
}: {
directory: string | undefined
query?: Query | undefined
token: string
}): Promise<Parameters<typeof adaptData>[0]> {
const rootFolderID = '0'
const cursor =
typeof query?.['cursor'] === 'string' ? query?.['cursor'] : undefined
return getClient({ token })
.get(`folders/${directory || rootFolderID}/items`, {
searchParams: {
fields: BOX_FILES_FIELDS,
offset: query.cursor,
offset: cursor,
limit: 1000,
},
responseType: 'json',
})
.json()
.json<Parameters<typeof adaptData>[0]>()
}
/**
* Adapter for API https://developer.box.com/reference/
*/
export default class Box extends Provider {
constructor(options) {
export default class Box extends Provider<BoxUserSession> {
constructor(options: ConstructorParameters<typeof Provider>[0]) {
super(options)
// needed for the thumbnails fetched via companion
this.needsCookieAuth = true
}
static get oauthProvider() {
static override get oauthProvider() {
return 'box'
}
/**
* Lists files and folders from Box API
*
* @param {object} options
* @param {string} options.directory
* @param {any} options.query
* @param {{ accessToken: string }} options.providerUserSession
* @param {unknown} options.companion
* @param options
* @param options.directory
* @param options.query
* @param options.providerUserSession
* @param options.companion
*/
async list({
override async list({
directory,
providerUserSession: { accessToken: token },
query,
companion,
}) {
}: {
directory?: string | undefined
providerUserSession: BoxUserSession
query?: Query
companion: CompanionLike
}): Promise<ProviderListResponse> {
return this.#withErrorHandling('provider.box.list.error', async () => {
const [userInfo, files] = await Promise.all([
getUserInfo({ token }),
@ -72,7 +105,13 @@ export default class Box extends Provider {
})
}
async download({ id, providerUserSession: { accessToken: token } }) {
override async download({
id,
providerUserSession: { accessToken: token },
}: {
id: string
providerUserSession: BoxUserSession
}): Promise<{ stream: Readable; size: number | undefined }> {
return this.#withErrorHandling('provider.box.download.error', async () => {
const stream = getClient({ token }).stream.get(`files/${id}/content`, {
responseType: 'json',
@ -83,7 +122,13 @@ export default class Box extends Provider {
})
}
async thumbnail({ id, providerUserSession: { accessToken: token } }) {
override async thumbnail({
id,
providerUserSession: { accessToken: token },
}: {
id: string
providerUserSession: BoxUserSession
}): Promise<{ stream: Readable; contentType: string }> {
return this.#withErrorHandling('provider.box.thumbnail.error', async () => {
const extension = 'jpg' // you can set this to png to more easily reproduce http 202 retry-after
@ -112,18 +157,37 @@ export default class Box extends Provider {
})
}
async size({ id, providerUserSession: { accessToken: token } }) {
override async size({
id,
providerUserSession: { accessToken: token },
}: {
id: string
providerUserSession: BoxUserSession
}): Promise<number> {
return this.#withErrorHandling('provider.box.size.error', async () => {
const { size } = await getClient({ token })
const file = await getClient({ token })
.get(`files/${id}`, { responseType: 'json' })
.json()
return parseInt(size, 10)
.json<Record<string, unknown>>()
const sizeValue = file['size']
const sizeStr =
typeof sizeValue === 'string'
? sizeValue
: typeof sizeValue === 'number'
? `${sizeValue}`
: '0'
return parseInt(sizeStr, 10)
})
}
logout({ companion, providerUserSession: { accessToken: token } }) {
override async logout({
companion,
providerUserSession: { accessToken: token },
}: {
companion: CompanionLike
providerUserSession: BoxUserSession
}): Promise<{ revoked: true }> {
return this.#withErrorHandling('provider.box.logout.error', async () => {
const { key, secret } = companion.options.providerOptions.box
const { key, secret } = companion.options.providerOptions['box']!
await getClient({ token }).post('oauth2/revoke', {
prefixUrl: 'https://api.box.com',
form: {
@ -138,13 +202,17 @@ export default class Box extends Provider {
})
}
async #withErrorHandling(tag, fn) {
async #withErrorHandling<T>(tag: string, fn: () => Promise<T>): Promise<T> {
return withProviderErrorHandling({
fn,
tag,
providerName: Box.oauthProvider,
isAuthError: (response) => response.statusCode === 401,
getJsonErrorMessage: (body) => body?.message,
getJsonErrorMessage: (body) => {
if (!isRecord(body)) return undefined
const msg = body['message']
return typeof msg === 'string' ? msg : undefined
},
})
}
}

242
packages/@uppy/companion/src/server/provider/credentials.js
View file

@ -1,242 +0,0 @@
import { htmlEscape } from 'escape-goat'
import got from 'got'
import * as tokenService from '../helpers/jwt.js'
import * as oAuthState from '../helpers/oauth-state.js'
import { getRedirectPath, getURLBuilder } from '../helpers/utils.js'
import logger from '../logger.js'
// biome-ignore lint/correctness/noUnusedImports: It is used as a type
import Provider from './Provider.js'
/**
* @param {string} url
* @param {string} providerName
* @param {object|null} credentialRequestParams - null asks for default credentials.
*/
async function fetchKeys(url, providerName, credentialRequestParams) {
try {
const { credentials } = await got
.post(url, {
json: { provider: providerName, parameters: credentialRequestParams },
})
.json()
if (!credentials) throw new Error('Received no remote credentials')
return credentials
} catch (err) {
logger.error(err, 'credentials.fetch.fail')
throw err
}
}
/**
* Fetches for a providers OAuth credentials. If the config for that provider allows fetching
* of the credentials via http, and the `credentialRequestParams` argument is provided, the oauth
* credentials will be fetched via http. Otherwise, the credentials provided via companion options
* will be used instead.
*
* @param {string} providerName the name of the provider whose oauth keys we want to fetch (e.g onedrive)
* @param {object} companionOptions the companion options object
* @param {object} credentialRequestParams the params that should be sent if an http request is required.
*/
async function fetchProviderKeys(
providerName,
companionOptions,
credentialRequestParams,
) {
let providerConfig = companionOptions.providerOptions[providerName]
if (!providerConfig) {
providerConfig = companionOptions.customProviders[providerName]?.config
}
if (!providerConfig) {
return null
}
if (!providerConfig.credentialsURL) {
return providerConfig
}
// If a default key is configured, do not ask the credentials endpoint for it.
// In a future version we could make this an XOR thing, providing either an endpoint or global keys,
// but not both.
if (!credentialRequestParams && providerConfig.key) {
return providerConfig
}
return fetchKeys(
providerConfig.credentialsURL,
providerName,
credentialRequestParams || null,
)
}
/**
* Returns a request middleware function that can be used to pre-fetch a provider's
* Oauth credentials before the request is passed to the Oauth handler (https://github.com/simov/grant in this case).
*
* @param {Record<string, typeof Provider>} providers provider classes enabled for this server
* @param {object} companionOptions companion options object
* @returns {import('express').RequestHandler}
*/
export const getCredentialsOverrideMiddleware = (
providers,
companionOptions,
) => {
return async (req, res, next) => {
try {
const { oauthProvider, override } = req.params
const [providerName] = Object.keys(providers).filter(
(name) => providers[name].oauthProvider === oauthProvider,
)
if (!providerName) {
next()
return
}
if (!companionOptions.providerOptions[providerName]?.credentialsURL) {
next()
return
}
const grantDynamic = oAuthState.getGrantDynamicFromRequest(req)
// only use state via session object if user isn't making intial "connect" request.
// override param indicates subsequent requests from the oauth flow
const state = override ? grantDynamic.state : req.query.state
if (!state) {
next()
return
}
const preAuthToken = oAuthState.getFromState(
state,
'preAuthToken',
companionOptions.secret,
)
if (!preAuthToken) {
next()
return
}
let payload
try {
payload = tokenService.verifyEncryptedToken(
preAuthToken,
companionOptions.preAuthSecret,
)
} catch (_err) {
next()
return
}
const credentials = await fetchProviderKeys(
providerName,
companionOptions,
payload,
)
// Besides the key and secret the fetched credentials can also contain `origins`,
// which is an array of strings of allowed origins to prevent any origin from getting the OAuth
// token through window.postMessage (see comment in connect.js).
// postMessage happens in send-token.js, which is a different request, so we need to put the allowed origins
// on the encrypted session state to access it later there.
if (
Array.isArray(credentials.origins) &&
credentials.origins.length > 0
) {
const decodedState = oAuthState.decodeState(
state,
companionOptions.secret,
)
decodedState.customerDefinedAllowedOrigins = credentials.origins
const newState = oAuthState.encodeState(
decodedState,
companionOptions.secret,
)
// @ts-expect-error untyped
req.session.grant = {
// @ts-expect-error untyped
...req.session.grant,
dynamic: {
// @ts-expect-error untyped
...req.session.grant?.dynamic,
state: newState,
},
}
}
res.locals.grant = {
dynamic: {
key: credentials.key,
secret: credentials.secret,
origins: credentials.origins,
},
}
if (credentials.transloadit_gateway) {
const redirectPath = getRedirectPath(providerName)
const fullRedirectPath = getURLBuilder(companionOptions)(
redirectPath,
true,
true,
)
const redirectUri = new URL(
fullRedirectPath,
credentials.transloadit_gateway,
).toString()
logger.info('Using redirect URI from transloadit_gateway', redirectUri)
res.locals.grant.dynamic.redirect_uri = redirectUri
}
next()
} catch (keyErr) {
res.send(`
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
</head>
<body>
<h1>Could not fetch credentials</h1>
<p>
This is probably an Uppy configuration issue. Check that your Transloadit key is correct, and that the configured <code>credentialsName</code> for this remote provider matches the name you gave it in the Template Credentials setup on the Transloadit side.
</p>
<p>Internal error message: ${htmlEscape(keyErr.message)}</p>
</body>
</html>
`)
}
}
}
/**
* Returns a request scoped function that can be used to get a provider's oauth credentials
* through out the lifetime of the request.
*
* @param {string} providerName the name of the provider attached to the scope of the request
* @param {object} companionOptions the companion options object
* @param {object} req the express request object for the said request
* @returns {(providerName: string, companionOptions: object, credentialRequestParams?: object) => Promise}
*/
export const getCredentialsResolver = (providerName, companionOptions, req) => {
const credentialsResolver = () => {
const encodedCredentialsParams = req.header('uppy-credentials-params')
let credentialRequestParams = null
if (encodedCredentialsParams) {
try {
credentialRequestParams = JSON.parse(
atob(encodedCredentialsParams),
).params
} catch (error) {
logger.error(error, 'credentials.resolve.fail', req.id)
}
}
return fetchProviderKeys(
providerName,
companionOptions,
credentialRequestParams,
)
}
return credentialsResolver
}

266
packages/@uppy/companion/src/server/provider/credentials.ts Normal file
View file

@ -0,0 +1,266 @@
import { htmlEscape } from 'escape-goat'
import type { NextFunction, Request, Response } from 'express'
import got from 'got'
import type { CredentialsFetchResponse } from '../../schemas/companion.js'
import type { CompanionRuntimeOptions } from '../../types/companion-options.js'
import type { CompanionExpressLocals } from '../../types/express.js'
import * as tokenService from '../helpers/jwt.js'
import * as oAuthState from '../helpers/oauth-state.js'
import { isRecord, toError } from '../helpers/type-guards.js'
import { getRedirectPath, getURLBuilder } from '../helpers/utils.js'
import logger from '../logger.js'
import type Provider from './Provider.js'
/**
* @param url
* @param providerName
* @param credentialRequestParams - null asks for default credentials.
*/
async function fetchKeys(
url: string,
providerName: string,
credentialRequestParams: unknown | null,
) {
try {
const { credentials } = await got
.post(url, {
json: { provider: providerName, parameters: credentialRequestParams },
})
.json<{ credentials?: CredentialsFetchResponse }>()
if (!isRecord(credentials))
throw new Error('Received no remote credentials')
return credentials
} catch (err) {
logger.error(err, 'credentials.fetch.fail')
throw err
}
}
/**
* Fetches for a providers OAuth credentials. If the config for that provider allows fetching
* of the credentials via http, and the `credentialRequestParams` argument is provided, the oauth
* credentials will be fetched via http. Otherwise, the credentials provided via companion options
* will be used instead.
*
* @param providerName the name of the provider whose oauth keys we want to fetch (e.g onedrive)
* @param companionOptions the companion options object
* @param credentialRequestParams the params that should be sent if an http request is required.
*/
async function fetchProviderKeys(
providerName: string,
companionOptions: CompanionRuntimeOptions,
credentialRequestParams: unknown,
): Promise<CredentialsFetchResponse | null> {
let providerConfig = companionOptions.providerOptions[providerName]
if (!providerConfig) {
providerConfig = companionOptions.customProviders?.[providerName]?.config
}
if (!providerConfig) {
return null
}
// If a default key is configured, do not ask the credentials endpoint for it.
// In a future version we could make this an XOR thing, providing either an endpoint or global keys,
// but not both.
const key = providerConfig.key
if (!credentialRequestParams && key != null) {
return providerConfig
}
const credentialsURL = providerConfig.credentialsURL
if (credentialsURL == null) {
return providerConfig
}
return fetchKeys(
credentialsURL,
providerName,
credentialRequestParams || null,
)
}
/**
* Returns a request middleware function that can be used to pre-fetch a provider's
* Oauth credentials before the request is passed to the Oauth handler (https://github.com/simov/grant in this case).
*
* @param providers provider classes enabled for this server
* @param companionOptions companion options object
*/
export const getCredentialsOverrideMiddleware = (
providers: Record<string, typeof Provider>,
companionOptions: CompanionRuntimeOptions,
) => {
return async (
req: Request,
res: Response<unknown, CompanionExpressLocals>,
next: NextFunction,
) => {
try {
const { oauthProvider, override } = req.params
if (oauthProvider == null || oauthProvider.length === 0) {
next()
return
}
const [providerName] = Object.keys(providers).filter(
(name) => providers[name]?.oauthProvider === oauthProvider,
)
if (
!providerName ||
!companionOptions.providerOptions[providerName]?.credentialsURL
) {
next()
return
}
const grantDynamic = oAuthState.getGrantDynamicFromRequest(req)
// only use state via session object if user isn't making intial "connect" request.
// override param indicates subsequent requests from the oauth flow
const state = override ? grantDynamic.state : req.query['state']
if (!state || typeof state !== 'string') {
next()
return
}
const { secret, preAuthSecret } = companionOptions
if (preAuthSecret == null) {
next()
return
}
const preAuthToken = oAuthState.getFromState(
state,
'preAuthToken',
secret,
)
if (preAuthToken == null) {
next()
return
}
let payload: unknown
try {
payload = tokenService.verifyEncryptedToken(preAuthToken, preAuthSecret)
} catch (_err) {
next()
return
}
const credentials = await fetchProviderKeys(
providerName,
companionOptions,
payload,
)
if (!credentials) {
next()
return
}
// Besides the key and secret the fetched credentials can also contain `origins`,
// which is an array of strings of allowed origins to prevent any origin from getting the OAuth
// token through window.postMessage (see comment in connect.js).
// postMessage happens in send-token.js, which is a different request, so we need to put the allowed origins
// on the encrypted session state to access it later there.
const { origins } = credentials
if (Array.isArray(origins) && origins.length > 0) {
const decodedState = oAuthState.decodeState(state, secret)
decodedState['customerDefinedAllowedOrigins'] = origins
const newState = oAuthState.encodeState(decodedState, secret)
if (req.session != null) {
req.session.grant = {
...req.session.grant,
dynamic: {
...req.session.grant?.dynamic,
state: newState,
},
}
}
}
const dynamic: Record<string, unknown> = {}
const fetchedKey = credentials.key
const fetchedSecret = credentials.secret
if (fetchedKey != null) dynamic['key'] = fetchedKey
if (fetchedSecret != null) dynamic['secret'] = fetchedSecret
if (origins != null) dynamic['origins'] = origins
res.locals['grant'] = { dynamic }
const gateway = credentials.transloadit_gateway
if (typeof gateway === 'string') {
const redirectPath = getRedirectPath(providerName)
const fullRedirectPath = getURLBuilder(companionOptions)(
redirectPath,
true,
true,
)
const redirectUri = new URL(fullRedirectPath, gateway).toString()
logger.info('Using redirect URI from transloadit_gateway', redirectUri)
const grant = res.locals['grant']
if (isRecord(grant) && isRecord(grant['dynamic'])) {
grant['dynamic']['redirect_uri'] = redirectUri
}
}
next()
} catch (keyErr) {
const error = toError(keyErr)
res.send(`
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
</head>
<body>
<h1>Could not fetch credentials</h1>
<p>
This is probably an Uppy configuration issue. Check that your Transloadit key is correct, and that the configured <code>credentialsName</code> for this remote provider matches the name you gave it in the Template Credentials setup on the Transloadit side.
</p>
<p>Internal error message: ${htmlEscape(error.message)}</p>
</body>
</html>
`)
}
}
}
/**
* Returns a request scoped function that can be used to get a provider's oauth credentials
* through out the lifetime of the request.
*
* @param providerName the name of the provider attached to the scope of the request
* @param companionOptions the companion options object
* @param req the express request object for the said request
*/
export const getCredentialsResolver = (
providerName: string,
companionOptions: CompanionRuntimeOptions,
req: Request,
): (() => Promise<CredentialsFetchResponse | null>) => {
const credentialsResolver = () => {
const encodedCredentialsParams = req.header('uppy-credentials-params')
let credentialRequestParams: unknown = null
if (encodedCredentialsParams) {
try {
const parsed: unknown = JSON.parse(atob(encodedCredentialsParams))
credentialRequestParams =
isRecord(parsed) && Object.hasOwn(parsed, 'params')
? parsed['params']
: null
} catch (error) {
logger.error(error, 'credentials.resolve.fail', req.id)
}
}
return fetchProviderKeys(
providerName,
companionOptions,
credentialRequestParams,
)
}
return credentialsResolver
}

73
packages/@uppy/companion/src/server/provider/dropbox/adapter.js
View file

@ -1,73 +0,0 @@
import querystring from 'node:querystring'
import mime from 'mime-types'
const isFolder = (item) => {
return item['.tag'] === 'folder'
}
const getItemSize = (item) => {
return item.size
}
const getItemIcon = (item) => {
return item['.tag']
}
const getItemSubList = (item) => {
return item.entries
}
const getItemName = (item) => {
return item.name || ''
}
const getMimeType = (item) => {
return mime.lookup(getItemName(item)) || null
}
const getItemId = (item) => {
return item.id
}
const getItemRequestPath = (item) => {
return encodeURIComponent(item.path_lower)
}
const getItemModifiedDate = (item) => {
return item.server_modified
}
const getItemThumbnailUrl = (item) => {
return `/dropbox/thumbnail/${getItemRequestPath(item)}`
}
const getNextPagePath = (data) => {
if (!data.has_more) {
return null
}
const query = { cursor: data.cursor }
return `?${querystring.stringify(query)}`
}
const adaptData = (res, email, buildURL) => {
const items = getItemSubList(res).map((item) => ({
isFolder: isFolder(item),
icon: getItemIcon(item),
name: getItemName(item),
mimeType: getMimeType(item),
id: getItemId(item),
thumbnail: buildURL(getItemThumbnailUrl(item), true),
requestPath: getItemRequestPath(item),
modifiedDate: getItemModifiedDate(item),
size: getItemSize(item),
}))
items.sort((a, b) => a.name.localeCompare(b.name, 'en-US', { numeric: true }))
return {
username: email,
items,
nextPagePath: getNextPagePath(res),
}
}
export default adaptData

108
packages/@uppy/companion/src/server/provider/dropbox/adapter.ts Normal file
View file

@ -0,0 +1,108 @@
import querystring from 'node:querystring'
import mime from 'mime-types'
import type { BuildUrl } from '../../../types/express.js'
type DropboxItem = {
'.tag'?: string
size?: number
name?: string
id: string
path_lower: string
server_modified?: string
}
type DropboxListResponse = {
entries: DropboxItem[]
has_more: boolean
cursor?: string
}
const isFolder = (item: DropboxItem): boolean => {
return item['.tag'] === 'folder'
}
const getItemSize = (item: DropboxItem): number | undefined => {
return item.size
}
const getItemIcon = (item: DropboxItem): string | undefined => {
return item['.tag']
}
const getItemSubList = (item: DropboxListResponse): DropboxItem[] => {
return item.entries
}
const getItemName = (item: DropboxItem): string => {
return item.name || ''
}
const getMimeType = (item: DropboxItem): string | null => {
const mt = mime.lookup(getItemName(item))
return typeof mt === 'string' ? mt : null
}
const getItemId = (item: DropboxItem): string => {
return item.id
}
const getItemRequestPath = (item: DropboxItem): string => {
return encodeURIComponent(item.path_lower)
}
const getItemModifiedDate = (item: DropboxItem): string | undefined => {
return item.server_modified
}
const getItemThumbnailUrl = (item: DropboxItem): string => {
return `/dropbox/thumbnail/${getItemRequestPath(item)}`
}
const getNextPagePath = (data: DropboxListResponse): string | null => {
if (!data.has_more) {
return null
}
const query = { cursor: data.cursor }
return `?${querystring.stringify(query)}`
}
const adaptData = (
res: DropboxListResponse,
email: string | undefined,
buildURL: BuildUrl,
): {
username: string | undefined
items: Array<{
isFolder: boolean
icon: string | undefined
name: string
mimeType: string | null
id: string
thumbnail: string
requestPath: string
modifiedDate: string | undefined
size: number | undefined
}>
nextPagePath: string | null
} => {
const items = getItemSubList(res).map((item) => ({
isFolder: isFolder(item),
icon: getItemIcon(item),
name: getItemName(item),
mimeType: getMimeType(item),
id: getItemId(item),
thumbnail: buildURL(getItemThumbnailUrl(item), true),
requestPath: getItemRequestPath(item),
modifiedDate: getItemModifiedDate(item),
size: getItemSize(item),
}))
items.sort((a, b) => a.name.localeCompare(b.name, 'en-US', { numeric: true }))
return {
username: email,
items,
nextPagePath: getNextPagePath(res),
}
}
export default adaptData

205
packages/@uppy/companion/src/server/provider/dropbox/index.js → packages/@uppy/companion/src/server/provider/dropbox/index.ts
View file

@ -2,29 +2,57 @@
//
// This function is simple and has OK performance compared to more
// complicated ones: http://jsperf.com/json-escape-unicode/4
import type { Readable } from 'node:stream'
import got from 'got'
import type { BuildUrl } from '../../../types/express.js'
import { MAX_AGE_REFRESH_TOKEN } from '../../helpers/jwt.js'
import { isRecord } from '../../helpers/type-guards.js'
import { prepareStream } from '../../helpers/utils.js'
import logger from '../../logger.js'
import Provider from '../Provider.js'
import Provider, {
type ProviderListResponse,
type ProviderSearchResponse,
type Query,
} from '../Provider.js'
import { withProviderErrorHandling } from '../providerErrors.js'
import adaptData from './adapter.js'
const charsToEncode = /[\u007f-\uffff]/g
function httpHeaderSafeJson(v) {
function httpHeaderSafeJson(v: unknown): string {
return JSON.stringify(v).replace(charsToEncode, (c) => {
return `\\u${`000${c.charCodeAt(0).toString(16)}`.slice(-4)}`
})
}
async function getUserInfo({ client }) {
return client
.post('users/get_current_account', { responseType: 'json' })
.json()
type DropboxClient = ReturnType<typeof got.extend>
type DropboxUserInfo = {
email?: string
root_info?: {
root_namespace_id?: string
home_namespace_id?: string
}
}
async function getClient({ token, namespaced }) {
const makeClient = (namespace) =>
async function getUserInfo({
client,
}: {
client: DropboxClient
}): Promise<DropboxUserInfo> {
return client
.post('users/get_current_account', { responseType: 'json' })
.json<DropboxUserInfo>()
}
async function getClient({
token,
namespaced,
}: {
token: string
namespaced: boolean
}): Promise<{ client: DropboxClient; userInfo: DropboxUserInfo }> {
const makeClient = (namespace?: string) =>
got.extend({
prefixUrl: 'https://api.dropboxapi.com/2',
headers: {
@ -48,17 +76,14 @@ async function getClient({ token, namespaced }) {
// https://www.dropboxforum.com/discussions/101000014/how-to-list-the-contents-of-a-team-folder/258310
// https://developers.dropbox.com/dbx-team-files-guide#namespaces
// https://www.dropbox.com/developers/reference/path-root-header-modes
const rootInfo = userInfo.root_info
if (
namespaced &&
userInfo.root_info != null &&
userInfo.root_info.root_namespace_id !==
userInfo.root_info.home_namespace_id
rootInfo != null &&
rootInfo.root_namespace_id !== rootInfo.home_namespace_id
) {
logger.debug(
'using root_namespace_id',
userInfo.root_info.root_namespace_id,
)
client = makeClient(userInfo.root_info.root_namespace_id)
logger.debug('using root_namespace_id', rootInfo.root_namespace_id)
client = makeClient(rootInfo.root_namespace_id)
}
return {
@ -72,19 +97,34 @@ const getOauthClient = () =>
prefixUrl: 'https://api.dropboxapi.com/oauth2',
})
async function list({ client, directory, query }) {
if (query.cursor) {
async function list({
client,
directory,
query,
}: {
client: DropboxClient
directory?: string | undefined
query?: Query | undefined
}): Promise<Parameters<typeof adaptData>[0]> {
if (query?.['cursor']) {
return client
.post('files/list_folder/continue', {
json: { cursor: query.cursor },
json: { cursor: query['cursor'] },
responseType: 'json',
})
.json()
.json<Parameters<typeof adaptData>[0]>()
}
const searchParams: Record<string, string> = {}
if (query != null) {
for (const [k, v] of Object.entries(query)) {
if (typeof v === 'string') searchParams[k] = v
}
}
return client
.post('files/list_folder', {
searchParams: query,
searchParams,
json: {
path: `${directory || ''}`,
include_non_downloadable_files: false,
@ -93,13 +133,27 @@ async function list({ client, directory, query }) {
},
responseType: 'json',
})
.json()
.json<Parameters<typeof adaptData>[0]>()
}
async function fetchSearchEntries({ client, query }) {
async function fetchSearchEntries({
client,
query,
}: {
client: DropboxClient
query: { q: string; path?: string; [k: string]: unknown }
}): Promise<Parameters<typeof adaptData>[0]> {
const scopePath =
typeof query.path === 'string' ? decodeURIComponent(query.path) : undefined
type DropboxSearchResponse = {
matches: Array<{
metadata: { metadata: Parameters<typeof adaptData>[0]['entries'][number] }
}>
has_more: boolean
cursor?: string
}
const searchRes = await client
.post('files/search_v2', {
json: {
@ -113,37 +167,45 @@ async function fetchSearchEntries({ client, query }) {
},
responseType: 'json',
})
.json()
.json<DropboxSearchResponse>()
const entries = searchRes.matches.map((m) => m.metadata.metadata)
return {
entries,
has_more: searchRes.has_more,
cursor: searchRes.cursor,
...(searchRes.cursor != null && { cursor: searchRes.cursor }),
}
}
interface DropboxUserSession {
accessToken: string
}
/**
* Adapter for API https://www.dropbox.com/developers/documentation/http/documentation
*/
export default class Dropbox extends Provider {
constructor(options) {
export default class Dropbox extends Provider<DropboxUserSession> {
constructor(options: ConstructorParameters<typeof Provider>[0]) {
super(options)
this.needsCookieAuth = true
}
static get oauthProvider() {
static override get oauthProvider() {
return 'dropbox'
}
static get authStateExpiry() {
static override get authStateExpiry() {
return MAX_AGE_REFRESH_TOKEN
}
/**
* Search entries
*/
async search(options) {
override async search(options: {
providerUserSession: DropboxUserSession
query: { q: string; path?: string; [k: string]: unknown }
companion: { buildURL: BuildUrl }
}): Promise<ProviderSearchResponse> {
return this.#withErrorHandling(
'provider.dropbox.search.error',
async () => {
@ -153,7 +215,6 @@ export default class Dropbox extends Provider {
})
const stats = await fetchSearchEntries({ client, query: options.query })
console.log(stats)
const { email } = userInfo
// we don't really need email, but let's mimic `list` response shape for consistency
return adaptData(stats, email, options.companion.buildURL)
@ -164,20 +225,35 @@ export default class Dropbox extends Provider {
/**
* List folder entries
*/
async list(options) {
override async list(options: {
directory?: string | undefined
providerUserSession: DropboxUserSession
query?: Query
companion: { buildURL?: BuildUrl }
}): Promise<ProviderListResponse> {
return this.#withErrorHandling('provider.dropbox.list.error', async () => {
const { client, userInfo } = await getClient({
token: options.providerUserSession.accessToken,
namespaced: true,
})
if (options.companion.buildURL == null) {
throw new Error('companion.buildURL is required for Dropbox provider')
}
const stats = await list({ ...options, client })
const { email } = userInfo
return adaptData(stats, email, options.companion.buildURL)
})
}
async download({ id, providerUserSession: { accessToken: token } }) {
override async download({
id,
providerUserSession: { accessToken: token },
}: {
id: string
providerUserSession: DropboxUserSession
}): Promise<{ stream: Readable; size: number | undefined }> {
return this.#withErrorHandling(
'provider.dropbox.download.error',
async () => {
@ -199,7 +275,13 @@ export default class Dropbox extends Provider {
)
}
async thumbnail({ id, providerUserSession: { accessToken: token } }) {
override async thumbnail({
id,
providerUserSession: { accessToken: token },
}: {
id: string
providerUserSession: DropboxUserSession
}): Promise<{ stream: Readable; contentType: string }> {
return this.#withErrorHandling(
'provider.dropbox.thumbnail.error',
async () => {
@ -224,21 +306,36 @@ export default class Dropbox extends Provider {
)
}
async size({ id, providerUserSession: { accessToken: token } }) {
override async size({
id,
providerUserSession: { accessToken: token },
}: {
id: string
providerUserSession: DropboxUserSession
}): Promise<number> {
return this.#withErrorHandling('provider.dropbox.size.error', async () => {
const { size } = await (
await getClient({ token, namespaced: true })
).client
const meta = await (await getClient({ token, namespaced: true })).client
.post('files/get_metadata', {
json: { path: id },
responseType: 'json',
})
.json()
return parseInt(size, 10)
.json<{ size?: unknown }>()
const sizeValue = meta?.size
const sizeStr =
typeof sizeValue === 'string'
? sizeValue
: typeof sizeValue === 'number'
? String(sizeValue)
: ''
return parseInt(sizeStr, 10)
})
}
async logout({ providerUserSession: { accessToken: token } }) {
override async logout({
providerUserSession: { accessToken: token },
}: {
providerUserSession: DropboxUserSession
}): Promise<{ revoked: true }> {
return this.#withErrorHandling(
'provider.dropbox.logout.error',
async () => {
@ -251,11 +348,19 @@ export default class Dropbox extends Provider {
)
}
async refreshToken({ clientId, clientSecret, refreshToken }) {
override async refreshToken({
clientId,
clientSecret,
refreshToken,
}: {
clientId: string | undefined
clientSecret: string | undefined
refreshToken: string
}): Promise<{ accessToken: string }> {
return this.#withErrorHandling(
'provider.dropbox.token.refresh.error',
async () => {
const { access_token: accessToken } = await getOauthClient()
const tokenRes = await getOauthClient()
.post('token', {
form: {
refresh_token: refreshToken,
@ -264,19 +369,27 @@ export default class Dropbox extends Provider {
client_secret: clientSecret,
},
})
.json()
.json<{ access_token?: unknown }>()
const accessToken = tokenRes.access_token
if (typeof accessToken !== 'string' || accessToken.length === 0) {
throw new Error('Missing access_token')
}
return { accessToken }
},
)
}
async #withErrorHandling(tag, fn) {
async #withErrorHandling<T>(tag: string, fn: () => Promise<T>): Promise<T> {
return withProviderErrorHandling({
fn,
tag,
providerName: Dropbox.oauthProvider,
isAuthError: (response) => response.statusCode === 401,
getJsonErrorMessage: (body) => body?.error_summary,
getJsonErrorMessage: (body) => {
if (!isRecord(body)) return undefined
const summary = body['error_summary']
return typeof summary === 'string' ? summary : undefined
},
})
}
}

118
packages/@uppy/companion/src/server/provider/error.js
View file

@ -1,118 +0,0 @@
/**
* ProviderApiError is error returned when an adapter encounters
* an http error while communication with its corresponding provider
*/
export class ProviderApiError extends Error {
/**
* @param {string} message error message
* @param {number} statusCode the http status code from the provider api
*/
constructor(message, statusCode) {
super(`HTTP ${statusCode}: ${message}`) // Include statusCode to make it easier to debug
this.name = 'ProviderApiError'
this.statusCode = statusCode
this.isAuthError = false
}
}
export class ProviderUserError extends ProviderApiError {
/**
* @param {object} json arbitrary JSON.stringify-able object that will be passed to the client
*/
constructor(json) {
super('User error', undefined)
this.name = 'ProviderUserError'
this.json = json
}
}
/**
* AuthError is error returned when an adapter encounters
* an authorization error while communication with its corresponding provider
* this signals to the client that the access token is invalid and needs to be
* refreshed or the user needs to re-authenticate
*/
export class ProviderAuthError extends ProviderApiError {
constructor() {
super('invalid access token detected by Provider', 401)
this.name = 'AuthError'
this.isAuthError = true
}
}
export function parseHttpError(err) {
if (err?.name === 'HTTPError') {
return {
statusCode: err.response?.statusCode,
body: err.response?.body,
}
}
if (err?.name === 'HttpError') {
return {
statusCode: err.statusCode,
body: err.responseJson,
}
}
return undefined
}
/**
* Convert an error instance to an http response if possible
*
* @param {Error | ProviderApiError} err the error instance to convert to an http json response
* @returns {object | undefined} an object with a code and json field if the error can be converted to a response
*/
function errorToResponse(err) {
// @ts-ignore
if (err?.isAuthError) {
return { code: 401, json: { message: err.message } }
}
if (err?.name === 'ValidationError') {
return { code: 400, json: { message: err.message } }
}
if (err?.name === 'ProviderUserError') {
// @ts-ignore
return { code: 400, json: err.json }
}
if (err?.name === 'ProviderApiError') {
// @ts-ignore
if (err.statusCode >= 500) {
// bad gateway i.e the provider APIs gateway
return { code: 502, json: { message: err.message } }
}
// @ts-ignore
if (err.statusCode === 429) {
return { code: 429, json: { message: err.message } }
}
// @ts-ignore
if (err.statusCode >= 400) {
// 424 Failed Dependency
return { code: 424, json: { message: err.message } }
}
}
const httpError = parseHttpError(err)
if (httpError) {
// We proxy the response purely for ease of debugging
return {
code: 500,
json: { statusCode: httpError.statusCode, body: httpError.body },
}
}
return undefined
}
export function respondWithError(err, res) {
const errResp = errorToResponse(err)
if (errResp) {
res.status(errResp.code).json(errResp.json)
return true
}
return false
}

141
packages/@uppy/companion/src/server/provider/error.ts Normal file
View file

@ -0,0 +1,141 @@
import { isRecord } from '../helpers/type-guards.js'
type HttpErrorLike = {
statusCode: number | undefined
body: unknown
}
/**
* Error thrown when an adapter encounters an HTTP error while communicating
* with its corresponding provider.
*/
export class ProviderApiError extends Error {
statusCode: number | undefined
isAuthError: boolean
constructor(message: string, statusCode: number | undefined) {
super(`HTTP ${statusCode}: ${message}`) // Include statusCode to make it easier to debug
this.name = 'ProviderApiError'
this.statusCode = statusCode
this.isAuthError = false
}
}
/**
* Error thrown when the provider response should be forwarded to the client
* as-is (e.g. user-facing validation errors).
*/
export class ProviderUserError extends ProviderApiError {
json: unknown // arbitrary JSON.stringify-able object that will be passed to the client
constructor(json: unknown) {
super('User error', undefined)
this.name = 'ProviderUserError'
this.json = json
}
}
/**
* Error thrown when an adapter encounters an authorization error while
* communicating with its provider. This signals to the client that the access
* token is invalid and needs to be refreshed or the user needs to re-authenticate.
*/
export class ProviderAuthError extends ProviderApiError {
constructor() {
super('invalid access token detected by Provider', 401)
this.name = 'AuthError'
this.isAuthError = true
}
}
export function parseHttpError(err: unknown): HttpErrorLike | undefined {
if (!isRecord(err)) return undefined
const name = err['name']
if (name === 'HTTPError') {
const responseCandidate = err['response']
const response = isRecord(responseCandidate) ? responseCandidate : undefined
const statusCode =
response && typeof response['statusCode'] === 'number'
? response['statusCode']
: undefined
const body = response ? response['body'] : undefined
return { statusCode, body }
}
if (name === 'HttpError') {
const statusCode =
typeof err['statusCode'] === 'number' ? err['statusCode'] : undefined
const body = err['responseJson']
return { statusCode, body }
}
return undefined
}
/**
* Convert an error instance to an HTTP response if possible.
*/
function errorToResponse(
err: unknown,
): { code: number; json: Record<string, unknown> } | undefined {
if (!isRecord(err)) return undefined
if (err['isAuthError'] === true) {
return { code: 401, json: { message: err['message'] } }
}
const name = err['name']
if (name === 'ValidationError') {
return { code: 400, json: { message: err['message'] } }
}
if (name === 'ProviderUserError') {
const json = err['json'] as Record<string, unknown>
return {
code: 400,
json,
}
}
if (name === 'ProviderApiError') {
const statusCode =
typeof err['statusCode'] === 'number' ? err['statusCode'] : undefined
if (statusCode != null && statusCode >= 500) {
// bad gateway i.e the provider APIs gateway
return { code: 502, json: { message: err['message'] } }
}
if (statusCode === 429) {
return { code: 429, json: { message: err['message'] } }
}
if (statusCode != null && statusCode >= 400) {
// 424 Failed Dependency
return { code: 424, json: { message: err['message'] } }
}
}
const httpError = parseHttpError(err)
if (httpError) {
// We proxy the response purely for ease of debugging
return {
code: 500,
json: { statusCode: httpError.statusCode, body: httpError.body },
}
}
return undefined
}
export function respondWithError(
err: unknown,
res: { status: (n: number) => { json: (v: unknown) => void } },
): boolean {
const errResp = errorToResponse(err)
if (errResp) {
res.status(errResp.code).json(errResp.json)
return true
}
return false
}

75
packages/@uppy/companion/src/server/provider/facebook/adapter.js
View file

@ -1,75 +0,0 @@
import querystring from 'node:querystring'
const isFolder = (item) => {
return !!item.type
}
export const sortImages = (images) => {
// sort in ascending order of dimension
return images.slice().sort((a, b) => a.width - b.width)
}
const getItemIcon = (item) => {
if (isFolder(item)) {
return 'folder'
}
return sortImages(item.images)[0].source
}
const getItemSubList = (item) => {
return item.data
}
const getItemName = (item) => {
return item.name || `${item.id} ${item.created_time}`
}
const getMimeType = (item) => {
return isFolder(item) ? null : 'image/jpeg'
}
const getItemId = (item) => {
return `${item.id}`
}
const getItemRequestPath = (item) => {
return `${item.id}`
}
const getItemModifiedDate = (item) => {
return item.created_time
}
const getItemThumbnailUrl = (item) => {
return isFolder(item) ? null : sortImages(item.images)[0].source
}
const getNextPagePath = (data, currentQuery, currentPath) => {
if (!data.paging || !data.paging.cursors) {
return null
}
const query = { ...currentQuery, cursor: data.paging.cursors.after }
return `${currentPath || ''}?${querystring.stringify(query)}`
}
export const adaptData = (res, username, directory, currentQuery) => {
const data = { username, items: [] }
const items = getItemSubList(res)
items.forEach((item) => {
data.items.push({
isFolder: isFolder(item),
icon: getItemIcon(item),
name: getItemName(item),
mimeType: getMimeType(item),
size: null,
id: getItemId(item),
thumbnail: getItemThumbnailUrl(item),
requestPath: getItemRequestPath(item),
modifiedDate: getItemModifiedDate(item),
})
})
data.nextPagePath = getNextPagePath(res, currentQuery, directory)
return data
}

110
packages/@uppy/companion/src/server/provider/facebook/adapter.ts Normal file
View file

@ -0,0 +1,110 @@
import querystring from 'node:querystring'
type FacebookImage = { width: number; source: string }
type FacebookAlbum = {
type?: string
id: string | number
name?: string
created_time?: string
}
type FacebookPhoto = {
id: string | number
name?: string
created_time?: string
images: FacebookImage[]
}
type FacebookItem = FacebookAlbum | FacebookPhoto
export type FacebookListResponse = {
data: FacebookItem[]
paging?: {
cursors?: {
after?: string
}
}
}
const isFolder = (item: FacebookItem): item is FacebookAlbum => {
return 'type' in item && typeof item.type === 'string'
}
export const sortImages = (images: FacebookImage[]): FacebookImage[] => {
// sort in ascending order of dimension
return images.slice().sort((a, b) => a.width - b.width)
}
const getItemIcon = (item: FacebookItem): string | undefined => {
if (isFolder(item)) {
return 'folder'
}
return sortImages(item.images)[0]?.source
}
const getItemSubList = (item: FacebookListResponse): FacebookItem[] => {
return item.data
}
const getItemName = (item: FacebookItem): string => {
return item.name || `${item.id} ${item.created_time}`
}
const getMimeType = (item: FacebookItem): string | null => {
return isFolder(item) ? null : 'image/jpeg'
}
const getItemId = (item: FacebookItem): string => {
return `${item.id}`
}
const getItemRequestPath = (item: FacebookItem): string => {
return `${item.id}`
}
const getItemModifiedDate = (item: FacebookItem): string | undefined => {
return item.created_time
}
const getItemThumbnailUrl = (item: FacebookItem): string | null => {
return isFolder(item) ? null : (sortImages(item.images)[0]?.source ?? null)
}
const getNextPagePath = (
data: FacebookListResponse,
currentQuery: Record<string, string>,
currentPath: string | undefined,
): string | null => {
if (!data.paging?.cursors) {
return null
}
const after = data.paging.cursors.after
if (after == null || after.length === 0) return null
const query = { ...currentQuery, cursor: after }
return `${currentPath || ''}?${querystring.stringify(query)}`
}
export const adaptData = (
res: FacebookListResponse,
username: string | undefined,
directory: string | undefined,
currentQuery: Record<string, string>,
) => {
const items = getItemSubList(res).map((item) => ({
isFolder: isFolder(item),
icon: getItemIcon(item),
name: getItemName(item),
mimeType: getMimeType(item),
size: null,
id: getItemId(item),
thumbnail: getItemThumbnailUrl(item),
requestPath: getItemRequestPath(item),
modifiedDate: getItemModifiedDate(item),
}))
const nextPagePath = getNextPagePath(res, currentQuery, directory)
return { username, items, nextPagePath }
}

150
packages/@uppy/companion/src/server/provider/facebook/index.js
View file

@ -1,150 +0,0 @@
import crypto from 'node:crypto'
import got from 'got'
import { HttpError, prepareStream } from '../../helpers/utils.js'
import logger from '../../logger.js'
import Provider from '../Provider.js'
import { withProviderErrorHandling } from '../providerErrors.js'
import { adaptData, sortImages } from './adapter.js'
async function runRequestBatch({ secret, token, requests }) {
// https://developers.facebook.com/docs/facebook-login/security/#appsecret
// couldn't get `appsecret_time` working, but it seems to be working without it
// const time = Math.floor(Date.now() / 1000)
const appSecretProof = crypto
.createHmac('sha256', secret)
// .update(`${token}|${time}`)
.update(token)
.digest('hex')
const form = {
access_token: token,
appsecret_proof: appSecretProof,
// appsecret_time: String(time),
batch: JSON.stringify(requests),
}
const responsesRaw = await got
.post('https://graph.facebook.com', { form })
.json()
const responses = responsesRaw.map((response) => ({
...response,
body: JSON.parse(response.body),
}))
const errorResponse = responses.find((response) => response.code !== 200)
if (errorResponse) {
throw new HttpError({
statusCode: errorResponse.code,
responseJson: errorResponse.body,
})
}
return responses
}
async function getMediaUrl({ secret, token, id }) {
const [{ body }] = await runRequestBatch({
secret,
token,
requests: [
{
method: 'GET',
relative_url: `${id}?${new URLSearchParams({ fields: 'images' }).toString()}`,
},
],
})
const sortedImages = sortImages(body.images)
return sortedImages[sortedImages.length - 1].source
}
/**
* Adapter for API https://developers.facebook.com/docs/graph-api/using-graph-api/
*/
export default class Facebook extends Provider {
static get oauthProvider() {
return 'facebook'
}
async list({
directory,
providerUserSession: { accessToken: token },
query = { cursor: null },
}) {
return this.#withErrorHandling('provider.facebook.list.error', async () => {
const qs = { fields: 'name,cover_photo,created_time,type' }
if (query.cursor) qs.after = query.cursor
let path = 'me/albums'
if (directory) {
path = `${directory}/photos`
qs.fields = 'icon,images,name,width,height,created_time'
}
const [response1, response2] = await runRequestBatch({
secret: this.secret,
token,
requests: [
{
method: 'GET',
relative_url: `me?${new URLSearchParams({ fields: 'email' }).toString()}`,
},
{ method: 'GET', relative_url: `${path}?${new URLSearchParams(qs)}` },
],
})
const { email } = response1.body
const list = response2.body
return adaptData(list, email, directory, query)
})
}
async download({ id, providerUserSession: { accessToken: token } }) {
return this.#withErrorHandling(
'provider.facebook.download.error',
async () => {
const url = await getMediaUrl({ secret: this.secret, token, id })
const stream = got.stream.get(url, { responseType: 'json' })
const { size } = await prepareStream(stream)
return { stream, size }
},
)
}
async thumbnail() {
// not implementing this because a public thumbnail from facebook will be used instead
logger.error(
'call to thumbnail is not implemented',
'provider.facebook.thumbnail.error',
)
throw new Error('call to thumbnail is not implemented')
}
async logout({ providerUserSession: { accessToken: token } }) {
return this.#withErrorHandling(
'provider.facebook.logout.error',
async () => {
await runRequestBatch({
secret: this.secret,
token,
requests: [{ method: 'DELETE', relative_url: 'me/permissions' }],
})
return { revoked: true }
},
)
}
async #withErrorHandling(tag, fn) {
return withProviderErrorHandling({
fn,
tag,
providerName: Facebook.oauthProvider,
isAuthError: (response) =>
typeof response.body === 'object' && response.body?.error?.code === 190, // Invalid OAuth 2.0 Access Token
getJsonErrorMessage: (body) => body?.error?.message,
})
}
}

265
packages/@uppy/companion/src/server/provider/facebook/index.ts Normal file
View file

@ -0,0 +1,265 @@
import crypto from 'node:crypto'
import type { Readable } from 'node:stream'
import got from 'got'
import { isRecord } from '../../helpers/type-guards.js'
import { HttpError, prepareStream } from '../../helpers/utils.js'
import logger from '../../logger.js'
import Provider, {
type CompanionLike,
type ProviderListResponse,
type Query,
} from '../Provider.js'
import { withProviderErrorHandling } from '../providerErrors.js'
import { adaptData, type FacebookListResponse, sortImages } from './adapter.js'
type FacebookBatchRequest = {
method: string
relative_url: string
}
type FacebookBatchResponse = { code: number; body: unknown }
interface FacebookUserSession {
accessToken: string
}
async function runRequestBatch({
secret,
token,
requests,
}: {
secret: string
token: string
requests: FacebookBatchRequest[]
}): Promise<FacebookBatchResponse[]> {
// https://developers.facebook.com/docs/facebook-login/security/#appsecret
// couldn't get `appsecret_time` working, but it seems to be working without it
// const time = Math.floor(Date.now() / 1000)
const appSecretProof = crypto
.createHmac('sha256', secret)
// .update(`${token}|${time}`)
.update(token)
.digest('hex')
const form = {
access_token: token,
appsecret_proof: appSecretProof,
// appsecret_time: String(time),
batch: JSON.stringify(requests),
}
const responsesRaw = await got
.post('https://graph.facebook.com', { form })
.json<{ code: number; body: string }[]>()
const responses = responsesRaw.map((response) => ({
...response,
body: (() => {
const parsed: unknown = JSON.parse(response.body)
return parsed
})(),
}))
const errorResponse = responses.find((response) => response.code !== 200)
if (errorResponse) {
throw new HttpError({
statusCode: errorResponse.code,
responseJson: errorResponse.body,
})
}
return responses
}
async function getMediaUrl({
secret,
token,
id,
}: {
secret: string
token: string
id: string
}): Promise<string> {
const batch = await runRequestBatch({
secret,
token,
requests: [
{
method: 'GET',
relative_url: `${id}?${new URLSearchParams({ fields: 'images' }).toString()}`,
},
],
})
const first = batch[0]
if (!first) throw new Error('Unexpected Facebook response: missing body')
const body = first.body
const imagesValue = isRecord(body) ? body['images'] : undefined
const isFbImage = (
value: unknown,
): value is { width: number; source: string } =>
isRecord(value) &&
typeof value['width'] === 'number' &&
typeof value['source'] === 'string'
const images = Array.isArray(imagesValue) ? imagesValue.filter(isFbImage) : []
const sortedImages = sortImages(images)
const largest = sortedImages[sortedImages.length - 1]
if (!largest) {
throw new Error('Unexpected Facebook response: missing images')
}
return largest.source
}
/**
* Adapter for API https://developers.facebook.com/docs/graph-api/using-graph-api/
*/
export default class Facebook extends Provider<FacebookUserSession> {
static override get oauthProvider() {
return 'facebook'
}
override async list({
directory,
providerUserSession: { accessToken: token },
query,
companion,
}: {
directory?: string | undefined
providerUserSession: FacebookUserSession
query?: Query | undefined
companion: CompanionLike
}): Promise<ProviderListResponse> {
return this.#withErrorHandling('provider.facebook.list.error', async () => {
const qs: Record<string, string> = {
fields: 'name,cover_photo,created_time,type',
}
const cursor = query?.['cursor'] ?? null
if (typeof cursor === 'string') {
qs['after'] = cursor
}
let path = 'me/albums'
if (directory) {
path = `${directory}/photos`
qs['fields'] = 'icon,images,name,width,height,created_time'
}
const { secret } = (await companion.getProviderCredentials?.())!
if (secret == null) {
throw new Error('Facebook provider secret is not configured')
}
const responses = await runRequestBatch({
secret,
token,
requests: [
{
method: 'GET',
relative_url: `me?${new URLSearchParams({ fields: 'email' }).toString()}`,
},
{ method: 'GET', relative_url: `${path}?${new URLSearchParams(qs)}` },
],
})
const response1 = responses[0]
const response2 = responses[1]
if (!response1 || !response2) {
throw new Error('Unexpected Facebook response: missing batch result')
}
const { email } = response1.body as { email?: string }
const list = response2.body as FacebookListResponse
const currentQuery: Record<string, string> = {}
if (typeof cursor === 'string') currentQuery['cursor'] = cursor
return adaptData(list, email, directory, currentQuery)
})
}
override async download({
companion,
id,
providerUserSession: { accessToken: token },
}: {
companion: CompanionLike
id: string
providerUserSession: FacebookUserSession
}): Promise<{ stream: Readable; size: number | undefined }> {
return this.#withErrorHandling(
'provider.facebook.download.error',
async () => {
const { secret } = (await companion.getProviderCredentials?.())!
if (secret == null) {
throw new Error('Facebook provider secret is not configured')
}
const url = await getMediaUrl({ secret, token, id })
const stream = got.stream.get(url, { responseType: 'json' })
const { size } = await prepareStream(stream)
return { stream, size }
},
)
}
override async thumbnail(): Promise<{
stream: Readable
contentType: string
}> {
// not implementing this because a public thumbnail from facebook will be used instead
logger.error(
'call to thumbnail is not implemented',
'provider.facebook.thumbnail.error',
)
throw new Error('call to thumbnail is not implemented')
}
override async logout({
companion,
providerUserSession: { accessToken: token },
}: {
companion: CompanionLike
providerUserSession: FacebookUserSession
}): Promise<{ revoked: true }> {
return this.#withErrorHandling(
'provider.facebook.logout.error',
async () => {
const { secret } = (await companion.getProviderCredentials?.())!
if (secret == null) {
throw new Error('Facebook provider secret is not configured')
}
await runRequestBatch({
secret,
token,
requests: [{ method: 'DELETE', relative_url: 'me/permissions' }],
})
return { revoked: true }
},
)
}
async #withErrorHandling<T>(tag: string, fn: () => Promise<T>): Promise<T> {
return withProviderErrorHandling({
fn,
tag,
providerName: Facebook.oauthProvider,
isAuthError: (response) =>
isRecord(response.body) &&
isRecord(response.body['error']) &&
response.body['error']['code'] === 190, // Invalid OAuth 2.0 Access Token
getJsonErrorMessage: (body) => {
if (!isRecord(body)) return undefined
const error = body['error']
if (!isRecord(error)) return undefined
const message = error['message']
return typeof message === 'string' ? message : undefined
},
})
}
}

133
packages/@uppy/companion/src/server/provider/google/drive/adapter.js → packages/@uppy/companion/src/server/provider/google/drive/adapter.ts
View file

@ -1,43 +1,82 @@
import querystring from 'node:querystring'
import type { Query } from '../../Provider.js'
const getUsername = (data) => {
return data.user.emailAddress
export type DriveItem = {
kind?: string
mimeType?: string
files?: DriveItem[]
id: string
name?: string
size?: string
thumbnailLink?: string
iconLink?: string
backgroundImageLink?: string
modifiedTime?: string
shortcutDetails?: { targetMimeType?: string }
imageMediaMetadata?: {
height?: number
width?: number
rotation?: number
date?: string
}
videoMediaMetadata?: {
height?: number
width?: number
durationMillis?: number
}
}
export const isGsuiteFile = (mimeType) => {
return mimeType?.startsWith('application/vnd.google')
export type DriveListResponse = {
files?: DriveItem[]
nextPageToken?: string
}
export type DriveSharedDrivesResponse = {
drives?: DriveItem[]
nextPageToken?: unknown
}
export type DriveAbout = { user?: { emailAddress?: string } }
const getUsername = (data: DriveAbout) => {
return data.user?.emailAddress
}
const isSharedDrive = (item) => {
export const isGsuiteFile = (mimeType: string | undefined) =>
mimeType?.startsWith('application/vnd.google')
const isSharedDrive = (item: DriveItem): boolean => {
return item.kind === 'drive#drive'
}
const isFolder = (item) => {
const isFolder = (item: DriveItem): boolean => {
return (
item.mimeType === 'application/vnd.google-apps.folder' ||
isSharedDrive(item)
)
}
export const isShortcut = (mimeType) => {
export const isShortcut = (mimeType: string | undefined): boolean => {
return mimeType === 'application/vnd.google-apps.shortcut'
}
const getItemSize = (item) => {
return parseInt(item.size, 10)
const getItemSize = (item: DriveItem): number => {
const size = item.size != null ? parseInt(item.size, 10) : NaN
return Number.isFinite(size) ? size : 0
}
const getItemIcon = (item) => {
const getItemIcon = (item: DriveItem): string | undefined => {
if (isSharedDrive(item)) {
const size = '=w16-h16-n'
const sizeParamRegex = /=[-whncsp0-9]*$/
return item.backgroundImageLink.match(sizeParamRegex)
? item.backgroundImageLink.replace(sizeParamRegex, size)
: `${item.backgroundImageLink}${size}`
const background = item.backgroundImageLink
if (background == null) return undefined
return background.match(sizeParamRegex)
? background.replace(sizeParamRegex, size)
: `${background}${size}`
}
if (
item.thumbnailLink &&
item.mimeType != null &&
!item.mimeType.startsWith('application/vnd.google')
) {
const smallerThumbnailLink = item.thumbnailLink.replace('s220', 's40')
@ -47,7 +86,7 @@ const getItemIcon = (item) => {
return item.iconLink
}
const getItemSubList = (item) => {
const getItemSubList = (item: DriveListResponse): DriveItem[] => {
const allowedGSuiteTypes = [
'application/vnd.google-apps.document',
'application/vnd.google-apps.drawing',
@ -57,17 +96,18 @@ const getItemSubList = (item) => {
'application/vnd.google-apps.shortcut',
]
return item.files.filter((i) => {
const files = item.files ?? []
return files.filter((i) => {
return (
isFolder(i) ||
!isGsuiteFile(i.mimeType) ||
allowedGSuiteTypes.includes(i.mimeType)
(i.mimeType != null && allowedGSuiteTypes.includes(i.mimeType))
)
})
}
const getItemName = (item) => {
const extensionMaps = {
const getItemName = (item: DriveItem): string => {
const extensionMaps: Record<string, string> = {
'application/vnd.google-apps.document': '.docx',
'application/vnd.google-apps.drawing': '.png',
'application/vnd.google-apps.script': '.json',
@ -75,7 +115,7 @@ const getItemName = (item) => {
'application/vnd.google-apps.presentation': '.ppt',
}
const extension = extensionMaps[item.mimeType]
const extension = item.mimeType ? extensionMaps[item.mimeType] : undefined
if (extension && item.name && !item.name.endsWith(extension)) {
return item.name + extension
}
@ -83,8 +123,8 @@ const getItemName = (item) => {
return item.name ? item.name : '/'
}
export const getGsuiteExportType = (mimeType) => {
const typeMaps = {
export const getGsuiteExportType = (mimeType: string): string => {
const typeMaps: Record<string, string> = {
'application/vnd.google-apps.document':
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.google-apps.drawing': 'image/png',
@ -99,37 +139,41 @@ export const getGsuiteExportType = (mimeType) => {
return typeMaps[mimeType] || 'application/pdf'
}
function getMimeType2(mimeType) {
function getMimeType2(mimeType: string | undefined) {
if (isGsuiteFile(mimeType)) {
return getGsuiteExportType(mimeType)
return getGsuiteExportType(mimeType ?? '')
}
return mimeType
}
const getMimeType = (item) => {
const getMimeType = (item: DriveItem) => {
if (isShortcut(item.mimeType)) {
return getMimeType2(item.shortcutDetails.targetMimeType)
return getMimeType2(item.shortcutDetails?.targetMimeType)
}
return getMimeType2(item.mimeType)
}
const getItemId = (item) => {
const getItemId = (item: DriveItem) => {
return item.id
}
const getItemRequestPath = (item) => {
const getItemRequestPath = (item: DriveItem) => {
return item.id
}
const getItemModifiedDate = (item) => {
const getItemModifiedDate = (item: DriveItem) => {
return item.modifiedTime
}
const getItemThumbnailUrl = (item) => {
const getItemThumbnailUrl = (item: DriveItem) => {
return item.thumbnailLink
}
const getNextPagePath = (data, currentQuery, currentPath) => {
const getNextPagePath = (
data: DriveListResponse,
currentQuery: Query | undefined,
currentPath: string | undefined,
): string | null => {
if (!data.nextPageToken) {
return null
}
@ -137,32 +181,33 @@ const getNextPagePath = (data, currentQuery, currentPath) => {
return `${currentPath}?${querystring.stringify(query)}`
}
const getImageHeight = (item) => item.imageMediaMetadata?.height
const getImageHeight = (item: DriveItem) => item.imageMediaMetadata?.height
const getImageWidth = (item) => item.imageMediaMetadata?.width
const getImageWidth = (item: DriveItem) => item.imageMediaMetadata?.width
const getImageRotation = (item) => item.imageMediaMetadata?.rotation
const getImageRotation = (item: DriveItem) => item.imageMediaMetadata?.rotation
const getImageDate = (item) => item.imageMediaMetadata?.date
const getImageDate = (item: DriveItem) => item.imageMediaMetadata?.date
const getVideoHeight = (item) => item.videoMediaMetadata?.height
const getVideoHeight = (item: DriveItem) => item.videoMediaMetadata?.height
const getVideoWidth = (item) => item.videoMediaMetadata?.width
const getVideoWidth = (item: DriveItem) => item.videoMediaMetadata?.width
const getVideoDurationMillis = (item) => item.videoMediaMetadata?.durationMillis
const getVideoDurationMillis = (item: DriveItem) =>
item.videoMediaMetadata?.durationMillis
// Hopefully this name will not be used by Google
export const VIRTUAL_SHARED_DIR = 'shared-with-me'
export const adaptData = (
listFilesResp,
sharedDrivesResp,
directory,
query,
showSharedWithMe,
about,
listFilesResp: DriveListResponse,
sharedDrivesResp: DriveSharedDrivesResponse | null | undefined,
directory: string | undefined,
query: Query | undefined,
showSharedWithMe: boolean,
about: DriveAbout,
) => {
const adaptItem = (item) => ({
const adaptItem = (item: DriveItem) => ({
isFolder: isFolder(item),
icon: getItemIcon(item),
name: getItemName(item),

143
packages/@uppy/companion/src/server/provider/google/drive/index.js → packages/@uppy/companion/src/server/provider/google/drive/index.ts
View file

@ -1,13 +1,21 @@
import type { Readable } from 'node:stream'
import got from 'got'
import { MAX_AGE_REFRESH_TOKEN } from '../../../helpers/jwt.js'
import { isRecord } from '../../../helpers/type-guards.js'
import { prepareStream } from '../../../helpers/utils.js'
import logger from '../../../logger.js'
import { ProviderAuthError } from '../../error.js'
import Provider from '../../Provider.js'
import Provider, {
type ProviderListResponse,
type Query,
} from '../../Provider.js'
import { withGoogleErrorHandling } from '../../providerErrors.js'
import { logout, refreshToken } from '../index.js'
import {
adaptData,
type DriveAbout,
type DriveListResponse,
type DriveSharedDrivesResponse,
getGsuiteExportType,
isGsuiteFile,
isShortcut,
@ -18,7 +26,7 @@ import {
// first run a download with mockAccessTokenExpiredError = true
// then when you want to test expiry, set to mockAccessTokenExpiredError to the logged access token
// This will trigger companion/nodemon to restart, and it will respond with a simulated invalid token response
const mockAccessTokenExpiredError = undefined
const mockAccessTokenExpiredError: string | true | undefined = undefined
// const mockAccessTokenExpiredError = true
// const mockAccessTokenExpiredError = ''
@ -28,7 +36,7 @@ const DRIVE_FILES_FIELDS = `kind,nextPageToken,incompleteSearch,files(${DRIVE_FI
// using wildcard to get all 'drive' fields because specifying fields seems no to work for the /drives endpoint
const SHARED_DRIVE_FIELDS = '*'
const getClient = ({ token }) =>
const getClient = ({ token }: { token: string }) =>
got.extend({
prefixUrl: 'https://www.googleapis.com/drive/v3',
headers: {
@ -36,31 +44,58 @@ const getClient = ({ token }) =>
},
})
async function getStats({ id, token }) {
type DriveFileStats = {
mimeType?: unknown
id?: unknown
exportLinks?: unknown
shortcutDetails?: unknown
} & Record<string, unknown>
async function getStats({ id, token }: { id: string; token: string }) {
const client = getClient({ token })
const getStatsInner = async (statsOfId) =>
const getStatsInner = async (statsOfId: string) =>
client
.get(`files/${encodeURIComponent(statsOfId)}`, {
searchParams: { fields: DRIVE_FILE_FIELDS, supportsAllDrives: true },
responseType: 'json',
})
.json()
.json<DriveFileStats>()
const stats = await getStatsInner(id)
// If it is a shortcut, we need to get stats again on the target
if (isShortcut(stats.mimeType))
return getStatsInner(stats.shortcutDetails.targetId)
const mimeType =
typeof stats.mimeType === 'string' ? stats.mimeType : undefined
if (mimeType && isShortcut(mimeType)) {
const shortcutDetails = stats.shortcutDetails
if (
isRecord(shortcutDetails) &&
typeof shortcutDetails['targetId'] === 'string'
) {
return getStatsInner(shortcutDetails['targetId'])
}
}
return stats
}
export async function streamGoogleFile({ token, id: idIn }) {
export async function streamGoogleFile({
token,
id: idIn,
}: {
token: string
id: string
}) {
const client = getClient({ token })
const { mimeType, id, exportLinks } = await getStats({ id: idIn, token })
const stats = await getStats({ id: idIn, token })
const mimeType = typeof stats.mimeType === 'string' ? stats.mimeType : ''
const id = typeof stats.id === 'string' ? stats.id : `${idIn}`
const exportLinks = isRecord(stats.exportLinks)
? stats.exportLinks
: undefined
let stream
let stream: Readable
if (isGsuiteFile(mimeType)) {
const mimeType2 = getGsuiteExportType(mimeType)
@ -73,7 +108,10 @@ export async function streamGoogleFile({ token, id: idIn }) {
// Error message: "This file is too large to be exported.".
// Issue logged in Google APIs: https://github.com/googleapis/google-api-nodejs-client/issues/3446
// Implemented based on the answer from StackOverflow: https://stackoverflow.com/a/59168288
const mimeTypeExportLink = exportLinks?.[mimeType2]
const mimeTypeExportLink =
exportLinks && typeof exportLinks[mimeType2] === 'string'
? exportLinks[mimeType2]
: undefined
if (mimeTypeExportLink) {
stream = got.stream.get(mimeTypeExportLink, {
headers: {
@ -98,36 +136,64 @@ export async function streamGoogleFile({ token, id: idIn }) {
return { stream, size }
}
interface DriveUserSession {
accessToken: string
}
/**
* Adapter for API https://developers.google.com/drive/api/v3/
*/
export class Drive extends Provider {
static get oauthProvider() {
export class Drive extends Provider<DriveUserSession> {
static override get oauthProvider() {
return 'googledrive'
}
static get authStateExpiry() {
static override get authStateExpiry() {
return MAX_AGE_REFRESH_TOKEN
}
async list(options) {
// Define these as real methods (not prototype assignment), so we don't risk
// instance fields shadowing the prototype in downlevel transpiles.
override logout(args: Parameters<typeof logout>[0]) {
return logout(args)
}
override refreshToken(args: {
redirectUri: string | undefined
clientId: string | undefined
clientSecret: string | undefined
refreshToken: string
}) {
return refreshToken(args)
}
override async list({
directory: directoryIn,
providerUserSession: { accessToken: token },
query,
}: {
directory?: string | undefined
providerUserSession: DriveUserSession
query?: Query | undefined
}): Promise<ProviderListResponse> {
return withGoogleErrorHandling(
Drive.oauthProvider,
'provider.drive.list.error',
async () => {
const directory = options.directory || 'root'
const query = options.query || {}
const {
providerUserSession: { accessToken: token },
} = options
const cursor =
typeof query?.['cursor'] === 'string' ? query['cursor'] : undefined
const directory = directoryIn || 'root'
const isRoot = directory === 'root'
const isVirtualSharedDirRoot = directory === VIRTUAL_SHARED_DIR
const client = getClient({ token })
async function fetchSharedDrives(pageToken = null) {
const shouldListSharedDrives = isRoot && !query.cursor
async function fetchSharedDrives(
pageToken: string | null = null,
): Promise<DriveSharedDrivesResponse | undefined> {
const shouldListSharedDrives = isRoot && !cursor
if (!shouldListSharedDrives) return undefined
const response = await client
@ -139,15 +205,21 @@ export class Drive extends Provider {
},
responseType: 'json',
})
.json()
.json<DriveSharedDrivesResponse>()
const { nextPageToken } = response
const nextPageToken =
typeof response.nextPageToken === 'string'
? response.nextPageToken
: undefined
if (nextPageToken) {
const nextResponse = await fetchSharedDrives(nextPageToken)
if (!nextResponse) return response
return {
...nextResponse,
drives: [...response.drives, ...nextResponse.drives],
drives: [
...(response.drives ?? []),
...(nextResponse.drives ?? []),
],
}
}
@ -162,7 +234,7 @@ export class Drive extends Provider {
const searchParams = {
fields: DRIVE_FILES_FIELDS,
pageToken: query.cursor,
pageToken: cursor,
q,
// We can only do a page size of 1000 because we do not request permissions in DRIVE_FILES_FIELDS.
// Otherwise we are limited to 100. Instead we get the user info from `this.user()`
@ -174,7 +246,7 @@ export class Drive extends Provider {
return client
.get('files', { searchParams, responseType: 'json' })
.json()
.json<DriveListResponse>()
}
async function fetchAbout() {
@ -182,7 +254,7 @@ export class Drive extends Provider {
return client
.get('about', { searchParams, responseType: 'json' })
.json()
.json<DriveAbout>()
}
const [sharedDrives, filesResponse, about] = await Promise.all([
@ -196,14 +268,20 @@ export class Drive extends Provider {
sharedDrives,
directory,
query,
isRoot && !query.cursor, // we can only show it on the first page request, or else we will have duplicates of it
isRoot && !cursor, // we can only show it on the first page request, or else we will have duplicates of it
about,
)
},
)
}
async download({ id, providerUserSession: { accessToken: token } }) {
override async download({
id,
providerUserSession: { accessToken: token },
}: {
id: string
providerUserSession: DriveUserSession
}): Promise<{ stream: Readable; size: number | undefined }> {
if (mockAccessTokenExpiredError != null) {
logger.warn(`Access token: ${token}`)
@ -222,6 +300,3 @@ export class Drive extends Provider {
)
}
}
Drive.prototype.logout = logout
Drive.prototype.refreshToken = refreshToken

20
packages/@uppy/companion/src/server/provider/google/index.js → packages/@uppy/companion/src/server/provider/google/index.ts
View file

@ -14,12 +14,16 @@ export async function refreshToken({
clientId,
clientSecret,
refreshToken: theRefreshToken,
}) {
}: {
clientId: string | undefined
clientSecret: string | undefined
refreshToken: string
}): Promise<{ accessToken: string }> {
return withGoogleErrorHandling(
'google',
'provider.google.token.refresh.error',
async () => {
const { access_token: accessToken } = await getOauthClient()
const tokenRes = await getOauthClient()
.post('token', {
responseType: 'json',
form: {
@ -29,13 +33,21 @@ export async function refreshToken({
client_secret: clientSecret,
},
})
.json()
.json<{ access_token?: unknown }>()
const accessToken = tokenRes.access_token
if (typeof accessToken !== 'string' || accessToken.length === 0) {
throw new Error('Missing access_token')
}
return { accessToken }
},
)
}
export async function logout({ providerUserSession: { accessToken: token } }) {
export async function logout({
providerUserSession: { accessToken: token },
}: {
providerUserSession: { accessToken: string }
}): Promise<{ revoked: true }> {
return withGoogleErrorHandling(
'google',
'provider.google.logout.error',

202
packages/@uppy/companion/src/server/provider/index.js
View file

@ -1,202 +0,0 @@
/**
* @module provider
*/
import { getRedirectPath, getURLBuilder } from '../helpers/utils.js'
import * as logger from '../logger.js'
import box from './box/index.js'
import { getCredentialsResolver } from './credentials.js'
import dropbox from './dropbox/index.js'
import facebook from './facebook/index.js'
import { Drive } from './google/drive/index.js'
import instagram from './instagram/graph/index.js'
import onedrive from './onedrive/index.js'
// biome-ignore lint/correctness/noUnusedImports: It's used as a type
import Provider, { isOAuthProvider } from './Provider.js'
import unsplash from './unsplash/index.js'
import webdav from './webdav/index.js'
import zoom from './zoom/index.js'
/**
*
* @param {{server: object}} options
*/
const validOptions = (options) => {
return options.server.host && options.server.protocol
}
/**
* adds the desired provider module to the request object,
* based on the providerName parameter specified
*
* @param {Record<string, typeof Provider>} providers
*/
export function getProviderMiddleware(providers, grantConfig) {
/**
*
* @param {object} req
* @param {object} res
* @param {Function} next
* @param {string} providerName
*/
const middleware = (req, res, next, providerName) => {
const ProviderClass = providers[providerName]
if (ProviderClass && validOptions(req.companion.options)) {
const { allowLocalUrls, providerOptions } = req.companion.options
const { oauthProvider } = ProviderClass
let providerGrantConfig
if (isOAuthProvider(oauthProvider)) {
req.companion.getProviderCredentials = getCredentialsResolver(
providerName,
req.companion.options,
req,
)
providerGrantConfig = grantConfig[oauthProvider]
req.companion.providerGrantConfig = providerGrantConfig
}
const secret = providerOptions[providerName]?.secret
req.companion.provider = new ProviderClass({
secret,
providerName,
providerGrantConfig,
allowLocalUrls,
})
req.companion.providerName = providerName
req.companion.providerClass = ProviderClass
} else {
logger.warn(
'invalid provider options detected. Provider will not be loaded',
'provider.middleware.invalid',
req.id,
)
}
next()
}
return middleware
}
/**
* @returns {Record<string, typeof Provider>}
*/
export function getDefaultProviders() {
const providers = {
dropbox,
box,
drive: Drive,
facebook,
onedrive,
zoom,
instagram,
unsplash,
webdav,
}
return providers
}
/**
*
* @typedef {{'module': typeof Provider, config: Record<string,unknown>}} CustomProvider
*
* @param {Record<string, CustomProvider>} customProviders
* @param {Record<string, typeof Provider>} providers
* @param {object} grantConfig
*/
export function addCustomProviders(customProviders, providers, grantConfig) {
Object.keys(customProviders).forEach((providerName) => {
const customProvider = customProviders[providerName]
providers[providerName] = customProvider.module
const { oauthProvider } = customProvider.module
if (isOAuthProvider(oauthProvider)) {
grantConfig[oauthProvider] = {
...customProvider.config,
// todo: consider setting these options from a universal point also used
// by official providers. It'll prevent these from getting left out if the
// requirement changes.
callback: `/${providerName}/callback`,
transport: 'session',
}
}
})
}
/**
*
* @param {{server: object, providerOptions: object}} companionOptions
* @param {object} grantConfig
* @param {(a: string) => string} getOauthProvider
*/
export function addProviderOptions(
companionOptions,
grantConfig,
getOauthProvider,
) {
const { server, providerOptions } = companionOptions
if (!validOptions({ server })) {
logger.warn(
'invalid provider options detected. Providers will not be loaded',
'provider.options.invalid',
)
return
}
grantConfig.defaults = {
host: server.host,
protocol: server.protocol,
path: server.path,
}
const { oauthDomain } = server
const keys = Object.keys(providerOptions).filter((key) => key !== 'server')
keys.forEach((providerName) => {
const oauthProvider = getOauthProvider?.(providerName)
if (isOAuthProvider(oauthProvider) && grantConfig[oauthProvider]) {
// explicitly add providerOptions so users don't override other providerOptions.
grantConfig[oauthProvider].key = providerOptions[providerName].key
grantConfig[oauthProvider].secret = providerOptions[providerName].secret
if (providerOptions[providerName].credentialsURL) {
grantConfig[oauthProvider].dynamic = [
'key',
'secret',
'redirect_uri',
'origins',
]
}
const provider = getDefaultProviders()[providerName]
Object.assign(grantConfig[oauthProvider], provider.getExtraGrantConfig())
// override grant.js redirect uri with companion's custom redirect url
const isExternal = !!server.implicitPath
const redirectPath = getRedirectPath(providerName)
grantConfig[oauthProvider].redirect_uri = getURLBuilder(companionOptions)(
redirectPath,
isExternal,
)
if (oauthDomain) {
const fullRedirectPath = getURLBuilder(companionOptions)(
redirectPath,
isExternal,
true,
)
grantConfig[oauthProvider].redirect_uri =
`${server.protocol}://${oauthDomain}${fullRedirectPath}`
}
if (server.implicitPath) {
// no url builder is used for this because grant internally adds the path
grantConfig[oauthProvider].callback =
`${server.implicitPath}${grantConfig[oauthProvider].callback}`
} else if (server.path) {
grantConfig[oauthProvider].callback =
`${server.path}${grantConfig[oauthProvider].callback}`
}
}
})
}

Some files were not shown because too many files have changed in this diff Show more