Mirrors/uppy
1
0
Fork 0
mirror of https://github.com/transloadit/uppy.git synced 2026-01-23 02:25:07 +00:00
Code Issues Projects Releases Wiki Activity

Re-use types from the Transloadit node-sdk (#5992)

Browse source 
The schemas and types that we have in the Transloadit Node.js SDK v4 are
used in our API's system tests. We've also ran hundreds of thousands of
Assemblies through them, ever loosening them, until they all fit. This
means the schemas are fairly wide, but model the reality of our 15 year
old API. In the future we will make schema failures in the API fatal (as
already is the case with system tests), and we don't want to break
production traffic when we do. So we accept wider schemas than are
beautiful, and once the schemas control what is allowed in all places,
we gradually evolve the API and schemas towards being more pretty in
lockstep.

More on this in
https://transloadit.com/blog/2025/09/nodejs-sdk-v4/#our-approach-to-type-retrofitting

For uppy this means, we'll need a few more guards than we had with our
handrolled types, that actually assumed things that turned out to be not
true in all cases. Not all Assembly status responses have an id or a url
for one example. There are for instance particular errors (by Node,
Nginx, Haproxy) that would not return those. The added guards will
ensure we don't break deeply inside customer code.

This PR was completely written by gpt-5-codex, which means it was faster
and of higher quality than if I had handrolled it as a founder
unfamiliar with this codebase, but despite of that, please still review
my contribution with as much care as you would normally :)

---------

Co-authored-by: Mikael Finstad <finstaden@gmail.com>
Co-authored-by: Merlijn Vos <merlijn@soverin.net>
This commit is contained in:
Kevin van Zonneveld 2025-09-30 19:15:39 +02:00 • committed by GitHub
parent 34639ba768
commit 6f764122a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 1522 additions and 166 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.changeset/transloadit-types-from-sdk.md Normal file
View file

@ -0,0 +1,5 @@
---
"@uppy/transloadit": minor
---
Use the Transloadit Node.js SDK's exported Assembly typings, removing Uppy-handrolled ones

1
packages/@uppy/transloadit/package.json
View file

@ -55,6 +55,7 @@
"@uppy/core": "workspace:^",
"jsdom": "^26.1.0",
"msw": "^2.10.4",
"transloadit": "^4.0.2",
"typescript": "^5.8.3",
"vitest": "^3.2.4",
"whatwg-fetch": "^3.6.2"

76
packages/@uppy/transloadit/src/Assembly.ts
View file

@ -6,7 +6,12 @@ import {
NetworkError,
} from '@uppy/utils'
import Emitter from 'component-emitter'
import type { AssemblyResponse } from './index.js'
import {
type AssemblyFile,
type AssemblyResponse,
type AssemblyResult,
getAssemblyUrlSsl,
} from './index.js'
const ASSEMBLY_UPLOADING = 'ASSEMBLY_UPLOADING'
const ASSEMBLY_EXECUTING = 'ASSEMBLY_EXECUTING'
@ -24,7 +29,11 @@ const statusOrder = [ASSEMBLY_UPLOADING, ASSEMBLY_EXECUTING, ASSEMBLY_COMPLETED]
* …so that we can emit the 'executing' event even if the execution step was so
* fast that we missed it.
*/
function isStatus(status: string, test: string) {
function isStatus(status: unknown, test: string) {
if (typeof status !== 'string') {
return false
}
return statusOrder.indexOf(status) >= statusOrder.indexOf(test)
}
@ -101,16 +110,22 @@ class TransloaditAssembly extends Emitter {
})
this.#sse.addEventListener('assembly_upload_finished', (e) => {
const file = JSON.parse(e.data)
const file = JSON.parse(e.data) as AssemblyFile
this.status.uploads ??= []
this.status.uploads.push(file)
this.emit('upload', file)
})
this.#sse.addEventListener('assembly_result_finished', (e) => {
const [stepName, result] = JSON.parse(e.data)
// biome-ignore lint/suspicious/noAssignInExpressions: ...
;(this.status.results[stepName] ??= []).push(result)
this.emit('result', stepName, result)
const [stepName, rawResult] = JSON.parse(e.data) as [
string,
AssemblyResult,
]
this.status.results ??= {}
this.status.results[stepName] ??= []
this.status.results[stepName].push(rawResult)
this.emit('result', stepName, rawResult)
})
this.#sse.addEventListener('assembly_execution_progress', (e) => {
@ -165,9 +180,9 @@ class TransloaditAssembly extends Emitter {
try {
this.#previousFetchStatusStillPending = true
const response = await this.#fetchWithNetworkError(
this.status.assembly_ssl_url,
)
const statusUrl = getAssemblyUrlSsl(this.status)
const response = await this.#fetchWithNetworkError(statusUrl)
this.#previousFetchStatusStillPending = false
if (this.closed) return
@ -244,29 +259,38 @@ class TransloaditAssembly extends Emitter {
}
// Only emit if the upload is new (not in prev.uploads).
Object.keys(next.uploads)
.filter((upload) => !has(prev.uploads, upload))
.forEach((upload) => {
// @ts-ignore either the types are wrong or the tests are wrong.
// types think next.uploads is an array, but the tests pass an object.
this.emit('upload', next.uploads[upload])
})
const prevUploads = prev.uploads
const nextUploads = next.uploads
if (nextUploads != null && prevUploads != null) {
Object.keys(nextUploads)
.filter((upload) => !has(prevUploads, upload))
.forEach((upload) => {
// This is a bit confusing. Not sure why Object.keys was chosen here, because nextUploads is an Array. Object.keys returns strings for array keys ("0", "1", etc.). Typescript expects arrays to be indexed with a number, not a string nextUploads[0], even though JavaScript is fine with it, so we need to type assert here:
this.emit('upload', nextUploads[upload as unknown as number])
})
}
if (nowExecuting) {
this.emit('metadata')
}
// Find new results.
Object.keys(next.results).forEach((stepName) => {
const nextResults = next.results[stepName]
const prevResults = prev.results[stepName]
const nextResultsMap = next.results
const prevResultsMap = prev.results
if (nextResultsMap != null && prevResultsMap != null) {
Object.keys(nextResultsMap).forEach((stepName) => {
const nextResults = nextResultsMap[stepName] ?? []
const prevResults = prevResultsMap[stepName] ?? []
nextResults
.filter((n) => !prevResults || !prevResults.some((p) => p.id === n.id))
.forEach((result) => {
this.emit('result', stepName, result)
})
})
nextResults
.filter(
(n) => !prevResults || !prevResults.some((p) => p.id === n.id),
)
.forEach((result) => {
this.emit('result', stepName, result)
})
})
}
if (
isStatus(nextStatus, ASSEMBLY_COMPLETED) &&

20
packages/@uppy/transloadit/src/AssemblyWatcher.ts
View file

@ -49,21 +49,23 @@ class TransloaditAssemblyWatcher<
}
#onAssemblyComplete = (assembly: AssemblyResponse) => {
if (!this.#watching(assembly.assembly_id)) {
const assemblyId = assembly.assembly_id
if (assemblyId == null || !this.#watching(assemblyId)) {
return
}
this.#uppy.log(
`[Transloadit] AssemblyWatcher: Got Assembly finish ${assembly.assembly_id}`,
`[Transloadit] AssemblyWatcher: Got Assembly finish ${assemblyId}`,
)
this.emit('assembly-complete', assembly.assembly_id)
this.emit('assembly-complete', assemblyId)
this.#checkAllComplete()
}
#onAssemblyCancel = (assembly: AssemblyResponse) => {
if (!this.#watching(assembly.assembly_id)) {
const assemblyId = assembly.assembly_id
if (assemblyId == null || !this.#watching(assemblyId)) {
return
}
@ -71,16 +73,17 @@ class TransloaditAssemblyWatcher<
}
#onAssemblyError = (assembly: AssemblyResponse, error: Error) => {
if (!this.#watching(assembly.assembly_id)) {
const assemblyId = assembly.assembly_id
if (assemblyId == null || !this.#watching(assemblyId)) {
return
}
this.#uppy.log(
`[Transloadit] AssemblyWatcher: Got Assembly error ${assembly.assembly_id}`,
`[Transloadit] AssemblyWatcher: Got Assembly error ${assemblyId}`,
)
this.#uppy.log(error)
this.emit('assembly-error', assembly.assembly_id, error)
this.emit('assembly-error', assemblyId, error)
this.#checkAllComplete()
}
@ -90,7 +93,8 @@ class TransloaditAssemblyWatcher<
fileID: string,
error: Error,
) => {
if (!this.#watching(assembly.assembly_id)) {
const assemblyId = assembly.assembly_id
if (assemblyId == null || !this.#watching(assemblyId)) {
return
}

15
packages/@uppy/transloadit/src/Client.ts
View file

@ -6,9 +6,10 @@ import type {
WrapPromiseFunctionType,
} from '@uppy/utils'
import { fetchWithNetworkError } from '@uppy/utils'
import type {
AssemblyResponse,
OptionsWithRestructuredFields,
import {
type AssemblyResponse,
getAssemblyUrlSsl,
type OptionsWithRestructuredFields,
} from './index.js'
const ASSEMBLIES_ENDPOINT = '/assemblies'
@ -139,7 +140,8 @@ export default class Client<M extends Meta, B extends Body> {
file: UppyFile<M, B>,
): Promise<AssemblyResponse> {
const size = encodeURIComponent(file.size!)
const url = `${assembly.assembly_ssl_url}/reserve_file?size=${size}`
const assemblyUrl = getAssemblyUrlSsl(assembly)
const url = `${assemblyUrl}/reserve_file?size=${size}`
return this.#fetchJSON(url, {
method: 'POST',
headers: this.#headers,
@ -164,7 +166,8 @@ export default class Client<M extends Meta, B extends Body> {
const fieldname = 'file'
const qs = `size=${size}&filename=${filename}&fieldname=${fieldname}&s3Url=${uploadUrl}`
const url = `${assembly.assembly_ssl_url}/add_file?${qs}`
const assemblyUrl = getAssemblyUrlSsl(assembly)
const url = `${assemblyUrl}/add_file?${qs}`
return this.#fetchJSON(url, {
method: 'POST',
headers: this.#headers,
@ -177,7 +180,7 @@ export default class Client<M extends Meta, B extends Body> {
* Cancel a running Assembly.
*/
async cancelAssembly(assembly: AssemblyResponse): Promise<void> {
const url = assembly.assembly_ssl_url
const url = getAssemblyUrlSsl(assembly)
await this.#fetchWithNetworkError(url, {
method: 'DELETE',
headers: this.#headers,

237
packages/@uppy/transloadit/src/index.ts
View file

@ -9,111 +9,25 @@ import type {
import { BasePlugin } from '@uppy/core'
import Tus, { type TusDetailedError, type TusOpts } from '@uppy/tus'
import { ErrorWithCause, hasProperty, RateLimitedQueue } from '@uppy/utils'
import type {
AssemblyStatus,
AssemblyStatusResult,
AssemblyStatusUpload,
CreateAssemblyParams,
} from 'transloadit'
import packageJson from '../package.json' with { type: 'json' }
import Assembly from './Assembly.js'
import AssemblyWatcher from './AssemblyWatcher.js'
import Client, { type AssemblyError } from './Client.js'
import locale from './locale.js'
export interface AssemblyFile {
id: string
name: string
basename: string
ext: string
size: number
mime: string
type: string
field: string
md5hash: string
is_tus_file: boolean
original_md5hash: string
original_id: string
original_name: string
original_basename: string
original_path: string
url: string
ssl_url: string
tus_upload_url: string
meta: Record<string, any>
}
export interface AssemblyResult extends AssemblyFile {
cost: number
execTime: number
queue: string
queueTime: number
localId: string | null
user_meta?: Record<string, string>
}
export interface AssemblyResponse {
ok: string
message?: string
assembly_id: string
parent_id?: string
account_id: string
template_id?: string
instance: string
assembly_url: string
assembly_ssl_url: string
uppyserver_url: string
companion_url: string
websocket_url: string
tus_url: string
bytes_received: number
bytes_expected: number
upload_duration: number
client_agent?: string
client_ip?: string
client_referer?: string
transloadit_client: string
start_date: string
upload_meta_data_extracted: boolean
warnings: any[]
is_infinite: boolean
has_dupe_jobs: boolean
execution_start: string
execution_duration: number
execution_progress?: number
queue_duration: number
jobs_queue_duration: number
notify_start?: any
notify_url?: string
notify_status?: any
notify_response_code?: any
notify_duration?: any
last_job_completed?: string
fields: Record<string, any>
running_jobs: any[]
bytes_usage: number
executing_jobs: any[]
started_jobs: string[]
parent_assembly_status: any
params: string
template?: any
merged_params: string
uploads: AssemblyFile[]
results: Record<string, AssemblyResult[]>
build_id: string
error?: string
stderr?: string
stdout?: string
reason?: string
}
export interface AssemblyParameters {
auth: {
key: string
expires?: string
}
template_id?: string
steps?: { [step: string]: Record<string, unknown> }
fields?: { [name: string]: number | string }
notify_url?: string
}
export type AssemblyResponse = AssemblyStatus
export type AssemblyFile = AssemblyStatusUpload
export type AssemblyResult = AssemblyStatusResult & { localId: string | null }
export type AssemblyParameters = CreateAssemblyParams
export interface AssemblyOptions {
params?: AssemblyParameters | null
params?: AssemblyParameters | string | null
fields?: Record<string, string | number> | string[] | null
signature?: string | null
}
@ -229,14 +143,15 @@ const sendErrorToConsole = (originalErr: Error) => (err: Error) => {
console.error(error, originalErr)
}
function validateParams(params?: AssemblyParameters | null): void {
function validateParams(params?: AssemblyOptions['params']): void {
if (params == null) {
throw new Error('Transloadit: The `params` option is required.')
}
let parsed: AssemblyParameters
if (typeof params === 'string') {
try {
params = JSON.parse(params)
parsed = JSON.parse(params) as AssemblyParameters
} catch (err) {
// Tell the user that this is not an Uppy bug!
throw new ErrorWithCause(
@ -244,9 +159,11 @@ function validateParams(params?: AssemblyParameters | null): void {
{ cause: err },
)
}
} else {
parsed = params
}
if (!params!.auth || !params!.auth.key) {
if (!parsed.auth || !parsed.auth.key) {
throw new Error(
'Transloadit: The `params.auth.key` option is required. ' +
'You can find your Transloadit API key at https://transloadit.com/c/template-credentials',
@ -254,6 +171,46 @@ function validateParams(params?: AssemblyParameters | null): void {
}
}
function ensureAssemblyId(status: AssemblyResponse): string {
if (!status.assembly_id) {
console.warn('Assembly status is missing `assembly_id`.', status)
throw new Error('Transloadit: Assembly status is missing `assembly_id`.')
}
return status.assembly_id
}
function ensureUrl(
label: string,
...candidates: Array<string | undefined>
): string {
for (const value of candidates) {
if (typeof value === 'string' && value.length > 0) {
return value
}
}
throw new Error(`Transloadit: Assembly status is missing ${label}.`)
}
export function getAssemblyUrl(
assembly: Pick<AssemblyResponse, 'assembly_ssl_url' | 'assembly_url'>,
): string {
return ensureUrl(
'`assembly_url`',
assembly.assembly_url,
assembly.assembly_ssl_url,
)
}
export function getAssemblyUrlSsl(
assembly: Pick<AssemblyResponse, 'assembly_ssl_url' | 'assembly_url'>,
): string {
return ensureUrl(
'`assembly_ssl_url`',
assembly.assembly_ssl_url,
assembly.assembly_url,
)
}
const COMPANION_URL = 'https://api2.transloadit.com/companion'
// Regex matching acceptable postMessage() origins for authentication feedback from companion.
const COMPANION_ALLOWED_HOSTS = /\.transloadit\.com$/
@ -352,16 +309,21 @@ export default class Transloadit<
*/
#attachAssemblyMetadata(file: UppyFile<M, B>, status: AssemblyResponse) {
// Add the metadata parameters Transloadit needs.
const assemblyUrl = getAssemblyUrl(status)
const tusEndpoint = ensureUrl('`tus_url`', status.tus_url)
const assemblyId = ensureAssemblyId(status)
const meta = {
...file.meta,
assembly_url: status.assembly_url,
// @TODO(tim-kos), can we safely bump this to assembly_ssl_url / getAssemblyUrlSsl?
assembly_url: assemblyUrl,
filename: file.name,
fieldname: 'file',
}
// Add Assembly-specific Tus endpoint.
const tus = {
...file.tus,
endpoint: status.tus_url,
endpoint: tusEndpoint,
// Include X-Request-ID headers for better debugging.
addRequestId: true,
}
@ -372,7 +334,11 @@ export default class Transloadit<
// people can also self-host them while still using Transloadit for encoding.
let { remote } = file
if (file.remote && TL_COMPANION.test(file.remote.companionUrl)) {
if (
file.remote &&
status.companion_url &&
TL_COMPANION.test(file.remote.companionUrl)
) {
const newHost = status.companion_url.replace(/\/$/, '')
const path = file.remote.url
.replace(file.remote.companionUrl, '')
@ -389,7 +355,7 @@ export default class Transloadit<
const newFile = {
...file,
transloadit: {
assembly: status.assembly_id,
assembly: assemblyId,
},
}
// Only configure the Tus plugin if we are uploading straight to Transloadit (the default).
@ -423,7 +389,7 @@ export default class Transloadit<
const assembly = new Assembly(newAssembly, this.#rateLimitedQueue)
const { status } = assembly
const assemblyID = status.assembly_id
const assemblyID = ensureAssemblyId(status)
const updatedFiles: Record<string, UppyFile<M, B>> = {}
files.forEach((file) => {
@ -582,9 +548,15 @@ export default class Transloadit<
#onResult(assemblyId: string, stepName: string, result: AssemblyResult) {
const state = this.getPluginState()
const file = state.files[result.original_id]
// The `file` may not exist if an import robot was used instead of a file upload.
result.localId = file ? file.id : null
if (!('id' in result)) {
console.warn('Result has no id', result)
return
}
if (typeof result.id !== 'string') {
console.warn('Result has no id of type string', result)
return
}
const entry = {
result,
@ -596,7 +568,12 @@ export default class Transloadit<
this.setPluginState({
results: [...state.results, entry],
})
this.uppy.emit('transloadit:result', stepName, result, this.getAssembly()!)
this.uppy.emit(
'transloadit:result',
stepName,
entry.result,
this.getAssembly()!,
)
}
/**
@ -604,7 +581,7 @@ export default class Transloadit<
* and emit it.
*/
#onAssemblyFinished(assembly: Assembly) {
const url = assembly.status.assembly_ssl_url
const url = getAssemblyUrl(assembly.status)
this.client.getAssemblyStatus(url).then((finalStatus) => {
assembly.status = finalStatus
this.uppy.emit('transloadit:complete', finalStatus)
@ -665,9 +642,9 @@ export default class Transloadit<
id: string
assembly: string
}[] = []
const { assembly_id: id } = previousAssembly
const id = ensureAssemblyId(previousAssembly)
previousAssembly.uploads.forEach((uploadedFile) => {
previousAssembly.uploads?.forEach((uploadedFile) => {
const file = this.#findFile(uploadedFile)
files[uploadedFile.id] = {
id: file!.id,
@ -677,13 +654,31 @@ export default class Transloadit<
})
const state = this.getPluginState()
Object.keys(previousAssembly.results).forEach((stepName) => {
for (const result of previousAssembly.results[stepName]) {
const restoredResults = previousAssembly.results ?? {}
Object.keys(restoredResults).forEach((stepName) => {
const stepResults = restoredResults[stepName] ?? []
for (const result of stepResults) {
if (!('id' in result)) {
console.warn('Result has no id', result)
continue
}
if (typeof result.id !== 'string') {
console.warn('Result has no id of type string', result)
continue
}
if (!('original_id' in result)) {
console.warn('Result has no original_id', result)
continue
}
if (typeof result.original_id !== 'string') {
console.warn('Result has no original_id of type string', result)
continue
}
const file = state.files[result.original_id]
result.localId = file ? file.id : null
results.push({
id: result.id,
result,
result: { ...result, localId: file ? file.id : null },
stepName,
assembly: id,
})
@ -698,7 +693,7 @@ export default class Transloadit<
// Set up the Assembly instances and AssemblyWatchers for existing Assemblies.
const restoreAssemblies = (ids: string[]) => {
this.#createAssemblyWatcher(previousAssembly.assembly_id)
this.#createAssemblyWatcher(ensureAssemblyId(previousAssembly))
this.#connectAssembly(this.assembly!, ids)
}
@ -722,7 +717,7 @@ export default class Transloadit<
#connectAssembly(assembly: Assembly, ids: UppyFile<M, B>['id'][]) {
const { status } = assembly
const id = status.assembly_id
const id = ensureAssemblyId(status)
this.assembly = assembly
assembly.on('upload', (file: AssemblyFile) => {
@ -814,7 +809,7 @@ export default class Transloadit<
const file = this.uppy.getFile(fileID)
this.uppy.emit('preprocess-complete', file)
})
this.#createAssemblyWatcher(assembly.status.assembly_id)
this.#createAssemblyWatcher(ensureAssemblyId(assembly.status))
this.#connectAssembly(assembly, fileIDs)
} catch (err) {
fileIDs.forEach((fileID) => {
@ -838,7 +833,9 @@ export default class Transloadit<
// Only use files without errors
.filter((file) => !file.error)
const assemblyID = this.assembly?.status.assembly_id
const assemblyID = this.assembly
? ensureAssemblyId(this.assembly.status)
: undefined
const closeSocketConnections = () => {
this.assembly?.close()

1334
yarn.lock
View file

File diff suppressed because it is too large Load diff