From 260329c3718ecf7c03de732cbfa5eede2145e483 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 29 Mar 2026 18:02:42 +0800 Subject: [PATCH] simplify and document minio tests --- .github/CONTRIBUTING.md | 4 + .github/workflows/ci.yml | 2 +- .../@uppy/aws-s3/tests/s3-client/config.ts | 14 + .../tests/s3-client/{docker.js => docker.ts} | 67 +- .../aws-s3/tests/s3-client/minio.test.ts | 629 ++++++++++++------ .../@uppy/aws-s3/tests/s3-client/setup.js | 75 --- .../@uppy/aws-s3/tests/s3-client/setup.ts | 48 ++ .../@uppy/aws-s3/tests/s3-client/teardown.ts | 30 - .../@uppy/aws-s3/tests/s3-client/test-util.ts | 270 -------- packages/@uppy/aws-s3/vitest.config.ts | 12 +- turbo.json | 2 +- 11 files changed, 525 insertions(+), 628 deletions(-) create mode 100644 packages/@uppy/aws-s3/tests/s3-client/config.ts rename packages/@uppy/aws-s3/tests/s3-client/{docker.js => docker.ts} (57%) delete mode 100644 packages/@uppy/aws-s3/tests/s3-client/setup.js create mode 100644 packages/@uppy/aws-s3/tests/s3-client/setup.ts delete mode 100644 packages/@uppy/aws-s3/tests/s3-client/teardown.ts delete mode 100644 packages/@uppy/aws-s3/tests/s3-client/test-util.ts diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 22ea1f915..91272bce3 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -47,6 +47,10 @@ yarn build - `yarn typecheck` - Run TypeScript type checking - `yarn check` - Run Biome linting and formatting +### Running minio tests + +Tests for `@uppy/aws-s3` need a minio server running inside Docker, and in order to run those tests you must set the environment variable `VITE_MINIO_CONFIG="test_user,test_password,http://localhost:9002/test-bucket,us-east-1"` when running the tests. + ### Headless components When adding a new component to `@uppy/components`, you have to run `yarn migrate:components` from root diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a67c13be7..e4c8bd147 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: COMPANION_DOMAIN: localhost:3020 COMPANION_PROTOCOL: http COMPANION_REDIS_URL: redis://localhost:6379 - BUCKET_ENV_MINIO: "minio,test_user,test_password,http://localhost:9002/test-bucket,us-east-1" + VITE_MINIO_CONFIG: "test_user,test_password,http://localhost:9002/test-bucket,us-east-1" types: name: Types diff --git a/packages/@uppy/aws-s3/tests/s3-client/config.ts b/packages/@uppy/aws-s3/tests/s3-client/config.ts new file mode 100644 index 000000000..4f1a93753 --- /dev/null +++ b/packages/@uppy/aws-s3/tests/s3-client/config.ts @@ -0,0 +1,14 @@ +export const accessKeyId = 'stsuser' +export const secretAccessKey = 'stspassword123' + +export function getConfig(env: Record) { + const config = env['VITE_MINIO_CONFIG'] + if (!config) { + return undefined + } + const [rootAccessKeyId, rootSecretAccessKey, endpoint, region] = + config.split(',') + // Use stsuser credentials for all tests (readwrite policy is sufficient) + // Root credentials are kept for Docker container startup + return { endpoint, region, rootAccessKeyId, rootSecretAccessKey } +} diff --git a/packages/@uppy/aws-s3/tests/s3-client/docker.js b/packages/@uppy/aws-s3/tests/s3-client/docker.ts similarity index 57% rename from packages/@uppy/aws-s3/tests/s3-client/docker.js rename to packages/@uppy/aws-s3/tests/s3-client/docker.ts index 4cb03c150..93ddaf276 100644 --- a/packages/@uppy/aws-s3/tests/s3-client/docker.js +++ b/packages/@uppy/aws-s3/tests/s3-client/docker.ts @@ -1,32 +1,23 @@ import { exec, spawn } from 'node:child_process' -import { resolve } from 'node:path' import { promisify } from 'node:util' -const CWD = resolve('.') - const execAsync = promisify(exec) -export async function getContainerName(serviceName) { - try { - // Find container by service name using docker ps - const { stdout } = await execAsync( - `docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`, - ) - const containerName = stdout.trim().split('\n')[0] // Get first matching container - if (!containerName) { - throw new Error(`No running container found for service: ${serviceName}`) - } - return containerName - } catch (error) { - throw new Error( - `Failed to find container for service ${serviceName}: ${error.message}`, - ) +export async function getContainerName(serviceName: string) { + // Find container by service name using docker ps + const { stdout } = await execAsync( + `docker ps --filter "label=com.docker.compose.service=${serviceName}" --format "{{.Names}}"`, + ) + const containerName = stdout.trim().split('\n')[0] // Get first matching container + if (!containerName) { + throw new Error(`No running container found for service: ${serviceName}`) } + return containerName } export async function execDockerCommand( - containerName, - command, + containerName: string, + command: string, timeoutMs = 10000, ) { // If it's a service name (like 'garage'), find the actual container name @@ -34,9 +25,7 @@ export async function execDockerCommand( if (['garage', 'minio', 'ceph'].includes(containerName)) { try { actualContainerName = await getContainerName(containerName) - console.log( - `Found container: ${actualContainerName} for service: ${containerName}`, - ) + // console.log(`Found container: ${actualContainerName} for service: ${containerName}`,) } catch (_error) { console.log(`Using container name as-is: ${containerName}`) } @@ -74,27 +63,27 @@ export async function execDockerCommand( } } -function run(cmd, args) { - return new Promise((res, rej) => { - const p = spawn(cmd, args, { cwd: CWD, stdio: 'inherit' }) +async function run(cmd: string, args: string[], env?: Record) { + // console.log('running command:', cmd, args.join(' ')) + return new Promise((res, rej) => { + const p = spawn(cmd, args, { + stdio: 'inherit', + env: { ...process.env, ...env }, + }) p.on('close', (code) => code === 0 ? res() : rej(new Error(`${cmd} ${args.join(' ')} exited ${code}`)), ) + p.on('error', (err) => rej(err)) }) } -export const composeUp = (file) => - run('docker', ['compose', '-f', file, 'up', '-d', '--force-recreate']) -export const composeUpWait = (file) => - run('docker', [ - 'compose', - '-f', - file, - 'up', - '-d', - '--force-recreate', - '--wait', - ]) -export const composeDown = (file) => + +export const composeUpWait = (file: string, env?: Record) => + run( + 'docker', + ['compose', '-f', file, 'up', '-d', '--force-recreate', '--wait'], + env, + ) +export const composeDown = (file: string) => run('docker', ['compose', '-f', file, 'down', '--remove-orphans', '-v']) diff --git a/packages/@uppy/aws-s3/tests/s3-client/minio.test.ts b/packages/@uppy/aws-s3/tests/s3-client/minio.test.ts index 07af1f82c..0e688af4c 100644 --- a/packages/@uppy/aws-s3/tests/s3-client/minio.test.ts +++ b/packages/@uppy/aws-s3/tests/s3-client/minio.test.ts @@ -1,245 +1,464 @@ import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts' -import { describe, expect, inject, it, vi } from 'vitest' +import { beforeAll, describe, expect, it } from 'vitest' import { S3mini } from '../../src/s3-client/S3.js' import { createSigV4Signer } from '../../src/s3-client/signer.js' +import type { UploadPart } from '../../src/s3-client/types.js' import { randomBytes } from '../test-utils/browser-crypto.js' -import type { BucketConfigs } from './teardown.js' -import { beforeRun, cleanupTestBeforeAll } from './test-util.js' +import { accessKeyId, getConfig, secretAccessKey } from './config.js' -const name = 'minio' +// @ts-expect-error todo +const config = getConfig(import.meta.env) -// Get bucket configs from globalSetup via Vitest inject -const bucketConfigs: BucketConfigs = inject('bucketConfigs' as never) || [] -const raw = bucketConfigs.find((c) => c.provider === name) - ? [ - bucketConfigs.find((c) => c.provider === name)!.provider, - bucketConfigs.find((c) => c.provider === name)!.accessKeyId, - bucketConfigs.find((c) => c.provider === name)!.secretAccessKey, - bucketConfigs.find((c) => c.provider === name)!.endpoint, - bucketConfigs.find((c) => c.provider === name)!.region, - ] - : null +const suiteName = 'MinIO S3 client tests' -/** Create STS client using @aws-sdk/client-sts */ -function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) { - return new STSClient({ +if (config) { + const { endpoint, region } = config + const presigner = createSigV4Signer({ + accessKeyId, + secretAccessKey, region, endpoint, - credentials: { accessKeyId, secretAccessKey }, - }) -} - -/** Get temporary credentials via AssumeRole */ -async function assumeRole(stsClient, { durationSeconds = 900 } = {}) { - const response = await stsClient.send( - new AssumeRoleCommand({ - RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN - RoleSessionName: 'uppy-test', - DurationSeconds: durationSeconds, - }), - ) - const creds = response.Credentials - return { - AccessKeyId: creds.AccessKeyId, - SecretAccessKey: creds.SecretAccessKey, - SessionToken: creds.SessionToken, - Expiration: creds.Expiration?.toISOString() || creds.Expiration, - } -} - -const minioSpecific = (bucket) => { - vi.setConfig({ testTimeout: 120_000 }) - - const presigner = createSigV4Signer({ - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - endpoint: bucket.endpoint, }) const s3client = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, + endpoint, + region, signRequest: presigner, }) - cleanupTestBeforeAll(s3client) + const EIGHT_MB = 8 * 1024 * 1024 + const key_bin = 'test-multipart.bin' + + const large_buffer = randomBytes(EIGHT_MB * 3.2) + const content = 'some content' + const key = 'first-test-object.txt' + const key_list_parts = 'test-list-parts.bin' + const key_abort_multipart = 'test-abort-multipart.bin' + + const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart] + + beforeAll(async () => { + for (const key of FILE_KEYS) { + try { + await s3client.deleteObject(key) + } catch { + // intentionally ignore the errors as the objects don't exists. still feels hacky tbh + } + } + }) + + /** Create STS client using @aws-sdk/client-sts */ + function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) { + return new STSClient({ + region, + endpoint, + credentials: { accessKeyId, secretAccessKey }, + }) + } + + /** Get temporary credentials via AssumeRole */ + async function assumeRole( + stsClient: STSClient, + { durationSeconds = 900 } = {}, + ) { + const response = await stsClient.send( + new AssumeRoleCommand({ + RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN + RoleSessionName: 'uppy-test', + DurationSeconds: durationSeconds, + }), + ) + const creds = response.Credentials! + return { + AccessKeyId: creds.AccessKeyId, + SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, + Expiration: creds.Expiration?.toISOString() || creds.Expiration, + } + } // ===== signRequest tests ===== - it('simple putObject upload', async () => { - const key = 'presigned-test-file.txt' - const fileContents = new TextEncoder().encode( - 'Hello from pre-signed URL test.', - ) + describe(suiteName, () => { + it('simple putObject upload', async () => { + const key = 'presigned-test-file.txt' + const fileContents = new TextEncoder().encode( + 'Hello from pre-signed URL test.', + ) - const result = await s3client.putObject(key, fileContents, 'text/plain') + const result = await s3client.putObject(key, fileContents, 'text/plain') - expect(result.ok).toBe(true) - expect(result.location).toBeDefined() - expect(result.location).toContain(key) - // location should be a clean URL without query string (no signing params) - expect(result.location).not.toContain('X-Amz-Signature') - expect(result.location).not.toContain('?') - }) - - it('multipart upload with signRequest', async () => { - const key = `presigned-multipart-${Date.now()}.bin` - const partSize = 5 * 1024 * 1024 // 5MB - const part = randomBytes(partSize) - - const uploadId = await s3client.createMultipartUpload( - key, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - - const uploaded = await s3client.uploadPart(key, uploadId, part, 1) - expect(uploaded.etag).toBeDefined() - - const result = await s3client.completeMultipartUpload(key, uploadId, [ - uploaded, - ]) - expect(result.etag).toBeDefined() - expect(result.location).toBeDefined() - expect(result.location).toContain(key) - expect(result.key).toBe(key) - - await s3client.deleteObject(key) - }) - - // ===== getCredentials tests (STS) ===== - - describe('getCredentials (STS)', () => { - const stsEndpoint = new URL(bucket.endpoint).origin - - it('should upload using getCredentials callback', async () => { - const testKey = `sts-getcreds-${Date.now()}.txt` - - const s3 = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - getCredentials: async () => { - const stsClient = createSTSClient({ - endpoint: stsEndpoint, - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - }) - const creds = await assumeRole(stsClient) - return { - credentials: { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }, - bucket: new URL(bucket.endpoint).pathname.slice(1), - region: bucket.region, - } - }, - }) - - try { - const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain') - expect(result.ok).toBe(true) - expect(result.location).toBeDefined() - expect(result.location).toContain(testKey) - expect(result.location).not.toContain('X-Amz-Signature') - expect(result.location).not.toContain('?') - } finally { - await s3.deleteObject(testKey) - } + expect(result.ok).toBe(true) + expect(result.location).toBeDefined() + expect(result.location).toContain(key) + // location should be a clean URL without query string (no signing params) + expect(result.location).not.toContain('X-Amz-Signature') + expect(result.location).not.toContain('?') }) - it('should cache credentials across multiple requests', async () => { - let fetchCount = 0 - - const s3 = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - getCredentials: async () => { - fetchCount++ - const stsClient = createSTSClient({ - endpoint: stsEndpoint, - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - }) - const creds = await assumeRole(stsClient) - return { - credentials: { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }, - bucket: new URL(bucket.endpoint).pathname.slice(1), - region: bucket.region, - } - }, - }) - - await s3.putObject( - `sts-cache-1-${Date.now()}.txt`, - 'File 1', - 'text/plain', - ) - await s3.putObject( - `sts-cache-2-${Date.now()}.txt`, - 'File 2', - 'text/plain', - ) - expect(fetchCount).toBe(1) - }) - - it('should perform multipart upload with getCredentials', async () => { - const s3 = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - getCredentials: async () => { - const stsClient = createSTSClient({ - endpoint: stsEndpoint, - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - }) - const creds = await assumeRole(stsClient) - return { - credentials: { - accessKeyId: creds.AccessKeyId, - secretAccessKey: creds.SecretAccessKey, - sessionToken: creds.SessionToken, - expiration: creds.Expiration, - }, - bucket: new URL(bucket.endpoint).pathname.slice(1), - region: bucket.region, - } - }, - }) - - const key = `sts-multipart-${Date.now()}.bin` + it('multipart upload with signRequest', async () => { + const key = `presigned-multipart-${Date.now()}.bin` const partSize = 5 * 1024 * 1024 // 5MB const part = randomBytes(partSize) - const uploadId = await s3.createMultipartUpload( + const uploadId = await s3client.createMultipartUpload( key, 'application/octet-stream', ) expect(uploadId).toBeDefined() - const uploaded = await s3.uploadPart(key, uploadId, part, 1) + const uploaded = await s3client.uploadPart(key, uploadId, part, 1) expect(uploaded.etag).toBeDefined() - const result = await s3.completeMultipartUpload(key, uploadId, [uploaded]) + const result = await s3client.completeMultipartUpload(key, uploadId, [ + uploaded, + ]) expect(result.etag).toBeDefined() expect(result.location).toBeDefined() expect(result.location).toContain(key) expect(result.key).toBe(key) - await s3.deleteObject(key) + await s3client.deleteObject(key) + }) + + // ===== getCredentials tests (STS) ===== + + describe('getCredentials (STS)', () => { + const stsEndpoint = new URL(endpoint).origin + + it('should upload using getCredentials callback', async () => { + const testKey = `sts-getcreds-${Date.now()}.txt` + + const s3 = new S3mini({ + endpoint, + region, + getCredentials: async () => { + const stsClient = createSTSClient({ + endpoint: stsEndpoint, + accessKeyId, + secretAccessKey, + region, + }) + const creds = await assumeRole(stsClient) + return { + credentials: { + accessKeyId: creds.AccessKeyId!, + secretAccessKey: creds.SecretAccessKey!, + sessionToken: creds.SessionToken!, + expiration: creds.Expiration as string, + }, + bucket: new URL(endpoint).pathname.slice(1), + region, + } + }, + }) + + try { + const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain') + expect(result.ok).toBe(true) + expect(result.location).toBeDefined() + expect(result.location).toContain(testKey) + expect(result.location).not.toContain('X-Amz-Signature') + expect(result.location).not.toContain('?') + } finally { + await s3.deleteObject(testKey) + } + }) + + it('should cache credentials across multiple requests', async () => { + let fetchCount = 0 + + const s3 = new S3mini({ + endpoint, + region, + getCredentials: async () => { + fetchCount++ + const stsClient = createSTSClient({ + endpoint: stsEndpoint, + accessKeyId, + secretAccessKey, + region, + }) + const creds = await assumeRole(stsClient) + return { + credentials: { + accessKeyId: creds.AccessKeyId!, + secretAccessKey: creds.SecretAccessKey!, + sessionToken: creds.SessionToken!, + expiration: creds.Expiration as string, + }, + bucket: new URL(endpoint).pathname.slice(1), + region, + } + }, + }) + + await s3.putObject( + `sts-cache-1-${Date.now()}.txt`, + 'File 1', + 'text/plain', + ) + await s3.putObject( + `sts-cache-2-${Date.now()}.txt`, + 'File 2', + 'text/plain', + ) + expect(fetchCount).toBe(1) + }) + + it('should perform multipart upload with getCredentials', async () => { + const s3 = new S3mini({ + endpoint, + region, + getCredentials: async () => { + const stsClient = createSTSClient({ + endpoint: stsEndpoint, + accessKeyId, + secretAccessKey, + region, + }) + const creds = await assumeRole(stsClient) + return { + credentials: { + accessKeyId: creds.AccessKeyId!, + secretAccessKey: creds.SecretAccessKey!, + sessionToken: creds.SessionToken!, + expiration: creds.Expiration as string, + }, + bucket: new URL(endpoint).pathname.slice(1), + region, + } + }, + }) + + const key = `sts-multipart-${Date.now()}.bin` + const partSize = 5 * 1024 * 1024 // 5MB + const part = randomBytes(partSize) + + const uploadId = await s3.createMultipartUpload( + key, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + + const uploaded = await s3.uploadPart(key, uploadId, part, 1) + expect(uploaded.etag).toBeDefined() + + const result = await s3.completeMultipartUpload(key, uploadId, [ + uploaded, + ]) + expect(result.etag).toBeDefined() + expect(result.location).toBeDefined() + expect(result.location).toContain(key) + expect(result.key).toBe(key) + + await s3.deleteObject(key) + }) + }) + + it('instantiates s3client', () => { + expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation + }) + + // we don't need an explicit eTag method as we already get eTag in the putOject response + it('putObject uploads successfully and returns ETag', async () => { + const response = await s3client.putObject(key, content, 'text/plain') + expect(response.status).toBe(200) + expect(response.headers.get('etag')).toBeDefined() + await s3client.deleteObject(key) + }) + + it('putObject handles binary data', async () => { + const binaryData = new Uint8Array(6).fill(0xff) + + const response = await s3client.putObject( + key, + binaryData, + 'application/octet-stream', + ) + + expect(response).toBeDefined() + expect(response.status).toBe(200) + expect(response.headers.get('etag')).toBeDefined() + + // cleanup + + await s3client.deleteObject(key) + }) + + // test createMultipartUpload + + it('createMultipartUpload returns a valid uploadId', async () => { + const uploadId = await s3client.createMultipartUpload( + key_bin, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + expect(typeof uploadId).toBe('string') + expect(uploadId.length).toBeGreaterThan(0) + + // cleanup + await s3client.abortMultipartUpload(key, uploadId) + }) + + // test uploadPart + + it('uploadPart returns partNumber and Etag', async () => { + const partData = randomBytes(EIGHT_MB) + + const uploadId = await s3client.createMultipartUpload( + key_bin, + 'application/octet-stream', + ) + + // upload part + const partResult = await s3client.uploadPart( + key_bin, + uploadId, + partData, + 1, + ) + + expect(partResult).toBeDefined() + expect(partResult.partNumber).toBe(1) + expect(partResult.etag).toBeDefined() + expect(typeof partResult.etag).toBe('string') + expect(partResult.etag.length).toBe(32) + + // cleanup + await s3client.abortMultipartUpload(key, uploadId) + }) + + // end to end multipart flow + + it('completeMultipartUpload assembles parts correctly', async () => { + // key - key_bin + const partSize = EIGHT_MB + const totalParts = Math.ceil(large_buffer.byteLength / partSize) + + const uploadId = await s3client.createMultipartUpload( + key_bin, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + + // upload all parts + const uploadPromises: Promise[] = [] + for (let i = 0; i < totalParts; i++) { + const partBuffer = large_buffer.subarray( + i * partSize, + (i + 1) * partSize, + ) + uploadPromises.push( + s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1), + ) + } + const uploadResponses = await Promise.all(uploadPromises) + + // verify all parts uploaded succesfully + + expect(uploadResponses.length).toBe(totalParts) + + uploadResponses.forEach((response, index) => { + expect(response.partNumber).toBe(index + 1) + expect(response.etag).toBeDefined() + }) + + // create multipart upload + + const parts = uploadResponses.map((response) => ({ + partNumber: response.partNumber, + etag: response.etag, + })) + + const completeResponse = await s3client.completeMultipartUpload( + key_bin, + uploadId, + parts, + ) + + expect(completeResponse).toBeDefined() + expect(typeof completeResponse).toBe('object') + expect(completeResponse.etag).toBeDefined() + expect(typeof completeResponse.etag).toBe('string') + expect(completeResponse.etag.length).toBe(32 + 2) + + // cleanup + + await s3client.deleteObject(key_bin) + }) + + it('abortMultipartUpload cancels upload successfully', async () => { + // start upload + const uploadId = await s3client.createMultipartUpload( + key_abort_multipart, + 'application/octet-stream', + ) + + expect(uploadId).toBeDefined() + + const partData = randomBytes(EIGHT_MB) + await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1) + + // abort + const abortResult = await s3client.abortMultipartUpload( + key_abort_multipart, + uploadId, + ) + + expect(abortResult).toBeDefined() + expect(abortResult.status).toBe('Aborted') + expect(abortResult.key).toBe(key_abort_multipart) + expect(abortResult.uploadId).toBe(uploadId) + }) + + it('listParts returns uploaded parts correctly', async () => { + const partSize = EIGHT_MB + + const uploadId = await s3client.createMultipartUpload( + key_list_parts, + 'application/octet-stream', + ) + expect(uploadId).toBeDefined() + + const part1Data = randomBytes(partSize) + const part2Data = randomBytes(partSize) + + const part1Result = await s3client.uploadPart( + key_list_parts, + uploadId, + part1Data, + 1, + ) + const part2Result = await s3client.uploadPart( + key_list_parts, + uploadId, + part2Data, + 2, + ) + + const parts = await s3client.listParts(uploadId, key_list_parts) + + expect(parts).toBeInstanceOf(Array) + expect(parts.length).toBe(2) + + // verify part 1 + expect(parts[0].partNumber).toBe(1) + expect(parts[0].etag).toBe(part1Result.etag) + + // verify part 2 + expect(parts[1].partNumber).toBe(2) + expect(parts[1].etag).toBe(part2Result.etag) + + // cleanup abort upload + await s3client.abortMultipartUpload(key, uploadId) }) }) +} else { + console.warn( + 'Skipping MinIO S3 client tests: missing env variable VITE_MINIO_CONFIG', + ) + describe.skip(suiteName, () => { + it('noop', () => {}) + }) } - -beforeRun(raw, name, minioSpecific) diff --git a/packages/@uppy/aws-s3/tests/s3-client/setup.js b/packages/@uppy/aws-s3/tests/s3-client/setup.js deleted file mode 100644 index fbcf4a030..000000000 --- a/packages/@uppy/aws-s3/tests/s3-client/setup.js +++ /dev/null @@ -1,75 +0,0 @@ -import * as dotenv from 'dotenv' - -dotenv.config() - -import { join } from 'node:path' - -import { composeUp, composeUpWait, execDockerCommand } from './docker.js' - -const composeFiles = { - minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'), -} - -const bucketConfigs = Object.keys(process.env) - .filter((k) => k.startsWith('BUCKET_ENV_')) - .map((k) => { - const [provider, rootAccessKeyId, rootSecretAccessKey, endpoint, region] = - process.env[k].split(',') - // Use stsuser credentials for all tests (readwrite policy is sufficient) - // Root credentials are kept for Docker container startup - return { - provider, - accessKeyId: 'stsuser', - secretAccessKey: 'stspassword123', - rootAccessKeyId, - rootSecretAccessKey, - endpoint, - region, - } - }) - -export default async function setup({ provide }) { - // Provide bucket configs to browser tests via Vitest's inject mechanism - provide('bucketConfigs', bucketConfigs) - - for (const cfg of bucketConfigs) { - const composeFile = composeFiles[cfg.provider] - if (!composeFile) continue - console.log(`⏫ starting ${cfg.provider} image …`) - switch (cfg.provider) { - case 'minio': - process.env.MINIO_ROOT_USER = cfg.rootAccessKeyId - process.env.MINIO_ROOT_PASSWORD = cfg.rootSecretAccessKey - await composeUpWait(composeFile) - // biome-ignore lint/correctness/noSwitchDeclarations: no other switch blocks - const bucketName = new URL(cfg.endpoint).pathname.split('/')[1] - if (bucketName) { - await execDockerCommand( - 'minio', - `mc mb local/${bucketName} --ignore-existing`, - 30000, - ) - // Create a user with readwrite policy for STS testing - // The root user cannot AssumeRole for itself - needs a regular user - try { - await execDockerCommand( - 'minio', - `mc admin user add local stsuser stspassword123`, - 15000, - ) - await execDockerCommand( - 'minio', - `mc admin policy attach local readwrite --user stsuser`, - 15000, - ) - } catch (err) { - // User may already exist, that's fine - console.log('STS user setup:', err.message || 'error occurred') - } - } - break - default: - await composeUp(composeFile) - } - } -} diff --git a/packages/@uppy/aws-s3/tests/s3-client/setup.ts b/packages/@uppy/aws-s3/tests/s3-client/setup.ts new file mode 100644 index 000000000..2f52e586d --- /dev/null +++ b/packages/@uppy/aws-s3/tests/s3-client/setup.ts @@ -0,0 +1,48 @@ +import path from 'node:path' +import { getConfig } from './config' +import { composeDown, composeUpWait, execDockerCommand } from './docker' + +const composeFile = path.join(__dirname, 'compose.minio.yaml') + +const config = getConfig(process.env) + +export async function setup() { + if (config == null) return + + const { endpoint, rootAccessKeyId, rootSecretAccessKey } = config + + console.log('⏫ starting minio image', composeFile) + process.env.MINIO_ROOT_USER = rootAccessKeyId + process.env.MINIO_ROOT_PASSWORD = rootSecretAccessKey + await composeUpWait(composeFile) + const bucketName = new URL(endpoint).pathname.split('/')[1] + await execDockerCommand( + 'minio', + `mc mb local/${bucketName} --ignore-existing`, + 30000, + ) + // Create a user with readwrite policy for STS testing + // The root user cannot AssumeRole for itself - needs a regular user + try { + await execDockerCommand( + 'minio', + `mc admin user add local stsuser stspassword123`, + ) + await execDockerCommand( + 'minio', + `mc admin policy attach local readwrite --user stsuser`, + ) + } catch (err) { + // User may already exist, that's fine + console.log('STS user setup:', err.message || 'error occurred') + } + console.log(`✅ minio is ready`) +} + +export async function teardown() { + if (config == null) return + + console.log(`⏬ stopping minio image …`) + await composeDown(composeFile) + console.log(`✅ minio stopped and cleaned up`) +} diff --git a/packages/@uppy/aws-s3/tests/s3-client/teardown.ts b/packages/@uppy/aws-s3/tests/s3-client/teardown.ts deleted file mode 100644 index 2753fa7d2..000000000 --- a/packages/@uppy/aws-s3/tests/s3-client/teardown.ts +++ /dev/null @@ -1,30 +0,0 @@ -import * as dotenv from 'dotenv' - -dotenv.config() - -import { join } from 'node:path' -import { composeDown } from './docker.js' - -const composeFiles = { - minio: join(process.cwd(), 'tests', 's3-client', 'compose.minio.yaml'), -} - -const bucketConfigs = Object.keys(process.env) - .filter((k) => k.startsWith('BUCKET_ENV_')) - .map((k) => { - const [provider, accessKeyId, secretAccessKey, endpoint, region] = - process.env[k]!.split(',') - return { provider, accessKeyId, secretAccessKey, endpoint, region } - }) - -export type BucketConfigs = typeof bucketConfigs - -export default async () => { - for (const cfg of bucketConfigs) { - const composeFile = composeFiles[cfg.provider] - if (!composeFile) continue // ignore unknown providers - - console.log(`⏬ stopping ${cfg.provider} …`) - await composeDown(composeFile) // `docker compose -f … down` - } -} diff --git a/packages/@uppy/aws-s3/tests/s3-client/test-util.ts b/packages/@uppy/aws-s3/tests/s3-client/test-util.ts deleted file mode 100644 index df6b7d6b6..000000000 --- a/packages/@uppy/aws-s3/tests/s3-client/test-util.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { beforeAll, describe, expect, it, vi } from 'vitest' -import { S3mini, type UploadPart } from '../../src/s3-client/index.js' -import { createSigV4Signer } from '../../src/s3-client/signer.js' -import { randomBytes } from '../test-utils/browser-crypto.js' - -export const beforeRun = (raw, name, providerSpecific) => { - if (!raw) { - console.error( - 'No credentials found. Please set the BUCKET_ENV_ environment variables.', - ) - describe.skip(name, () => { - it('skipped', () => { - expect(true).toBe(true) - }) - }) - } else { - console.log('Running tests for bucket:', name) - const credentials = { - provider: raw[0], - accessKeyId: raw[1], - secretAccessKey: raw[2], - endpoint: raw[3], - region: raw[4], - } - describe(`:::: ${credentials.provider} ::::`, () => { - expect(credentials.provider).toBe(name) - expect(credentials.accessKeyId).toBeDefined() - expect(credentials.secretAccessKey).toBeDefined() - expect(credentials.endpoint).toBeDefined() - expect(credentials.region).toBeDefined() - testRunner(credentials) - if (providerSpecific) { - providerSpecific(credentials) - } - }) - } -} - -const EIGHT_MB = 8 * 1024 * 1024 -const key_bin = 'test-multipart.bin' - -const large_buffer = randomBytes(EIGHT_MB * 3.2) -const content = 'some content' -const key = 'first-test-object.txt' -const key_list_parts = 'test-list-parts.bin' -const key_abort_multipart = 'test-abort-multipart.bin' - -const FILE_KEYS = [key, key_bin, key_list_parts, key_abort_multipart] - -export const cleanupTestBeforeAll = (s3client) => { - beforeAll(async () => { - for (const key of FILE_KEYS) { - try { - await s3client.deleteObject(key) - } catch { - // intentionally ignore the errors as the objects don't exists. still feels hacky tbh - } - } - }) -} - -// --- 2 ■ A separate describe makes test output nicer ----------------------- -export const testRunner = (bucket) => { - vi.setConfig({ testTimeout: 120_000 }) - - const presigner = createSigV4Signer({ - accessKeyId: bucket.accessKeyId, - secretAccessKey: bucket.secretAccessKey, - region: bucket.region, - endpoint: bucket.endpoint, - }) - - const s3client = new S3mini({ - endpoint: bucket.endpoint, - region: bucket.region, - signRequest: presigner, - }) - - cleanupTestBeforeAll(s3client) - - it('instantiates s3client', () => { - expect(s3client).toBeInstanceOf(S3mini) // ← updated expectation - }) - - // we don't need an explicit eTag method as we already get eTag in the putOject response - it('putObject uploads successfully and returns ETag', async () => { - const response = await s3client.putObject(key, content, 'text/plain') - expect(response.status).toBe(200) - expect(response.headers.get('etag')).toBeDefined() - await s3client.deleteObject(key) - }) - - it('putObject handles binary data', async () => { - const binaryData = new Uint8Array(6).fill(0xff) - - const response = await s3client.putObject( - key, - binaryData, - 'application/octet-stream', - ) - - expect(response).toBeDefined() - expect(response.status).toBe(200) - expect(response.headers.get('etag')).toBeDefined() - - // cleanup - - await s3client.deleteObject(key) - }) - - // test createMultipartUpload - - it('createMultipartUpload returns a valid uploadId', async () => { - const uploadId = await s3client.createMultipartUpload( - key_bin, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - expect(typeof uploadId).toBe('string') - expect(uploadId.length).toBeGreaterThan(0) - - // cleanup - await s3client.abortMultipartUpload(key, uploadId) - }) - - // test uploadPart - - it('uploadPart returns partNumber and Etag', async () => { - const partData = randomBytes(EIGHT_MB) - - const uploadId = await s3client.createMultipartUpload( - key_bin, - 'application/octet-stream', - ) - - // upload part - const partResult = await s3client.uploadPart(key_bin, uploadId, partData, 1) - - expect(partResult).toBeDefined() - expect(partResult.partNumber).toBe(1) - expect(partResult.etag).toBeDefined() - expect(typeof partResult.etag).toBe('string') - expect(partResult.etag.length).toBe(32) - - // cleanup - await s3client.abortMultipartUpload(key, uploadId) - }) - - // end to end multipart flow - - it('completeMultipartUpload assembles parts correctly', async () => { - // key - key_bin - const partSize = EIGHT_MB - const totalParts = Math.ceil(large_buffer.byteLength / partSize) - - const uploadId = await s3client.createMultipartUpload( - key_bin, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - - // upload all parts - const uploadPromises: Promise[] = [] - for (let i = 0; i < totalParts; i++) { - const partBuffer = large_buffer.subarray(i * partSize, (i + 1) * partSize) - uploadPromises.push( - s3client.uploadPart(key_bin, uploadId, partBuffer, i + 1), - ) - } - const uploadResponses = await Promise.all(uploadPromises) - - // verify all parts uploaded succesfully - - expect(uploadResponses.length).toBe(totalParts) - - uploadResponses.forEach((response, index) => { - expect(response.partNumber).toBe(index + 1) - expect(response.etag).toBeDefined() - }) - - // create multipart upload - - const parts = uploadResponses.map((response) => ({ - partNumber: response.partNumber, - etag: response.etag, - })) - - const completeResponse = await s3client.completeMultipartUpload( - key_bin, - uploadId, - parts, - ) - - expect(completeResponse).toBeDefined() - expect(typeof completeResponse).toBe('object') - expect(completeResponse.etag).toBeDefined() - expect(typeof completeResponse.etag).toBe('string') - expect(completeResponse.etag.length).toBe(32 + 2) - - // cleanup - - await s3client.deleteObject(key_bin) - }) - - it('abortMultipartUpload cancels upload successfully', async () => { - // start upload - const uploadId = await s3client.createMultipartUpload( - key_abort_multipart, - 'application/octet-stream', - ) - - expect(uploadId).toBeDefined() - - const partData = randomBytes(EIGHT_MB) - await s3client.uploadPart(key_abort_multipart, uploadId, partData, 1) - - // abort - const abortResult = await s3client.abortMultipartUpload( - key_abort_multipart, - uploadId, - ) - - expect(abortResult).toBeDefined() - expect(abortResult.status).toBe('Aborted') - expect(abortResult.key).toBe(key_abort_multipart) - expect(abortResult.uploadId).toBe(uploadId) - }) - - it('listParts returns uploaded parts correctly', async () => { - const partSize = EIGHT_MB - - const uploadId = await s3client.createMultipartUpload( - key_list_parts, - 'application/octet-stream', - ) - expect(uploadId).toBeDefined() - - const part1Data = randomBytes(partSize) - const part2Data = randomBytes(partSize) - - const part1Result = await s3client.uploadPart( - key_list_parts, - uploadId, - part1Data, - 1, - ) - const part2Result = await s3client.uploadPart( - key_list_parts, - uploadId, - part2Data, - 2, - ) - - const parts = await s3client.listParts(uploadId, key_list_parts) - - expect(parts).toBeInstanceOf(Array) - expect(parts.length).toBe(2) - - // verify part 1 - expect(parts[0].partNumber).toBe(1) - expect(parts[0].etag).toBe(part1Result.etag) - - // verify part 2 - expect(parts[1].partNumber).toBe(2) - expect(parts[1].etag).toBe(part2Result.etag) - - // cleanup abort upload - await s3client.abortMultipartUpload(key, uploadId) - }) -} diff --git a/packages/@uppy/aws-s3/vitest.config.ts b/packages/@uppy/aws-s3/vitest.config.ts index 74e01e7ea..37b4bd06a 100644 --- a/packages/@uppy/aws-s3/vitest.config.ts +++ b/packages/@uppy/aws-s3/vitest.config.ts @@ -3,21 +3,19 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { testTimeout: 120_000, - globalSetup: ['./tests/s3-client/setup.js'], - setupFiles: ['./test-setup.mjs'], + globalSetup: ['tests/s3-client/setup.ts'], projects: [ { test: { - name: 'aws-s3-node', - include: ['tests/index.test.ts', 'tests/createSignedURL.test.ts'], + name: 's3-jsdom', + include: ['**/*.test.ts'], environment: 'jsdom', }, }, { test: { - name: 's3-client-browser', - include: ['tests/s3-client/*.test.{js,ts}'], - exclude: ['tests/s3-client/_shared.test.js'], + name: 's3-browser', + include: ['**/minio.test.ts'], browser: { enabled: true, provider: 'playwright', diff --git a/turbo.json b/turbo.json index c36552ab0..a3386bfe3 100644 --- a/turbo.json +++ b/turbo.json @@ -42,7 +42,7 @@ "test": { "dependsOn": ["^test"], "cache": false, - "passThroughEnv": ["BUCKET_ENV_MINIO"] + "passThroughEnv": ["VITE_MINIO_CONFIG"] }, "test:e2e": { "dependsOn": ["^test:e2e"],