mirror of
https://github.com/transloadit/uppy.git
synced 2026-07-17 16:50:22 +00:00
@uppy/aws-s3: add support for tempCredentials in s3mini (#6110)
- added `getCredentials` callback in `S3Config` - simplified signer test util ( `sigv4-signer.js` ) and moved to `src/signer.ts` - all tests run in a new non root minio user ( required for sts ) , which gets created before the tests start see `tests/s3-client/setup.js` - tests added for sts , using `@aws-sdk/client-sts` for getting creds. - credential caching inspired from the current implementation in `@aws-s3` plugin , see `#getTemporarySecurityCredentials` inside `@uppy/aws-s3/src/index.ts` --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
parent
db9506a57c
commit
bd8d51b02d
9 changed files with 1381 additions and 168 deletions
|
|
@ -43,6 +43,7 @@
|
|||
},
|
||||
"devDependencies": {
|
||||
"@aws-sdk/client-s3": "^3.362.0",
|
||||
"@aws-sdk/client-sts": "^3.362.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.362.0",
|
||||
"@uppy/core": "workspace:^",
|
||||
"jsdom": "^26.1.0",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
*/
|
||||
|
||||
import * as C from './consts.js'
|
||||
import { createSigV4Signer } from './signer.js'
|
||||
import type * as IT from './types.js'
|
||||
import * as U from './utils.js'
|
||||
|
||||
|
|
@ -12,6 +13,7 @@ import * as U from './utils.js'
|
|||
* Supports simple uploads, multipart uploads, and object deletion.
|
||||
*
|
||||
* @example
|
||||
* // Option 1: With signRequest callback
|
||||
* const s3 = new S3mini({
|
||||
* endpoint: 'https://s3.amazonaws.com/my-bucket',
|
||||
* signRequest: async ({ method, url, headers }) => {
|
||||
|
|
@ -19,6 +21,15 @@ import * as U from './utils.js'
|
|||
* },
|
||||
* });
|
||||
*
|
||||
* // Option 2: With getCredentials callback (client-side signing)
|
||||
* const s3 = new S3mini({
|
||||
* endpoint: 'https://s3.amazonaws.com/my-bucket',
|
||||
* getCredentials: async () => {
|
||||
* const resp = await fetch('/api/s3/credentials');
|
||||
* return resp.json(); // { credentials, bucket, region }
|
||||
* },
|
||||
* });
|
||||
*
|
||||
* await s3.putObject('file.txt', 'Hello, World!');
|
||||
*/
|
||||
class S3mini {
|
||||
|
|
@ -26,38 +37,92 @@ class S3mini {
|
|||
readonly region: string
|
||||
readonly requestSizeInBytes: number
|
||||
readonly requestAbortTimeout?: number
|
||||
readonly fetch: typeof fetch
|
||||
readonly signRequest: IT.signRequestFn
|
||||
|
||||
private readonly getCredentials?: IT.getCredentialsFn
|
||||
private cachedCredentials?: IT.CredentialsResponse
|
||||
private cachedCredentialsPromise?: Promise<IT.CredentialsResponse>
|
||||
private signRequest!: IT.signRequestFn
|
||||
|
||||
constructor({
|
||||
endpoint,
|
||||
signRequest,
|
||||
getCredentials,
|
||||
region = 'auto',
|
||||
requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES,
|
||||
requestAbortTimeout = undefined,
|
||||
fetch = globalThis.fetch,
|
||||
}: IT.S3Config) {
|
||||
this._validateConstructorParams(endpoint, signRequest)
|
||||
this._validateConstructorParams(endpoint, signRequest, getCredentials)
|
||||
this.endpoint = new URL(this._ensureValidUrl(endpoint))
|
||||
this.signRequest = signRequest
|
||||
this.region = region
|
||||
this.requestSizeInBytes = requestSizeInBytes
|
||||
this.requestAbortTimeout = requestAbortTimeout
|
||||
// Bind fetch to globalThis to preserve correct 'this' context in browsers
|
||||
// Without this, calling this.fetch() throws "Illegal invocation"
|
||||
this.fetch = fetch.bind(globalThis)
|
||||
|
||||
if (signRequest) {
|
||||
this.signRequest = signRequest
|
||||
} else if (getCredentials) {
|
||||
this.getCredentials = getCredentials
|
||||
this.signRequest = this._createCredentialBasedSigner()
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates a signer that fetches/caches credentials and signs requests. */
|
||||
private _createCredentialBasedSigner(): IT.signRequestFn {
|
||||
return async (request: IT.signableRequest): Promise<IT.signedHeaders> => {
|
||||
const creds = await this._getCachedCredentials()
|
||||
const signer = createSigV4Signer({
|
||||
accessKeyId: creds.credentials.accessKeyId,
|
||||
secretAccessKey: creds.credentials.secretAccessKey,
|
||||
sessionToken: creds.credentials.sessionToken,
|
||||
region: creds.region || this.region,
|
||||
})
|
||||
return signer(request)
|
||||
}
|
||||
}
|
||||
|
||||
/** Gets cached credentials or fetches new ones. */
|
||||
private async _getCachedCredentials(): Promise<IT.CredentialsResponse> {
|
||||
// Return Cached Credentials if available
|
||||
if (this.cachedCredentials != null) {
|
||||
return this.cachedCredentials
|
||||
}
|
||||
|
||||
// Cache the promise so concurrent calls wait for the same fetch
|
||||
if (this.cachedCredentialsPromise == null) {
|
||||
this.cachedCredentialsPromise = this.getCredentials!()
|
||||
.then((creds) => {
|
||||
this.cachedCredentials = creds
|
||||
return creds
|
||||
})
|
||||
.finally(() => {
|
||||
// Clear promise cache after resolution to allow future retries
|
||||
this.cachedCredentialsPromise = undefined
|
||||
})
|
||||
}
|
||||
|
||||
return this.cachedCredentialsPromise
|
||||
}
|
||||
|
||||
private _validateConstructorParams(
|
||||
endpoint: string,
|
||||
signRequest: IT.signRequestFn,
|
||||
signRequest?: IT.signRequestFn,
|
||||
getCredentials?: IT.getCredentialsFn,
|
||||
): void {
|
||||
if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
|
||||
throw new TypeError(C.ERROR_ENDPOINT_REQUIRED)
|
||||
}
|
||||
|
||||
if (typeof signRequest !== 'function') {
|
||||
throw new TypeError('signRequest is not passed')
|
||||
if (!signRequest && !getCredentials) {
|
||||
throw new TypeError(
|
||||
'Either signRequest or getCredentials must be provided',
|
||||
)
|
||||
}
|
||||
|
||||
if (signRequest && typeof signRequest !== 'function') {
|
||||
throw new TypeError('signRequest must be a function')
|
||||
}
|
||||
|
||||
if (getCredentials && typeof getCredentials !== 'function') {
|
||||
throw new TypeError('getCredentials must be a function')
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -219,13 +284,41 @@ class S3mini {
|
|||
headers: baseHeaders,
|
||||
})
|
||||
|
||||
return this._sendRequest(
|
||||
finalUrl.toString(),
|
||||
method,
|
||||
signedHeaders,
|
||||
body,
|
||||
tolerated,
|
||||
)
|
||||
try {
|
||||
return await this._sendRequest(
|
||||
finalUrl.toString(),
|
||||
method,
|
||||
signedHeaders,
|
||||
body,
|
||||
tolerated,
|
||||
)
|
||||
} catch (err) {
|
||||
// If expired token error and using getCredentials, clear cache and retry once
|
||||
if (
|
||||
this.getCredentials &&
|
||||
err instanceof U.S3ServiceError &&
|
||||
err.code &&
|
||||
['ExpiredToken', 'InvalidAccessKeyId'].includes(err.code)
|
||||
) {
|
||||
// Clear timer and cache
|
||||
this.clearCachedCredentials()
|
||||
|
||||
// Retry with fresh credentials
|
||||
const freshSignedHeaders = await this.signRequest({
|
||||
method,
|
||||
url: finalUrl.toString(),
|
||||
headers: baseHeaders,
|
||||
})
|
||||
return this._sendRequest(
|
||||
finalUrl.toString(),
|
||||
method,
|
||||
freshSignedHeaders,
|
||||
body,
|
||||
tolerated,
|
||||
)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/** Uploads an object to the S3-compatible service. */
|
||||
|
|
@ -471,6 +564,15 @@ class S3mini {
|
|||
return res.status === 200 || res.status === 204
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears cached credentials.
|
||||
* Call this method when you need to force a credential refresh on the next request.
|
||||
*/
|
||||
public clearCachedCredentials(): void {
|
||||
this.cachedCredentials = undefined
|
||||
this.cachedCredentialsPromise = undefined
|
||||
}
|
||||
|
||||
private async _sendRequest(
|
||||
url: string,
|
||||
method: IT.HttpMethod,
|
||||
|
|
@ -479,7 +581,7 @@ class S3mini {
|
|||
toleratedStatusCodes: number[] = [],
|
||||
): Promise<Response> {
|
||||
try {
|
||||
const res = await this.fetch(url, {
|
||||
const res = await fetch(url, {
|
||||
method,
|
||||
headers,
|
||||
body: ['GET', 'HEAD'].includes(method) ? undefined : body,
|
||||
|
|
|
|||
111
packages/@uppy/aws-s3/src/s3-client/signer.ts
Normal file
111
packages/@uppy/aws-s3/src/s3-client/signer.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
/**
|
||||
* AWS Signature Version 4 Signer for S3-compatible services.
|
||||
* @see https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-signing.html
|
||||
*/
|
||||
|
||||
import * as C from './consts.js'
|
||||
import type { signableRequest, signedHeaders } from './types.js'
|
||||
import * as U from './utils.js'
|
||||
|
||||
export interface SignerConfig {
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
sessionToken?: string
|
||||
region: string
|
||||
service?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a SigV4 signer for S3 requests.
|
||||
* Returns a function that can sign requests with Authorization headers.
|
||||
*/
|
||||
export function createSigV4Signer(config: SignerConfig) {
|
||||
const {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
sessionToken,
|
||||
region,
|
||||
service = C.S3_SERVICE,
|
||||
} = config
|
||||
|
||||
let signingKeyDate: string | null = null
|
||||
let signingKey: ArrayBuffer | null = null
|
||||
|
||||
async function getSignatureKey(dateStamp: string): Promise<ArrayBuffer> {
|
||||
const kDate = await U.hmac(`AWS4${secretAccessKey}`, dateStamp)
|
||||
const kRegion = await U.hmac(kDate, region)
|
||||
const kService = await U.hmac(kRegion, service)
|
||||
return U.hmac(kService, C.AWS_REQUEST_TYPE)
|
||||
}
|
||||
|
||||
return async function signRequest(
|
||||
request: signableRequest,
|
||||
): Promise<signedHeaders> {
|
||||
const { method, url, headers } = request
|
||||
const parsedUrl = new URL(url)
|
||||
|
||||
const now = new Date()
|
||||
const shortDate = now.toISOString().slice(0, 10).replace(/-/g, '')
|
||||
const fullDatetime = `${shortDate}T${now.toISOString().slice(11, 19).replace(/:/g, '')}Z`
|
||||
const scope = `${shortDate}/${region}/${service}/${C.AWS_REQUEST_TYPE}`
|
||||
|
||||
// S3 supports both hashed / unhashed payload no need for hashed payload now as
|
||||
// we're not talking to sts, also it takes up CPU
|
||||
const payloadHash = C.UNSIGNED_PAYLOAD
|
||||
|
||||
const headersToSign: Record<string, string> = {
|
||||
...headers,
|
||||
[C.HEADER_AMZ_CONTENT_SHA256]: payloadHash,
|
||||
[C.HEADER_AMZ_DATE]: fullDatetime,
|
||||
[C.HEADER_HOST]: parsedUrl.host,
|
||||
...(sessionToken ? { 'x-amz-security-token': sessionToken } : {}),
|
||||
}
|
||||
|
||||
const ignored = new Set([
|
||||
'authorization',
|
||||
'content-length',
|
||||
'content-type',
|
||||
'user-agent',
|
||||
])
|
||||
const sorted = Object.entries(headersToSign)
|
||||
.filter(([k]) => !ignored.has(k.toLowerCase()))
|
||||
.sort(([a], [b]) => a.toLowerCase().localeCompare(b.toLowerCase()))
|
||||
|
||||
const canonicalHeaders = sorted
|
||||
.map(([k, v]) => `${k.toLowerCase()}:${v.trim()}`)
|
||||
.join('\n')
|
||||
const signedHeaderNames = sorted.map(([k]) => k.toLowerCase()).join(';')
|
||||
|
||||
const queryString = [...parsedUrl.searchParams.entries()]
|
||||
.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
|
||||
.sort()
|
||||
.join('&')
|
||||
|
||||
const canonicalRequest = [
|
||||
method,
|
||||
parsedUrl.pathname,
|
||||
queryString,
|
||||
canonicalHeaders,
|
||||
'',
|
||||
signedHeaderNames,
|
||||
payloadHash,
|
||||
].join('\n')
|
||||
|
||||
const stringToSign = [
|
||||
C.AWS_ALGORITHM,
|
||||
fullDatetime,
|
||||
scope,
|
||||
U.hexFromBuffer(await U.sha256(canonicalRequest)),
|
||||
].join('\n')
|
||||
|
||||
if (shortDate !== signingKeyDate || !signingKey) {
|
||||
signingKeyDate = shortDate
|
||||
signingKey = await getSignatureKey(shortDate)
|
||||
}
|
||||
|
||||
const signature = U.hexFromBuffer(await U.hmac(signingKey, stringToSign))
|
||||
const authorization = `${C.AWS_ALGORITHM} Credential=${accessKeyId}/${scope}, SignedHeaders=${signedHeaderNames}, Signature=${signature}`
|
||||
|
||||
return { ...headersToSign, authorization }
|
||||
}
|
||||
}
|
||||
|
|
@ -12,22 +12,63 @@ export type signedHeaders = Record<string, string>
|
|||
/** Function that signs a request and returns the signed headers */
|
||||
export type signRequestFn = (request: signableRequest) => Promise<signedHeaders>
|
||||
|
||||
/** Configuration options for S3mini client */
|
||||
export interface S3Config {
|
||||
/**
|
||||
* Temporary security credentials from STS or similar service.
|
||||
* These are used with getCredentials callback for client-side signing.
|
||||
*/
|
||||
export interface TemporaryCredentials {
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
sessionToken: string
|
||||
/** ISO 8601 date string when credentials expire */
|
||||
expiration?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Response from getCredentials callback.
|
||||
* Includes temporary credentials plus region info.
|
||||
*/
|
||||
export interface CredentialsResponse {
|
||||
credentials: TemporaryCredentials
|
||||
region: string
|
||||
}
|
||||
|
||||
/** Function that retrieves temporary credentials */
|
||||
export type getCredentialsFn = (options?: {
|
||||
signal?: AbortSignal
|
||||
}) => Promise<CredentialsResponse>
|
||||
|
||||
/** Base configuration shared by both signing approaches */
|
||||
type S3ConfigBase = {
|
||||
/** Endpoint URL of the S3-compatible service (e.g., 'https://s3.amazonaws.com/bucket-name') */
|
||||
endpoint: string
|
||||
/** Function to sign requests. Called for each S3 API request. */
|
||||
signRequest: signRequestFn
|
||||
/** AWS region. Defaults to 'auto'. */
|
||||
region?: string
|
||||
/** Request size in bytes for multipart uploads. Defaults to 8MB. */
|
||||
requestSizeInBytes?: number
|
||||
/** Timeout in ms after which a request should be aborted. */
|
||||
requestAbortTimeout?: number
|
||||
/** Custom fetch implementation. Defaults to globalThis.fetch. */
|
||||
fetch?: typeof fetch
|
||||
}
|
||||
|
||||
/** Config when using signRequest callback (region optional) */
|
||||
type S3ConfigWithSignRequest = S3ConfigBase & {
|
||||
/** Function to sign requests. Called for each S3 API request. */
|
||||
signRequest: signRequestFn
|
||||
getCredentials?: never
|
||||
}
|
||||
|
||||
/** Config when using getCredentials callback (region required for signing) */
|
||||
type S3ConfigWithGetCredentials = Omit<S3ConfigBase, 'region'> & {
|
||||
signRequest?: never
|
||||
/** Function to retrieve temporary credentials for client-side signing. */
|
||||
getCredentials: getCredentialsFn
|
||||
/** AWS region. Required for signing with getCredentials. */
|
||||
region: string
|
||||
}
|
||||
|
||||
/** Configuration options for S3mini client */
|
||||
export type S3Config = S3ConfigWithSignRequest | S3ConfigWithGetCredentials
|
||||
|
||||
export interface SSECHeaders {
|
||||
'x-amz-server-side-encryption-customer-algorithm': string
|
||||
'x-amz-server-side-encryption-customer-key': string
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
import { S3mini } from '../../src/s3-client/index.js'
|
||||
import { createSigV4Signer } from '../../src/s3-client/signer.js'
|
||||
import { randomBytes } from '../test-utils/browser-crypto.js'
|
||||
import { createSigV4Signer } from '../test-utils/sigv4-signer.js'
|
||||
|
||||
let _providerName
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { expect, inject, it, vi } from 'vitest'
|
||||
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts'
|
||||
import { describe, expect, inject, it, vi } from 'vitest'
|
||||
import { S3mini } from '../../src/s3-client/S3.js'
|
||||
import { createSigV4Signer } from '../../src/s3-client/signer.js'
|
||||
import { sha1Base64 } from '../test-utils/browser-crypto.js'
|
||||
import { createSigV4Signer } from '../test-utils/sigv4-signer.js'
|
||||
import { beforeRun, cleanupTestBeforeAll } from './_shared.test.js'
|
||||
|
||||
const name = 'minio'
|
||||
|
|
@ -18,6 +19,33 @@ const raw = bucketConfigs.find((c) => c.provider === name)
|
|||
]
|
||||
: null
|
||||
|
||||
/** Create STS client using @aws-sdk/client-sts */
|
||||
function createSTSClient({ endpoint, accessKeyId, secretAccessKey, region }) {
|
||||
return new STSClient({
|
||||
region,
|
||||
endpoint,
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
})
|
||||
}
|
||||
|
||||
/** Get temporary credentials via AssumeRole */
|
||||
async function assumeRole(stsClient, { durationSeconds = 900 } = {}) {
|
||||
const response = await stsClient.send(
|
||||
new AssumeRoleCommand({
|
||||
RoleArn: 'aws:iam::000000000000:role/test-role', // MinIO doesn't validate ARN
|
||||
RoleSessionName: 'uppy-test',
|
||||
DurationSeconds: durationSeconds,
|
||||
}),
|
||||
)
|
||||
const creds = response.Credentials
|
||||
return {
|
||||
AccessKeyId: creds.AccessKeyId,
|
||||
SecretAccessKey: creds.SecretAccessKey,
|
||||
SessionToken: creds.SessionToken,
|
||||
Expiration: creds.Expiration?.toISOString() || creds.Expiration,
|
||||
}
|
||||
}
|
||||
|
||||
const minioSpecific = (bucket) => {
|
||||
vi.setConfig({ testTimeout: 120_000 })
|
||||
|
||||
|
|
@ -35,6 +63,8 @@ const minioSpecific = (bucket) => {
|
|||
|
||||
cleanupTestBeforeAll(s3client)
|
||||
|
||||
// ===== signRequest tests =====
|
||||
|
||||
it('put object with valid x-amz-checksum-sha1 header', async () => {
|
||||
const fileContents = new TextEncoder().encode('Some file contents.')
|
||||
const fileHash = await sha1Base64(fileContents)
|
||||
|
|
@ -77,6 +107,138 @@ const minioSpecific = (bucket) => {
|
|||
expect(err.code).toBe('XAmzContentChecksumMismatch')
|
||||
}
|
||||
})
|
||||
|
||||
// ===== getCredentials tests (STS) =====
|
||||
|
||||
describe('getCredentials (STS)', () => {
|
||||
const stsEndpoint = new URL(bucket.endpoint).origin
|
||||
|
||||
it('should upload using getCredentials callback', async () => {
|
||||
const testKey = `sts-getcreds-${Date.now()}.txt`
|
||||
|
||||
const s3 = new S3mini({
|
||||
endpoint: bucket.endpoint,
|
||||
region: bucket.region,
|
||||
getCredentials: async () => {
|
||||
const stsClient = createSTSClient({
|
||||
endpoint: stsEndpoint,
|
||||
accessKeyId: bucket.accessKeyId,
|
||||
secretAccessKey: bucket.secretAccessKey,
|
||||
region: bucket.region,
|
||||
})
|
||||
const creds = await assumeRole(stsClient)
|
||||
return {
|
||||
credentials: {
|
||||
accessKeyId: creds.AccessKeyId,
|
||||
secretAccessKey: creds.SecretAccessKey,
|
||||
sessionToken: creds.SessionToken,
|
||||
expiration: creds.Expiration,
|
||||
},
|
||||
bucket: new URL(bucket.endpoint).pathname.slice(1),
|
||||
region: bucket.region,
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
try {
|
||||
const result = await s3.putObject(testKey, 'Hello STS!', 'text/plain')
|
||||
expect(result.ok).toBe(true)
|
||||
} finally {
|
||||
await s3.deleteObject(testKey)
|
||||
}
|
||||
})
|
||||
|
||||
it('should cache credentials across multiple requests', async () => {
|
||||
let fetchCount = 0
|
||||
|
||||
const s3 = new S3mini({
|
||||
endpoint: bucket.endpoint,
|
||||
region: bucket.region,
|
||||
getCredentials: async () => {
|
||||
fetchCount++
|
||||
const stsClient = createSTSClient({
|
||||
endpoint: stsEndpoint,
|
||||
accessKeyId: bucket.accessKeyId,
|
||||
secretAccessKey: bucket.secretAccessKey,
|
||||
region: bucket.region,
|
||||
})
|
||||
const creds = await assumeRole(stsClient)
|
||||
return {
|
||||
credentials: {
|
||||
accessKeyId: creds.AccessKeyId,
|
||||
secretAccessKey: creds.SecretAccessKey,
|
||||
sessionToken: creds.SessionToken,
|
||||
expiration: creds.Expiration,
|
||||
},
|
||||
bucket: new URL(bucket.endpoint).pathname.slice(1),
|
||||
region: bucket.region,
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
await s3.putObject(
|
||||
`sts-cache-1-${Date.now()}.txt`,
|
||||
'File 1',
|
||||
'text/plain',
|
||||
)
|
||||
await s3.putObject(
|
||||
`sts-cache-2-${Date.now()}.txt`,
|
||||
'File 2',
|
||||
'text/plain',
|
||||
)
|
||||
expect(fetchCount).toBe(1)
|
||||
})
|
||||
|
||||
it('should perform multipart upload with getCredentials', async () => {
|
||||
const s3 = new S3mini({
|
||||
endpoint: bucket.endpoint,
|
||||
region: bucket.region,
|
||||
getCredentials: async () => {
|
||||
const stsClient = createSTSClient({
|
||||
endpoint: stsEndpoint,
|
||||
accessKeyId: bucket.accessKeyId,
|
||||
secretAccessKey: bucket.secretAccessKey,
|
||||
region: bucket.region,
|
||||
})
|
||||
const creds = await assumeRole(stsClient)
|
||||
return {
|
||||
credentials: {
|
||||
accessKeyId: creds.AccessKeyId,
|
||||
secretAccessKey: creds.SecretAccessKey,
|
||||
sessionToken: creds.SessionToken,
|
||||
expiration: creds.Expiration,
|
||||
},
|
||||
bucket: new URL(bucket.endpoint).pathname.slice(1),
|
||||
region: bucket.region,
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
const key = `sts-multipart-${Date.now()}.bin`
|
||||
const partSize = 5 * 1024 * 1024 // 5MB
|
||||
|
||||
const part = new Uint8Array(partSize)
|
||||
for (let i = 0; i < part.length; i += 65536) {
|
||||
crypto.getRandomValues(
|
||||
new Uint8Array(part.buffer, i, Math.min(65536, part.length - i)),
|
||||
)
|
||||
}
|
||||
|
||||
const uploadId = await s3.getMultipartUploadId(
|
||||
key,
|
||||
'application/octet-stream',
|
||||
)
|
||||
expect(uploadId).toBeDefined()
|
||||
|
||||
const uploaded = await s3.uploadPart(key, uploadId, part, 1)
|
||||
expect(uploaded.etag).toBeDefined()
|
||||
|
||||
const result = await s3.completeMultipartUpload(key, uploadId, [uploaded])
|
||||
expect(result.etag).toBeDefined()
|
||||
|
||||
await s3.deleteObject(key)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
beforeRun(raw, name, minioSpecific)
|
||||
|
|
|
|||
|
|
@ -13,9 +13,19 @@ const composeFiles = {
|
|||
const bucketConfigs = Object.keys(process.env)
|
||||
.filter((k) => k.startsWith('BUCKET_ENV_'))
|
||||
.map((k) => {
|
||||
const [provider, accessKeyId, secretAccessKey, endpoint, region] =
|
||||
const [provider, rootAccessKeyId, rootSecretAccessKey, endpoint, region] =
|
||||
process.env[k].split(',')
|
||||
return { provider, accessKeyId, secretAccessKey, endpoint, region }
|
||||
// Use stsuser credentials for all tests (readwrite policy is sufficient)
|
||||
// Root credentials are kept for Docker container startup
|
||||
return {
|
||||
provider,
|
||||
accessKeyId: 'stsuser',
|
||||
secretAccessKey: 'stspassword123',
|
||||
rootAccessKeyId,
|
||||
rootSecretAccessKey,
|
||||
endpoint,
|
||||
region,
|
||||
}
|
||||
})
|
||||
|
||||
export default async function setup({ provide }) {
|
||||
|
|
@ -28,8 +38,8 @@ export default async function setup({ provide }) {
|
|||
console.log(`⏫ starting ${cfg.provider} image …`)
|
||||
switch (cfg.provider) {
|
||||
case 'minio':
|
||||
process.env.MINIO_ROOT_USER = cfg.accessKeyId
|
||||
process.env.MINIO_ROOT_PASSWORD = cfg.secretAccessKey
|
||||
process.env.MINIO_ROOT_USER = cfg.rootAccessKeyId
|
||||
process.env.MINIO_ROOT_PASSWORD = cfg.rootSecretAccessKey
|
||||
await composeUpWait(composeFile)
|
||||
// biome-ignore lint/correctness/noSwitchDeclarations: no other switch blocks
|
||||
const bucketName = new URL(cfg.endpoint).pathname.split('/')[1]
|
||||
|
|
@ -39,6 +49,23 @@ export default async function setup({ provide }) {
|
|||
`mc mb local/${bucketName} --ignore-existing`,
|
||||
30000,
|
||||
)
|
||||
// Create a user with readwrite policy for STS testing
|
||||
// The root user cannot AssumeRole for itself - needs a regular user
|
||||
try {
|
||||
await execDockerCommand(
|
||||
'minio',
|
||||
`mc admin user add local stsuser stspassword123`,
|
||||
15000,
|
||||
)
|
||||
await execDockerCommand(
|
||||
'minio',
|
||||
`mc admin policy attach local readwrite --user stsuser`,
|
||||
15000,
|
||||
)
|
||||
} catch (err) {
|
||||
// User may already exist, that's fine
|
||||
console.log('STS user setup:', err.message || 'error occurred')
|
||||
}
|
||||
}
|
||||
break
|
||||
default:
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
import {
|
||||
AWS_ALGORITHM,
|
||||
AWS_REQUEST_TYPE,
|
||||
HEADER_AMZ_CONTENT_SHA256,
|
||||
HEADER_AMZ_DATE,
|
||||
HEADER_HOST,
|
||||
S3_SERVICE,
|
||||
UNSIGNED_PAYLOAD,
|
||||
} from '../../src/s3-client/consts'
|
||||
import { hexFromBuffer, hmac, sha256 } from '../../src/s3-client/utils'
|
||||
/**
|
||||
* get the url
|
||||
* create the headers object which we need to sign
|
||||
* create the sortedHeaders and ignored headers
|
||||
* build canonicalHeaders
|
||||
* build query string
|
||||
* build cannonical request
|
||||
* build string to sign
|
||||
*/
|
||||
|
||||
// ! WIP
|
||||
export function createSigV4Signer({ accessKeyId, secretAccessKey, region }) {
|
||||
//
|
||||
let signingKeyDate
|
||||
let signingKey
|
||||
|
||||
const getSignatureKey = async (dateStamp) => {
|
||||
const kDate = await hmac(`AWS4${secretAccessKey}`, dateStamp)
|
||||
const kRegion = await hmac(kDate, region)
|
||||
const kService = await hmac(kRegion, S3_SERVICE)
|
||||
return await hmac(kService, AWS_REQUEST_TYPE)
|
||||
}
|
||||
|
||||
return async function signRequest({ method, url, headers }) {
|
||||
const parsedUrl = new URL(url)
|
||||
|
||||
const d = new Date()
|
||||
const year = d.getUTCFullYear()
|
||||
const month = String(d.getUTCMonth() + 1).padStart(2, '0')
|
||||
const day = String(d.getUTCDate()).padStart(2, '0')
|
||||
|
||||
const shortDatetime = `${year}${month}${day}`
|
||||
const fullDatetime = `${shortDatetime}T${String(d.getUTCHours()).padStart(2, '0')}${String(
|
||||
d.getUTCMinutes(),
|
||||
).padStart(2, '0')}${String(d.getUTCSeconds()).padStart(2, '0')}Z`
|
||||
const credentialScope = `${shortDatetime}/${region}/${S3_SERVICE}/${AWS_REQUEST_TYPE}`
|
||||
|
||||
// Headers to sign
|
||||
|
||||
const signedHeadersObj = {
|
||||
...headers,
|
||||
[HEADER_AMZ_CONTENT_SHA256]: UNSIGNED_PAYLOAD,
|
||||
[HEADER_AMZ_DATE]: fullDatetime,
|
||||
[HEADER_HOST]: parsedUrl.host,
|
||||
}
|
||||
|
||||
const ignoredHeaders = new Set([
|
||||
'authorization',
|
||||
'content-length',
|
||||
'content-type',
|
||||
'user-agent',
|
||||
])
|
||||
|
||||
let canonicalHeaders = ''
|
||||
let signedHeaders = ''
|
||||
|
||||
const sortedHeaders = Object.entries(signedHeadersObj)
|
||||
.filter(([key]) => !ignoredHeaders.has(key.toLocaleLowerCase()))
|
||||
.sort(([a], [b]) => a.toLowerCase().localeCompare(b.toLowerCase()))
|
||||
|
||||
for (const [key, value] of sortedHeaders) {
|
||||
const lowerKey = key.toLowerCase()
|
||||
if (canonicalHeaders) {
|
||||
canonicalHeaders += '\n'
|
||||
signedHeaders += ';'
|
||||
}
|
||||
canonicalHeaders += `${lowerKey}:${String(value).trim()}`
|
||||
signedHeaders += lowerKey
|
||||
}
|
||||
|
||||
const queryParams = {}
|
||||
parsedUrl.searchParams.forEach((value, key) => {
|
||||
queryParams[key] = value
|
||||
})
|
||||
|
||||
const canonicalQueryString = Object.keys(queryParams)
|
||||
.map(
|
||||
(key) =>
|
||||
`${encodeURIComponent(key)}=${encodeURIComponent(queryParams[key])}`,
|
||||
)
|
||||
.sort()
|
||||
.join('&')
|
||||
|
||||
// canonical request
|
||||
|
||||
const canonicalRequest = [
|
||||
method,
|
||||
parsedUrl.pathname,
|
||||
canonicalQueryString,
|
||||
canonicalHeaders,
|
||||
'',
|
||||
signedHeaders,
|
||||
UNSIGNED_PAYLOAD,
|
||||
].join('\n')
|
||||
|
||||
// build string to sign
|
||||
|
||||
const hashedCanonical = hexFromBuffer(await sha256(canonicalRequest))
|
||||
|
||||
const stringToSign = [
|
||||
'AWS4-HMAC-SHA256',
|
||||
fullDatetime,
|
||||
credentialScope,
|
||||
hashedCanonical,
|
||||
].join('\n')
|
||||
|
||||
// get key
|
||||
|
||||
if (shortDatetime !== signingKeyDate || !signingKey) {
|
||||
signingKeyDate = shortDatetime
|
||||
signingKey = await getSignatureKey(shortDatetime)
|
||||
}
|
||||
|
||||
// get signature
|
||||
const signature = hexFromBuffer(await hmac(signingKey, stringToSign))
|
||||
|
||||
// auth header
|
||||
|
||||
const authorization = `${AWS_ALGORITHM} Credential=${accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`
|
||||
|
||||
return {
|
||||
...signedHeadersObj,
|
||||
authorization: authorization,
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue