From d846e3510c69e0862144fc1ba019491390e1de2f Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 5 Aug 2026 20:22:53 +0200 Subject: [PATCH 01/14] CLDSRV-965: add checksum toggle to config --- config.json | 3 +++ lib/Config.js | 54 ++++++++++++++++++++++++++++++++++++++++++++ lib/server.js | 5 ++++ tests/unit/Config.js | 44 ++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+) diff --git a/config.json b/config.json index 7d0f7d9efe..6846ead615 100644 --- a/config.json +++ b/config.json @@ -152,6 +152,9 @@ "multiObjectDelete": 2097152, "bucketPutPolicy": 20480 }, + "integrityChecks": { + "enabled": true + }, "serverAccessLogs": { "mode": "DISABLED", "outputFile": "/logs/server-access.log", diff --git a/lib/Config.js b/lib/Config.js index 3c9887fed1..af21a103be 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -584,6 +584,58 @@ function parseServerAccessLogs(config) { return res; } +/** + * Parse the `integrityChecks` config section. + * + * `enabled` is a kill switch for `x-amz-checksum-*` digests, defaulting to true. + * Setting it to false stops CloudServer computing them at all — the point is to + * reclaim the CPU spent hashing every payload byte, so nothing is compared and + * no checksum is stored. Objects and MPU parts then carry no checksum metadata, + * the same state as objects predating checksum support. + * + * The headers are then ignored outright rather than merely unenforced: a + * malformed or unsupported `x-amz-checksum-*` value is accepted instead of + * rejected, and neither CreateMultipartUpload nor CompleteMultipartUpload + * validates `x-amz-checksum-algorithm`/`-type`. + * + * Content-MD5 and x-amz-content-sha256 are unaffected and remain enforced. + * + * Safe to disable part-way through a multipart upload, but not to re-enable: + * parts uploaded while disabled carry no digest, so a later CompleteMPU cannot + * compose the final checksum and fails. + * + * @param {object} config - raw parsed config file contents + * @return {{enabled: boolean}} the parsed integrityChecks section + */ +function parseIntegrityChecks(config) { + const res = { enabled: true }; + + if (config && config.integrityChecks) { + assert( + typeof config.integrityChecks === 'object' && !Array.isArray(config.integrityChecks), + 'bad config: integrityChecks must be an object', + ); + + if ('enabled' in config.integrityChecks) { + assert( + typeof config.integrityChecks.enabled === 'boolean', + 'bad config: integrityChecks.enabled must be a boolean', + ); + res.enabled = config.integrityChecks.enabled; + } + } + + if (process.env.S3_INTEGRITY_CHECKS_ENABLED !== undefined) { + assert( + ['true', 'false'].includes(process.env.S3_INTEGRITY_CHECKS_ENABLED), + "bad config: S3_INTEGRITY_CHECKS_ENABLED must be 'true' or 'false'", + ); + res.enabled = process.env.S3_INTEGRITY_CHECKS_ENABLED === 'true'; + } + + return res; +} + /** * Reads from a config file and returns the content as a config object */ @@ -1829,6 +1881,7 @@ class Config extends EventEmitter { this.apiBodySizeLimits[apiKey] = limit; } } + this.integrityChecks = parseIntegrityChecks(config); this.serverAccessLogs = parseServerAccessLogs(config); /** * S3C-10336: PutObject max size of 5GB is new in 9.5.1 @@ -2201,4 +2254,5 @@ module.exports = { azureGetStorageAccountName, azureGetLocationCredentials, parseSupportedLifecycleRules, + parseIntegrityChecks, }; diff --git a/lib/server.js b/lib/server.js index a3ccdc29fa..bd45191371 100644 --- a/lib/server.js +++ b/lib/server.js @@ -419,6 +419,11 @@ class S3Server { } } + logger.info('integrityChecks config', { config: _config.integrityChecks }); + if (!_config.integrityChecks.enabled) { + logger.warn('x-amz-checksum-* digests are disabled: not computed, not validated, and not stored'); + } + try { logger.info('ServerAccessLogger config', { config: _config.serverAccessLogs }); if ( diff --git a/tests/unit/Config.js b/tests/unit/Config.js index 4ecdbe4bbb..a7d801e0a9 100644 --- a/tests/unit/Config.js +++ b/tests/unit/Config.js @@ -7,6 +7,7 @@ const { azureGetLocationCredentials, locationConstraintAssert, parseSupportedLifecycleRules, + parseIntegrityChecks, ConfigObject, } = require('../../lib/Config'); @@ -908,6 +909,49 @@ describe('Config', () => { }); }); + describe('parse integrity checks', () => { + afterEach(() => { + delete process.env.S3_INTEGRITY_CHECKS_ENABLED; + }); + + it('should default to enabled when not configured', () => { + assert.deepStrictEqual(parseIntegrityChecks(null), { enabled: true }); + assert.deepStrictEqual(parseIntegrityChecks({}), { enabled: true }); + }); + + it('should read the configured value', () => { + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: false } }), { enabled: false }); + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: true } }), { enabled: true }); + }); + + it('should throw if integrityChecks is not an object', () => { + assert.throws(() => parseIntegrityChecks({ integrityChecks: 'yes' }), /must be an object/); + assert.throws(() => parseIntegrityChecks({ integrityChecks: [true] }), /must be an object/); + }); + + it('should throw if enabled is not a boolean', () => { + assert.throws(() => parseIntegrityChecks({ integrityChecks: { enabled: 'false' } }), /must be a boolean/); + assert.throws(() => parseIntegrityChecks({ integrityChecks: { enabled: 0 } }), /must be a boolean/); + }); + + it('should let S3_INTEGRITY_CHECKS_ENABLED override the config file', () => { + process.env.S3_INTEGRITY_CHECKS_ENABLED = 'false'; + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: true } }), { enabled: false }); + process.env.S3_INTEGRITY_CHECKS_ENABLED = 'true'; + assert.deepStrictEqual(parseIntegrityChecks({ integrityChecks: { enabled: false } }), { enabled: true }); + }); + + it('should throw on a non-boolean S3_INTEGRITY_CHECKS_ENABLED', () => { + process.env.S3_INTEGRITY_CHECKS_ENABLED = 'nope'; + assert.throws(() => parseIntegrityChecks(null), /S3_INTEGRITY_CHECKS_ENABLED/); + }); + + it('should expose integrityChecks on the config object', () => { + const config = new ConfigObject(); + assert.deepStrictEqual(config.integrityChecks, { enabled: true }); + }); + }); + describe('serverHeader', () => { let sandbox; let readFileStub; From 2275d854ddbadda84db5378065441b9be0eaf4e3 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 5 Aug 2026 21:14:42 +0200 Subject: [PATCH 02/14] CLDSRV-965: extract CompleteMPU checksum-type validation to a helper --- .../apiUtils/integrity/validateChecksums.js | 54 +++++++++ lib/api/completeMultipartUpload.js | 49 +++------ .../apiUtils/integrity/validateChecksums.js | 103 ++++++++++++++++++ 3 files changed, 170 insertions(+), 36 deletions(-) diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 99f083aeb1..6ba47dca66 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -37,6 +37,10 @@ const errMPUTypeInvalid = errorInstances.InvalidRequest.customizeDescription( const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription( 'The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.', ); +const errMPUTypeNotConfigured = errorInstances.InvalidRequest.customizeDescription( + 'The upload was not created with a checksum mode. ' + + 'The complete request must not include a x-amz-checksum-type header.', +); // TODO(S3C-11278): Update with 'MD5', 'SHA512', 'XXHASH128', 'XXHASH3', 'XXHASH64' when they are introduced. // https://scality.atlassian.net/browse/S3C-11278 @@ -111,6 +115,8 @@ const ChecksumError = Object.freeze({ MPUAlgoNotSupported: 'MPUAlgoNotSupported', MPUTypeInvalid: 'MPUTypeInvalid', MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo', + MPUTypeNotConfigured: 'MPUTypeNotConfigured', + MPUTypeModeMismatch: 'MPUTypeModeMismatch', MPUInvalidCombination: 'MPUInvalidCombination', CopyChecksumAlgoNotSupported: 'CopyChecksumAlgoNotSupported', ContentSHA256Missing: 'ContentSHA256Missing', @@ -558,6 +564,13 @@ function arsenalErrorFromChecksumError(err) { return errMPUTypeInvalid; case ChecksumError.MPUTypeWithoutAlgo: return errMPUTypeWithoutAlgo; + case ChecksumError.MPUTypeNotConfigured: + return errMPUTypeNotConfigured; + case ChecksumError.MPUTypeModeMismatch: + return errorInstances.InvalidRequest.customizeDescription( + `The upload was created using the ${err.details.type} checksum mode. ` + + 'The complete request must use the same checksum mode.', + ); case ChecksumError.MPUInvalidCombination: return errorInstances.InvalidRequest.customizeDescription( `The ${err.details.type} checksum type cannot be used ` + @@ -758,6 +771,46 @@ function getChecksumDataFromMPUHeaders(headers) { return { algorithm: algo, type: defaultChecksumType[algo], isDefault: false }; } +/** + * Validate the x-amz-checksum-type header on a CompleteMultipartUpload request + * against the checksum type the MPU was created with. + * + * x-amz-checksum-algorithm is deliberately not validated: AWS ignores a mismatch + * on that header for CompleteMultipartUpload. + * + * @param {object} headers - request headers (lowercased keys) + * @param {string|undefined} mpuChecksumType - the checksum type recorded on the + * MPU at CreateMPU time; falsy for a legacy MPU predating type tracking + * @param {boolean} isExternal - external-backend MPU. Those record no checksum + * config (CLDSRV-964), so an absent type means "not tracked here" rather than + * a legacy upload, and the header is ignored instead of rejected. + * @returns {{error: string, details: object}|null} null when valid + */ +function validateCompleteMPUChecksumType(headers, mpuChecksumType, isExternal) { + const headerType = headers['x-amz-checksum-type']; + if (!headerType) { + return null; + } + + const headerTypeUpper = headerType.toUpperCase(); + if (!validMPUTypes.has(headerTypeUpper)) { + return { error: ChecksumError.MPUTypeInvalid, details: { type: headerType } }; + } + + if (!mpuChecksumType) { + if (isExternal) { + return null; + } + return { error: ChecksumError.MPUTypeNotConfigured, details: { type: headerType } }; + } + + if (headerTypeUpper !== mpuChecksumType.toUpperCase()) { + return { error: ChecksumError.MPUTypeModeMismatch, details: { type: mpuChecksumType } }; + } + + return null; +} + // ============================================================================= // MPU final-object checksum computation // ============================================================================= @@ -876,6 +929,7 @@ module.exports = { algorithms, checksumedMethods, getChecksumDataFromMPUHeaders, + validateCompleteMPUChecksumType, computeCompositeMPUChecksum, computeFullObjectMPUChecksum, validateCompleteMultipartUploadChecksum, diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index feb1a09bc1..d5a4474b8a 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -32,6 +32,7 @@ const { computeCompositeMPUChecksum, computeFullObjectMPUChecksum, validateCompleteMultipartUploadChecksum, + validateCompleteMPUChecksumType, } = require('./apiUtils/integrity/validateChecksums'); const versionIdUtils = versioning.VersionID; @@ -336,42 +337,18 @@ function completeMultipartUpload(authInfo, request, log, callback) { log.error('error validating request', { error: err }); return next(err, destBucket); } - // Validate x-amz-checksum-type header (if present) matches - // the checksum type the MPU was created with. - // x-amz-checksum-algorithm is not validated: AWS ignores - // a mismatch on this header for CompleteMultipartUpload. - const headerType = request.headers['x-amz-checksum-type']; - if (headerType) { - const headerTypeUpper = headerType.toUpperCase(); - if (headerTypeUpper !== 'COMPOSITE' && headerTypeUpper !== 'FULL_OBJECT') { - const typeErr = errorInstances.InvalidRequest.customizeDescription( - 'Value for x-amz-checksum-type header is invalid.', - ); - return next(typeErr, destBucket); - } - const mpuType = storedMetadata.checksumType; - if (!mpuType) { - // External-backend MPUs record no checksum config - // (CLDSRV-964): ignore the header, like every other - // checksum input on CompleteMPU for external backends. - const mpuLocation = storedMetadata.controllingLocationConstraint; - const isExternalMpu = - !!constants.externalBackends[config.getLocationConstraintType(mpuLocation)]; - if (!isExternalMpu) { - // Legacy MPU created before checksumType was tracked. - const typeErr = errorInstances.InvalidRequest.customizeDescription( - 'The upload was not created with a checksum mode. ' + - 'The complete request must not include a x-amz-checksum-type header.', - ); - return next(typeErr, destBucket); - } - } else if (headerTypeUpper !== mpuType.toUpperCase()) { - const typeErr = errorInstances.InvalidRequest.customizeDescription( - `The upload was created using the ${mpuType} checksum mode. ` + - 'The complete request must use the same checksum mode.', - ); - return next(typeErr, destBucket); - } + // External-backend MPUs record no checksum config (CLDSRV-964), + // so an absent stored type means "not tracked" rather than legacy. + const mpuLocation = storedMetadata.controllingLocationConstraint; + const isExternalMpu = + !!constants.externalBackends[config.getLocationConstraintType(mpuLocation)]; + const typeErr = validateCompleteMPUChecksumType( + request.headers, + storedMetadata.checksumType, + isExternalMpu, + ); + if (typeErr) { + return next(arsenalErrorFromChecksumError(typeErr), destBucket); } return next(null, destBucket, objMD, mpuBucket, storedMetadata); }, diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index f456739152..2b5f2f9009 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -14,6 +14,7 @@ const { arsenalErrorFromChecksumError, getChecksumDataFromMPUHeaders, validateCompleteMultipartUploadChecksum, + validateCompleteMPUChecksumType, getCopyObjectChecksumAlgorithm, } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); @@ -1465,3 +1466,105 @@ describe('validateMethodChecksumNoChunking x-amz-content-sha256', () => { assert.ifError(result); }); }); + +describe('validateCompleteMPUChecksumType', () => { + describe('when the header is absent', () => { + it('should return null whatever the MPU checksum type is', () => { + assert.strictEqual(validateCompleteMPUChecksumType({}, 'COMPOSITE'), null); + assert.strictEqual(validateCompleteMPUChecksumType({}, 'FULL_OBJECT'), null); + assert.strictEqual(validateCompleteMPUChecksumType({}, undefined), null); + }); + + it('should return null for an empty header value', () => { + assert.strictEqual(validateCompleteMPUChecksumType({ 'x-amz-checksum-type': '' }, 'COMPOSITE'), null); + }); + }); + + describe('when the header matches the MPU checksum type', () => { + ['COMPOSITE', 'FULL_OBJECT'].forEach(type => { + it(`should return null for ${type}`, () => { + assert.strictEqual(validateCompleteMPUChecksumType({ 'x-amz-checksum-type': type }, type), null); + }); + }); + + it('should compare case-insensitively on both sides', () => { + const lowerHeader = { 'x-amz-checksum-type': 'composite' }; + assert.strictEqual(validateCompleteMPUChecksumType(lowerHeader, 'COMPOSITE'), null); + const upperHeader = { 'x-amz-checksum-type': 'FULL_OBJECT' }; + assert.strictEqual(validateCompleteMPUChecksumType(upperHeader, 'full_object'), null); + }); + }); + + describe('when the header value is not a valid checksum type', () => { + it('should return MPUTypeInvalid', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'BOGUS' }, 'COMPOSITE'); + assert.strictEqual(result.error, ChecksumError.MPUTypeInvalid); + assert.strictEqual(result.details.type, 'BOGUS'); + }); + + it('should take precedence over an unset MPU checksum type', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'BOGUS' }, undefined); + assert.strictEqual(result.error, ChecksumError.MPUTypeInvalid); + }); + + it('should map to InvalidRequest (400)', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'BOGUS' }, 'COMPOSITE'); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.description, 'Value for x-amz-checksum-type header is invalid.'); + }); + }); + + describe('when the MPU was created without a checksum type', () => { + it('should return MPUTypeNotConfigured', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, undefined); + assert.strictEqual(result.error, ChecksumError.MPUTypeNotConfigured); + }); + + it('should map to InvalidRequest (400) describing the legacy upload', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, undefined); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + assert.strictEqual( + err.description, + 'The upload was not created with a checksum mode. ' + + 'The complete request must not include a x-amz-checksum-type header.', + ); + }); + + it('should ignore the header for an external-backend MPU', () => { + // External backends record no checksum config (CLDSRV-964), so an + // absent type means "not tracked here", not a legacy upload. + const headers = { 'x-amz-checksum-type': 'COMPOSITE' }; + assert.strictEqual(validateCompleteMPUChecksumType(headers, undefined, true), null); + }); + + it('should still reject a mismatch on an external-backend MPU that has a type', () => { + const headers = { 'x-amz-checksum-type': 'COMPOSITE' }; + const result = validateCompleteMPUChecksumType(headers, 'FULL_OBJECT', true); + assert.strictEqual(result.error, ChecksumError.MPUTypeModeMismatch); + }); + }); + + describe('when the header does not match the MPU checksum type', () => { + it('should return MPUTypeModeMismatch carrying the MPU type', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, 'FULL_OBJECT'); + assert.strictEqual(result.error, ChecksumError.MPUTypeModeMismatch); + assert.strictEqual(result.details.type, 'FULL_OBJECT'); + }); + + it('should map to InvalidRequest (400) naming the mode the MPU was created with', () => { + const result = validateCompleteMPUChecksumType({ 'x-amz-checksum-type': 'COMPOSITE' }, 'FULL_OBJECT'); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + assert.strictEqual( + err.description, + 'The upload was created using the FULL_OBJECT checksum mode. ' + + 'The complete request must use the same checksum mode.', + ); + }); + }); +}); From 0949b18d727abc64c4c3706c2550aa554d68fa22 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 5 Aug 2026 22:17:01 +0200 Subject: [PATCH 03/14] CLDSRV-965: add areChecksumsEnabled helper --- .../apiUtils/integrity/validateChecksums.js | 14 ++++++ .../apiUtils/integrity/validateChecksums.js | 44 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 6ba47dca66..8efed39802 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -3,6 +3,7 @@ const crypto = require('crypto'); const { crc32: crtCrc32, crc32c: crtCrc32c } = require('aws-crt').checksums; const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const { errors: ArsenalErrors, errorInstances } = require('arsenal'); +const { config } = require('../../../Config'); const { combinePartCrcs } = require('./crcCombine'); const { supportedSignatureChecksums, unsupportedSignatureChecksums } = require('../../../../constants'); @@ -126,6 +127,18 @@ const ChecksumError = Object.freeze({ const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; +/** + * When false, x-amz-checksum-* headers are ignored: no digest is calculated, + * compared, or stored. + * + * Content-MD5 and x-amz-content-sha256 are not impacted. + * + * @return {boolean} true when checksums are enabled. + */ +function areChecksumsEnabled() { + return config.integrityChecks?.enabled !== false; +} + function uint32ToBase64(num) { const buf = Buffer.alloc(4); buf.writeUInt32BE(num, 0); @@ -934,4 +947,5 @@ module.exports = { computeFullObjectMPUChecksum, validateCompleteMultipartUploadChecksum, getCopyObjectChecksumAlgorithm, + areChecksumsEnabled, }; diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index 2b5f2f9009..b8399a7c67 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -16,7 +16,9 @@ const { validateCompleteMultipartUploadChecksum, validateCompleteMPUChecksumType, getCopyObjectChecksumAlgorithm, + areChecksumsEnabled, } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { config } = require('../../../../../lib/Config'); const { errors: ArsenalErrors } = require('arsenal'); describe('validateChecksumsNoChunking MD5', () => { @@ -1568,3 +1570,45 @@ describe('validateCompleteMPUChecksumType', () => { }); }); }); + +describe('areChecksumsEnabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + it('should be enabled with the shipped config', () => { + assert.strictEqual(areChecksumsEnabled(), true); + }); + + it('should be disabled only when enabled is exactly false', () => { + config.integrityChecks = { enabled: false }; + assert.strictEqual(areChecksumsEnabled(), false); + }); + + it('should be enabled when enabled is true', () => { + config.integrityChecks = { enabled: true }; + assert.strictEqual(areChecksumsEnabled(), true); + }); + + it('should default to enabled when the section or the key is missing', () => { + config.integrityChecks = undefined; + assert.strictEqual(areChecksumsEnabled(), true); + config.integrityChecks = {}; + assert.strictEqual(areChecksumsEnabled(), true); + }); + + it('should not treat a falsy non-false value as disabled', () => { + // Guards the `!== false` comparison: only an explicit boolean false + // turns checksums off, so a mis-typed config cannot silently disable them. + [0, '', null, 'false'].forEach(value => { + config.integrityChecks = { enabled: value }; + assert.strictEqual(areChecksumsEnabled(), true, `enabled: ${JSON.stringify(value)}`); + }); + }); +}); From b4e2b216a45bdc8f130e75f2dea5ffec2ca48b75 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 5 Aug 2026 22:23:16 +0200 Subject: [PATCH 04/14] CLDSRV-965: add checksum toggle to buffered endpoints --- .../apiUtils/integrity/validateChecksums.js | 3 + .../apiUtils/integrity/validateChecksums.js | 60 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 8efed39802..9d84c9152c 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -706,6 +706,9 @@ async function validateMethodChecksumNoChunking(request, body, log) { } if (request.apiMethod in checksumedMethods) { + if (!areChecksumsEnabled()) { + return md5OnlyValidationFunc(request, body, log); + } return await defaultValidationFunc(request, body, log); } diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index b8399a7c67..b2fec1353e 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -1612,3 +1612,63 @@ describe('areChecksumsEnabled', () => { }); }); }); + +describe('validateMethodChecksumNoChunking with checksums disabled', () => { + const body = 'Hello, World!'; + const sigV4Header = 'AWS4-HMAC-SHA256 Credential=x'; + const correctMd5 = crypto.createHash('md5').update(body).digest('base64'); + const wrongMd5 = crypto.createHash('md5').update('other').digest('base64'); + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + config.integrityChecks = { enabled: false }; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + const run = headers => + validateMethodChecksumNoChunking({ apiMethod: 'bucketPutCors', headers }, body, new DummyRequestLogger()); + + it('should ignore a wrong x-amz-checksum- for every checksumed method', async () => { + for (const method of Object.keys(checksumedMethods)) { + const request = { apiMethod: method, headers: { 'x-amz-checksum-crc32': 'AAAAAA==' } }; + const result = await validateMethodChecksumNoChunking(request, body, new DummyRequestLogger()); + assert.ifError(result, `${method} should not reject`); + } + }); + + it('should ignore x-amz-checksum-* headers that would otherwise be rejected', async () => { + // Malformed, unsupported, multiple, and a mismatched sdk-algorithm all + // become no-ops: the header is not looked at once checksums are off. + assert.ifError(await run({ 'x-amz-checksum-crc32': 'not-base64!!' })); + assert.ifError(await run({ 'x-amz-checksum-md5': 'AAAAAA==' })); + assert.ifError(await run({ 'x-amz-checksum-crc32': 'AAAAAA==', 'x-amz-checksum-crc32c': 'AAAAAA==' })); + assert.ifError(await run({ 'x-amz-checksum-crc32': 'AAAAAA==', 'x-amz-sdk-checksum-algorithm': 'SHA256' })); + }); + + it('should still enforce Content-MD5', async () => { + const result = await run({ 'content-md5': wrongMd5 }); + assert.strictEqual(result.message, 'BadDigest'); + assert.ifError(await run({ 'content-md5': correctMd5 })); + }); + + it('should still enforce Content-MD5 alongside an ignored x-amz-checksum', async () => { + const result = await run({ 'content-md5': wrongMd5, 'x-amz-checksum-crc32': 'AAAAAA==' }); + assert.strictEqual(result.message, 'BadDigest'); + }); + + it('should still enforce x-amz-content-sha256', async () => { + const wrongHex = crypto.createHash('sha256').update('other').digest('hex'); + const result = await run({ authorization: sigV4Header, 'x-amz-content-sha256': wrongHex }); + assert.strictEqual(result.message, 'XAmzContentSHA256Mismatch'); + }); + + it('should reject a wrong x-amz-checksum- again once re-enabled', async () => { + config.integrityChecks = { enabled: true }; + const result = await run({ 'x-amz-checksum-crc32': 'AAAAAA==' }); + assert.strictEqual(result.message, 'BadDigest'); + }); +}); From b945b92f2694f23568a183bdb0cbf7482785356f Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 5 Aug 2026 22:30:32 +0200 Subject: [PATCH 05/14] CLDSRV-965: add checksum toggle to CompleteMultipartUpload --- lib/api/completeMultipartUpload.js | 9 ++- tests/unit/api/multipartUpload.js | 93 ++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index d5a4474b8a..edb571e441 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -33,6 +33,7 @@ const { computeFullObjectMPUChecksum, validateCompleteMultipartUploadChecksum, validateCompleteMPUChecksumType, + areChecksumsEnabled, } = require('./apiUtils/integrity/validateChecksums'); const versionIdUtils = versioning.VersionID; @@ -447,7 +448,10 @@ function completeMultipartUpload(authInfo, request, log, callback) { type: storedMetadata.checksumType, isDefault: storedMetadata.checksumIsDefault, }; - const checksumErr = validatePerPartChecksums(jsonList, storedParts, splitter, mpuChecksum); + let checksumErr = null; + if (areChecksumsEnabled()) { + checksumErr = validatePerPartChecksums(jsonList, storedParts, splitter, mpuChecksum); + } if (checksumErr) { log.debug('per-part checksum validation failed', { error: checksumErr, @@ -586,7 +590,8 @@ function completeMultipartUpload(authInfo, request, log, callback) { // - if no filteredPartsObj then there is no per-part info to compute from (aws_s3/gcp/ingestion // return no filteredPartsObj; azure returns filteredPartsObj, but its parts store no checksum) // - if completeObjData is present it means the MPU was completed by an external backend - if (!filteredPartsObj || completeObjData) { + // - if checksums are disabled the parts carry no digest to compose from + if (!filteredPartsObj || completeObjData || !areChecksumsEnabled()) { return continueProcessParts(null); } computeFinalChecksum( diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index 1128cd4bb7..30ef26ccd2 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -4792,3 +4792,96 @@ describe('CompleteMultipartUpload final checksum on azure-style external backend await _assertNoChecksumInResult(xml); }); }); + +describe('CompleteMultipartUpload with checksums disabled', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // Starts a CRC64NVME (FULL_OBJECT) MPU and uploads one part, returning what CompleteMPU needs. + // Runs with checksums enabled so the part carries a stored digest — this + // commit only gates CompleteMPU, UploadPart still computes as usual. + async function initiateAndUploadPart() { + config.integrityChecks = { enabled: true }; + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc64nvme' }; + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + let storedChecksum; + for (const [key, value] of metadata.keyMaps.get(mpuBucket)) { + if (key.startsWith(uploadId) && !key.startsWith('overview')) { + storedChecksum = value.checksumValue; + } + } + assert(storedChecksum, 'part should have a stored checksum to validate against'); + + return { uploadId, eTag, storedChecksum }; + } + + function completeRequestWithPartChecksum(uploadId, eTag, checksumValue) { + const post = [ + '', + '', + '1', + `"${eTag}"`, + `${checksumValue}`, + '', + '', + ]; + return { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId }, + post, + actionImplicitDenies: false, + }; + } + + const complete = request => + util.promisify(cb => completeMultipartUpload(authInfo, request, log, (err, xml) => cb(err, xml)))(); + + it('should reject a mismatched per-part checksum while enabled', async () => { + const { uploadId, eTag } = await initiateAndUploadPart(); + const request = completeRequestWithPartChecksum(uploadId, eTag, 'AQIDBAUGBwg='); + await assert.rejects(complete(request), { message: 'InvalidPart' }); + }); + + it('should accept a mismatched per-part checksum once disabled', async () => { + const { uploadId, eTag } = await initiateAndUploadPart(); + config.integrityChecks = { enabled: false }; + const request = completeRequestWithPartChecksum(uploadId, eTag, 'AQIDBAUGBwg='); + const xml = await complete(request); + assert.match(xml, / { + const { uploadId, eTag, storedChecksum } = await initiateAndUploadPart(); + + const enabledXml = await complete(completeRequestWithPartChecksum(uploadId, eTag, storedChecksum)); + assert.match(enabledXml, //, 'enabled should return a final checksum'); + + const second = await initiateAndUploadPart(); + config.integrityChecks = { enabled: false }; + const disabledXml = await complete( + completeRequestWithPartChecksum(second.uploadId, second.eTag, second.storedChecksum), + ); + assert.doesNotMatch(disabledXml, //, 'disabled should omit the final checksum'); + }); +}); From 0db5a7c779c1f0a0bfdb0efcfdbb3023e43bd65c Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 5 Aug 2026 22:57:18 +0200 Subject: [PATCH 06/14] CLDSRV-965: add checksum toggle to CopyObject --- lib/api/objectCopy.js | 10 ++- tests/unit/api/objectCopy.js | 114 +++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/lib/api/objectCopy.js b/lib/api/objectCopy.js index 19d043181e..153aa21f03 100644 --- a/lib/api/objectCopy.js +++ b/lib/api/objectCopy.js @@ -29,6 +29,7 @@ const { algorithms, arsenalErrorFromChecksumError, getCopyObjectChecksumAlgorithm, + areChecksumsEnabled, } = require('./apiUtils/integrity/validateChecksums'); const ChecksumTransform = require('../auth/streamingV4/ChecksumTransform'); const ChecksumWritable = require('../auth/streamingV4/ChecksumWritable'); @@ -75,6 +76,9 @@ function _orphanedDataLocations(dataToDelete, newDataGetInfo) { * @returns {boolean} */ function _shouldRecomputeChecksum(headers, sourceObjMD) { + if (!areChecksumsEnabled()) { + return false; + } const requestedAlgo = headers['x-amz-checksum-algorithm']?.toLowerCase(); if ( sourceObjMD.checksum?.checksumType === 'FULL_OBJECT' && @@ -410,7 +414,7 @@ function _prepMetadata( storeMetadataParams.defaultRetention = defaultRetentionConfig; } - if (sourceObjMD.checksum && !_shouldRecomputeChecksum(headers, sourceObjMD)) { + if (areChecksumsEnabled() && sourceObjMD.checksum && !_shouldRecomputeChecksum(headers, sourceObjMD)) { storeMetadataParams.checksum = { algorithm: sourceObjMD.checksum.checksumAlgorithm, value: sourceObjMD.checksum.checksumValue, @@ -503,7 +507,9 @@ function objectCopy(authInfo, request, sourceBucket, sourceObject, sourceVersion monitoring.promMetrics('PUT', destBucketName, err.code, 'copyObject'); return callback(err); } - const { error: checksumAlgoErr, algorithm: requestedAlgo } = getCopyObjectChecksumAlgorithm(request.headers); + const { error: checksumAlgoErr, algorithm: requestedAlgo } = areChecksumsEnabled() + ? getCopyObjectChecksumAlgorithm(request.headers) + : { error: null, algorithm: null }; if (checksumAlgoErr) { const err = arsenalErrorFromChecksumError(checksumAlgoErr); log.debug('invalid x-amz-checksum-algorithm', { error: checksumAlgoErr }); diff --git a/tests/unit/api/objectCopy.js b/tests/unit/api/objectCopy.js index 5af59326ab..25b7b850ed 100644 --- a/tests/unit/api/objectCopy.js +++ b/tests/unit/api/objectCopy.js @@ -1924,3 +1924,117 @@ describe('objectCopy source size limit', () => { }); }); }); + +describe('objectCopy with checksums disabled', () => { + const sourceChecksum = { + checksumAlgorithm: 'crc32', + checksumValue: 'AAAAAA==', + checksumType: 'FULL_OBJECT', + }; + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + async.series( + [ + next => bucketPut(authInfo, putDestBucketRequest, log, next), + next => bucketPut(authInfo, putSourceBucketRequest, log, next), + next => + objectPut( + authInfo, + versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData[0]), + undefined, + log, + next, + ), + ], + done, + ); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + sinon.restore(); + cleanup(); + }); + + // Runs a copy with checksums disabled and hands the destination metadata back. + function copyDisabled(headers, cb) { + config.integrityChecks = { enabled: false }; + const req = _createObjectCopyRequest(destBucketName, headers); + return objectCopy(authInfo, req, sourceBucketName, objectKey, undefined, log, (err, xml) => { + assert.ifError(err); + return metadata.getObjectMD(destBucketName, objectKey, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + cb(xml, md); + }); + }); + } + + it('should not store a checksum when the source has none', done => { + setSourceChecksum(null, err => { + assert.ifError(err); + copyDisabled(undefined, (xml, md) => { + assert.strictEqual(md.checksum, undefined); + assert.doesNotMatch(xml, / { + setSourceChecksum(sourceChecksum, err => { + assert.ifError(err); + copyDisabled(undefined, (xml, md) => { + assert.strictEqual(md.checksum, undefined, 'destination must not inherit the source checksum'); + done(); + }); + }); + }); + + it('should ignore x-amz-checksum-algorithm instead of recomputing', done => { + setSourceChecksum(sourceChecksum, err => { + assert.ifError(err); + // sha256 differs from the source's crc32, which would normally force a + // recompute — the expensive path this flag exists to avoid. + copyDisabled({ 'x-amz-checksum-algorithm': 'SHA256' }, (xml, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should not reject an invalid x-amz-checksum-algorithm', done => { + setSourceChecksum(null, err => { + assert.ifError(err); + copyDisabled({ 'x-amz-checksum-algorithm': 'NOT-AN-ALGO' }, (xml, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should still reject an invalid x-amz-checksum-algorithm when enabled', done => { + config.integrityChecks = { enabled: true }; + const req = _createObjectCopyRequest(destBucketName, { 'x-amz-checksum-algorithm': 'NOT-AN-ALGO' }); + objectCopy(authInfo, req, sourceBucketName, objectKey, undefined, log, err => { + assert(err, 'should reject'); + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should recompute again once re-enabled', done => { + config.integrityChecks = { enabled: true }; + setSourceChecksum(null, err => { + assert.ifError(err); + const req = _createObjectCopyRequest(destBucketName); + objectCopy(authInfo, req, sourceBucketName, objectKey, undefined, log, (err, xml) => { + assert.ifError(err); + assert.match(xml, //, 'enabled should recompute the default checksum'); + done(); + }); + }); + }); +}); From d84027246ee66cb2bbc4046a2d5c4d7c12456c3d Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 6 Aug 2026 00:41:10 +0200 Subject: [PATCH 07/14] CLDSRV-965: add checksum toggle to the object data path --- .../apiUtils/object/createAndStoreObject.js | 8 +- lib/api/apiUtils/object/prepareStream.js | 4 +- lib/routes/veeam/utils.js | 11 +- .../unit/api/apiUtils/object/prepareStream.js | 65 +++++++ tests/unit/api/objectPut.js | 183 ++++++++++++++++++ tests/unit/routes/veeam-utils.js | 59 ++++++ 6 files changed, 322 insertions(+), 8 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 24d915acd0..82f26ccfc5 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -19,6 +19,7 @@ const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError, validateXAmzContentSHA256, + areChecksumsEnabled, } = require('../integrity/validateChecksums'); const { externalBackends, versioningNotImplBackends } = constants; @@ -43,6 +44,11 @@ function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { if (contentSHA256Err) { return callback(arsenalErrorFromChecksumError(contentSHA256Err)); } + // Nothing is computed or stored when checksums are disabled, so a zero-byte + // object ends up with no checksum metadata like every other object. + if (!areChecksumsEnabled()) { + return callback(null); + } const checksumData = getChecksumDataFromHeaders(headers) || defaultChecksumData; if (checksumData.error) { return callback(arsenalErrorFromChecksumError(checksumData)); @@ -316,7 +322,7 @@ function createAndStoreObject( } } - const headerChecksum = getChecksumDataFromHeaders(request.headers); + const headerChecksum = areChecksumsEnabled() ? getChecksumDataFromHeaders(request.headers) : null; if (headerChecksum && headerChecksum.error) { return next(arsenalErrorFromChecksumError(headerChecksum)); } diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index 38f1a07836..e0a70c990a 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -2,7 +2,7 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); const ContentSHA256Transform = require('../../../auth/streamingV4/ContentSHA256Transform'); -const { parseContentSHA256, ContentSHA256Type } = require('../integrity/validateChecksums'); +const { parseContentSHA256, ContentSHA256Type, areChecksumsEnabled } = require('../integrity/validateChecksums'); const { errors, errorInstances, jsutil } = require('arsenal'); const { unsupportedSignatureChecksums } = require('../../../../constants'); @@ -80,7 +80,7 @@ function pipeChecksumStreams(inputStream, primary, secondary, onStreamError, log */ function prepareStream(request, streamingV4Params, checksums, log, errCb) { const xAmzContentSHA256 = request.headers['x-amz-content-sha256']; - const { primary = null, secondary = null } = checksums || {}; + const { primary = null, secondary = null } = (areChecksumsEnabled() && checksums) || {}; switch (xAmzContentSHA256) { case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': { diff --git a/lib/routes/veeam/utils.js b/lib/routes/veeam/utils.js index cf3fa0f7a3..3634f18a34 100644 --- a/lib/routes/veeam/utils.js +++ b/lib/routes/veeam/utils.js @@ -10,6 +10,7 @@ const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError, defaultChecksumData, + areChecksumsEnabled, } = require('../../api/apiUtils/integrity/validateChecksums'); const UtilizationService = require('../../utilization/instance'); const metadata = require('../../metadata/wrapper'); @@ -49,7 +50,7 @@ async function receiveData(request, log) { `maximum allowed content-length is ${ContentLengthThreshold} bytes`, ); } - const headerChecksum = getChecksumDataFromHeaders(request.headers); + const headerChecksum = areChecksumsEnabled() ? getChecksumDataFromHeaders(request.headers) : null; if (headerChecksum && headerChecksum.error) { throw arsenalErrorFromChecksumError(headerChecksum); } @@ -89,12 +90,12 @@ async function receiveData(request, log) { // Checksum transforms only compute digests while streaming: validation // against the expected values (header or trailer) must be done once the // stream is fully consumed. - // `checksums.primary` is always set above, so primaryChecksumStream is the - // end of the pipeline here; validate it explicitly rather than relying on - // `prepared.stream` happening to be that transform. + // Validate the primary stream explicitly rather than relying on + // `prepared.stream` happening to be that transform. It is absent when + // checksums are disabled, in which case there is nothing to validate. const checksumErr = (prepared.contentSHA256Stream && prepared.contentSHA256Stream.validateChecksum()) || - prepared.primaryChecksumStream.validateChecksum(); + (prepared.primaryChecksumStream && prepared.primaryChecksumStream.validateChecksum()); if (checksumErr) { log.debug('failed checksum validation', { error: checksumErr }); throw arsenalErrorFromChecksumError(checksumErr); diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index 5a582f03ef..efabb3d0b2 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -10,6 +10,7 @@ const TrailingChecksumTransform = require('../../../../../lib/auth/streamingV4/t const { DummyRequestLogger } = require('../../../helpers'); const DummyRequest = require('../../../DummyRequest'); const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { config } = require('../../../../../lib/Config'); const log = new DummyRequestLogger(); const defaultChecksums = { primary: defaultChecksumData, secondary: null }; @@ -419,3 +420,67 @@ describe('prepareStream', () => { }); }); }); + +describe('prepareStream with checksums disabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + config.integrityChecks = { enabled: false }; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + // Disabling must land in exactly the state a caller requesting no checksum + // already produces, which the 'no checksum requested' suite above pins down. + it('should build no ChecksumTransform even when checksums are requested', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const checksums = { primary: defaultChecksumData, secondary: { algorithm: 'crc32', isTrailer: false } }; + const result = prepareStream(request, null, checksums, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream, request, 'the request should pass through untouched'); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + }); + + it('should drop the checksum transforms on the chunked upload path', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, defaultChecksums, log, () => {}); + assert(result.stream instanceof V4Transform, 'v4 chunk decoding must still happen'); + assert.strictEqual(result.primaryChecksumStream, null); + assert.strictEqual(result.secondaryChecksumStream, null); + }); + + it('should drop the checksum transforms on the trailer path', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert(result.stream instanceof TrailingChecksumTransform, 'trailer framing must still be parsed'); + assert.strictEqual(result.primaryChecksumStream, null); + }); + + it('should still validate a literal x-amz-content-sha256 payload hash', done => { + // x-amz-content-sha256 is SigV4 load-bearing and out of scope for the flag. + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }, bodyData); + const result = prepareStream(request, null, defaultChecksums, log, done); + assert.strictEqual(result.primaryChecksumStream, null); + assert(result.contentSHA256Stream instanceof ContentSHA256Transform); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.contentSHA256Stream.validateChecksum(), null); + done(); + }); + result.stream.on('error', done); + }); + + it('should build the transforms again once re-enabled', () => { + config.integrityChecks = { enabled: true }; + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert(result.primaryChecksumStream instanceof ChecksumTransform); + }); +}); diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 3d3f29bc8f..29b81d965c 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -1411,3 +1411,186 @@ describe('objectPut with objectKeyByteLimit', () => { }); }); }); + +describe('objectPut with checksums disabled', () => { + const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, testPutBucketRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + const putRequest = headers => + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...headers }, + url: '/', + }, + postBody, + ); + + function putDisabled(headers, cb) { + config.integrityChecks = { enabled: false }; + return objectPut(authInfo, putRequest(headers), undefined, log, (err, resHeaders) => { + assert.ifError(err); + return metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + cb(resHeaders, md); + }); + }); + } + + it('should store no checksum metadata when none is requested', done => { + putDisabled(undefined, (resHeaders, md) => { + // Enabled, this stores the implicit crc64nvme default. + assert.strictEqual(md.checksum, undefined); + assert.strictEqual(resHeaders['x-amz-checksum-crc64nvme'], undefined); + done(); + }); + }); + + it('should ignore a client-supplied checksum rather than storing it', done => { + putDisabled({ 'x-amz-checksum-sha256': sha256Value }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + assert.strictEqual(resHeaders['x-amz-checksum-sha256'], undefined); + done(); + }); + }); + + it('should accept a client-supplied checksum that does not match the body', done => { + const wrong = crypto.createHash('sha256').update('not the body').digest('base64'); + putDisabled({ 'x-amz-checksum-sha256': wrong }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should accept a malformed checksum value', done => { + // Enabled, this is InvalidRequest. + putDisabled({ 'x-amz-checksum-crc32': 'not-base64!' }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should accept an unsupported checksum algorithm', done => { + // Enabled, this is InvalidRequest. + putDisabled({ 'x-amz-checksum-md5': 'AAAAAA==' }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should accept multiple checksum headers', done => { + // Enabled, this is InvalidRequest. + putDisabled({ 'x-amz-checksum-sha256': sha256Value, 'x-amz-checksum-crc32': 'AAAAAA==' }, (resHeaders, md) => { + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + + it('should still reject a mismatched Content-MD5', done => { + config.integrityChecks = { enabled: false }; + // checkHashMatchMD5 reads request.contentMD5, not the header. + const request = putRequest(); + request.contentMD5 = crypto.createHash('md5').update('not the body').digest('base64'); + objectPut(authInfo, request, undefined, log, err => { + assert(err, 'should reject'); + assert.strictEqual(err.message, 'BadDigest'); + done(); + }); + }); + + it('should store no checksum for a zero-byte object', done => { + config.integrityChecks = { enabled: false }; + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + Buffer.alloc(0), + ); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should ignore a wrong checksum on a zero-byte object', done => { + config.integrityChecks = { enabled: false }; + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'host': `${bucketName}.s3.amazonaws.com`, + 'x-amz-checksum-crc32': 'AAAAAA==', + }, + url: '/', + }, + Buffer.alloc(0), + ); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert.strictEqual(md.checksum, undefined); + done(); + }); + }); + }); + + it('should store the zero-byte checksum again once re-enabled', done => { + config.integrityChecks = { enabled: true }; + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + Buffer.alloc(0), + ); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert(md.checksum, 'enabled should store the empty-body checksum'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + done(); + }); + }); + }); + + it('should store the checksum again once re-enabled', done => { + config.integrityChecks = { enabled: true }; + objectPut(authInfo, putRequest(), undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (mdErr, md) => { + assert.ifError(mdErr); + assert(md.checksum, 'enabled should store the default checksum'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + done(); + }); + }); + }); +}); diff --git a/tests/unit/routes/veeam-utils.js b/tests/unit/routes/veeam-utils.js index 6e31dd5035..0960f96be6 100644 --- a/tests/unit/routes/veeam-utils.js +++ b/tests/unit/routes/veeam-utils.js @@ -5,6 +5,7 @@ const { Readable } = require('stream'); const UtilizationService = require('../../../lib/utilization/instance'); const metadata = require('../../../lib/metadata/wrapper'); const { fetchCapacityMetrics, buildVeeamFileData, receiveData } = require('../../../lib/routes/veeam/utils'); +const { config } = require('../../../lib/Config'); const { DummyRequestLogger } = require('../helpers'); describe('fetchCapacityMetrics', () => { @@ -385,4 +386,62 @@ describe('receiveData', () => { ); await assert.rejects(receiveData(request, log), err => err.is.InvalidArgument); }); + + describe('with checksums disabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + config.integrityChecks = { enabled: false }; + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + }); + + it('should accept a malformed x-amz-checksum header', async () => { + // Enabled, this is InvalidRequest. + const request = makeRequest( + payload, + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc64nvme': 'not-base64!', + }, + payload.length, + ); + const data = await receiveData(request, log); + assert.strictEqual(data, payload); + }); + + it('should ignore a mismatched x-amz-checksum header', async () => { + // Enabled, this is BadDigest. + const request = makeRequest( + payload, + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-sha256': crypto.createHash('sha256').update('other').digest('base64'), + }, + payload.length, + ); + const data = await receiveData(request, log); + assert.strictEqual(data, payload); + }); + + it('should still strip an unvalidated trailing checksum from the body', async () => { + // Enabled, this digest is a BadDigest mismatch. + const body = chunkedBody('AAAAAAAAAAA='); + const request = makeRequest( + body, + { + 'content-length': `${body.length}`, + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc64nvme', + 'x-amz-decoded-content-length': `${payload.length}`, + }, + payload.length, + ); + const data = await receiveData(request, log); + assert.strictEqual(data, payload); + }); + }); }); From 9d68467c65f73a8f8c5b9b0818c9fa4a83a1cb13 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 6 Aug 2026 01:18:55 +0200 Subject: [PATCH 08/14] CLDSRV-965: add checksum toggle to CreateMultipartUpload --- lib/api/completeMultipartUpload.js | 24 +++-- lib/api/initiateMultipartUpload.js | 15 +-- tests/unit/api/multipartUpload.js | 161 +++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 16 deletions(-) diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index edb571e441..be85005f2f 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -340,16 +340,20 @@ function completeMultipartUpload(authInfo, request, log, callback) { } // External-backend MPUs record no checksum config (CLDSRV-964), // so an absent stored type means "not tracked" rather than legacy. - const mpuLocation = storedMetadata.controllingLocationConstraint; - const isExternalMpu = - !!constants.externalBackends[config.getLocationConstraintType(mpuLocation)]; - const typeErr = validateCompleteMPUChecksumType( - request.headers, - storedMetadata.checksumType, - isExternalMpu, - ); - if (typeErr) { - return next(arsenalErrorFromChecksumError(typeErr), destBucket); + // MPUs created while checksums were disabled record none either, + // so the header is ignored rather than validated. + if (areChecksumsEnabled()) { + const mpuLocation = storedMetadata.controllingLocationConstraint; + const isExternalMpu = + !!constants.externalBackends[config.getLocationConstraintType(mpuLocation)]; + const typeErr = validateCompleteMPUChecksumType( + request.headers, + storedMetadata.checksumType, + isExternalMpu, + ); + if (typeErr) { + return next(arsenalErrorFromChecksumError(typeErr), destBucket); + } } return next(null, destBucket, objMD, mpuBucket, storedMetadata); }, diff --git a/lib/api/initiateMultipartUpload.js b/lib/api/initiateMultipartUpload.js index a5331f890e..a216290096 100644 --- a/lib/api/initiateMultipartUpload.js +++ b/lib/api/initiateMultipartUpload.js @@ -24,6 +24,7 @@ const { updateEncryption } = require('./apiUtils/bucket/updateEncryption'); const { getChecksumDataFromMPUHeaders, arsenalErrorFromChecksumError, + areChecksumsEnabled, } = require('./apiUtils/integrity/validateChecksums'); const { config } = require('../Config'); const kms = require('../kms/wrapper'); @@ -87,8 +88,8 @@ function initiateMultipartUpload(authInfo, request, log, callback) { log.debug('invalid x-amz-website-redirect-location' + `value ${websiteRedirectHeader}`, { error: err }); return callback(err); } - const checksumConfig = getChecksumDataFromMPUHeaders(request.headers); - if (checksumConfig.error) { + const checksumConfig = areChecksumsEnabled() ? getChecksumDataFromMPUHeaders(request.headers) : null; + if (checksumConfig && checksumConfig.error) { const checksumErr = arsenalErrorFromChecksumError(checksumConfig); log.debug('checksum header validation failed', { error: checksumErr, method: 'initiateMultipartUpload' }); monitoring.promMetrics('PUT', bucketName, checksumErr.code, 'initiateMultipartUpload'); @@ -150,9 +151,11 @@ function initiateMultipartUpload(authInfo, request, log, callback) { initiatorDisplayName, splitter: constants.splitter, }; - metadataStoreParams.checksumAlgorithm = checksumConfig.algorithm; - metadataStoreParams.checksumType = checksumConfig.type; - metadataStoreParams.checksumIsDefault = checksumConfig.isDefault; + if (checksumConfig) { + metadataStoreParams.checksumAlgorithm = checksumConfig.algorithm; + metadataStoreParams.checksumType = checksumConfig.type; + metadataStoreParams.checksumIsDefault = checksumConfig.isDefault; + } const tagging = request.headers['x-amz-tagging']; if (tagging) { metadataStoreParams.tagging = tagging; @@ -224,7 +227,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) { // Only respond the headers if the user sent them and // the MPU can honor them (not an external backend). - if (!checksumConfig.isDefault && !isExternalLocation) { + if (checksumConfig && !checksumConfig.isDefault && !isExternalLocation) { // eslint-disable-next-line no-param-reassign corsHeaders['x-amz-checksum-algorithm'] = checksumConfig.algorithm.toUpperCase(); // eslint-disable-next-line no-param-reassign diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index 30ef26ccd2..ba844fd53a 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -4885,3 +4885,164 @@ describe('CompleteMultipartUpload with checksums disabled', () => { assert.doesNotMatch(disabledXml, //, 'disabled should omit the final checksum'); }); }); + +describe('initiateMultipartUpload with checksums disabled', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + config.integrityChecks = { enabled: false }; + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + const initiate = headers => util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + + async function uploadIdFrom(xml) { + return (await parseStringPromise(xml)).InitiateMultipartUploadResult.UploadId[0]; + } + + function storedOverview(uploadId) { + for (const [key, value] of metadata.keyMaps.get(mpuBucket)) { + if (key.startsWith('overview') && value.uploadId === uploadId) { + return value; + } + } + return null; + } + + it('should record no checksum config on the MPU', async () => { + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const overview = storedOverview(await uploadIdFrom(await initiate(headers))); + assert.strictEqual(overview.checksumAlgorithm, undefined); + assert.strictEqual(overview.checksumType, undefined); + assert.strictEqual(overview.checksumIsDefault, undefined); + }); + + it('should not reject an unsupported x-amz-checksum-algorithm', async () => { + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'NOT-AN-ALGO' }; + await assert.doesNotReject(initiate(headers)); + }); + + it('should not reject x-amz-checksum-type without an algorithm', async () => { + const headers = { ...initiateRequest.headers, 'x-amz-checksum-type': 'FULL_OBJECT' }; + await assert.doesNotReject(initiate(headers)); + }); + + it('should complete an MPU whose parts carry no checksum', async () => { + // The end-to-end case CreateMPU previously broke: an explicit algorithm + // was recorded, so CompleteMPU demanded per-part digests that never existed. + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const uploadId = await uploadIdFrom(await initiate(headers)); + + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + const completeRequest = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + const xml = await util.promisify(cb => + completeMultipartUpload(authInfo, completeRequest, log, (err, res) => cb(err, res)), + )(); + assert.match(xml, / { + const uploadId = await uploadIdFrom(await initiate(initiateRequest.headers)); + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + const completeRequest = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + completeRequest.headers = { ...completeRequest.headers, 'x-amz-checksum-type': 'FULL_OBJECT' }; + await assert.doesNotReject( + util.promisify(cb => completeMultipartUpload(authInfo, completeRequest, log, (err, res) => cb(err, res)))(), + ); + }); + + it('should record the checksum config again once re-enabled', async () => { + config.integrityChecks = { enabled: true }; + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const overview = storedOverview(await uploadIdFrom(await initiate(headers))); + assert.strictEqual(overview.checksumAlgorithm, 'crc32'); + assert.strictEqual(overview.checksumType, 'COMPOSITE'); + }); +}); + +describe('CompleteMPU x-amz-checksum-type validation vs the checksum flag', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // Creates an MPU under `createEnabled`, uploads one part, and returns a + // CompleteMPU request carrying `headerType` as x-amz-checksum-type. + async function buildComplete(createEnabled, createHeaders, headerType) { + config.integrityChecks = { enabled: createEnabled }; + const headers = { ...initiateRequest.headers, ...createHeaders }; + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + const request = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + request.headers = { ...request.headers, 'x-amz-checksum-type': headerType }; + return request; + } + + const complete = request => + util.promisify(cb => completeMultipartUpload(authInfo, request, log, (err, xml) => cb(err, xml)))(); + + describe('when disabled', () => { + it('should ignore a type header on an MPU that recorded none', async () => { + const request = await buildComplete(false, { 'x-amz-checksum-algorithm': 'crc32' }, 'COMPOSITE'); + config.integrityChecks = { enabled: false }; + await assert.doesNotReject(complete(request)); + }); + + it('should ignore an invalid type value', async () => { + const request = await buildComplete(false, {}, 'NOT-A-TYPE'); + config.integrityChecks = { enabled: false }; + await assert.doesNotReject(complete(request)); + }); + + it('should ignore a type that mismatches one recorded while enabled', async () => { + // MPU created with checksums on, completed after the flag was flipped. + const request = await buildComplete(true, { 'x-amz-checksum-algorithm': 'crc64nvme' }, 'COMPOSITE'); + config.integrityChecks = { enabled: false }; + await assert.doesNotReject(complete(request)); + }); + }); + + describe('when enabled', () => { + it('should still reject a type header on an MPU that recorded none', async () => { + const request = await buildComplete(false, {}, 'FULL_OBJECT'); + config.integrityChecks = { enabled: true }; + await assert.rejects(complete(request), { message: 'InvalidRequest' }); + }); + + it('should still reject an invalid type value', async () => { + const request = await buildComplete(true, { 'x-amz-checksum-algorithm': 'crc64nvme' }, 'NOT-A-TYPE'); + config.integrityChecks = { enabled: true }; + await assert.rejects(complete(request), { message: 'InvalidRequest' }); + }); + + it('should still reject a type that mismatches the recorded one', async () => { + const request = await buildComplete(true, { 'x-amz-checksum-algorithm': 'crc64nvme' }, 'COMPOSITE'); + config.integrityChecks = { enabled: true }; + await assert.rejects(complete(request), { message: 'InvalidRequest' }); + }); + }); +}); From 1a579c1a6f0a057f1f05de7cb24f787d601a233b Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 6 Aug 2026 01:35:06 +0200 Subject: [PATCH 09/14] CLDSRV-965: add checksum toggle to UploadPartCopy --- lib/api/objectPutCopyPart.js | 7 +- tests/unit/api/objectCopyPart.js | 107 +++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/lib/api/objectPutCopyPart.js b/lib/api/objectPutCopyPart.js index ccba422c71..5d7fc53691 100644 --- a/lib/api/objectPutCopyPart.js +++ b/lib/api/objectPutCopyPart.js @@ -16,7 +16,7 @@ const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { initializeInternalLogRequestQueue, queueInternalLogRequest } = require('../utilities/serverAccessLogger'); -const { algorithms } = require('./apiUtils/integrity/validateChecksums'); +const { algorithms, areChecksumsEnabled } = require('./apiUtils/integrity/validateChecksums'); const { buildSourcePartsStream, computeChecksumFromDataLocator } = require('./apiUtils/object/sourceChecksum'); const { config } = require('../Config'); const kms = require('../kms/wrapper'); @@ -28,6 +28,9 @@ const { BackendInfo } = models; const skipError = new Error('skip'); function _shouldRecomputeChecksum(request, sourceChecksum, algo) { + if (!areChecksumsEnabled()) { + return false; + } if (request.headers['x-amz-copy-source-range']) { return true; } @@ -564,7 +567,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket, sourceObject, reqVer // Reuse the source's stored checksum, or none for a legacy or // external-backend MPU. const partChecksum = - algo && !destIsExternal + algo && !destIsExternal && areChecksumsEnabled() ? { algorithm: algo, value: sourceObjMD.checksum.checksumValue } : undefined; if (isSkip) { diff --git a/tests/unit/api/objectCopyPart.js b/tests/unit/api/objectCopyPart.js index cbcf713d40..80dd8b3169 100644 --- a/tests/unit/api/objectCopyPart.js +++ b/tests/unit/api/objectCopyPart.js @@ -457,3 +457,110 @@ describe('objectPutCopyPart._copyPartStreamingWithChecksum', () => { }); }); }); + +describe('objectPutCopyPart with checksums disabled', () => { + const { _shouldRecomputeChecksum } = objectPutCopyPart; + const objData = Buffer.from('foo', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + sinon.spy(metadataswitch, 'putObjectMD'); + async.waterfall( + [ + cb => bucketPut(authInfo, putDestBucketRequest, log, e => cb(e)), + cb => bucketPut(authInfo, putSourceBucketRequest, log, e => cb(e)), + cb => + objectPut( + authInfo, + versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData), + undefined, + log, + e => cb(e), + ), + ], + done, + ); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + sinon.restore(); + cleanup(); + }); + + describe('_shouldRecomputeChecksum', () => { + it('should never recompute, whatever the source or range', () => { + config.integrityChecks = { enabled: false }; + const withRange = { headers: { 'x-amz-copy-source-range': 'bytes=0-1' } }; + const noRange = { headers: {} }; + const composite = { checksumType: 'COMPOSITE', checksumAlgorithm: 'crc32' }; + // Each of these returns true when enabled. + assert.strictEqual(_shouldRecomputeChecksum(withRange, composite, 'crc32'), false); + assert.strictEqual(_shouldRecomputeChecksum(noRange, undefined, 'crc32'), false); + assert.strictEqual(_shouldRecomputeChecksum(noRange, composite, 'crc32'), false); + }); + + it('should recompute again once re-enabled', () => { + config.integrityChecks = { enabled: true }; + assert.strictEqual(_shouldRecomputeChecksum({ headers: {} }, undefined, 'crc32'), true); + }); + }); + + describe('part metadata', () => { + function copyPartDisabled({ sourceChecksum, headers } = {}) { + return new Promise((resolve, reject) => { + const initReq = _createInitiateRequest(destBucketName, { + 'x-amz-checksum-algorithm': 'CRC32', + }); + // Create the MPU with checksums on so an algorithm is recorded, + // then disable: the flag must be honoured at copy time. + config.integrityChecks = { enabled: true }; + return initiateMultipartUpload(authInfo, initReq, log, (err, res) => { + if (err) { + return reject(err); + } + return parseString(res, (parseErr, json) => { + if (parseErr) { + return reject(parseErr); + } + const uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + if (sourceChecksum) { + metadata.keyMaps.get(sourceBucketName).get(objectKey).checksum = sourceChecksum; + } else { + delete metadata.keyMaps.get(sourceBucketName).get(objectKey).checksum; + } + config.integrityChecks = { enabled: false }; + const req = _createObjectCopyPartRequest(destBucketName, uploadId, headers); + return objectPutCopyPart(authInfo, req, sourceBucketName, objectKey, undefined, log, copyErr => + copyErr ? reject(copyErr) : resolve(metadataswitch.putObjectMD.lastCall.args[2]), + ); + }); + }); + }); + } + + it('should store no checksum when the source has one to reuse', async () => { + const omVal = await copyPartDisabled({ + sourceChecksum: { checksumType: 'FULL_OBJECT', checksumAlgorithm: 'crc32', checksumValue: 'AAAAAA==' }, + }); + assert.strictEqual(omVal.checksumAlgorithm, undefined); + assert.strictEqual(omVal.checksumValue, undefined); + }); + + it('should store no checksum when the source has none', async () => { + // Would previously recompute; must not dereference the absent source checksum. + const omVal = await copyPartDisabled(); + assert.strictEqual(omVal.checksumAlgorithm, undefined); + assert.strictEqual(omVal.checksumValue, undefined); + }); + + it('should store no checksum for a ranged copy', async () => { + // A range always forces a recompute when enabled. + const omVal = await copyPartDisabled({ headers: { 'x-amz-copy-source-range': 'bytes=0-1' } }); + assert.strictEqual(omVal.checksumAlgorithm, undefined); + assert.strictEqual(omVal.checksumValue, undefined); + }); + }); +}); From 38f420c078e694c23436e8b57f25d1c1d512e822 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 6 Aug 2026 01:45:39 +0200 Subject: [PATCH 10/14] CLDSRV-965: add checksum toggle to UploadPart --- lib/api/objectPutPart.js | 13 +-- tests/unit/api/multipartUpload.js | 36 +++++++++ tests/unit/api/objectPutPartChecksum.js | 100 ++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 5 deletions(-) diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index 780c1e9b8e..c314c3bd2b 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -19,7 +19,11 @@ const { BackendInfo } = models; const writeContinue = require('../utilities/writeContinue'); const { parseObjectEncryptionHeaders } = require('./apiUtils/bucket/bucketEncryption'); const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol'); -const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError } = require('./apiUtils/integrity/validateChecksums'); +const { + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, + areChecksumsEnabled, +} = require('./apiUtils/integrity/validateChecksums'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { storeServerAccessLogInfo } = require('../metadata/metadataUtils'); @@ -373,7 +377,8 @@ function objectPutPart(authInfo, request, streamingV4Params, log, cb) { }; const backendInfo = new BackendInfo(config, objectLocationConstraint); - const headerChecksum = getChecksumDataFromHeaders(request.headers); + const checksumsEnabled = areChecksumsEnabled(); + const headerChecksum = checksumsEnabled ? getChecksumDataFromHeaders(request.headers) : null; if (headerChecksum && headerChecksum.error) { return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket); } @@ -392,9 +397,7 @@ function objectPutPart(authInfo, request, streamingV4Params, log, cb) { return next(checksumTypeMismatchErr(mpuChecksumAlgo, headerChecksum.algorithm), destinationBucket); } - // A COMPOSITE MPU's final checksum is composed from the per-part - // checksums, so every part must carry one. - if (!headerChecksum && mpuChecksumType === 'COMPOSITE') { + if (checksumsEnabled && !headerChecksum && mpuChecksumType === 'COMPOSITE') { return next(checksumTypeMismatchErr(mpuChecksumAlgo, 'null'), destinationBucket); } diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index ba844fd53a..4ea1e69c26 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -5046,3 +5046,39 @@ describe('CompleteMPU x-amz-checksum-type validation vs the checksum flag', () = }); }); }); + +describe('MPU created with checksums, then disabled mid-upload', () => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // crc32 defaults to COMPOSITE, the combination that previously wedged: + // UploadPart demanded a per-part checksum that CompleteMPU would never use. + it('should complete an explicit COMPOSITE MPU flipped to disabled', async () => { + config.integrityChecks = { enabled: true }; + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': 'crc32' }; + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + + config.integrityChecks = { enabled: false }; + const partRequest = _createPutPartRequest(uploadId, 1, partBody); + const eTag = await util.promisify(objectPutPart)(authInfo, partRequest, undefined, log); + + const completeRequest = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + const xml = await util.promisify(cb => + completeMultipartUpload(authInfo, completeRequest, log, (err, res) => cb(err, res)), + )(); + assert.match(xml, / { }); }); }); + +describe('objectPutPart with checksums disabled', () => { + let originalIntegrityChecks; + + beforeEach(() => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // Creates the MPU with checksums enabled so an algorithm is recorded, then + // disables before uploading: the mid-flight case where the flag is flipped + // between CreateMPU and UploadPart. + function uploadPartAfterDisabling(initiateHeaders, partHeaders, cb) { + config.integrityChecks = { enabled: true }; + return initiateMPU(initiateHeaders, (err, uploadId) => { + assert.ifError(err); + config.integrityChecks = { enabled: false }; + const request = makePutPartRequest(uploadId, 1, partBody, partHeaders); + return objectPutPart(authInfo, request, undefined, log, (putErr, resHeaders) => + cb(putErr, uploadId, resHeaders), + ); + }); + } + + it('should accept a part with no checksum on a COMPOSITE MPU', done => { + // Enabled, this is rejected: a COMPOSITE MPU requires a per-part checksum. + uploadPartAfterDisabling({ 'x-amz-checksum-algorithm': 'crc32' }, {}, (err, uploadId) => { + assert.ifError(err); + const part = getPartMetadata(uploadId); + assert.strictEqual(part.checksumValue, undefined); + done(); + }); + }); + + it('should accept a part whose algorithm differs from the MPU', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-sha256': 'YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=' }, + (err, uploadId) => { + assert.ifError(err); + assert.strictEqual(getPartMetadata(uploadId).checksumValue, undefined); + done(); + }, + ); + }); + + it('should accept a part whose checksum does not match the body', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-crc32': 'AAAAAA==' }, + (err, uploadId) => { + assert.ifError(err); + assert.strictEqual(getPartMetadata(uploadId).checksumValue, undefined); + done(); + }, + ); + }); + + it('should not echo a checksum back in the response', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-crc32': 'AAAAAA==' }, + (err, uploadId, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders['x-amz-checksum-crc32'], undefined); + done(); + }, + ); + }); + + it('should not reject a malformed per-part checksum header', done => { + uploadPartAfterDisabling( + { 'x-amz-checksum-algorithm': 'crc32' }, + { 'x-amz-checksum-crc32': 'not-base64!!' }, + err => { + assert.ifError(err); + done(); + }, + ); + }); + + it('should reject a part with no checksum on a COMPOSITE MPU once re-enabled', done => { + config.integrityChecks = { enabled: true }; + initiateMPU({ 'x-amz-checksum-algorithm': 'crc32' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody, {}); + objectPutPart(authInfo, request, undefined, log, putErr => { + assert(putErr, 'should reject'); + assert.strictEqual(putErr.message, 'InvalidRequest'); + done(); + }); + }); + }); +}); From 82188477ad4e7befd33bd74272995459acc235da Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 6 Aug 2026 02:22:30 +0200 Subject: [PATCH 11/14] CLDSRV-965: add checksums disabled functional tests --- .github/docker/docker-compose.yaml | 1 + .github/workflows/tests.yaml | 49 +++ package.json | 1 + .../checksumsDisabled/checksumsDisabled.js | 365 ++++++++++++++++++ 4 files changed, 416 insertions(+) create mode 100644 tests/functional/checksumsDisabled/checksumsDisabled.js diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index ffd1172b94..ec860ba00f 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -47,6 +47,7 @@ services: - S3QUOTA - QUOTA_ENABLE_INFLIGHTS - S3_VERSION_ID_ENCODING_TYPE + - S3_INTEGRITY_CHECKS_ENABLED - S3_SERVER_ACCESS_LOGS_MODE=ENABLED - S3_ENABLE_SERVER_ACCESS_LOGS=true - RATE_LIMIT_SERVICE_USER_ARN=arn:aws:iam::123456789013:root diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f96a040dc7..6fe10a3fe4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -558,6 +558,55 @@ jobs: source: /tmp/artifacts if: always() + # Runs the server with x-amz-checksum-* computation turned off and asserts the + # resulting behaviour. Needs its own job because every other functional suite + # runs against a server with checksums enabled. + checksums-disabled-tests: + runs-on: ubuntu-24.04 + needs: build + env: + S3BACKEND: mem + S3VAULT: mem + CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage + MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} + MPU_TESTING: 'yes' + S3_INTEGRITY_CHECKS_ENABLED: 'false' + JOB_NAME: ${{ github.job }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup CI services + run: docker compose up -d + working-directory: .github/docker + - name: Run checksums-disabled tests + run: |- + set -o pipefail; + bash wait_for_local_port.bash 8000 40 + yarn run ft_checksums_disabled | tee /tmp/artifacts/${{ github.job }}/tests.log + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: checksums-disabled-tests + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() + # Configure and run as Integration run S3C tests s3c-ft-tests: strategy: diff --git a/package.json b/package.json index 8b99acb926..939de713a8 100644 --- a/package.json +++ b/package.json @@ -121,6 +121,7 @@ "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support --exit", "ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/versioning/ --exit", "ft_awssdk_external_backends": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/multipleBackend --exit", + "ft_checksums_disabled": "cd tests/functional/checksumsDisabled && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 120000 *.js --exit", "ft_mixed_bucket_format_version": "cd tests/functional/metadata && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json MixedVersionFormat.js --exit", "ft_management": "cd tests/functional/report && yarn test", "ft_backbeat": "cd tests/functional/backbeat && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", diff --git a/tests/functional/checksumsDisabled/checksumsDisabled.js b/tests/functional/checksumsDisabled/checksumsDisabled.js new file mode 100644 index 0000000000..b70d92e18b --- /dev/null +++ b/tests/functional/checksumsDisabled/checksumsDisabled.js @@ -0,0 +1,365 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const { + S3Client, + CreateBucketCommand, + DeleteBucketCommand, + PutObjectCommand, + GetObjectCommand, + HeadObjectCommand, + CopyObjectCommand, + GetObjectAttributesCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + UploadPartCopyCommand, + CompleteMultipartUploadCommand, + AbortMultipartUploadCommand, + ListPartsCommand, + PutBucketTaggingCommand, + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); + +const getConfig = require('../aws-node-sdk/test/support/config'); +const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); + +/* + * These tests require CloudServer to be running with checksums DISABLED: + * + * S3_INTEGRITY_CHECKS_ENABLED=false + * + * or `integrityChecks: { enabled: false }` in config.json. They assert the + * opposite of the normal checksum suites, so they are deliberately kept out of + * tests/functional/aws-node-sdk/test/ — `yarn ft_awssdk` runs that tree against + * a server with checksums on, where every assertion here would fail. + * + * Run with: yarn ft_checksums_disabled + */ + +const bucket = `checksums-disabled-${Date.now()}`; +const body = Buffer.from('I am the body of an object', 'utf8'); +const bodyMd5 = crypto.createHash('md5').update(body).digest('base64'); +// A syntactically valid CRC32 that does not match `body`. +const wrongCrc32 = 'AAAAAA=='; +// 5MB, the minimum size for a non-final MPU part. +const partBody = Buffer.alloc(5 * 1024 * 1024, 'a'); + +// Every checksum field the SDK may surface on a response. +const CHECKSUM_FIELDS = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + 'ChecksumType', +]; + +function assertNoChecksum(res, context) { + CHECKSUM_FIELDS.forEach(field => { + assert.strictEqual(res[field], undefined, `${context}: expected no ${field}, got ${res[field]}`); + }); +} + +describe('with checksums disabled', () => { + let s3; + let bucketUtil; + + before(async () => { + bucketUtil = new BucketUtility('default', {}); + s3 = new S3Client({ ...getConfig('default', {}), maxAttempts: 0 }); + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + + // Fail fast and loudly rather than emitting a wall of confusing + // assertion errors if the server was started with checksums enabled. + const probe = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'probe', Body: body })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'probe' })); + if (CHECKSUM_FIELDS.some(f => probe[f] !== undefined)) { + throw new Error( + 'This suite requires CloudServer running with checksums disabled ' + + '(S3_INTEGRITY_CHECKS_ENABLED=false); the server returned a checksum.', + ); + } + }); + + after(async () => { + await bucketUtil.empty(bucket); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + describe('PutObject', () => { + it('should not return a checksum when none is requested', async () => { + const res = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'plain', Body: body })); + assertNoChecksum(res, 'PutObject'); + }); + + it('should accept a checksum that does not match the body', async () => { + // Enabled, this is BadDigest. + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'wrong-checksum', + Body: body, + ChecksumCRC32: wrongCrc32, + }), + ); + assertNoChecksum(res, 'PutObject with a wrong checksum'); + }); + + it('should not return a checksum on GET, HEAD or GetObjectAttributes', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'readback', Body: body })); + + const get = await s3.send( + new GetObjectCommand({ Bucket: bucket, Key: 'readback', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(get, 'GetObject'); + await get.Body.transformToByteArray(); + + const head = await s3.send( + new HeadObjectCommand({ Bucket: bucket, Key: 'readback', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(head, 'HeadObject'); + + const attrs = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: 'readback', + ObjectAttributes: ['Checksum', 'ETag'], + }), + ); + assert.strictEqual(attrs.Checksum, undefined, 'GetObjectAttributes should report no Checksum'); + assert(attrs.ETag, 'GetObjectAttributes should still report an ETag'); + }); + + it('should still enforce Content-MD5', async () => { + const wrongMd5 = crypto.createHash('md5').update('not the body').digest('base64'); + await assert.rejects( + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'bad-md5', + Body: body, + ContentMD5: wrongMd5, + }), + ), + err => err.name === 'BadDigest' || err.Code === 'BadDigest', + ); + }); + + it('should accept a correct Content-MD5', async () => { + const res = await s3.send( + new PutObjectCommand({ Bucket: bucket, Key: 'good-md5', Body: body, ContentMD5: bodyMd5 }), + ); + assertNoChecksum(res, 'PutObject with a valid Content-MD5'); + }); + + it('should store no checksum for a zero-byte object', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'empty', Body: Buffer.alloc(0) })); + const head = await s3.send( + new HeadObjectCommand({ Bucket: bucket, Key: 'empty', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(head, 'HeadObject on a zero-byte object'); + }); + }); + + describe('CopyObject', () => { + it('should not carry a checksum to the destination', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'copy-src', Body: body })); + const res = await s3.send( + new CopyObjectCommand({ + Bucket: bucket, + Key: 'copy-dst', + CopySource: `/${bucket}/copy-src`, + }), + ); + assertNoChecksum(res.CopyObjectResult || {}, 'CopyObject'); + + const head = await s3.send( + new HeadObjectCommand({ Bucket: bucket, Key: 'copy-dst', ChecksumMode: 'ENABLED' }), + ); + assertNoChecksum(head, 'HeadObject on the copy'); + }); + + it('should ignore a requested checksum algorithm', async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: bucket, + Key: 'copy-dst-sha256', + CopySource: `/${bucket}/copy-src`, + ChecksumAlgorithm: 'SHA256', + }), + ); + assertNoChecksum(res.CopyObjectResult || {}, 'CopyObject with ChecksumAlgorithm'); + }); + }); + + describe('multipart upload', () => { + async function runMpu(key, createParams, partParams) { + const create = await s3.send( + new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ...createParams }), + ); + const uploadId = create.UploadId; + try { + const part = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 1, + Body: partBody, + ...partParams, + }), + ); + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: [{ ETag: part.ETag, PartNumber: 1 }] }, + }), + ); + return { create, part, complete, uploadId }; + } catch (err) { + await s3 + .send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId })) + .catch(() => {}); + throw err; + } + } + + it('should not echo a checksum algorithm from CreateMultipartUpload', async () => { + const { create, part, complete } = await runMpu('mpu-explicit', { + ChecksumAlgorithm: 'CRC32', + }); + assert.strictEqual(create.ChecksumAlgorithm, undefined); + assert.strictEqual(create.ChecksumType, undefined); + assertNoChecksum(part, 'UploadPart'); + assertNoChecksum(complete, 'CompleteMultipartUpload'); + }); + + it('should complete an MPU created with an explicit algorithm and no per-part checksums', async () => { + // Enabled, a CRC32 MPU is COMPOSITE and UploadPart would reject a + // part carrying no x-amz-checksum-crc32. + const { complete } = await runMpu('mpu-no-part-checksums', { ChecksumAlgorithm: 'CRC32' }); + assert(complete.ETag, 'CompleteMultipartUpload should succeed'); + assertNoChecksum(complete, 'CompleteMultipartUpload'); + }); + + it('should accept a per-part checksum that does not match the part', async () => { + const { complete } = await runMpu( + 'mpu-wrong-part-checksum', + { ChecksumAlgorithm: 'CRC32' }, + { ChecksumCRC32: wrongCrc32 }, + ); + assert(complete.ETag); + }); + + it('should report no checksum in ListParts', async () => { + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-listparts', + ChecksumAlgorithm: 'CRC32', + }), + ); + const part = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: 'mpu-listparts', + UploadId: create.UploadId, + PartNumber: 1, + Body: partBody, + }), + ); + assert(part.ETag); + + const list = await s3.send( + new ListPartsCommand({ Bucket: bucket, Key: 'mpu-listparts', UploadId: create.UploadId }), + ); + assert.strictEqual(list.ChecksumAlgorithm, undefined); + (list.Parts || []).forEach(p => assertNoChecksum(p, 'ListParts part')); + + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-listparts', + UploadId: create.UploadId, + }), + ); + }); + + it('should not store a checksum on UploadPartCopy', async () => { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'copypart-src', Body: partBody })); + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart', + ChecksumAlgorithm: 'CRC32', + }), + ); + const copied = await s3.send( + new UploadPartCopyCommand({ + Bucket: bucket, + Key: 'mpu-copypart', + UploadId: create.UploadId, + PartNumber: 1, + CopySource: `/${bucket}/copypart-src`, + }), + ); + assertNoChecksum(copied.CopyPartResult || {}, 'UploadPartCopy'); + + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart', + UploadId: create.UploadId, + MultipartUpload: { + Parts: [{ ETag: copied.CopyPartResult.ETag, PartNumber: 1 }], + }, + }), + ); + assertNoChecksum(complete, 'CompleteMultipartUpload after UploadPartCopy'); + }); + + it('should not store a checksum on a ranged UploadPartCopy', async () => { + // A copy-source range always forces a recompute when enabled. + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart-range', + ChecksumAlgorithm: 'CRC32', + }), + ); + const copied = await s3.send( + new UploadPartCopyCommand({ + Bucket: bucket, + Key: 'mpu-copypart-range', + UploadId: create.UploadId, + PartNumber: 1, + CopySource: `/${bucket}/copypart-src`, + CopySourceRange: `bytes=0-${partBody.length - 1}`, + }), + ); + assertNoChecksum(copied.CopyPartResult || {}, 'ranged UploadPartCopy'); + + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: 'mpu-copypart-range', + UploadId: create.UploadId, + }), + ); + }); + }); + + describe('buffered-body endpoints', () => { + it('should accept a wrong x-amz-checksum on PutBucketTagging', async () => { + // Enabled, this is BadDigest. + await s3.send( + new PutBucketTaggingCommand({ + Bucket: bucket, + Tagging: { TagSet: [{ Key: 'k', Value: 'v' }] }, + ChecksumCRC32: wrongCrc32, + }), + ); + }); + }); +}); From feb4abfae42b3ec4a3de6cc998c709cb107ff6eb Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 10 Aug 2026 18:45:44 +0200 Subject: [PATCH 12/14] CLDSRV-965: complete MPUs whose parts have no stored checksum --- lib/Config.js | 7 +- .../apiUtils/integrity/validateChecksums.js | 8 +- lib/api/completeMultipartUpload.js | 45 ++++-- lib/api/objectPutPart.js | 18 +++ .../apiUtils/integrity/validateChecksums.js | 14 ++ tests/unit/api/multipartUpload.js | 150 ++++++++++++++++-- 6 files changed, 217 insertions(+), 25 deletions(-) diff --git a/lib/Config.js b/lib/Config.js index af21a103be..97ffccbd38 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -600,9 +600,10 @@ function parseServerAccessLogs(config) { * * Content-MD5 and x-amz-content-sha256 are unaffected and remain enforced. * - * Safe to disable part-way through a multipart upload, but not to re-enable: - * parts uploaded while disabled carry no digest, so a later CompleteMPU cannot - * compose the final checksum and fails. + * Safe to toggle part-way through a multipart upload, and safe to roll out + * progressively across a fleet. Parts uploaded while disabled carry no digest, + * so CompleteMultipartUpload cannot compose a final checksum for that object: + * it completes the upload without one and logs an error naming the parts. * * @param {object} config - raw parsed config file contents * @return {{enabled: boolean}} the parsed integrityChecks section diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 9d84c9152c..91943f60fb 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -608,9 +608,11 @@ function arsenalErrorFromChecksumError(err) { * * @param {object} headers - request.headers (lowercased keys) * @param {object|null} finalChecksum - { algorithm, type, value } or null + * @param {boolean} [checksumUnavailable] - no final checksum could be composed + * because one or more parts carry no stored digest. * @returns {{error: string, details: object}|null} */ -function validateCompleteMultipartUploadChecksum(headers, finalChecksum) { +function validateCompleteMultipartUploadChecksum(headers, finalChecksum, checksumUnavailable) { // x-amz-checksum-type can be present in a completeMPU request so we drop it. // x-amz-checksum-algorithm is ignored by AWS in this context so we drop it. const valueHeaders = listChecksumValueHeaders(headers); @@ -644,6 +646,10 @@ function validateCompleteMultipartUploadChecksum(headers, finalChecksum) { return { error: ChecksumError.MalformedChecksum, details: { algorithm: foundAlgo, expected: foundValue } }; } + if (checksumUnavailable) { + return null; + } + if (!finalChecksum || finalChecksum.algorithm !== foundAlgo || finalChecksum.value !== foundValue) { return { error: ChecksumError.XAmzMismatch, diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index be85005f2f..766e4bc844 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -56,6 +56,10 @@ const allChecksumXmlTags = Object.values(checksumAlgorithms).map(algo => algo.xm * in the request body MUST include the matching Checksum field; * missing → InvalidRequest. * + * Parts with no stored ChecksumValue are exempt from both rules: they are + * uploaded by instances running with checksums disabled. Failing here would make + * such an MPU permanently uncompletable. + * * @param {object} jsonList - parsed CompleteMultipartUpload XML * @param {array} storedParts - parts as returned by services.getMPUparts * @param {string} mpuSplitter - splitter used in part keys @@ -105,18 +109,20 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu } } + const storedPart = storedByPartNumber.get(partNumber); + const storedValue = storedPart && storedPart.value && storedPart.value.ChecksumValue; + if (expectedTag && presentTags.includes(expectedTag)) { const providedValue = part[expectedTag][0]; - const storedPart = storedByPartNumber.get(partNumber); - const storedValue = storedPart && storedPart.value && storedPart.value.ChecksumValue; - if (!storedValue || providedValue !== storedValue) { + // If storedValue is false skip the check. The part was uploaded by a cloudserver with checksums disabled. + if (storedValue && providedValue !== storedValue) { return errorInstances.InvalidPart.customizeDescription( 'One or more of the specified parts could not be found. ' + 'The part may not have been uploaded, or the specified ' + "entity tag may not match the part's entity tag.", ); } - } else if (requireForEachPart) { + } else if (requireForEachPart && storedValue) { return errorInstances.InvalidRequest.customizeDescription( `The upload was created using a ${mpuAlgo} checksum. ` + 'The complete request must include the checksum for each ' + @@ -131,12 +137,17 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu * Compute the final-object checksum for a CompleteMultipartUpload from the * stored MPU configuration and per-part checksums. * - * Returns `{ result, error }`: + * Returns `{ result, error, missingPartChecksums }`: * - Success: `{ result: { algorithm, type, value }, error: null }` * - No MPU checksum configured: `{ result: null, error: null }` - * - "Should-not-happen" branch (missing stored ChecksumValue, unknown - * checksumType, compute primitive error): behavior depends on whether - * the client opted in to checksums — + * - One or more parts carry no stored ChecksumValue: `{ result: null, + * error: null, missingPartChecksums: true }`. Those parts were uploaded by + * an instance running with checksums disabled, so the object completes + * without a checksum rather than stranding the upload. The flag lets the + * caller skip validating a client-asserted final checksum, which it has + * nothing to compare against. + * - "Should-not-happen" branch (unknown checksumType, compute primitive + * error): behavior depends on whether the client opted in to checksums — * - Default MPU (`checksumIsDefault === true`): the client never * asked for a checksum; soft-fail with `{ result: null, error: null }` * so the response simply omits the checksum field. @@ -151,7 +162,8 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu * @param {string} mpuSplitter - splitter used in part keys * @param {string} uploadId - for log context * @param {object} log - werelogs logger - * @returns {Promise<{ result: ({algorithm, type, value}|null), error: (ArsenalError|null) }>} + * @returns {Promise<{ result: ({algorithm, type, value}|null), error: (ArsenalError|null), + * missingPartChecksums: (boolean|undefined) }>} */ async function computeFinalChecksum(storedParts, filteredPartList, storedMetadata, mpuSplitter, uploadId, log) { const algorithm = storedMetadata.checksumAlgorithm; @@ -185,7 +197,12 @@ async function computeFinalChecksum(storedParts, filteredPartList, storedMetadat } if (missingPartNumbers.length > 0) { - return failOrSkip('one or more MPU parts missing checksum value', { missingPartNumbers }); + log.error( + 'unable to calculate final object checksum, some parts have no stored checksum. ' + + 'Object will be created with no checksum', + { uploadId, algorithm, type, missingPartNumbers }, + ); + return { result: null, error: null, missingPartChecksums: true }; } let computed; @@ -606,7 +623,7 @@ function completeMultipartUpload(authInfo, request, log, callback) { uploadId, log, ).then( - ({ result, error }) => { + ({ result, error, missingPartChecksums }) => { if (error) { log.error('failing CompleteMPU due to final-object checksum error', { uploadId, @@ -616,7 +633,11 @@ function completeMultipartUpload(authInfo, request, log, callback) { } try { finalChecksum = result; - const checksumErr = validateCompleteMultipartUploadChecksum(request.headers, finalChecksum); + const checksumErr = validateCompleteMultipartUploadChecksum( + request.headers, + finalChecksum, + missingPartChecksums, + ); if (checksumErr) { log.debug('x-amz-checksum- header validation failed on CompleteMPU', { uploadId, diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index c314c3bd2b..a593dca3a7 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -378,6 +378,24 @@ function objectPutPart(authInfo, request, streamingV4Params, log, cb) { const backendInfo = new BackendInfo(config, objectLocationConstraint); const checksumsEnabled = areChecksumsEnabled(); + // The MPU recorded a checksum algorithm, so this part was meant to + // get a digest and the final object a checksum. Storing none means + // CompleteMultipartUpload will not be able to compose one. Logged + // for implicit-default MPUs too: the object still loses a checksum + // it would otherwise have had. + if (!checksumsEnabled && mpuChecksumAlgo) { + log.warn( + "checksums are disabled; MPU required part checksum won't be stored; " + + "final object won't get a checksum", + { + uploadId, + partNumber, + checksumAlgorithm: mpuChecksumAlgo, + checksumType: mpuChecksumType, + checksumIsDefault: !!mpuChecksumIsDefault, + }, + ); + } const headerChecksum = checksumsEnabled ? getChecksumDataFromHeaders(request.headers) : null; if (headerChecksum && headerChecksum.error) { return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index b2fec1353e..94d1d45bbe 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -1095,6 +1095,20 @@ describe('validateCompleteMultipartUploadChecksum', () => { assert.strictEqual(err.error, ChecksumError.XAmzMismatch); }); + it('should return null when the checksum could not be composed', () => { + // Parts uploaded by an instance with checksums disabled. The client's + // asserted value is likely correct; it is the server that has nothing + // to compare it against, so the assertion is not rejected. + const err = validateCompleteMultipartUploadChecksum({ 'x-amz-checksum-sha256': `${SHA256_A}-3` }, null, true); + assert.ifError(err); + }); + + it('should still shape-check the header when the checksum could not be composed', () => { + const err = validateCompleteMultipartUploadChecksum({ 'x-amz-checksum-crc32': 'not-base64!!' }, null, true); + assert(err); + assert.strictEqual(err.error, ChecksumError.MalformedChecksum); + }); + it('should return null when finalChecksum is null and no header present', () => { const err = validateCompleteMultipartUploadChecksum({ host: 'example.com' }, null); assert.ifError(err); diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index 4ea1e69c26..114bf3120b 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -3876,7 +3876,7 @@ describe('validatePerPartChecksums', () => { assert.ifError(err); }); - it('should return InvalidPart when stored part has no checksum but request does', () => { + it('should accept a submitted checksum when the stored part has none', () => { const mpuChecksum = { algorithm: 'sha256', type: 'COMPOSITE', @@ -3891,6 +3891,33 @@ describe('validatePerPartChecksums', () => { ], }; const err = validatePerPartChecksums(jsonList, stored, splitter, mpuChecksum); + assert.ifError(err); + }); + + it('should not require a per-part checksum when the stored part has none', () => { + // Mixed fleet: part 1 stored a digest, part 2 did not, so the client + // only has a checksum for part 1. + const mpuChecksum = { algorithm: 'sha256', type: 'COMPOSITE', isDefault: false }; + const stored = [ + makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] }), + makeStoredPart(2, null), + ]; + const jsonList = { + Part: [ + makeJsonPart(1, 'etag1', { ChecksumSHA256: SAMPLE_DIGESTS.sha256[0] }), + makeJsonPart(2, 'etag2'), + ], + }; + assert.ifError(validatePerPartChecksums(jsonList, stored, splitter, mpuChecksum)); + }); + + it('should still reject a mismatch when the stored part does have a checksum', () => { + const mpuChecksum = { algorithm: 'sha256', type: 'COMPOSITE', isDefault: false }; + const stored = [makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] })]; + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { ChecksumSHA256: SAMPLE_DIGESTS.sha256[1] })], + }; + const err = validatePerPartChecksums(jsonList, stored, splitter, mpuChecksum); assert(err); assert.strictEqual(err.message, 'InvalidPart'); }); @@ -4040,6 +4067,10 @@ describe('computeFinalChecksum', () => { assert.deepStrictEqual(got, { result: null, error: null }); } + function assertMissingPartChecksums(got) { + assert.deepStrictEqual(got, { result: null, error: null, missingPartChecksums: true }); + } + function assertInternalError(got) { assert.strictEqual(got.result, null); assert(got.error, 'expected an error on the result'); @@ -4159,10 +4190,10 @@ describe('computeFinalChecksum', () => { uploadId, log, ); - assertSoftNull(got); + assertMissingPartChecksums(got); }); - it('should return InternalError when an explicit-MPU part is missing ChecksumValue', async () => { + it('should soft-null when an explicit-MPU part is missing ChecksumValue', async () => { const stored = [ makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] }), makeStoredPart(2, null), @@ -4176,7 +4207,9 @@ describe('computeFinalChecksum', () => { uploadId, log, ); - assertInternalError(got); + // Parts uploaded while checksums were disabled cannot be recovered by the + // client, so the object completes without a checksum instead of failing. + assertMissingPartChecksums(got); }); it('should soft-null when checksumType is unknown on a default MPU', async () => { @@ -4501,13 +4534,13 @@ describe('CompleteMultipartUpload per-part validation on external backends', () const _complete = completeReq => util.promisify(cb => completeMultipartUpload(authInfo, completeReq, log, cb))(); describe('COMPOSITE MPU (no per-part checksum)', () => { - it('should reject on a local location', async () => { + it('should complete on a local location, without a final checksum', async () => { + // AWS rejects this, we don't: a part with no stored checksum may come + // from an instance with checksums toggled off, or from a mixed fleet. const { uploadId, eTag } = await _initiateExternalMpu({ type: 'COMPOSITE', locationType: 'scality' }); const completeReq = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); - await assert.rejects(_complete(completeReq), err => { - assert.match(err.message, /InvalidRequest/); - return true; - }); + const xml = await _complete(completeReq); + assert.doesNotMatch(xml, //, 'no final-object checksum should be returned'); }); it('should complete on an external location (checksum config not recorded)', async () => { @@ -5047,6 +5080,105 @@ describe('CompleteMPU x-amz-checksum-type validation vs the checksum flag', () = }); }); +describe('MPU with parts uploaded across a checksum toggle', () => { + // 5MB: the minimum size for a part that is not the last one. + const partBody = Buffer.alloc(5 * 1024 * 1024, 'a'); + const lastPartBody = Buffer.from('I am the last part\n', 'utf8'); + let originalIntegrityChecks; + + beforeEach(done => { + originalIntegrityChecks = config.integrityChecks; + cleanup(); + bucketPut(authInfo, bucketPutRequest, log, done); + }); + + afterEach(() => { + config.integrityChecks = originalIntegrityChecks; + cleanup(); + }); + + // A rolling upgrade or a progressive toggle means some instances store part + // checksums and some do not, so a single MPU can end up with a mix. The + // client's CompleteMPU body mirrors that: it carries a checksum for the + // parts whose UploadPart response returned one, and nothing for the others. + async function mixedPartsMpu(algo) { + config.integrityChecks = { enabled: true }; + const headers = { ...initiateRequest.headers, 'x-amz-checksum-algorithm': algo }; + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, { ...initiateRequest, headers }, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + + // part 1 lands on an instance with checksums on and gets a digest back + const digest = await algorithms[algo].digest(partBody); + const p1 = _createPutPartRequest(uploadId, 1, partBody); + p1.headers = { ...p1.headers, [`x-amz-checksum-${algo}`]: digest }; + const eTag1 = await util.promisify(objectPutPart)(authInfo, p1, undefined, log); + + // part 2 lands on an instance with checksums off and gets none + config.integrityChecks = { enabled: false }; + const eTag2 = await util.promisify(objectPutPart)( + authInfo, + _createPutPartRequest(uploadId, 2, lastPartBody), + undefined, + log, + ); + + // completion is served by an instance with checksums on + config.integrityChecks = { enabled: true }; + const tag = TAG_BY_ALGO[algo]; + const post = [ + '', + `1"${eTag1}"<${tag}>${digest}`, + `2"${eTag2}"`, + '', + ]; + return { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId }, + post, + actionImplicitDenies: false, + }; + } + + const complete = request => + util.promisify(cb => completeMultipartUpload(authInfo, request, log, (err, xml) => cb(err, xml)))(); + + it('should complete a COMPOSITE MPU with mixed parts, without a checksum', async () => { + const xml = await complete(await mixedPartsMpu('crc32')); + assert.match(xml, / { + // A FULL_OBJECT client may assert the whole-object checksum at + // CompleteMPU. It hashed the data itself, so its value is fine; we just + // have nothing to check it against. Returned BadDigest before the fix. + const algo = 'crc64nvme'; + const request = await mixedPartsMpu(algo); + const asserted = await algorithms[algo].digest(Buffer.concat([partBody, lastPartBody])); + request.headers = { ...request.headers, [`x-amz-checksum-${algo}`]: asserted }; + const xml = await complete(request); + assert.match(xml, / { + const request = await mixedPartsMpu('crc64nvme'); + request.headers = { ...request.headers, 'x-amz-checksum-crc64nvme': 'not-base64!!' }; + await assert.rejects(complete(request), { message: 'InvalidRequest' }); + }); + + it('should complete a FULL_OBJECT MPU with mixed parts, without a checksum', async () => { + const xml = await complete(await mixedPartsMpu('crc64nvme')); + assert.match(xml, / { const partBody = Buffer.from('I am a part\n', 'utf8'); let originalIntegrityChecks; From 758cb0d537e3e1cfda1e3e74ede27bddf7dc877a Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 10 Aug 2026 20:51:52 +0200 Subject: [PATCH 13/14] CLDSRV-965: prettier lint --- .github/docker/docker-compose.yaml | 20 +- tests/unit/api/objectPut.js | 1746 ++++++++++++++++------------ 2 files changed, 981 insertions(+), 785 deletions(-) diff --git a/.github/docker/docker-compose.yaml b/.github/docker/docker-compose.yaml index ec860ba00f..8880f765f2 100644 --- a/.github/docker/docker-compose.yaml +++ b/.github/docker/docker-compose.yaml @@ -1,7 +1,7 @@ services: cloudserver: image: ${CLOUDSERVER_IMAGE} - network_mode: "host" + network_mode: 'host' volumes: - /tmp/ssl:/ssl - /tmp/ssl-kmip:/tmp/ssl-kmip @@ -57,13 +57,13 @@ services: depends_on: - redis extra_hosts: - - "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1" - - "pykmip.local:127.0.0.1" + - 'bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1' + - 'pykmip.local:127.0.0.1' redis: image: redis:alpine - network_mode: "host" + network_mode: 'host' squid: - network_mode: "host" + network_mode: 'host' profiles: ['ci-proxy'] image: scality/ci-squid command: >- @@ -77,7 +77,7 @@ services: volumes: - /tmp/ssl:/ssl pykmip: - network_mode: "host" + network_mode: 'host' profiles: ['pykmip'] image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip} volumes: @@ -87,17 +87,17 @@ services: - ../pykmip/policy.json:/etc/pykmip/policies/policy.json - ../pykmip/server.conf:/etc/pykmip/server.conf localkms: - network_mode: "host" + network_mode: 'host' profiles: ['localkms'] image: ${KMS_IMAGE:-nsmithuk/local-kms:3.11.7} mongo: - network_mode: "host" + network_mode: 'host' profiles: ['mongo'] image: ${MONGODB_IMAGE} volumes: - /tmp/artifacts/${JOB_NAME}:/logs sproxyd: - network_mode: "host" + network_mode: 'host' profiles: ['sproxyd'] image: sproxyd-standalone build: ./sproxyd @@ -110,7 +110,7 @@ services: profiles: ['vault'] user: root command: sh -c "chmod 400 tests/utils/keyfile && yarn start > /artifacts/vault.log 2> /artifacts/vault-stderr.log" - network_mode: "host" + network_mode: 'host' volumes: - /tmp/artifacts/${JOB_NAME}:/artifacts - ./vault-config.json:/conf/config.json:ro diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 29b81d965c..985c5fd39e 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -11,25 +11,18 @@ const bucketPutACL = require('../../../lib/api/bucketPutACL'); const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const { parseTagFromQuery } = s3middleware.tagging; -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); const metadata = require('../metadataswitch'); const { data } = require('../../../lib/data/wrapper'); const objectPut = require('../../../lib/api/objectPut'); const { objectLockTestUtils } = require('../helpers'); const DummyRequest = require('../DummyRequest'); -const { - lastModifiedHeader, - maximumAllowedUploadSize, - objectLocationConstraintHeader, -} = require('../../../constants'); +const { lastModifiedHeader, maximumAllowedUploadSize, objectLocationConstraintHeader } = require('../../../constants'); const mpuUtils = require('../utils/mpuUtils'); const { fakeMetadataArchive } = require('../../functional/aws-node-sdk/test/utils/init'); const { config } = require('../../../lib/Config'); -const { - LOCATION_NAME_CRR, -} = require('../../constants'); +const { LOCATION_NAME_CRR } = require('../../constants'); const { ds } = storage.data.inMemory.datastore; @@ -53,7 +46,7 @@ const testPutBucketRequestLock = new DummyRequest({ bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'true', }, url: '/', @@ -63,21 +56,18 @@ const originalputObjectMD = metadata.putObjectMD; const objectName = 'objectName'; let testPutObjectRequest; -const enableVersioningRequest = - versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); -const suspendVersioningRequest = - versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Suspended'); +const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); +const suspendVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Suspended'); function testAuth(bucketOwner, authUser, bucketPutReq, log, cb) { bucketPut(bucketOwner, bucketPutReq, log, () => { bucketPutACL(bucketOwner, testPutBucketRequest, log, err => { assert.strictEqual(err, undefined); - objectPut(authUser, testPutObjectRequest, undefined, - log, (err, resHeaders) => { - assert.strictEqual(err, null); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - cb(); - }); + objectPut(authUser, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(err, null); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + cb(); + }); }); }); } @@ -88,8 +78,7 @@ describe('parseTagFromQuery', () => { const allowedChar = '+- =._:/'; const tests = [ { tagging: 'key1=value1', result: { key1: 'value1' } }, - { tagging: `key1=${encodeURIComponent(allowedChar)}`, - result: { key1: allowedChar } }, + { tagging: `key1=${encodeURIComponent(allowedChar)}`, result: { key1: allowedChar } }, { tagging: 'key1=value1=value2', error: invalidArgument }, { tagging: '=value1', error: invalidArgument }, { tagging: 'key1%=value1', error: invalidArgument }, @@ -120,13 +109,16 @@ describe('objectPut API', () => { beforeEach(() => { cleanup(); sinon.spy(metadata, 'putObjectMD'); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: '/', - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + postBody, + ); }); afterEach(() => { @@ -143,40 +135,38 @@ describe('objectPut API', () => { it('should return an error if user is not authorized', done => { const putAuthInfo = makeAuthInfo('accessKey2'); - bucketPut(putAuthInfo, testPutBucketRequest, - log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); - }); - }); - - it('should return error if the upload size exceeds the ' + - 'maximum allowed upload size for a single PUT request', done => { - testPutObjectRequest.parsedContentLength = maximumAllowedUploadSize + 1; - bucketPut(authInfo, testPutBucketRequest, log, () => { + bucketPut(putAuthInfo, testPutBucketRequest, log, () => { objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err.is.EntityTooLarge, true); + assert.strictEqual(err.is.AccessDenied, true); done(); }); }); }); + it( + 'should return error if the upload size exceeds the ' + 'maximum allowed upload size for a single PUT request', + done => { + testPutObjectRequest.parsedContentLength = maximumAllowedUploadSize + 1; + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.is.EntityTooLarge, true); + done(); + }); + }); + }, + ); + it('should put object if user has FULL_CONTROL grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testPutBucketRequest.headers['x-amz-grant-full-control'] = - `id=${authUser.getCanonicalID()}`; + testPutBucketRequest.headers['x-amz-grant-full-control'] = `id=${authUser.getCanonicalID()}`; testAuth(bucketOwner, authUser, testPutBucketRequest, log, done); }); it('should put object if user has WRITE grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testPutBucketRequest.headers['x-amz-grant-write'] = - `id=${authUser.getCanonicalID()}`; + testPutBucketRequest.headers['x-amz-grant-write'] = `id=${authUser.getCanonicalID()}`; testAuth(bucketOwner, authUser, testPutBucketRequest, log, done); }); @@ -190,60 +180,61 @@ describe('objectPut API', () => { }); it('should successfully put an object', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, - {}, log, (err, md) => { - assert(md); - assert - .strictEqual(md['content-md5'], correctMD5); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md['content-md5'], correctMD5); + done(); }); + }); }); }); const mockModes = ['GOVERNANCE', 'COMPLIANCE']; mockModes.forEach(mockMode => { it(`should put an object with valid date & ${mockMode} mode`, done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': mockDate, - 'x-amz-object-lock-mode': mockMode, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': mockDate, + 'x-amz-object-lock-mode': mockMode, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequestLock, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, headers) => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + const mode = md.retentionMode; + const retainUntilDate = md.retentionDate; assert.ifError(err); - assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - const mode = md.retentionMode; - const retainUntilDate = md.retentionDate; - assert.ifError(err); - assert(md); - assert.strictEqual(mode, mockMode); - assert.strictEqual(retainUntilDate, mockDate); - done(); - }); + assert(md); + assert.strictEqual(mode, mockMode); + assert.strictEqual(retainUntilDate, mockDate); + done(); }); + }); }); }); }); @@ -262,311 +253,323 @@ describe('objectPut API', () => { ]; testObjectLockConfigs.forEach(lockConfig => { const { testMode, type, val } = lockConfig; - it('should put an object with default retention if object does not ' + - 'have retention configuration but bucket has', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + it( + 'should put an object with default retention if object does not ' + + 'have retention configuration but bucket has', + done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); - const testObjLockRequest = { - bucketName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: objectLockTestUtils.generateXml(testMode, val, type), - }; + const testObjLockRequest = { + bucketName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: objectLockTestUtils.generateXml(testMode, val, type), + }; - bucketPut(authInfo, testPutBucketRequestLock, log, () => { - bucketPutObjectLock(authInfo, testObjLockRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, headers) => { + bucketPut(authInfo, testPutBucketRequestLock, log, () => { + bucketPutObjectLock(authInfo, testObjLockRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, headers) => { assert.ifError(err); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.ifError(err); - - const mode = md.retentionMode; - assert.strictEqual(mode, testMode); - - const retainDate = moment(md.retentionDate); - const days = type === 'Days' ? val : val * 365; - const { scaledMsPerDay } = config.getTimeOptions(); - const date = moment().add(days * scaledMsPerDay, 'ms'); - const dateDiff = retainDate.diff(date, 'ms'); - assert.ok(dateDiff < 10); - - done(); - }); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + + const mode = md.retentionMode; + assert.strictEqual(mode, testMode); + + const retainDate = moment(md.retentionDate); + const days = type === 'Days' ? val : val * 365; + const { scaledMsPerDay } = config.getTimeOptions(); + const date = moment().add(days * scaledMsPerDay, 'ms'); + const dateDiff = retainDate.diff(date, 'ms'); + assert.ok(dateDiff < 10); + + done(); + }); }); + }); }); - }); - }); + }, + ); }); it('should successfully put an object with legal hold ON', done => { - const request = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-legal-hold': 'ON', + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-legal-hold': 'ON', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequestLock, log, () => { objectPut(authInfo, request, undefined, log, (err, headers) => { assert.ifError(err); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert.ifError(err); - assert.strictEqual(md.legalHold, true); - done(); - }); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert.strictEqual(md.legalHold, true); + done(); + }); }); }); }); it('should successfully put an object with legal hold OFF', done => { - const request = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-legal-hold': 'OFF', + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-legal-hold': 'OFF', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequestLock, log, () => { objectPut(authInfo, request, undefined, log, (err, headers) => { assert.ifError(err); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert.ifError(err); - assert(md); - assert.strictEqual(md.legalHold, false); - done(); - }); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md); + assert.strictEqual(md.legalHold, false); + done(); + }); }); }); }); it('should successfully put an object with user metadata', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - // Note that Node will collapse common headers into one - // (e.g. "x-amz-meta-test: hi" and "x-amz-meta-test: - // there" becomes "x-amz-meta-test: hi, there") - // Here we are not going through an actual http - // request so will not collapse properly. - 'x-amz-meta-test': 'some metadata', - 'x-amz-meta-test2': 'some more metadata', - 'x-amz-meta-test3': 'even more metadata', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + // Note that Node will collapse common headers into one + // (e.g. "x-amz-meta-test: hi" and "x-amz-meta-test: + // there" becomes "x-amz-meta-test: hi, there") + // Here we are not going through an actual http + // request so will not collapse properly. + 'x-amz-meta-test': 'some metadata', + 'x-amz-meta-test2': 'some more metadata', + 'x-amz-meta-test3': 'even more metadata', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - assert.strictEqual(md['x-amz-meta-test'], - 'some metadata'); - assert.strictEqual(md['x-amz-meta-test2'], - 'some more metadata'); - assert.strictEqual(md['x-amz-meta-test3'], - 'even more metadata'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md['x-amz-meta-test'], 'some metadata'); + assert.strictEqual(md['x-amz-meta-test2'], 'some more metadata'); + assert.strictEqual(md['x-amz-meta-test3'], 'even more metadata'); + done(); }); + }); }); }); it('If testingMode=true and the last-modified header is given, should set last-modified accordingly', done => { const imposedLastModified = '2024-07-19'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - [lastModifiedHeader]: imposedLastModified, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + [lastModifiedHeader]: imposedLastModified, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { config.testingMode = true; - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - - const lastModified = md['last-modified']; - const lastModifiedDate = lastModified.split('T')[0]; - // last-modified date should be the one set by the last-modified header - assert.strictEqual(lastModifiedDate, imposedLastModified); - - // The header should be removed after being treated. - assert(md[lastModifiedHeader] === undefined); - - config.testingMode = false; - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + + const lastModified = md['last-modified']; + const lastModifiedDate = lastModified.split('T')[0]; + // last-modified date should be the one set by the last-modified header + assert.strictEqual(lastModifiedDate, imposedLastModified); + + // The header should be removed after being treated. + assert(md[lastModifiedHeader] === undefined); + + config.testingMode = false; + done(); }); + }); }); }); it('should not take into acccount the last-modified header when testingMode=false', done => { const imposedLastModified = '2024-07-19'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-meta-x-scal-last-modified': imposedLastModified, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-meta-x-scal-last-modified': imposedLastModified, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { config.testingMode = false; - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - assert.strictEqual(md['x-amz-meta-x-scal-last-modified'], - imposedLastModified); - const lastModified = md['last-modified']; - const lastModifiedDate = lastModified.split('T')[0]; - const currentTs = new Date().toJSON(); - const currentDate = currentTs.split('T')[0]; - assert.strictEqual(lastModifiedDate, currentDate); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md['x-amz-meta-x-scal-last-modified'], imposedLastModified); + const lastModified = md['last-modified']; + const lastModifiedDate = lastModified.split('T')[0]; + const currentTs = new Date().toJSON(); + const currentDate = currentTs.split('T')[0]; + assert.strictEqual(lastModifiedDate, currentDate); + done(); }); + }); }); }); it('should put an object with user metadata but no data', done => { const postBody = ''; const correctMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'content-length': '0', - 'x-amz-meta-test': 'some metadata', - 'x-amz-meta-test2': 'some more metadata', - 'x-amz-meta-test3': 'even more metadata', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': '0', + 'x-amz-meta-test': 'some metadata', + 'x-amz-meta-test2': 'some more metadata', + 'x-amz-meta-test3': 'even more metadata', + }, + parsedContentLength: 0, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'd41d8cd98f00b204e9800998ecf8427e', }, - parsedContentLength: 0, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'd41d8cd98f00b204e9800998ecf8427e', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(ds, []); - metadata.getObjectMD(bucketName, objectName, {}, log, - (err, md) => { - assert(md); - assert.strictEqual(md.location, null); - assert.strictEqual(md['x-amz-meta-test'], - 'some metadata'); - assert.strictEqual(md['x-amz-meta-test2'], - 'some more metadata'); - assert.strictEqual(md['x-amz-meta-test3'], - 'even more metadata'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(ds, []); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert(md); + assert.strictEqual(md.location, null); + assert.strictEqual(md['x-amz-meta-test'], 'some metadata'); + assert.strictEqual(md['x-amz-meta-test2'], 'some more metadata'); + assert.strictEqual(md['x-amz-meta-test3'], 'even more metadata'); + done(); }); + }); }); }); it('should not leave orphans in data when overwriting an object', done => { - const testPutObjectRequest2 = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }, Buffer.from('I am another body', 'utf8')); + const testPutObjectRequest2 = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + Buffer.from('I am another body', 'utf8'), + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, () => { - objectPut(authInfo, testPutObjectRequest2, undefined, - log, - () => { - // orphan objects don't get deleted - // until the next tick - // in memory - setImmediate(() => { - // Data store starts at index 1 - assert.strictEqual(ds[0], undefined); - assert.strictEqual(ds[1], undefined); - assert.deepStrictEqual(ds[2].value, - Buffer.from('I am another body', 'utf8')); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectPut(authInfo, testPutObjectRequest2, undefined, log, () => { + // orphan objects don't get deleted + // until the next tick + // in memory + setImmediate(() => { + // Data store starts at index 1 + assert.strictEqual(ds[0], undefined); + assert.strictEqual(ds[1], undefined); + assert.deepStrictEqual(ds[2].value, Buffer.from('I am another body', 'utf8')); + done(); }); }); + }); }); }); it('should not leave orphans in data when overwriting an multipart upload object', done => { bucketPut(authInfo, testPutBucketRequest, log, () => { - mpuUtils.createMPU(namespace, bucketName, objectName, log, - (err, testUploadId) => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD, - any, any, any, sinon.match({ oldReplayId: testUploadId }), any, any); - done(); - }); + mpuUtils.createMPU(namespace, bucketName, objectName, log, (err, testUploadId) => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD, + any, + any, + any, + sinon.match({ oldReplayId: testUploadId }), + any, + any, + ); + done(); }); + }); }); }); - it('should not put object with retention configuration if object lock ' + - 'is not enabled on the bucket', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': mockDate, - 'x-amz-object-lock-mode': 'GOVERNANCE', + it('should not put object with retention configuration if object lock ' + 'is not enabled on the bucket', done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': mockDate, + 'x-amz-object-lock-mode': 'GOVERNANCE', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { objectPut(authInfo, testPutObjectRequest, undefined, log, err => { @@ -578,233 +581,313 @@ describe('objectPut API', () => { }); it('should forward a 400 back to client on metadata 408 response', () => { - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; const originalPut = data.client.put; - data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => - cb({ httpCode: 408 }); + data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => cb({ httpCode: 408 }); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.code, 400); - data.client.put = originalPut; - data.switch(dataClient); - data.implName = prevDataImplName; - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.code, 400); + data.client.put = originalPut; + data.switch(dataClient); + data.implName = prevDataImplName; + }); }); }); it('should forward a 503 to the client for 4xx != 408', () => { - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; const originalPut = data.client.put; - data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => - cb({ httpCode: 412 }); + data.client.put = (hashedStream, valueSize, keyContext, backendInfo, log, cb) => cb({ httpCode: 412 }); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.code, 503); - data.client.put = originalPut; - data.switch(dataClient); - data.implName = prevDataImplName; - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.code, 503); + data.client.put = originalPut; + data.switch(dataClient); + data.implName = prevDataImplName; + }); }); }); it('should not put object with storage-class header not equal to STANDARD', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-storage-class': 'COLD', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-storage-class': 'COLD', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.is.InvalidStorageClass, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.is.InvalidStorageClass, true); + done(); + }); }); }); it('should pass overheadField to metadata.putObjectMD for a non-versioned request', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - contentMD5: correctMD5, - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + contentMD5: correctMD5, + }, + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); it('should pass overheadField to metadata.putObjectMD for a versioned request', done => { - const testPutObjectRequest = versioningTestUtils - .createPutObjectRequest(bucketName, objectName, Buffer.from('I am another body', 'utf8')); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest( + bucketName, + objectName, + Buffer.from('I am another body', 'utf8'), + ); bucketPut(authInfo, testPutBucketRequest, log, () => { bucketPutVersioning(authInfo, enableVersioningRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - } - ); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); }); it('should pass overheadField to metadata.putObjectMD for a version-suspended request', done => { - const testPutObjectRequest = versioningTestUtils - .createPutObjectRequest(bucketName, objectName, Buffer.from('I am another body', 'utf8')); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest( + bucketName, + objectName, + Buffer.from('I am another body', 'utf8'), + ); bucketPut(authInfo, testPutBucketRequest, log, () => { bucketPutVersioning(authInfo, suspendVersioningRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ overheadField: sinon.match.array }), any, any); - done(); - } - ); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ overheadField: sinon.match.array }), + any, + any, + ); + done(); + }); }); }); }); it('should not pass needOplogUpdate when writing new object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: true, - originOp: 's3:ReplaceArchivedObject', - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: true, + originOp: 's3:ReplaceArchivedObject', + }), + any, + any, + ); + }, + ], + done, + ); }); it('should pass needOplogUpdate to metadata when replacing archived object in version suspended bucket', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: true, - originOp: 's3:ReplaceArchivedObject', - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, suspendVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: true, + originOp: 's3:ReplaceArchivedObject', + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not set bucketOwnerId if requester owns the bucket', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, - objectName, - sinon.match({ bucketOwnerId: sinon.match.typeOf('undefined') }), - any, - any, - any - ); - done(); - } + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + sinon.match({ bucketOwnerId: sinon.match.typeOf('undefined') }), + any, + any, + any, ); + done(); + }); }); }); it('should set bucketOwnerId if requester does not own the bucket', done => { const authInfo2 = makeAuthInfo('accessKey2'); - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + postBody, + ); const testPutPolicyRequest = new DummyRequest({ bucketName, @@ -828,57 +911,61 @@ describe('objectPut API', () => { bucketPut(authInfo, testPutBucketRequest, log, () => { bucketPutPolicy(authInfo, testPutPolicyRequest, log, err => { assert.ifError(err); - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.ifError(err); - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, - objectName, - sinon.match({ bucketOwnerId: authInfo.canonicalId }), - any, - any, - any - ); - done(); - } - ); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + sinon.match({ bucketOwnerId: authInfo.canonicalId }), + any, + any, + any, + ); + done(); + }); }); }); }); it('should fail to put object when setting a crr location as the locationConstraint', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - [objectLocationConstraintHeader]: LOCATION_NAME_CRR, + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + [objectLocationConstraintHeader]: LOCATION_NAME_CRR, + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert(err.is.InvalidArgument); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert(err.is.InvalidArgument); + done(); + }); }); }); it('should store sha256 checksum in metadata when x-amz-checksum-sha256 header is provided', done => { const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64'); - const request = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - host: `${bucketName}.s3.amazonaws.com`, - 'x-amz-checksum-sha256': sha256Value, + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-checksum-sha256': sha256Value, + }, + url: '/', }, - url: '/', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, err => { assert.ifError(err); @@ -918,14 +1005,17 @@ describe('objectPut API', () => { it('should return crc64nvme response header for zero-byte object when no checksum header is provided', done => { const expectedCrc64nvme = 'AAAAAAAAAAA='; - const zeroBytePutRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${bucketName}/${objectName}`, - parsedContentLength: 0, - }, Buffer.alloc(0)); + const zeroBytePutRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${bucketName}/${objectName}`, + parsedContentLength: 0, + }, + Buffer.alloc(0), + ); bucketPut(authInfo, testPutBucketRequest, log, err => { assert.ifError(err); @@ -949,13 +1039,16 @@ describe('objectPut API with versioning', () => { beforeEach(() => { cleanup(); sinon.spy(metadata, 'putObjectMD'); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: '/', - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, + postBody, + ); }); afterEach(() => { @@ -963,197 +1056,241 @@ describe('objectPut API with versioning', () => { metadata.putObjectMD = originalputObjectMD; }); - const objData = ['foo0', 'foo1', 'foo2'].map(str => - Buffer.from(str, 'utf8')); - const testPutObjectRequests = objData.map(data => versioningTestUtils - .createPutObjectRequest(bucketName, objectName, data)); - - it('should delete latest version when creating new null version ' + - 'if latest version is null version', done => { - async.series([ - callback => bucketPut(authInfo, testPutBucketRequest, log, - callback), - // putting null version by putting obj before versioning configured - callback => objectPut(authInfo, testPutObjectRequests[0], undefined, - log, err => { - versioningTestUtils.assertDataStoreValues(ds, [objData[0]]); - callback(err); - }), - callback => bucketPutVersioning(authInfo, suspendVersioningRequest, - log, callback), - // creating new null version by putting obj after ver suspended - callback => objectPut(authInfo, testPutObjectRequests[1], - undefined, log, err => { - // wait until next tick since mem backend executes - // deletes in the next tick - setImmediate(() => { - // old null version should be deleted - versioningTestUtils.assertDataStoreValues(ds, - [undefined, objData[1]]); - callback(err); - }); - }), - // create another null version - callback => objectPut(authInfo, testPutObjectRequests[2], - undefined, log, err => { - setImmediate(() => { - // old null version should be deleted - versioningTestUtils.assertDataStoreValues(ds, - [undefined, undefined, objData[2]]); + const objData = ['foo0', 'foo1', 'foo2'].map(str => Buffer.from(str, 'utf8')); + const testPutObjectRequests = objData.map(data => + versioningTestUtils.createPutObjectRequest(bucketName, objectName, data), + ); + + it('should delete latest version when creating new null version ' + 'if latest version is null version', done => { + async.series( + [ + callback => bucketPut(authInfo, testPutBucketRequest, log, callback), + // putting null version by putting obj before versioning configured + callback => + objectPut(authInfo, testPutObjectRequests[0], undefined, log, err => { + versioningTestUtils.assertDataStoreValues(ds, [objData[0]]); callback(err); - }); - }), - ], done); + }), + callback => bucketPutVersioning(authInfo, suspendVersioningRequest, log, callback), + // creating new null version by putting obj after ver suspended + callback => + objectPut(authInfo, testPutObjectRequests[1], undefined, log, err => { + // wait until next tick since mem backend executes + // deletes in the next tick + setImmediate(() => { + // old null version should be deleted + versioningTestUtils.assertDataStoreValues(ds, [undefined, objData[1]]); + callback(err); + }); + }), + // create another null version + callback => + objectPut(authInfo, testPutObjectRequests[2], undefined, log, err => { + setImmediate(() => { + // old null version should be deleted + versioningTestUtils.assertDataStoreValues(ds, [undefined, undefined, objData[2]]); + callback(err); + }); + }), + ], + done, + ); }); describe('when null version is not the latest version', () => { - const objData = ['foo0', 'foo1', 'foo2'].map(str => - Buffer.from(str, 'utf8')); - const testPutObjectRequests = objData.map(data => versioningTestUtils - .createPutObjectRequest(bucketName, objectName, data)); + const objData = ['foo0', 'foo1', 'foo2'].map(str => Buffer.from(str, 'utf8')); + const testPutObjectRequests = objData.map(data => + versioningTestUtils.createPutObjectRequest(bucketName, objectName, data), + ); beforeEach(done => { - async.series([ - callback => bucketPut(authInfo, testPutBucketRequest, log, - callback), - // putting null version: put obj before versioning configured - callback => objectPut(authInfo, testPutObjectRequests[0], - undefined, log, callback), - callback => bucketPutVersioning(authInfo, - enableVersioningRequest, log, callback), - // put another version: - callback => objectPut(authInfo, testPutObjectRequests[1], - undefined, log, callback), - callback => bucketPutVersioning(authInfo, - suspendVersioningRequest, log, callback), - ], err => { - if (err) { - return done(err); - } - versioningTestUtils.assertDataStoreValues(ds, - objData.slice(0, 2)); - return done(); - }); + async.series( + [ + callback => bucketPut(authInfo, testPutBucketRequest, log, callback), + // putting null version: put obj before versioning configured + callback => objectPut(authInfo, testPutObjectRequests[0], undefined, log, callback), + callback => bucketPutVersioning(authInfo, enableVersioningRequest, log, callback), + // put another version: + callback => objectPut(authInfo, testPutObjectRequests[1], undefined, log, callback), + callback => bucketPutVersioning(authInfo, suspendVersioningRequest, log, callback), + ], + err => { + if (err) { + return done(err); + } + versioningTestUtils.assertDataStoreValues(ds, objData.slice(0, 2)); + return done(); + }, + ); }); - it('should still delete null version when creating new null version', - done => { - objectPut(authInfo, testPutObjectRequests[2], undefined, - log, err => { - assert.ifError(err, `Unexpected err: ${err}`); - setImmediate(() => { - // old null version should be deleted after putting - // new null version - versioningTestUtils.assertDataStoreValues(ds, - [undefined, objData[1], objData[2]]); - done(err); - }); + it('should still delete null version when creating new null version', done => { + objectPut(authInfo, testPutObjectRequests[2], undefined, log, err => { + assert.ifError(err, `Unexpected err: ${err}`); + setImmediate(() => { + // old null version should be deleted after putting + // new null version + versioningTestUtils.assertDataStoreValues(ds, [undefined, objData[1], objData[2]]); + done(err); }); + }); }); }); - it('should return BadDigest error and not leave orphans in data when ' + - 'contentMD5 and completedHash do not match', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - contentMD5: 'vnR+tLdVF79rPPfF+7YvOg==', - }, Buffer.from('I am another body', 'utf8')); - - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - err => { - assert.strictEqual(err.is.BadDigest, true); - // orphan objects don't get deleted - // until the next tick - // in memory - setImmediate(() => { - // Data store starts at index 1 - assert.strictEqual(ds[0], undefined); - assert.strictEqual(ds[1], undefined); - done(); + it( + 'should return BadDigest error and not leave orphans in data when ' + + 'contentMD5 and completedHash do not match', + done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + contentMD5: 'vnR+tLdVF79rPPfF+7YvOg==', + }, + Buffer.from('I am another body', 'utf8'), + ); + + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err.is.BadDigest, true); + // orphan objects don't get deleted + // until the next tick + // in memory + setImmediate(() => { + // Data store starts at index 1 + assert.strictEqual(ds[0], undefined); + assert.strictEqual(ds[1], undefined); + done(); + }); }); }); - }); - }); + }, + ); it('should set originOp when moving master-only document to a version document', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - // Old master-only document was moved to a proper version, with originOp overridden to prevent - // unexpected bucket notifications. - const calls = metadata.putObjectMD.getCalls(); - sinon.assert.calledWith(calls[calls.length - 2], - bucketName, objectName, sinon.match({ - originOp: 's3:StoreNullVersion', - }), any, any, any); - }, - async () => { - // New version document was created with the right originOp. - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, sinon.match({ - _data: { originOp: 's3:ObjectCreated:Put' }, - }), any, any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + // Old master-only document was moved to a proper version, with originOp overridden to prevent + // unexpected bucket notifications. + const calls = metadata.putObjectMD.getCalls(); + sinon.assert.calledWith( + calls[calls.length - 2], + bucketName, + objectName, + sinon.match({ + originOp: 's3:StoreNullVersion', + }), + any, + any, + any, + ); + }, + async () => { + // New version document was created with the right originOp. + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + sinon.match({ + _data: { originOp: 's3:ObjectCreated:Put' }, + }), + any, + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when writing new object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing object', done => { - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); it('should not pass needOplogUpdate when replacing archived object', done => { const archived = { - archiveInfo: { foo: 0, bar: 'stuff' } + archiveInfo: { foo: 0, bar: 'stuff' }, }; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - async () => { - sinon.assert.calledWith(metadata.putObjectMD.lastCall, - bucketName, objectName, any, sinon.match({ - needOplogUpdate: undefined, - originOp: undefined, - }), any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archived, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + async () => { + sinon.assert.calledWith( + metadata.putObjectMD.lastCall, + bucketName, + objectName, + any, + sinon.match({ + needOplogUpdate: undefined, + originOp: undefined, + }), + any, + any, + ); + }, + ], + done, + ); }); }); @@ -1163,10 +1300,16 @@ describe('objectPut API in ingestion bucket', () => { before(() => { // Setup multi-backend, this is required for ingestion - data.switch(new storage.data.MultipleBackendGateway({ - 'us-east-1': dataClient, - 'us-east-2': dataClient, - }, metadata, data.locStorageCheckFn)); + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); data.implName = 'multipleBackends'; }); @@ -1185,8 +1328,11 @@ describe('objectPut API in ingestion bucket', () => { const newPutObjectRequest = params => { const { location, versionID } = params || {}; - const r = versioningTestUtils - .createPutObjectRequest(bucketName, objectName, Buffer.from('I am another body', 'utf8')); + const r = versioningTestUtils.createPutObjectRequest( + bucketName, + objectName, + Buffer.from('I am another body', 'utf8'), + ); if (location) { r.headers[objectLocationConstraintHeader] = location; } @@ -1195,17 +1341,19 @@ describe('objectPut API in ingestion bucket', () => { } return r; }; - const newPutIngestBucketRequest = location => new DummyRequest({ - bucketName, - namespace, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: '/', - post: '' + - '' + - `${location}` + - '', - }); + const newPutIngestBucketRequest = location => + new DummyRequest({ + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + '' + + `${location}` + + '', + }); const archiveRestoreRequested = { archiveInfo: { foo: 0, bar: 'stuff' }, // opaque, can be anything... restoreRequestedAt: new Date().toString(), @@ -1220,13 +1368,17 @@ describe('objectPut API in ingestion bucket', () => { cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.strictEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.strictEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + ], + done, + ); }); it('should not use the versionID from the backend when writing in another location', done => { @@ -1237,16 +1389,26 @@ describe('objectPut API in ingestion bucket', () => { cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest({ - location: 'us-east-2', - }), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.notEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut( + authInfo, + newPutObjectRequest({ + location: 'us-east-2', + }), + undefined, + log, + (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.notEqual(headers['x-amz-version-id'], versionID); + next(err); + }, + ), + ], + done, + ); }); it('should not use the versionID from the backend when it is not a valid versionID', done => { @@ -1257,24 +1419,32 @@ describe('objectPut API in ingestion bucket', () => { cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.notEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.notEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + ], + done, + ); }); it('should not use the versionID from the backend when it is not provided', done => { - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.ok(headers['x-amz-version-id']); + next(err); + }), + ], + done, + ); }); it('should add versionID to backend putObject when restoring object', done => { @@ -1282,31 +1452,39 @@ describe('objectPut API in ingestion bucket', () => { const restoredVersionID = versioning.VersionID.encode(versioning.VersionID.generateVersionId('0', '')); // Use a "mock" data location, simulating a write to an ingest location - sinon.stub(dataClient, 'put') - .onCall(0).callsFake((writeStream, size, keyContext, reqUids, cb) => { + sinon + .stub(dataClient, 'put') + .onCall(0) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // First call: regular object creation, should not pass extra metadata header assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], undefined); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }) - .onCall(1).callsFake((writeStream, size, keyContext, reqUids, cb) => { + .onCall(1) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // Second call: "restored" data, should pass extra metadata header assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], versionID); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, restoredVersionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.strictEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), - next => objectPut(authInfo, newPutObjectRequest({ versionID }), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.strictEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), + next => + objectPut(authInfo, newPutObjectRequest({ versionID }), undefined, log, (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID + next(err); + }), + ], + done, + ); }); it('should not add versionID to backend putObject when restoring object to another location', done => { @@ -1314,34 +1492,48 @@ describe('objectPut API in ingestion bucket', () => { const restoredVersionID = versioning.VersionID.encode(versioning.VersionID.generateVersionId('0', '')); // Use a "mock" data location, simulating a write to an ingest location - sinon.stub(dataClient, 'put') - .onCall(0).callsFake((writeStream, size, keyContext, reqUids, cb) => { + sinon + .stub(dataClient, 'put') + .onCall(0) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // First call: regular object creation, should not pass extra metadata header assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], undefined); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, versionID, size, 'md5'); }) - .onCall(1).callsFake((writeStream, size, keyContext, reqUids, cb) => { + .onCall(1) + .callsFake((writeStream, size, keyContext, reqUids, cb) => { // Second call: "restored" data, should not pass extra metadata header (different location) assert.strictEqual(keyContext.metaHeaders['x-amz-meta-scal-version-id'], undefined); cb(null, `${keyContext.bucketName}/${keyContext.objectKey}`, restoredVersionID, size, 'md5'); }); - async.series([ - next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), - next => objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { - assert.strictEqual(headers['x-amz-version-id'], versionID); - next(err); - }), - next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), - next => objectPut(authInfo, newPutObjectRequest({ - versionID, - location: 'us-east-2', - }), undefined, log, (err, headers) => { - assert.ok(headers['x-amz-version-id']); - assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID - next(err); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log, next), + next => + objectPut(authInfo, newPutObjectRequest(), undefined, log, (err, headers) => { + assert.strictEqual(headers['x-amz-version-id'], versionID); + next(err); + }), + next => fakeMetadataArchive(bucketName, objectName, versionID, archiveRestoreRequested, next), + next => + objectPut( + authInfo, + newPutObjectRequest({ + versionID, + location: 'us-east-2', + }), + undefined, + log, + (err, headers) => { + assert.ok(headers['x-amz-version-id']); + assert.strictEqual(headers['x-amz-version-id'], versionID); // keep the same versionID + next(err); + }, + ), + ], + done, + ); }); }); @@ -1356,13 +1548,17 @@ describe('objectPut with objectKeyByteLimit', () => { config.objectKeyByteLimit = originalObjectKeyByteLimit; }); - const createTestPutObjectRequest = longKey => new DummyRequest({ - bucketName, - namespace, - objectKey: longKey, - headers: {}, - url: `/${bucketName}/${longKey}`, - }, postBody); + const createTestPutObjectRequest = longKey => + new DummyRequest( + { + bucketName, + namespace, + objectKey: longKey, + headers: {}, + url: `/${bucketName}/${longKey}`, + }, + postBody, + ); it('should reject object key longer than 915 bytes by default', done => { const longKey = 'a'.repeat(916); @@ -1541,7 +1737,7 @@ describe('objectPut with checksums disabled', () => { namespace, objectKey: objectName, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-checksum-crc32': 'AAAAAA==', }, url: '/', From bb10e461df2c4073c1491c02e5fa355080da9e78 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 10 Aug 2026 20:52:22 +0200 Subject: [PATCH 14/14] CLDSRV-965: bump package.json to 9.4.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 939de713a8..fd8521637d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@zenko/cloudserver", - "version": "9.4.0-preview.7", + "version": "9.4.1", "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", "main": "index.js", "engines": {