diff --git a/lib/utilization/scuba/wrapper.js b/lib/utilization/scuba/wrapper.js index e8a6c9a435..fd5f244f6b 100644 --- a/lib/utilization/scuba/wrapper.js +++ b/lib/utilization/scuba/wrapper.js @@ -71,7 +71,7 @@ class ScubaClientImpl extends ScubaClient { try { return await super.getLatestMetrics(metricsClass, resourceName, mergedOptions, body); } catch (err) { - code = err.statusCode || 500; + code = err.response?.status || err.code || 500; throw err; } finally { const responseTimeInNs = Number(process.hrtime.bigint() - requestStartTime); diff --git a/tests/sur/quota.js b/tests/sur/quota.js index c221b8512e..95e84372eb 100644 --- a/tests/sur/quota.js +++ b/tests/sur/quota.js @@ -1,5 +1,6 @@ const async = require('async'); const assert = require('assert'); +const http = require('http'); const getConfig = require('../functional/aws-node-sdk/test/support/config'); const { Scuba: MockScuba, inflightFlushFrequencyMS } = require('../utilities/mock/Scuba'); const sendRequest = require('../functional/aws-node-sdk/test/quota/tooling').sendRequest; @@ -28,6 +29,28 @@ const { let mockScuba = null; let s3Client = null; const quota = { quota: 1000 }; +const retrievalCountMetric = 's3_cloudserver_quota_metrics_retrieval_duration_seconds_count'; + +function getMetrics(cb) { + return http + .get({ host: '127.0.0.1', path: '/metrics', port: s3Config.metricsPort }, res => { + const body = []; + res.on('data', chunk => body.push(chunk)); + res.on('end', () => cb(null, body.join(''))); + }) + .on('error', cb); +} + +function parseMetricCount(metrics, name, labels) { + const line = metrics + .split('\n') + .find( + l => + l.startsWith(`${name}{`) && + Object.entries(labels).every(([key, value]) => l.includes(`${key}="${value}"`)), + ); + return line ? parseInt(line.slice(line.indexOf('} ') + 2), 10) : 0; +} function wait(timeoutMs, cb) { if (s3Config.isQuotaInflightEnabled()) { @@ -43,51 +66,62 @@ function createBucket(bucket, locked, cb) { if (locked) { config.ObjectLockEnabledForBucket = true; } - return s3Client.send(new CreateBucketCommand(config)) + return s3Client + .send(new CreateBucketCommand(config)) .then(data => cb(null, data)) .catch(cb); } function configureBucketVersioning(bucket, cb) { - return s3Client.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { - Status: 'Enabled', - }, - })) + return s3Client + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ) .then(data => cb(null, data)) .catch(cb); } function putObjectLockConfiguration(bucket, cb) { - return s3Client.send(new PutObjectLockConfigurationCommand({ - Bucket: bucket, - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { - DefaultRetention: { - Mode: 'GOVERNANCE', - Days: 1, + return s3Client + .send( + new PutObjectLockConfigurationCommand({ + Bucket: bucket, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { + DefaultRetention: { + Mode: 'GOVERNANCE', + Days: 1, + }, + }, }, - }, - }, - })) + }), + ) .then(data => cb(null, data)) .catch(cb); } function deleteBucket(bucket, cb) { - return s3Client.send(new DeleteBucketCommand({ Bucket: bucket })) + return s3Client + .send(new DeleteBucketCommand({ Bucket: bucket })) .then(data => cb(null, data)) .catch(cb); } function putObject(bucket, key, size, cb) { - return s3Client.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: Buffer.alloc(size), - })) + return s3Client + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: Buffer.alloc(size), + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, size); @@ -111,10 +145,11 @@ function putObjectWithCustomHeader(bucket, key, size, vID, cb) { args.request.headers['x-scal-s3-version-id'] = vID; return next(args); }, - { step: 'build' } + { step: 'build' }, ); - return s3Client.send(command) + return s3Client + .send(command) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, 0); @@ -125,11 +160,14 @@ function putObjectWithCustomHeader(bucket, key, size, vID, cb) { } function copyObject(bucket, key, sourceSize, cb) { - return s3Client.send(new CopyObjectCommand({ - Bucket: bucket, - CopySource: `${bucket}/${key}`, - Key: `${key}-copy`, - })) + return s3Client + .send( + new CopyObjectCommand({ + Bucket: bucket, + CopySource: `${bucket}/${key}`, + Key: `${key}-copy`, + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, sourceSize); @@ -140,10 +178,13 @@ function copyObject(bucket, key, sourceSize, cb) { } function deleteObject(bucket, key, size, cb) { - return s3Client.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })) + return s3Client + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) .then(() => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); @@ -154,11 +195,14 @@ function deleteObject(bucket, key, size, cb) { } function deleteVersionID(bucket, key, versionId, size, cb) { - return s3Client.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - })) + return s3Client + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); @@ -176,62 +220,77 @@ function objectMPU(bucket, key, parts, partSize, callback) { Bucket: bucket, Key: key, }; - return async.waterfall([ - next => s3Client.send(new CreateMultipartUploadCommand(initiateMPUParams)) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(next), - next => - async.mapLimit(partNumbers, 1, (partNumber, callback) => { - const uploadPartParams = { + return async.waterfall( + [ + next => + s3Client + .send(new CreateMultipartUploadCommand(initiateMPUParams)) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + next => + async.mapLimit( + partNumbers, + 1, + (partNumber, callback) => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: partNumber + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => callback(null, data.ETag)) + .catch(callback); + }, + (err, results) => { + if (err) { + return next(err); + } + ETags = results; + return next(); + }, + ), + next => { + const params = { Bucket: bucket, Key: key, - PartNumber: partNumber + 1, + MultipartUpload: { + Parts: partNumbers.map(n => ({ + ETag: ETags[n], + PartNumber: n + 1, + })), + }, UploadId: uploadId, - Body: Buffer.alloc(partSize), }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => callback(null, data.ETag)) - .catch(callback); - }, (err, results) => { - if (err) { - return next(err); - } - ETags = results; - return next(); - }), - next => { - const params = { - Bucket: bucket, - Key: key, - MultipartUpload: { - Parts: partNumbers.map(n => ({ - ETag: ETags[n], - PartNumber: n + 1, - })), - }, - UploadId: uploadId, - }; - return s3Client.send(new CompleteMultipartUploadCommand(params)) - .then(data => next(null, data)) - .catch(next); + return s3Client + .send(new CompleteMultipartUploadCommand(params)) + .then(data => next(null, data)) + .catch(next); + }, + ], + err => { + if (!err && !s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, parts * partSize); + } + return callback(err, uploadId); }, - ], err => { - if (!err && !s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, parts * partSize); - } - return callback(err, uploadId); - }); + ); } function abortMPU(bucket, key, uploadId, size, callback) { - return s3Client.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })) + return s3Client + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ) .then(data => { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); @@ -253,89 +312,101 @@ function uploadPartCopy(bucket, key, partNumber, partSize, sleepDuration, keyToC if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, parts * partSize); } - return async.waterfall([ - next => s3Client.send(new CreateMultipartUploadCommand(initiateMPUParams)) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(next), - next => { - const uploadPartParams = { - Bucket: bucket, - Key: key, - PartNumber: partNumber + 1, - UploadId: uploadId, - Body: Buffer.alloc(partSize), - }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => { - ETags[partNumber] = data.ETag; - return next(); - }) - .catch(next); - }, - next => wait(sleepDuration, next), - next => { - const copyPartParams = { - Bucket: bucket, - CopySource: `${bucket}/${keyToCopy}`, - Key: `${key}-copy`, - PartNumber: partNumber + 1, - UploadId: uploadId, - }; - return s3Client.send(new UploadPartCopyCommand(copyPartParams)) - .then(data => { - ETags[partNumber] = data.CopyPartResult.ETag; - return next(null, data.CopyPartResult.ETag); - }) - .catch(next); - }, - next => { - const params = { - Bucket: bucket, - Key: key, - MultipartUpload: { - Parts: partNumbers.map(n => ({ - ETag: ETags[n], - PartNumber: n + 1, - })), - }, - UploadId: uploadId, - }; - return s3Client.send(new CompleteMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); + return async.waterfall( + [ + next => + s3Client + .send(new CreateMultipartUploadCommand(initiateMPUParams)) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + next => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: partNumber + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => { + ETags[partNumber] = data.ETag; + return next(); + }) + .catch(next); + }, + next => wait(sleepDuration, next), + next => { + const copyPartParams = { + Bucket: bucket, + CopySource: `${bucket}/${keyToCopy}`, + Key: `${key}-copy`, + PartNumber: partNumber + 1, + UploadId: uploadId, + }; + return s3Client + .send(new UploadPartCopyCommand(copyPartParams)) + .then(data => { + ETags[partNumber] = data.CopyPartResult.ETag; + return next(null, data.CopyPartResult.ETag); + }) + .catch(next); + }, + next => { + const params = { + Bucket: bucket, + Key: key, + MultipartUpload: { + Parts: partNumbers.map(n => ({ + ETag: ETags[n], + PartNumber: n + 1, + })), + }, + UploadId: uploadId, + }; + return s3Client + .send(new CompleteMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + ], + err => { + if (err && !s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, -(parts * partSize)); + } + return callback(err, uploadId); }, - ], err => { - if (err && !s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, -(parts * partSize)); - } - return callback(err, uploadId); - }); + ); } function restoreObject(bucket, key, size, callback) { - return s3Client.send(new RestoreObjectCommand({ - Bucket: bucket, - Key: key, - RestoreRequest: { - Days: 1, - }, - })).then(data => { - if (!s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, size); - } - return callback(null, data); - }) - .catch(callback); + return s3Client + .send( + new RestoreObjectCommand({ + Bucket: bucket, + Key: key, + RestoreRequest: { + Days: 1, + }, + }), + ) + .then(data => { + if (!s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, size); + } + return callback(null, data); + }) + .catch(callback); } function multiObjectDelete(bucket, keys, size, callback) { if (!s3Config.isQuotaInflightEnabled()) { mockScuba.incrementBytesForBucket(bucket, -size); } - const deleteObjectsParams = keys.map(key => ({ Key: key })); + const deleteObjectsParams = keys.map(key => ({ Key: key })); const command = new DeleteObjectsCommand({ Bucket: bucket, Delete: { @@ -343,8 +414,9 @@ function multiObjectDelete(bucket, keys, size, callback) { Quiet: false, }, }); - - return s3Client.send(command) + + return s3Client + .send(command) .then(data => { callback(null, data); }) @@ -356,53 +428,113 @@ function multiObjectDelete(bucket, keys, size, callback) { }); } -(process.env.S3METADATA === 'mongodb' ? describe : describe.skip)('quota evaluation with scuba metrics', - function t() { - this.timeout(30000); - const scuba = new MockScuba(); - const putQuotaVerb = 'PUT'; - const config = { - accessKey: memCredentials.default.accessKey, - secretKey: memCredentials.default.secretKey, - }; - mockScuba = scuba; - - before(done => { - const config = getConfig('default', { - maxRetries: 0, - }); - - s3Client = new S3Client({ - ...config, - // Disable ALL automatic checksum handling - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - checksumDisabled: true, - disableRequestCompression: true, - // Force the client to not add automatic headers - useGlobalEndpoint: false, - }); +(process.env.S3METADATA === 'mongodb' ? describe : describe.skip)('quota evaluation with scuba metrics', function t() { + this.timeout(30000); + const scuba = new MockScuba(); + const putQuotaVerb = 'PUT'; + const config = { + accessKey: memCredentials.default.accessKey, + secretKey: memCredentials.default.secretKey, + }; + mockScuba = scuba; - scuba.start(); - metadata.setup(err => wait(2000, () => done(err))); + before(done => { + const config = getConfig('default', { + maxRetries: 0, }); - afterEach(() => { - scuba.reset(); + s3Client = new S3Client({ + ...config, + // Disable ALL automatic checksum handling + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + checksumDisabled: true, + disableRequestCompression: true, + // Force the client to not add automatic headers + useGlobalEndpoint: false, }); - after(() => { - scuba.stop(); - }); + scuba.start(); + metadata.setup(err => wait(2000, () => done(err))); + }); + + afterEach(() => { + scuba.reset(); + }); + + after(() => { + scuba.stop(); + }); - it('should return QuotaExceeded when trying to PutObject in a bucket with quota', done => { - const bucket = 'quota-test-bucket1'; - const key = 'quota-test-object'; - const size = 1024; - return async.series([ + it('should label the metrics retrieval histogram with the status returned by scuba', done => { + const bucket = 'quota-test-bucket-retrieval-code'; + const key = 'quota-test-object'; + const size = 1024; + const notFound = { code: '404', class: 'bucket' }; + const serverError = { code: '500', class: 'bucket' }; + let before404 = 0; + let before500 = 0; + return async.series( + [ + next => + getMetrics((err, metrics) => { + if (err) { + return next(err); + } + before404 = parseMetricCount(metrics, retrievalCountMetric, notFound); + before500 = parseMetricCount(metrics, retrievalCountMetric, serverError); + return next(); + }), next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => { + scuba.failMetrics(404); + return putObject(bucket, key, size, () => { + scuba.failMetrics(null); + return next(); + }); + }, + next => + getMetrics((err, metrics) => { + if (err) { + return next(err); + } + try { + assert( + parseMetricCount(metrics, retrievalCountMetric, notFound) > before404, + 'expected the retrieval histogram to record a 404 observation', + ); + assert.strictEqual( + parseMetricCount(metrics, retrievalCountMetric, serverError), + before500, + 'a 404 from scuba must not be labelled 500', + ); + return next(); + } catch (assertError) { + return next(assertError); + } + }), + next => deleteObject(bucket, key, size, () => next()), + next => deleteBucket(bucket, next), + ], + done, + ); + }); + + it('should return QuotaExceeded when trying to PutObject in a bucket with quota', done => { + const bucket = 'quota-test-bucket1'; + const key = 'quota-test-object'; + const size = 1024; + return async.series( + [ + next => createBucket(bucket, false, next), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), next => { putObject(bucket, key, size, err => { try { @@ -414,80 +546,99 @@ function multiObjectDelete(bucket, keys, size, callback) { }); }, next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to copyObject in a versioned bucket with quota', done => { - const bucket = 'quota-test-bucket12'; - const key = 'quota-test-object'; - const size = 900; - let vID = null; - return async.series([ + it('should return QuotaExceeded when trying to copyObject in a versioned bucket with quota', done => { + const bucket = 'quota-test-bucket12'; + const key = 'quota-test-object'; + const size = 900; + let vID = null; + return async.series( + [ next => createBucket(bucket, false, next), next => configureBucketVersioning(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, data) => { - assert.ifError(err); - vID = data.VersionId; - return next(); - }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => copyObject(bucket, key, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, data) => { + assert.ifError(err); + vID = data.VersionId; return next(); - } catch (assertError) { - return next(assertError); - } - }), + }), + next => wait(inflightFlushFrequencyMS * 2, next), + next => + copyObject(bucket, key, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => deleteVersionID(bucket, key, vID, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to CopyObject in a bucket with quota', done => { - const bucket = 'quota-test-bucket2'; - const key = 'quota-test-object'; - const size = 900; - return async.series([ + it('should return QuotaExceeded when trying to CopyObject in a bucket with quota', done => { + const bucket = 'quota-test-bucket2'; + const key = 'quota-test-object'; + const size = 900; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), next => putObject(bucket, key, size, next), next => wait(inflightFlushFrequencyMS * 2, next), - next => copyObject(bucket, key, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + copyObject(bucket, key, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to complete MPU in a bucket with quota', done => { - const bucket = 'quota-test-bucket3'; - const key = 'quota-test-object'; - const parts = 5; - const partSize = 1024 * 1024 * 6; - let uploadId = null; - return async.series([ + it('should return QuotaExceeded when trying to complete MPU in a bucket with quota', done => { + const bucket = 'quota-test-bucket3'; + const key = 'quota-test-object'; + const parts = 5; + const partSize = 1024 * 1024 * 6; + let uploadId = null; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => objectMPU(bucket, key, parts, partSize, (err, _uploadId) => { - uploadId = _uploadId; - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + objectMPU(bucket, key, parts, partSize, (err, _uploadId) => { + uploadId = _uploadId; + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => abortMPU(bucket, key, uploadId, 0, next), next => wait(inflightFlushFrequencyMS * 2, next), next => { @@ -495,130 +646,187 @@ function multiObjectDelete(bucket, keys, size, callback) { return next(); }, next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not return QuotaExceeded if the quota is not exceeded', done => { - const bucket = 'quota-test-bucket4'; - const key = 'quota-test-object'; - const size = 300; - return async.series([ + it('should not return QuotaExceeded if the quota is not exceeded', done => { + const bucket = 'quota-test-bucket4'; + const key = 'quota-test-object'; + const size = 300; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not evaluate quotas if the backend is not available', done => { - scuba.stop(); - const bucket = 'quota-test-bucket5'; - const key = 'quota-test-object'; - const size = 1024; - return async.series([ + it('should not evaluate quotas if the backend is not available', done => { + scuba.stop(); + const bucket = 'quota-test-bucket5'; + const key = 'quota-test-object'; + const size = 1024; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], err => { + ], + err => { assert.ifError(err); scuba.start(); return wait(2000, done); - }); - }); + }, + ); + }); - it('should return QuotaExceeded when trying to copy a part in a bucket with quota', done => { - const bucket = 'quota-test-bucket6'; - const key = 'quota-test-object-copy'; - const keyToCopy = 'quota-test-existing'; - const parts = 5; - const partSize = 1024 * 1024 * 6; - let uploadId = null; - return async.series([ + it('should return QuotaExceeded when trying to copy a part in a bucket with quota', done => { + const bucket = 'quota-test-bucket6'; + const key = 'quota-test-object-copy'; + const keyToCopy = 'quota-test-existing'; + const parts = 5; + const partSize = 1024 * 1024 * 6; + let uploadId = null; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: Math.round(partSize * 2.5) }), config) - .then(() => next()).catch(err => next(err)), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: Math.round(partSize * 2.5) }), + config, + ) + .then(() => next()) + .catch(err => next(err)), next => putObject(bucket, keyToCopy, partSize, next), - next => uploadPartCopy(bucket, key, parts, partSize, inflightFlushFrequencyMS * 2, keyToCopy, - (err, _uploadId) => { - uploadId = _uploadId; - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + uploadPartCopy( + bucket, + key, + parts, + partSize, + inflightFlushFrequencyMS * 2, + keyToCopy, + (err, _uploadId) => { + uploadId = _uploadId; + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }, + ), next => abortMPU(bucket, key, uploadId, parts * partSize, next), next => deleteObject(bucket, keyToCopy, partSize, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should return QuotaExceeded when trying to restore an object in a bucket with quota', done => { - const bucket = 'quota-test-bucket7'; - const key = 'quota-test-object'; - const size = 900; - let vID = null; - return async.series([ + it('should return QuotaExceeded when trying to restore an object in a bucket with quota', done => { + const bucket = 'quota-test-bucket7'; + const key = 'quota-test-object'; + const size = 900; + let vID = null; + return async.series( + [ next => createBucket(bucket, false, next), next => configureBucketVersioning(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, data) => { - assert.ifError(err); - vID = data.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - }, next), - next => wait(inflightFlushFrequencyMS * 2, next), - next => restoreObject(bucket, key, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, data) => { + assert.ifError(err); + vID = data.VersionId; return next(); - } catch (assertError) { - return next(assertError); - } - }), + }), + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + }, + next, + ), + next => wait(inflightFlushFrequencyMS * 2, next), + next => + restoreObject(bucket, key, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => deleteVersionID(bucket, key, vID, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not update the inflights if the quota check is passing but the object is already restored', done => { - const bucket = 'quota-test-bucket14'; - const key = 'quota-test-object'; - const size = 100; - let vID = null; - return async.series([ + it('should not update the inflights if the quota check is passing but the object is already restored', done => { + const bucket = 'quota-test-bucket14'; + const key = 'quota-test-object'; + const size = 100; + let vID = null; + return async.series( + [ next => createBucket(bucket, false, next), next => configureBucketVersioning(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, data) => { - assert.ifError(err); - vID = data.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - restoreRequestedAt: new Date(0).toString(), - restoreCompletedAt: new Date(0).toString() + 1, - restoreRequestedDays: 5, - }, next), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, data) => { + assert.ifError(err); + vID = data.VersionId; + return next(); + }), + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + restoreRequestedAt: new Date(0).toString(), + restoreCompletedAt: new Date(0).toString() + 1, + restoreRequestedDays: 5, + }, + next, + ), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size); @@ -632,34 +840,42 @@ function multiObjectDelete(bucket, keys, size, callback) { }, next => deleteVersionID(bucket, key, vID, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should allow writes after deleting data with quotas', done => { - const bucket = 'quota-test-bucket8'; - const key = 'quota-test-object'; - const size = 400; - return async.series([ + it('should allow writes after deleting data with quotas', done => { + const bucket = 'quota-test-bucket8'; + const key = 'quota-test-object'; + const size = 400; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, `${key}1`, size, err => { - assert.ifError(err); - return next(); - }), - next => putObject(bucket, `${key}2`, size, err => { - assert.ifError(err); - return next(); - }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}3`, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, `${key}1`, size, err => { + assert.ifError(err); return next(); - } catch (assertError) { - return next(assertError); - } - }), + }), + next => + putObject(bucket, `${key}2`, size, err => { + assert.ifError(err); + return next(); + }), + next => wait(inflightFlushFrequencyMS * 2, next), + next => + putObject(bucket, `${key}3`, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size * 2); @@ -668,38 +884,52 @@ function multiObjectDelete(bucket, keys, size, callback) { next => wait(inflightFlushFrequencyMS * 2, next), next => deleteObject(bucket, `${key}2`, size, next), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}4`, size, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}4`, size, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, `${key}1`, size, next), next => deleteObject(bucket, `${key}3`, size, next), next => deleteObject(bucket, `${key}4`, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should allow writes after deleting data with quotas below the current number of inflights', done => { - const bucket = 'quota-test-bucket8'; - const key = 'quota-test-object'; - const size = 400; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } - return async.series([ + it('should allow writes after deleting data with quotas below the current number of inflights', done => { + const bucket = 'quota-test-bucket8'; + const key = 'quota-test-object'; + const size = 400; + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } + return async.series( + [ next => createBucket(bucket, false, next), // Set the quota to 10 * size (4000) - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: 10 * size }), config).then(() => next()).catch(err => next(err)), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: 10 * size }), + config, + ) + .then(() => next()) + .catch(err => next(err)), // Simulate previous operations since last metrics update (4000 bytes) - next => putObject(bucket, `${key}1`, 5 * size, err => { - assert.ifError(err); - return next(); - }), - next => putObject(bucket, `${key}2`, 5 * size, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}1`, 5 * size, err => { + assert.ifError(err); + return next(); + }), + next => + putObject(bucket, `${key}2`, 5 * size, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), // After metrics update, set the inflights to 0 (simulate end of metrics update) next => { @@ -708,14 +938,15 @@ function multiObjectDelete(bucket, keys, size, callback) { }, // Here we have 0 inflight but the stored bytes are 4000 (equal to the quota) // Should reject new write with QuotaExceeded (4000 + 400) - next => putObject(bucket, `${key}3`, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + putObject(bucket, `${key}3`, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), // Should still have 0 as inflight next => { @@ -726,37 +957,45 @@ function multiObjectDelete(bucket, keys, size, callback) { // Now delete one object (2000 bytes), it should let us write again next => deleteObject(bucket, `${key}1`, size, next), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}4`, 5 * size, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}4`, 5 * size, err => { + assert.ifError(err); + return next(); + }), // Cleanup next => deleteObject(bucket, `${key}2`, size, next), next => deleteObject(bucket, `${key}4`, size, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not increase the inflights when the object is being rewritten with a smaller object', done => { - const bucket = 'quota-test-bucket9'; - const key = 'quota-test-object'; - const size = 400; - return async.series([ + it('should not increase the inflights when the object is being rewritten with a smaller object', done => { + const bucket = 'quota-test-bucket9'; + const key = 'quota-test-object'; + const size = 400; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, key, size - 100, err => { - assert.ifError(err); - if (!s3Config.isQuotaInflightEnabled()) { - mockScuba.incrementBytesForBucket(bucket, -size); - } - return next(); - }), + next => + putObject(bucket, key, size - 100, err => { + assert.ifError(err); + if (!s3Config.isQuotaInflightEnabled()) { + mockScuba.incrementBytesForBucket(bucket, -size); + } + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size - 100); @@ -764,16 +1003,21 @@ function multiObjectDelete(bucket, keys, size, callback) { }, next => deleteObject(bucket, key, size, next), next => deleteBucket(bucket, next), - ], done); - }); - it('should decrease the inflights when performing multi object delete', done => { - const bucket = 'quota-test-bucket10'; - const key = 'quota-test-object'; - const size = 400; - return async.series([ + ], + done, + ); + }); + it('should decrease the inflights when performing multi object delete', done => { + const bucket = 'quota-test-bucket10'; + const key = 'quota-test-object'; + const size = 400; + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), next => { putObject(bucket, `${key}1`, size, err => { assert.ifError(err); @@ -787,11 +1031,11 @@ function multiObjectDelete(bucket, keys, size, callback) { }); }, next => wait(inflightFlushFrequencyMS * 2, next), - next => + next => multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 2, err => { assert.ifError(err); return next(); - }), + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), 0); @@ -800,162 +1044,212 @@ function multiObjectDelete(bucket, keys, size, callback) { next => { deleteBucket(bucket, next); }, - ], done); - }); + ], + done, + ); + }); - it('should allow writes after multi-deleting data with quotas below the current number of inflights', done => { - const bucket = 'quota-test-bucket10'; - const key = 'quota-test-object'; - const size = 400; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } - return async.series([ + it('should allow writes after multi-deleting data with quotas below the current number of inflights', done => { + const bucket = 'quota-test-bucket10'; + const key = 'quota-test-object'; + const size = 400; + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: size * 10 }), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, `${key}1`, size * 5, err => { - assert.ifError(err); - return next(); - }), - next => putObject(bucket, `${key}2`, size * 5, err => { - assert.ifError(err); - return next(); - }), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: size * 10 }), + config, + ) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, `${key}1`, size * 5, err => { + assert.ifError(err); + return next(); + }), + next => + putObject(bucket, `${key}2`, size * 5, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { scuba.setInflightAsCapacity(bucket); return next(); }, - next => putObject(bucket, `${key}3`, size, err => { - try { - assert.strictEqual(err.name, 'QuotaExceeded'); - return next(); - } catch (assertError) { - return next(assertError); - } - }), + next => + putObject(bucket, `${key}3`, size, err => { + try { + assert.strictEqual(err.name, 'QuotaExceeded'); + return next(); + } catch (assertError) { + return next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), 0); return next(); }, - next => multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 10, err => { - assert.ifError(err); - return next(); - }), + next => + multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 10, err => { + assert.ifError(err); + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), - next => putObject(bucket, `${key}4`, size * 5, err => { - assert.ifError(err); - return next(); - }), + next => + putObject(bucket, `${key}4`, size * 5, err => { + assert.ifError(err); + return next(); + }), next => deleteObject(bucket, `${key}4`, size * 5, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should not update the inflights if the API errored after evaluating quotas (deletion)', done => { - const bucket = 'quota-test-bucket11'; - const key = 'quota-test-object'; - const size = 100; - let vID = null; - return async.series([ + it('should not update the inflights if the API errored after evaluating quotas (deletion)', done => { + const bucket = 'quota-test-bucket11'; + const key = 'quota-test-object'; + const size = 100; + let vID = null; + return async.series( + [ next => createBucket(bucket, true, next), next => putObjectLockConfiguration(bucket, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, val) => { - assert.ifError(err); - vID = val.VersionId; - return next(); - }), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, val) => { + assert.ifError(err); + vID = val.VersionId; + return next(); + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size); return next(); }, - next => deleteVersionID(bucket, key, vID, size, err => { - try { - assert.strictEqual(err.name, 'AccessDenied'); - next(); - } catch (assertError) { - next(assertError); - } - }), + next => + deleteVersionID(bucket, key, vID, size, err => { + try { + assert.strictEqual(err.name, 'AccessDenied'); + next(); + } catch (assertError) { + next(assertError); + } + }), next => wait(inflightFlushFrequencyMS * 2, next), next => { assert.strictEqual(scuba.getInflightsForBucket(bucket), size); return next(); }, - ], done); - }); + ], + done, + ); + }); - it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header', - done => { - const bucket = 'quota-test-bucket13'; - const key = 'quota-test-object'; - const size = 100; - let vID = null; - return async.series([ - next => createBucket(bucket, true, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, val) => { + // eslint-disable-next-line max-len + it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header', done => { + const bucket = 'quota-test-bucket13'; + const key = 'quota-test-object'; + const size = 100; + let vID = null; + return async.series( + [ + next => createBucket(bucket, true, next), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, val) => { assert.ifError(err); vID = val.VersionId; return next(); }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 7, - }, next), - // Simulate the real restore - next => putObjectWithCustomHeader(bucket, key, size, vID, err => { + next => wait(inflightFlushFrequencyMS * 2, next), + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 7, + }, + next, + ), + // Simulate the real restore + next => + putObjectWithCustomHeader(bucket, key, size, vID, err => { assert.ifError(err); return next(); }), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => deleteVersionID(bucket, key, vID, size, next), - next => deleteBucket(bucket, next), - ], done); - }); + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => deleteVersionID(bucket, key, vID, size, next), + next => deleteBucket(bucket, next), + ], + done, + ); + }); - it('should allow a restore if the quota is full but the objet fits with its reserved storage space', - done => { - const bucket = 'quota-test-bucket15'; - const key = 'quota-test-object'; - const size = 1000; - let vID = null; - return async.series([ - next => createBucket(bucket, true, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), config).then(() => next()).catch(err => next(err)), - next => putObject(bucket, key, size, (err, val) => { + it('should allow a restore if the quota is full but the objet fits with its reserved storage space', done => { + const bucket = 'quota-test-bucket15'; + const key = 'quota-test-object'; + const size = 1000; + let vID = null; + return async.series( + [ + next => createBucket(bucket, true, next), + next => + sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), config) + .then(() => next()) + .catch(err => next(err)), + next => + putObject(bucket, key, size, (err, val) => { assert.ifError(err); vID = val.VersionId; return next(); }), - next => wait(inflightFlushFrequencyMS * 2, next), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => fakeMetadataArchive(bucket, key, vID, { - archiveInfo: {}, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 7, - }, next), - // Put an object, the quota should be exceeded - next => putObject(bucket, `${key}-2`, size, err => { + next => wait(inflightFlushFrequencyMS * 2, next), + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => + fakeMetadataArchive( + bucket, + key, + vID, + { + archiveInfo: {}, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 7, + }, + next, + ), + // Put an object, the quota should be exceeded + next => + putObject(bucket, `${key}-2`, size, err => { try { assert.strictEqual(err.name, 'QuotaExceeded'); return next(); @@ -963,58 +1257,78 @@ function multiObjectDelete(bucket, keys, size, callback) { return next(assertError); } }), - next => { - assert.strictEqual(scuba.getInflightsForBucket(bucket), size); - return next(); - }, - next => deleteVersionID(bucket, key, vID, size, next), - next => deleteBucket(bucket, next), - ], done); - }); + next => { + assert.strictEqual(scuba.getInflightsForBucket(bucket), size); + return next(); + }, + next => deleteVersionID(bucket, key, vID, size, next), + next => deleteBucket(bucket, next), + ], + done, + ); + }); - it('should reduce inflights when completing MPU with fewer parts than uploaded', done => { - const bucket = 'quota-test-bucket-mpu1'; - const key = 'quota-test-object'; - const parts = 3; - const partSize = 5 * 1024 * 1024; - const totalSize = parts * partSize; - const usedParts = 2; - let uploadId = null; - const ETags = []; + it('should reduce inflights when completing MPU with fewer parts than uploaded', done => { + const bucket = 'quota-test-bucket-mpu1'; + const key = 'quota-test-object'; + const parts = 3; + const partSize = 5 * 1024 * 1024; + const totalSize = parts * partSize; + const usedParts = 2; + let uploadId = null; + const ETags = []; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } - return async.series([ + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: totalSize * 2 }), config) - .then(() => next()).catch(err => next(err)), - next => s3Client.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(err => next(err)), - next => async.timesSeries(parts, (n, cb) => { - const uploadPartParams = { - Bucket: bucket, - Key: key, - PartNumber: n + 1, - UploadId: uploadId, - Body: Buffer.alloc(partSize), - }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: totalSize * 2 }), + config, + ) + .then(() => next()) + .catch(err => next(err)), + next => + s3Client + .send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ) .then(data => { - ETags[n] = data.ETag; - return cb(); + uploadId = data.UploadId; + return next(); }) - .catch(cb); - }, next), + .catch(err => next(err)), + next => + async.timesSeries( + parts, + (n, cb) => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: n + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => { + ETags[n] = data.ETag; + return cb(); + }) + .catch(cb); + }, + next, + ), next => wait(inflightFlushFrequencyMS * 2, next), next => { // Verify all parts are counted in inflights @@ -1034,7 +1348,8 @@ function multiObjectDelete(bucket, keys, size, callback) { }, UploadId: uploadId, }; - return s3Client.send(new CompleteMultipartUploadCommand(params)) + return s3Client + .send(new CompleteMultipartUploadCommand(params)) .then(() => next()) .catch(err => next(err)); }, @@ -1047,47 +1362,67 @@ function multiObjectDelete(bucket, keys, size, callback) { }, next => deleteObject(bucket, key, usedParts * partSize, next), next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); + }); - it('should reduce inflights when aborting MPU', done => { - const bucket = 'quota-test-bucket-mpu2'; - const key = 'quota-test-object'; - const parts = 3; - const partSize = 5 * 1024 * 1024; - const totalSize = parts * partSize; - let uploadId = null; + it('should reduce inflights when aborting MPU', done => { + const bucket = 'quota-test-bucket-mpu2'; + const key = 'quota-test-object'; + const parts = 3; + const partSize = 5 * 1024 * 1024; + const totalSize = parts * partSize; + let uploadId = null; - if (!s3Config.isQuotaInflightEnabled()) { - return done(); - } + if (!s3Config.isQuotaInflightEnabled()) { + return done(); + } - return async.series([ + return async.series( + [ next => createBucket(bucket, false, next), - next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify({ quota: totalSize * 2 }), config) - .then(() => next()).catch(err => next(err)), - next => s3Client.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(err => next(err)), - next => async.timesSeries(parts, (n, cb) => { - const uploadPartParams = { - Bucket: bucket, - Key: key, - PartNumber: n + 1, - UploadId: uploadId, - Body: Buffer.alloc(partSize), - }; - return s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => cb(null, data)) - .catch(cb); - }, next), + next => + sendRequest( + putQuotaVerb, + '127.0.0.1:8000', + `/${bucket}/?quota=true`, + JSON.stringify({ quota: totalSize * 2 }), + config, + ) + .then(() => next()) + .catch(err => next(err)), + next => + s3Client + .send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(err => next(err)), + next => + async.timesSeries( + parts, + (n, cb) => { + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: n + 1, + UploadId: uploadId, + Body: Buffer.alloc(partSize), + }; + return s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => cb(null, data)) + .catch(cb); + }, + next, + ), next => wait(inflightFlushFrequencyMS * 2, next), next => { // Verify all parts are counted in inflights @@ -1102,6 +1437,8 @@ function multiObjectDelete(bucket, keys, size, callback) { return next(); }, next => deleteBucket(bucket, next), - ], done); - }); + ], + done, + ); }); +}); diff --git a/tests/unit/quotas/scuba/wrapper.js b/tests/unit/quotas/scuba/wrapper.js index aa796c8fe9..1355c84699 100644 --- a/tests/unit/quotas/scuba/wrapper.js +++ b/tests/unit/quotas/scuba/wrapper.js @@ -1,6 +1,7 @@ const assert = require('assert'); const sinon = require('sinon'); const { ScubaClientImpl } = require('../../../../lib/utilization/scuba/wrapper'); +const monitoring = require('../../../../lib/utilities/monitoringHandler'); const { default: ScubaClient } = require('scubaclient'); describe('ScubaClientImpl', () => { @@ -139,5 +140,57 @@ describe('ScubaClientImpl', () => { assert.strictEqual(forwardedOptions.headers['X-Existing'], 'v'); assert.strictEqual(forwardedOptions.headers['X-Scal-Request-Uids'], 'req1:req2'); }); + + describe('retrieval duration metric', () => { + let observe; + let labels; + let originalMetric; + + beforeEach(() => { + observe = sinon.spy(); + labels = sinon.stub().returns({ observe }); + originalMetric = monitoring.utilizationMetricsRetrievalDuration; + monitoring.utilizationMetricsRetrievalDuration = { labels }; + }); + + afterEach(() => { + monitoring.utilizationMetricsRetrievalDuration = originalMetric; + }); + + const observedCode = async error => { + if (error) { + sinon.stub(ScubaClient.prototype, 'getLatestMetrics').rejects(error); + await assert.rejects(client.getUtilizationMetrics('bucket', 'k', null, {}, {})); + } else { + sinon.stub(ScubaClient.prototype, 'getLatestMetrics').resolves({}); + await client.getUtilizationMetrics('bucket', 'k', null, {}, {}); + } + assert(observe.calledOnce); + return labels.getCall(0).args[0].code; + }; + + it('should label a successful retrieval with 200', async () => { + assert.strictEqual(await observedCode(null), 200); + }); + + it('should label an axios error with its response status', async () => { + const err = new Error('Not Found'); + err.response = { status: 404 }; + err.code = 'ERR_BAD_REQUEST'; + + assert.strictEqual(await observedCode(err), 404); + }); + + it('should label a transport failure with the error code', async () => { + const err = new Error('connect ECONNREFUSED'); + err.code = 'ECONNREFUSED'; + + assert.strictEqual(await observedCode(err), 'ECONNREFUSED'); + }); + + it('should label an error carrying no status with 500', async () => { + assert.strictEqual(await observedCode(new Error('boom')), 500); + }); + }); }); }); diff --git a/tests/utilities/mock/Scuba.js b/tests/utilities/mock/Scuba.js index 7dbfa49c72..8f20050d48 100644 --- a/tests/utilities/mock/Scuba.js +++ b/tests/utilities/mock/Scuba.js @@ -8,6 +8,7 @@ class Scuba { constructor() { this._server = null; this._port = 8100; + this._failMetricsStatus = null; this._data = { bucket: new Map(), }; @@ -27,6 +28,9 @@ class Scuba { }); this._app.post('/metrics/bucket/:bucket/latest', (req, res) => { + if (this._failMetricsStatus) { + return res.status(this._failMetricsStatus).end(); + } let bucketName = req.params.bucket; if (!this.supportsInflight) { bucketName = req.params.bucket?.split('_')[0]; @@ -42,7 +46,8 @@ class Scuba { }); const immediateInflights = req.body?.action === 'objectRestore' ? 0 : inflight; return res.json({ - bytesTotal: (this._data.bucket.get(bucketName)?.current || 0) + + bytesTotal: + (this._data.bucket.get(bucketName)?.current || 0) + (this._data.bucket.get(bucketName)?.nonCurrent || 0) + (this._data.bucket.get(bucketName)?.inflight || 0) + immediateInflights, @@ -88,11 +93,16 @@ class Scuba { } reset() { + this._failMetricsStatus = null; this._data = { bucket: new Map(), }; } + failMetrics(statusCode) { + this._failMetricsStatus = statusCode; + } + stop() { this._server.close(); } @@ -116,7 +126,7 @@ class Scuba { let inflightCount = 0; this._data.bucket.forEach((value, key) => { if (!this.supportsInflight && key === bucketName) { - inflightCount += (value.current + value.nonCurrent); + inflightCount += value.current + value.nonCurrent; } else if (this.supportsInflight && key.startsWith(`${bucketName}_`)) { inflightCount += value.inflight; }