Skip to content

Register quota metrics unconditionally #6253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
delthas wants to merge 2 commits into from
+155 −111
Open

Register quota metrics unconditionally #6253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2535370
Prettier reformat monitoringHandler and its tests
delthas Aug 12, 2026
33f77af
Register quota metrics unconditionally
delthas Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 88 additions & 97 deletions lib/utilities/monitoringHandler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,118 +65,112 @@ const httpResponseSizeBytes = new client.Summary({
help: 'Cloudserver HTTP response sizes in bytes',
});

let quotaEvaluationDuration;
let utilizationMetricsRetrievalDuration;
let utilizationServiceAvailable;
let bucketsWithQuota;
let accountsWithQuota;
let requestWithQuotaMetricsUnavailable;

if (config.isQuotaEnabled) {
quotaEvaluationDuration = new client.Histogram({
name: 's3_cloudserver_quota_evaluation_duration_seconds',
help: 'Duration of the quota evaluation operation',
labelNames: ['action', 'code', 'type'],
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1],
});
const quotaEvaluationDuration = new client.Histogram({
name: 's3_cloudserver_quota_evaluation_duration_seconds',
help: 'Duration of the quota evaluation operation',
labelNames: ['action', 'code', 'type'],
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1],
});

utilizationMetricsRetrievalDuration = new client.Histogram({
name: 's3_cloudserver_quota_metrics_retrieval_duration_seconds',
help: 'Duration of the utilization metrics retrieval operation',
labelNames: ['code', 'class'],
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5],
});
const utilizationMetricsRetrievalDuration = new client.Histogram({
name: 's3_cloudserver_quota_metrics_retrieval_duration_seconds',
help: 'Duration of the utilization metrics retrieval operation',
labelNames: ['code', 'class'],
buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5],
});

utilizationServiceAvailable = new client.Gauge({
name: 's3_cloudserver_quota_utilization_service_available',
help: 'Availability of the utilization service',
});
const utilizationServiceAvailable = new client.Gauge({
name: 's3_cloudserver_quota_utilization_service_available',
help: 'Availability of the utilization service',
});

bucketsWithQuota = new client.Gauge({
name: 's3_cloudserver_quota_buckets_count',
help: 'Total number of buckets quota',
});
const bucketsWithQuota = new client.Gauge({
name: 's3_cloudserver_quota_buckets_count',
help: 'Total number of buckets quota',
});

accountsWithQuota = new client.Gauge({
name: 's3_cloudserver_quota_accounts_count',
help: 'Total number of account quota',
});
const accountsWithQuota = new client.Gauge({
name: 's3_cloudserver_quota_accounts_count',
help: 'Total number of account quota',
});

requestWithQuotaMetricsUnavailable = new client.Counter({
name: 's3_cloudserver_quota_unavailable_count',
help: 'Total number of requests with quota metrics unavailable',
});
}
const requestWithQuotaMetricsUnavailable = new client.Counter({
name: 's3_cloudserver_quota_unavailable_count',
help: 'Total number of requests with quota metrics unavailable',
});

// Lifecycle duration metric, to track the completion of restore.
// This metric is used to track the time it takes to complete the lifecycle operation (restore).
// NOTE : this metric is the same as the one defined in Backbeat, and must keep the same name,
// labels and buckets.
const lifecycleDuration = new client.Histogram({
name: 's3_lifecycle_duration_seconds',
help: 'Duration of the lifecycle operation, calculated from the theoretical date to the end ' +
'of the operation',
help: 'Duration of the lifecycle operation, calculated from the theoretical date to the end ' + 'of the operation',
labelNames: ['type', 'location'],
buckets: [0.2, 1, 5, 30, 120, 600, 3600, 4 * 3600, 8 * 3600, 16 * 3600, 24 * 3600],
});

function promMetrics(method, bucketName, code, action,
newByteLength, oldByteLength, isVersionedObj,
numOfObjectsRemoved, ingestSize) {
function promMetrics(
method,
bucketName,
code,
action,
newByteLength,
oldByteLength,
isVersionedObj,
numOfObjectsRemoved,
ingestSize,
) {
let bytes;

switch (action) {
case 'putObject':
case 'copyObject':
case 'putObjectPart':
if (code === '200') {
bytes = newByteLength - (isVersionedObj ? 0 : oldByteLength);
httpRequestSizeBytes
.labels(method, action, code)
.observe(newByteLength);
dataDiskAvailable.dec(bytes);
dataDiskFree.dec(bytes);
if (ingestSize) {
numberOfIngestedObjects.inc();
dataIngested.inc(ingestSize);
case 'putObject':
case 'copyObject':
case 'putObjectPart':
if (code === '200') {
bytes = newByteLength - (isVersionedObj ? 0 : oldByteLength);
httpRequestSizeBytes.labels(method, action, code).observe(newByteLength);
dataDiskAvailable.dec(bytes);
dataDiskFree.dec(bytes);
if (ingestSize) {
numberOfIngestedObjects.inc();
dataIngested.inc(ingestSize);
}
numberOfObjects.inc();
}
numberOfObjects.inc();
}
break;
case 'createBucket':
if (code === '200') {
numberOfBuckets.inc();
}
break;
case 'getObject':
if (code === '200') {
httpResponseSizeBytes
.labels(method, action, code)
.observe(newByteLength);
}
break;
case 'deleteBucket':
case 'deleteBucketWebsite':
if (code === '200' || code === '204') {
numberOfBuckets.dec();
}
break;
case 'deleteObject':
case 'abortMultipartUpload':
case 'multiObjectDelete':
if (code === '200') {
dataDiskAvailable.inc(newByteLength);
dataDiskFree.inc(newByteLength);
const objs = numOfObjectsRemoved || 1;
numberOfObjects.dec(objs);
if (ingestSize) {
numberOfIngestedObjects.dec(objs);
dataIngested.dec(ingestSize);
break;
case 'createBucket':
if (code === '200') {
numberOfBuckets.inc();
}
}
break;
default:
break;
break;
case 'getObject':
if (code === '200') {
httpResponseSizeBytes.labels(method, action, code).observe(newByteLength);
}
break;
case 'deleteBucket':
case 'deleteBucketWebsite':
if (code === '200' || code === '204') {
numberOfBuckets.dec();
}
break;
case 'deleteObject':
case 'abortMultipartUpload':
case 'multiObjectDelete':
if (code === '200') {
dataDiskAvailable.inc(newByteLength);
dataDiskFree.inc(newByteLength);
const objs = numOfObjectsRemoved || 1;
numberOfObjects.dec(objs);
if (ingestSize) {
numberOfIngestedObjects.dec(objs);
dataIngested.dec(ingestSize);
}
}
break;
default:
break;
}
}

Expand All @@ -187,10 +181,8 @@ function crrCacheToProm(crrResults) {
numberOfBuckets.set(crrResults.getObjectCount.buckets || 0);
numberOfObjects.set(crrResults.getObjectCount.objects || 0);
}
if (config.isQuotaEnabled) {
bucketsWithQuota.set(crrResults?.getObjectCount?.bucketWithQuotaCount || 0);
accountsWithQuota.set(crrResults?.getVaultReport?.accountWithQuotaCount || 0);
}
bucketsWithQuota.set(crrResults?.getObjectCount?.bucketWithQuotaCount || 0);

@francoisferrand francoisferrand Aug 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit:

the first ? is useless - we are in an if (crrResults) block

Suggested change
bucketsWithQuota.set(crrResults?.getObjectCount?.bucketWithQuotaCount || 0);
bucketsWithQuota.set(crrResults.getObjectCount?.bucketWithQuotaCount || 0);

also this line maybe moved int he previous block, which already asserts crrResults.getObjectCount

accountsWithQuota.set(crrResults?.getVaultReport?.accountWithQuotaCount || 0);
if (crrResults.getDataDiskUsage) {
dataDiskAvailable.set(crrResults.getDataDiskUsage.available || 0);
dataDiskFree.set(crrResults.getDataDiskUsage.free || 0);
Expand All @@ -216,8 +208,7 @@ function writeResponse(res, error, results, cb) {
}

const registry = config.isCluster ? new client.AggregatorRegistry() : client.register;
const getMetrics = config.isCluster ?
registry.clusterMetrics.bind(registry) : registry.metrics.bind(registry);
const getMetrics = config.isCluster ? registry.clusterMetrics.bind(registry) : registry.metrics.bind(registry);

async function routeHandler(req, res, cb) {
if (req.method !== 'GET') {
Expand Down
81 changes: 67 additions & 14 deletions tests/unit/utils/monitoring.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,17 @@ const promclient = require('prom-client');
const sinon = require('sinon');

const monitoring = require('../../../lib/utilities/monitoringHandler');
const { config } = require('../../../lib/Config');

describe('Monitoring: endpoint', () => {
const sandbox = sinon.createSandbox();
const res = {
writeHead(/* result, headers */) { return this; },
write(/* body */) { return this; },
writeHead(/* result, headers */) {
return this;
},
write(/* body */) {
return this;
},
end(/* body */) {},
};
monitoring.collectDefaultMetrics();
Expand All @@ -23,9 +28,20 @@ describe('Monitoring: endpoint', () => {
});

async function fetchMetrics(req, res) {
await new Promise(resolve => monitoring.monitoringHandler(null, req, {
...res, end: (...body) => { res.end(...body); resolve(); }
}, null));
await new Promise(resolve =>
monitoring.monitoringHandler(
null,
req,
{
...res,
end: (...body) => {
res.end(...body);
resolve();
},
},
null,
),
);
}

it('should return an error is method is not GET', async () => {
Expand Down Expand Up @@ -80,29 +96,36 @@ describe('Monitoring: endpoint', () => {
});

function parseMetric(metrics, name, labels) {
const labelsString = Object.entries(labels).map(e => `${e[0]}="${e[1]}"`).join(',');
const labelsString = Object.entries(labels)
.map(e => `${e[0]}="${e[1]}"`)
.join(',');
const metric = metrics.match(new RegExp(`^${name}{${labelsString}} (.*)$`, 'm'));
return metric ? metric[1] : null;
}

function parseHttpRequestSize(metrics, action = 'putObject') {
const value = parseMetric(metrics, 's3_cloudserver_http_request_size_bytes_sum',
{ method: 'PUT', action, code: '200' });
const value = parseMetric(metrics, 's3_cloudserver_http_request_size_bytes_sum', {
method: 'PUT',
action,
code: '200',
});
return value ? parseInt(value, 10) : 0;
}

function parseHttpResponseSize(metrics, action = 'getObject') {
const value = parseMetric(metrics, 's3_cloudserver_http_response_size_bytes_sum',
{ method: 'GET', action, code: '200' });
const value = parseMetric(metrics, 's3_cloudserver_http_response_size_bytes_sum', {
method: 'GET',
action,
code: '200',
});
return value ? parseInt(value, 10) : 0;
}

it('should measure http requests size on putObject', async () => {
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
const requestSize = parseHttpRequestSize(res.end.args[0][0]);

monitoring.promMetrics('PUT', 'stuff', '200',
'putObject', 2357, 3572, false, null, 5723);
monitoring.promMetrics('PUT', 'stuff', '200', 'putObject', 2357, 3572, false, null, 5723);

await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
assert(parseHttpRequestSize(res.end.args[1][0]) === requestSize + 2357);
Expand All @@ -112,10 +135,40 @@ describe('Monitoring: endpoint', () => {
await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
const responseSize = parseHttpResponseSize(res.end.args[0][0]);

monitoring.promMetrics('GET', 'stuff', '200',
'getObject', 7532);
monitoring.promMetrics('GET', 'stuff', '200', 'getObject', 7532);

await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
assert(parseHttpResponseSize(res.end.args[1][0]) === responseSize + 7532);
});

const quotaMetricNames = [
's3_cloudserver_quota_evaluation_duration_seconds',
's3_cloudserver_quota_metrics_retrieval_duration_seconds',
's3_cloudserver_quota_utilization_service_available',
's3_cloudserver_quota_buckets_count',
's3_cloudserver_quota_accounts_count',
's3_cloudserver_quota_unavailable_count',
];

it('should register quota metrics even when quota is disabled', async () => {
assert.strictEqual(config.isQuotaEnabled(), false);

await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
const metrics = res.end.args[0][0];

quotaMetricNames.forEach(name => assert(metrics.includes(name), `${name} is not registered`));
Comment thread
delthas marked this conversation as resolved.
});

it('should report the quota counts from crrCacheToProm', async () => {
monitoring.crrCacheToProm({
getObjectCount: { buckets: 1, objects: 2, bucketWithQuotaCount: 3 },
getVaultReport: { accountWithQuotaCount: 4 },
});

await fetchMetrics({ method: 'GET', url: '/metrics' }, res);
const metrics = res.end.args[0][0];

assert(metrics.includes('\ns3_cloudserver_quota_buckets_count 3'));
assert(metrics.includes('\ns3_cloudserver_quota_accounts_count 4'));
});
});
Loading