feat: merge-train/spartan-v5

release-image/Dockerfile.dockerignore

@@ -18,6 +18,7 @@
 !/yarn-project/noir-protocol-circuits-types/artifacts/
 !/yarn-project/protocol-contracts/artifacts/
 !/yarn-project/standard-contracts/artifacts/
+!/yarn-project/standard-contracts/artifacts-historical/
 !/yarn-project/noir-contracts.js/artifacts/
 !/yarn-project/noir-test-contracts.js/artifacts/
 !/yarn-project/simulator/artifacts/

yarn-project/archiver/src/store/contract_instance_store.ts

@@ -1,4 +1,5 @@
 import type { Fr } from '@aztec/foundation/curves/bn254';
+import { first } from '@aztec/foundation/iterable';
 import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
 import { isProtocolContract } from '@aztec/protocol-contracts';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
@@ -134,19 +135,18 @@ export class ContractInstanceStore {
 
   async getCurrentContractInstanceClassId(address: AztecAddress, timestamp: UInt64, originalClassId: Fr): Promise<Fr> {
     // We need to find the last update before the given timestamp
-    const queryResult = await this.#contractInstanceUpdates
-      .valuesAsync({
+    const serializedUpdate = await first(
+      this.#contractInstanceUpdates.valuesAsync({
         reverse: true,
         start: this.getUpdateKey(address, 0n), // Make sure we only look at updates for this contract
         end: this.getUpdateKey(address, timestamp + 1n), // No update can match this key since it doesn't have a log index. We want the highest key <= timestamp
         limit: 1,
-      })
-      .next();
-    if (queryResult.done) {
+      }),
+    );
+    if (serializedUpdate === undefined) {
       return originalClassId;
     }
 
-    const serializedUpdate = queryResult.value;
     const update = SerializableContractInstanceUpdate.fromBuffer(serializedUpdate);
     if (timestamp < update.timestampOfChange) {
       return update.prevContractClassId.isZero() ? originalClassId : update.prevContractClassId;

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -16,6 +16,7 @@ import {
 import { compactArray, pick, unique } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
+import { first } from '@aztec/foundation/iterable';
 import { BadRequestError } from '@aztec/foundation/json-rpc';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { retryUntil } from '@aztec/foundation/retry';
@@ -455,11 +456,8 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb
   }
 
   public async getMaxPriorityFees(): Promise<GasFees> {
-    for await (const tx of this.p2pClient.iteratePendingTxs({ includeProof: false })) {
-      return tx.getGasSettings().maxPriorityFeesPerGas;
-    }
-
-    return GasFees.from({ feePerDaGas: 0n, feePerL2Gas: 0n });
+    const tx = await first(this.p2pClient.iteratePendingTxs({ includeProof: false }));
+    return tx ? tx.getGasSettings().maxPriorityFeesPerGas : GasFees.from({ feePerDaGas: 0n, feePerL2Gas: 0n });
   }
 
   /**

yarn-project/bb-prover/src/verifier/batch_chonk_verifier.ts

@@ -6,6 +6,7 @@ import { Timer } from '@aztec/foundation/timer';
 import { ProtocolCircuitVks } from '@aztec/noir-protocol-circuits-types/server/vks';
 import type { ClientProtocolCircuitVerifier, IVCProofVerificationResult } from '@aztec/stdlib/interfaces/server';
 import type { Tx } from '@aztec/stdlib/tx';
+import { getTelemetryClient } from '@aztec/telemetry-client';
 
 import { Unpackr } from 'msgpackr';
 import { execFile } from 'node:child_process';
@@ -16,6 +17,7 @@ import * as path from 'node:path';
 import { promisify } from 'node:util';
 
 import type { BBConfig } from '../config.js';
+import { IVCVerifierMetrics } from './queued_chonk_verifier.js';
 
 const execFileAsync = promisify(execFile);
 
@@ -54,6 +56,7 @@ export class BatchChonkVerifier implements ClientProtocolCircuitVerifier {
   private sendQueue: SerialQueue;
   private fifoReader: FifoFrameReader;
   private logger = createLogger('bb-prover:batch_chonk_verifier');
+  private metrics: IVCVerifierMetrics;
   /** Maps artifact name to VK index in the batch verifier. */
   private vkIndexMap = new Map<string, number>();
   /** Bound cleanup handler for process exit signals. */
@@ -69,6 +72,7 @@ export class BatchChonkVerifier implements ClientProtocolCircuitVerifier {
     this.fifoReader = new FifoFrameReader();
     this.sendQueue = new SerialQueue();
     this.sendQueue.start(1);
+    this.metrics = new IVCVerifierMetrics(getTelemetryClient(), `BatchChonkVerifier-${label}`);
   }
 
   /** Create and start a BatchChonkVerifier using the protocol circuit VKs. */
@@ -237,6 +241,7 @@ export class BatchChonkVerifier implements ClientProtocolCircuitVerifier {
     const totalDurationMs = pending.totalTimer.ms();
 
     const ivcResult: IVCProofVerificationResult = { valid, durationMs, totalDurationMs };
+    this.metrics.recordIVCVerification(ivcResult);
 
     if (!valid) {
       this.logger.warn(`Proof verification failed for request_id=${result.request_id}: ${result.error_message}`);

yarn-project/bb-prover/src/verifier/queued_chonk_verifier.ts

@@ -16,7 +16,8 @@ import {
 
 import { createHistogram } from 'node:perf_hooks';
 
-class IVCVerifierMetrics {
+/** Records verification timing and failure metrics for an IVC (chonk) proof verifier. */
+export class IVCVerifierMetrics {
   private ivcVerificationHistogram: Histogram;
   private ivcTotalVerificationHistogram: Histogram;
   private ivcFailureCount: UpDownCounter;

yarn-project/end-to-end/src/spartan/n_tps.test.ts

@@ -38,6 +38,7 @@ import {
   getRPCEndpoint,
   hasDeployedHelmRelease,
   installChaosMeshChart,
+  logGossipTxValidationMetrics,
   setupEnvironment,
   startPortForwardForPrometeheus,
   uninstallChaosMesh,
@@ -127,6 +128,15 @@ describe('sustained N TPS test', () => {
 
   afterAll(async () => {
     logger.info('Collecting benchmark metrics and cleaning up...');
+
+    // Log the gossip tx validation breakdown (per-stage timings, tx pool queue stats, chonk verifier
+    // timings) so slow gossip validations observed during the run can be attributed to a stage.
+    if (prometheusClient) {
+      await logGossipTxValidationMetrics(prometheusClient, config.NAMESPACE, TEST_DURATION_SECONDS + 60, logger).catch(
+        err => logger.warn(`Failed to scrape gossip validation metrics: ${err}`, { err }),
+      );
+    }
+
     if (process.env.BENCH_OUTPUT) {
       for (const topic of Object.values(TopicType)) {
         try {

yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts

@@ -35,6 +35,7 @@ import {
   type ServiceEndpoint,
   getEthereumEndpoint,
   getExternalIP,
+  logGossipTxValidationMetrics,
   setupEnvironment,
   startPortForwardForPrometeheus,
 } from './utils.js';
@@ -145,6 +146,15 @@ describe(`prove ${TARGET_TPS}TPS test`, () => {
         server: new URL(`http://127.0.0.1:${freshPromForward.port}`),
       });
 
+      // Log the gossip tx validation breakdown (per-stage timings, tx pool queue stats, chonk
+      // verifier timings) so slow gossip validations during the run can be attributed to a stage.
+      await logGossipTxValidationMetrics(
+        prometheusClient,
+        config.NAMESPACE,
+        epochDurationSeconds + SLOTS_BUFFER * slotDurationSeconds,
+        logger,
+      ).catch(err => logger.warn(`Failed to scrape gossip validation metrics: ${err}`, { err }));
+
       const endSnapshot = await captureMetricsSnapshot(prometheusClient, logger);
 
       // Helper to compute delta, clamping negative values to 0 (handles pod restarts)

yarn-project/end-to-end/src/spartan/utils/gossip_metrics.ts

@@ -0,0 +1,108 @@
+import type { Logger } from '@aztec/foundation/log';
+
+import type { PrometheusClient } from '../../quality_of_service/prometheus_client.js';
+
+/** Runs a PromQL query and returns each series as a label-record plus value. Returns [] on error. */
+async function queryVector(
+  prometheus: PrometheusClient,
+  query: string,
+  logger: Logger,
+): Promise<{ labels: Record<string, string>; value: number }[]> {
+  try {
+    const resp = await prometheus.queryRaw(query);
+    if (resp.status !== 'success' || resp.data.resultType !== 'vector') {
+      logger.warn(`Unexpected Prometheus response for query`, { query, resp });
+      return [];
+    }
+    return resp.data.result.map(({ metric, value }) => ({
+      labels: (metric ?? {}) as Record<string, string>,
+      value: parseFloat(value[1]),
+    }));
+  } catch (err) {
+    logger.warn(`Failed to run Prometheus query: ${err}`, { query });
+    return [];
+  }
+}
+
+/** Formats a grouped vector result as a { labelValue: value } record for structured logging. */
+function toRecord(series: { labels: Record<string, string>; value: number }[], label: string): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const { labels, value } of series) {
+    out[labels[label] ?? 'unknown'] = Math.round(value * 100) / 100;
+  }
+  return out;
+}
+
+/**
+ * Scrapes and logs gossip tx validation timing breakdowns from Prometheus: per-stage validation
+ * durations, tx pool serial queue wait/execution times, and chonk (IVC) proof verifier timings.
+ * Used by the spartan TPS benchmarks to attribute slow gossip validations to a specific stage.
+ */
+export async function logGossipTxValidationMetrics(
+  prometheus: PrometheusClient,
+  namespace: string,
+  windowSeconds: number,
+  logger: Logger,
+): Promise<void> {
+  const ns = `k8s_namespace_name="${namespace}"`;
+  const window = `[${Math.max(60, Math.ceil(windowSeconds))}s]`;
+
+  const stageQuantile = (perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_bucket{${ns}}${window})) by (le, aztec_p2p_tx_validation_stage))`;
+  const stageAvg = () =>
+    `sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_sum{${ns}}${window})) by (aztec_p2p_tx_validation_stage) / ` +
+    `sum(rate(aztec_p2p_gossip_tx_validation_stage_duration_milliseconds_count{${ns}}${window})) by (aztec_p2p_tx_validation_stage)`;
+  const validationQuantile = (perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(aztec_p2p_gossip_message_validation_duration_milliseconds_bucket{${ns}}${window})) by (le, aztec_gossip_topic_name))`;
+  const queueQuantile = (metric: string, perc: string) =>
+    `topk(10, histogram_quantile(${perc}, sum(rate(${metric}{${ns}}${window})) by (le, aztec_mempool_operation)))`;
+  const ivcQuantile = (metric: string, perc: string) =>
+    `histogram_quantile(${perc}, sum(rate(${metric}{${ns}}${window})) by (le))`;
+
+  const stageLabel = 'aztec_p2p_tx_validation_stage';
+  const topicLabel = 'aztec_gossip_topic_name';
+  const operationLabel = 'aztec_mempool_operation';
+
+  const [stageP50, stageP95, stageAvgs, validationP95, slowCount] = await Promise.all([
+    queryVector(prometheus, stageQuantile('0.50'), logger),
+    queryVector(prometheus, stageQuantile('0.95'), logger),
+    queryVector(prometheus, stageAvg(), logger),
+    queryVector(prometheus, validationQuantile('0.95'), logger),
+    queryVector(prometheus, `sum(aztec_p2p_gossip_slow_validation_count{${ns}}) by (${topicLabel})`, logger),
+  ]);
+
+  logger.info('Gossip tx validation stage timings (ms)', {
+    stageP50: toRecord(stageP50, stageLabel),
+    stageP95: toRecord(stageP95, stageLabel),
+    stageAvg: toRecord(stageAvgs, stageLabel),
+    validationP95ByTopic: toRecord(validationP95, topicLabel),
+    slowValidationCountByTopic: toRecord(slowCount, topicLabel),
+  });
+
+  const [queueWaitP95, queueExecutionP95, queueLengthMax] = await Promise.all([
+    queryVector(prometheus, queueQuantile('aztec_mempool_tx_pool_v2_queue_wait_milliseconds_bucket', '0.95'), logger),
+    queryVector(
+      prometheus,
+      queueQuantile('aztec_mempool_tx_pool_v2_queue_execution_milliseconds_bucket', '0.95'),
+      logger,
+    ),
+    queryVector(prometheus, `max(max_over_time(aztec_mempool_tx_pool_v2_queue_length{${ns}}${window}))`, logger),
+  ]);
+
+  logger.info('Tx pool serial queue stats (ms)', {
+    queueWaitP95: toRecord(queueWaitP95, operationLabel),
+    queueExecutionP95: toRecord(queueExecutionP95, operationLabel),
+    queueLengthMax: queueLengthMax[0]?.value,
+  });
+
+  const [ivcVerifyP95, ivcTotalP95] = await Promise.all([
+    queryVector(prometheus, ivcQuantile('aztec_ivc_verifier_time_milliseconds_bucket', '0.95'), logger),
+    queryVector(prometheus, ivcQuantile('aztec_ivc_verifier_total_time_milliseconds_bucket', '0.95'), logger),
+  ]);
+
+  // A large gap between total (queue + verify) and verify indicates a pile-up in the verifier queue.
+  logger.info('Chonk (IVC) proof verifier timings (ms)', {
+    verifyP95: ivcVerifyP95[0]?.value,
+    totalP95: ivcTotalP95[0]?.value,
+  });
+}

yarn-project/end-to-end/src/spartan/utils/index.ts

@@ -69,3 +69,6 @@ export { ChainHealth, type ChainHealthSnapshot } from './health.js';
 
 // Pod log extraction
 export { type BlockBuiltLogEntry, fetchBlockBuiltLogs } from './pod_logs.js';
+
+// Gossip validation metrics scraping
+export { logGossipTxValidationMetrics } from './gossip_metrics.js';

yarn-project/foundation/src/iterable/first.test.ts

@@ -0,0 +1,40 @@
+import { first } from './index.js';
+
+describe('first iterable', () => {
+  it('returns the first entry of a sync iterable', async () => {
+    await expect(first([3, 2, 1])).resolves.toEqual(3);
+  });
+
+  it('returns the first entry of an async iterable', async () => {
+    const generator = (async function* (): AsyncGenerator<number, void, undefined> {
+      yield* [3, 2, 1];
+    })();
+
+    await expect(first(generator)).resolves.toEqual(3);
+  });
+
+  it('returns undefined on an empty iterable', async () => {
+    await expect(first([])).resolves.toBeUndefined();
+    await expect(
+      first(
+        (async function* (): AsyncGenerator<number, void, undefined> {
+          /* yields nothing */
+        })(),
+      ),
+    ).resolves.toBeUndefined();
+  });
+
+  it('closes the underlying iterator after consuming the first entry', async () => {
+    let closed = false;
+    const generator = (async function* (): AsyncGenerator<number, void, undefined> {
+      try {
+        yield* [3, 2, 1];
+      } finally {
+        closed = true;
+      }
+    })();
+
+    await expect(first(generator)).resolves.toEqual(3);
+    expect(closed).toBe(true);
+  });
+});

yarn-project/foundation/src/iterable/first.ts

@@ -0,0 +1,14 @@
+/**
+ * Returns the first entry of an iterable, or undefined if it yields none, closing the underlying
+ * iterator either way. Use this instead of calling `.next()` once and abandoning the iterator:
+ * an abandoned generator never runs its finally blocks, so any resource it holds is leaked — e.g.
+ * a kv-store iterator's LMDB cursor, where enough leaks deadlock the store.
+ */
+export async function first<T>(
+  iterator: Iterable<T> | AsyncIterableIterator<T> | AsyncIterable<T> | IterableIterator<T>,
+): Promise<T | undefined> {
+  for await (const i of iterator) {
+    return i;
+  }
+  return undefined;
+}

yarn-project/foundation/src/iterable/index.ts

@@ -3,5 +3,6 @@ export * from './filter.js';
 export * from './sort.js';
 export * from './take.js';
 export * from './all.js';
+export * from './first.js';
 export * from './peek.js';
 export * from './toArray.js';

yarn-project/kv-store/src/deprecated/indexeddb/multi_map.ts

@@ -1,3 +1,5 @@
+import { first } from '@aztec/foundation/iterable';
+
 import { hash } from 'ohash';
 
 import type { Key, Value } from '../../interfaces/common.js';
@@ -35,7 +37,7 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
     // Instead, we iterate in reverse order to get the last inserted entry
     const index = this.db.index('keyCount');
     const rangeQuery = IDBKeyRange.upperBound([this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER]);
-    const maxEntry = (await index.iterate(rangeQuery, 'prevunique').next()).value;
+    const maxEntry = await first(index.iterate(rangeQuery, 'prevunique'));
     const count = maxEntry?.value?.keyCount ?? 0;
     await this.db.put({
       value: val,