feat: capture TXE server logs in CI and hexdump IPC response frames on decode errors

bootstrap.sh

@@ -173,9 +173,6 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 # Enable abbreviated output by default.
 export DENOISE=${DENOISE:-1}
 
-# Number of TXE servers to run when testing.
-export NUM_TXES=1
-
 # Number of jobs for make. Defaults to number of CPUs.
 # TODO: We should dial this back on consumer hardware, maybe to just 1.
 export MAKEFLAGS="-j${MAKE_JOBS:-$(get_num_cpus)}"
@@ -200,7 +197,7 @@ function cleanup {
     wait $make_pid
     make_pid=
   fi
-  stop_txes
+  stop_txe
 }
 trap cleanup EXIT
 
@@ -334,7 +331,7 @@ function pull_submodules {
   denoise "git submodule update --init --recursive --depth 1 --jobs 8 && git -C noir/noir-repo fetch --tags &>/dev/null"
 }
 
-function start_txes {
+function start_txe {
   # Until Kev's kzg lib stops using Tokio.
   export TOKIO_WORKER_THREADS=1
 
@@ -349,20 +346,20 @@ function start_txes {
     fi
   }
 
-  # Starting txe servers with incrementing port numbers.
-  # Base port is below the Linux ephemeral range (32768-60999) to avoid conflicts.
-  local txe_base_port=14730
-  for i in $(seq 0 $((NUM_TXES-1))); do
-    port=$((txe_base_port + i))
-    kill_port $port
-    dump_fail "LOG_LEVEL=info TXE_PORT=$port retry 'node --no-warnings ./yarn-project/txe/dest/bin/index.js'" &
-    txe_pids+="$! "
-  done
+  # Like the test engine: own session (so stop_txe can kill the whole group) with labeled
+  # output, and denoise so the server's full log lives in its own CI log. The previous
+  # dump_fail wrapper discarded TXE output whenever the process exited cleanly, which made
+  # server-side context for flaky TXE tests unrecoverable.
+  # Port is below the Linux ephemeral range (32768-60999) to avoid conflicts.
+  local txe_port=14730
+  kill_port $txe_port
+  setsid color_prefix "txe" "denoise \"LOG_LEVEL=info TXE_PORT=$txe_port retry 'node --no-warnings ./yarn-project/txe/dest/bin/index.js'\"" &
+  txe_pids+="$! "
 
   # Start the oracle test resolver for __oracle_test__-prefixed tests.
   local resolver_port=14830
   kill_port $resolver_port
-  dump_fail "LOG_LEVEL=error ORACLE_TEST_PORT=$resolver_port node --no-warnings ./yarn-project/txe/dest/bin/oracle_test_server.js" &
+  setsid color_prefix "oracle-resolver" "denoise \"LOG_LEVEL=error ORACLE_TEST_PORT=$resolver_port node --no-warnings ./yarn-project/txe/dest/bin/oracle_test_server.js\"" &
   txe_pids+="$! "
 
   wait_for_port() {
@@ -378,16 +375,18 @@ function start_txes {
       j=$((j+1))
     done
   }
-  for i in $(seq 0 $((NUM_TXES-1))); do
-    wait_for_port $((txe_base_port + i)) "TXE $i"
-  done
+  wait_for_port $txe_port "TXE"
   wait_for_port $resolver_port "oracle test resolver"
 }
 
-function stop_txes {
+function stop_txe {
   if [ -n "${txe_pids:-}" ]; then
     echo "Stopping TXE processes..."
-    kill -SIGTERM $txe_pids &>/dev/null || true
+    # Each pid is a setsid session leader; kill the group so node dies with its wrappers.
+    # denoise still publishes its log on SIGTERM.
+    for pid in $txe_pids; do
+      kill -SIGTERM -- -$pid &>/dev/null || true
+    done
     wait $txe_pids || true
     txe_pids=
   fi
@@ -429,11 +428,11 @@ function build_and_test {
 
     # If make succeeded, start txes and add tests that depend on them.
     if [ "$finished" == "$make_pid" ]; then
-      echo "Makefile build complete, starting TXEs and adding dependent tests..."
+      echo "Makefile build complete, starting TXE and adding dependent tests..."
       make_pid=
 
       # TODO: Handle this better so they can be run as part of the Makefile dependency tree.
-      start_txes
+      start_txe
       make noir-projects-txe-tests
 
       # Benches (full builds only). Uploadable runs (BENCH_UPLOAD=1 — the first instance of
@@ -458,7 +457,7 @@ function build_and_test {
     fi
   done
 
-  stop_txes
+  stop_txe
 
   # Benches (full builds only). Inline benches above are a breakage check only — the
   # dedicated box is the sole uploader. Wait on it here: fatal, matching the old inline

ipc-codegen/src/typescript_codegen.ts

@@ -23,6 +23,28 @@ import {
   dedupeStructsByName,
 } from "./naming.ts";
 
+// Emitted into both API files. Responses carry no request ids (correlation is positional), so
+// when a response fails to decode or fails a shape check, the raw frame is the only evidence of
+// what actually arrived — append a bounded hexdump so a single occurrence is diagnosable from
+// its error message alone. Mutates the original error to preserve its type and stack; the
+// createError (server error) path deliberately bypasses it, as those are well-formed frames
+// whose message already says everything.
+const FRAME_DUMP_HELPER = `function withFrameDump(err: unknown, raw: Uint8Array): Error {
+  const limit = 4096;
+  let hex = '';
+  const end = Math.min(raw.length, limit);
+  for (let i = 0; i < end; i++) {
+    hex += raw[i].toString(16).padStart(2, '0');
+  }
+  const size = raw.length > limit ? \`first \${limit} of \${raw.length} bytes\` : \`\${raw.length} bytes\`;
+  const dump = \`; response frame (\${size}): \${hex}\`;
+  if (err instanceof Error) {
+    err.message += dump;
+    return err;
+  }
+  return new Error(String(err) + dump);
+}`;
+
 export class TypeScriptCodegen {
   private errorTypeName: string = "ErrorResponse";
   /** Prefix to strip from command names when generating method names (e.g. "Bb" -> BbCircuitProve becomes circuitProve) */
@@ -446,14 +468,19 @@ ${syncApiMethods}
 
     return `  ${methodName}(command: ${cmdType}): Promise<${respType}> {
     const msgpackCommand = from${cmdType}(command);
-    return msgpackCall(this.backend, [["${command.name}", msgpackCommand]]).then(([variantName, result]: [string, any]) => {
+    return msgpackCall(this.backend, [["${command.name}", msgpackCommand]]).then(({ decoded, raw }) => {
+      const [variantName, result] = (Array.isArray(decoded) ? decoded : []) as [string, any];
       if (variantName === '${this.errorTypeName}') {
         throw this.createError(result.message || 'Unknown error from server');
       }
-      if (variantName !== '${command.responseType}') {
-        throw new Error(\`Expected variant name '${command.responseType}' but got '\${variantName}'\`);
+      try {
+        if (variantName !== '${command.responseType}') {
+          throw new Error(\`Expected variant name '${command.responseType}' but got '\${variantName}'\`);
+        }
+        return to${respType}(result);
+      } catch (err) {
+        throw withFrameDump(err, raw);
       }
-      return to${respType}(result);
     });
   }`;
   }
@@ -465,14 +492,19 @@ ${syncApiMethods}
 
     return `  ${methodName}(command: ${cmdType}): ${respType} {
     const msgpackCommand = from${cmdType}(command);
-    const [variantName, result] = msgpackCall(this.backend, [["${command.name}", msgpackCommand]]);
+    const { decoded, raw } = msgpackCall(this.backend, [["${command.name}", msgpackCommand]]);
+    const [variantName, result] = (Array.isArray(decoded) ? decoded : []) as [string, any];
     if (variantName === '${this.errorTypeName}') {
       throw this.createError(result.message || 'Unknown error from server');
     }
-    if (variantName !== '${command.responseType}') {
-      throw new Error(\`Expected variant name '${command.responseType}' but got '\${variantName}'\`);
+    try {
+      if (variantName !== '${command.responseType}') {
+        throw new Error(\`Expected variant name '${command.responseType}' but got '\${variantName}'\`);
+      }
+      return to${respType}(result);
+    } catch (err) {
+      throw withFrameDump(err, raw);
     }
-    return to${respType}(result);
   }`;
   }
 
@@ -496,10 +528,16 @@ export interface IpcClientAsync {
 
 export type IpcErrorFactory = (message: string) => Error;
 
-async function msgpackCall(backend: IpcClientAsync, input: any[]) {
+${FRAME_DUMP_HELPER}
+
+async function msgpackCall(backend: IpcClientAsync, input: any[]): Promise<{ decoded: any; raw: Uint8Array }> {
   const inputBuffer = new Encoder({ useRecords: false, variableMapSize: true }).pack(input);
-  const encodedResult = await backend.call(inputBuffer);
-  return new Decoder({ useRecords: false }).unpack(encodedResult);
+  const raw = await backend.call(inputBuffer);
+  try {
+    return { decoded: new Decoder({ useRecords: false }).unpack(raw), raw };
+  } catch (err) {
+    throw withFrameDump(err, raw);
+  }
 }
 
 export class AsyncApi implements AsyncApiBase {
@@ -537,10 +575,16 @@ export interface IpcClientSync {
 
 export type IpcErrorFactory = (message: string) => Error;
 
-function msgpackCall(backend: IpcClientSync, input: any[]) {
+${FRAME_DUMP_HELPER}
+
+function msgpackCall(backend: IpcClientSync, input: any[]): { decoded: any; raw: Uint8Array } {
   const inputBuffer = new Encoder({ useRecords: false, variableMapSize: true }).pack(input);
-  const encodedResult = backend.call(inputBuffer);
-  return new Decoder({ useRecords: false }).unpack(encodedResult);
+  const raw = backend.call(inputBuffer);
+  try {
+    return { decoded: new Decoder({ useRecords: false }).unpack(raw), raw };
+  } catch (err) {
+    throw withFrameDump(err, raw);
+  }
 }
 
 export class SyncApi implements SyncApiBase {

noir-projects/labs/aztec-nr/bootstrap.sh

@@ -30,11 +30,10 @@ function build {
 }
 
 function test_cmds {
-  i=0
+  # All TXE tests share the single TXE server (sessions are isolated per test).
+  local txe_port=14730
   $NARGO test --list-tests --silence-warnings | grep -v __oracle_test__ | sort | while read -r package test; do
-    # We assume there are 8 txe's running.
-    port=$((14730 + (i++ % ${NUM_TXES:-1})))
-    echo "$hash noir-projects/labs/scripts/run_test.sh aztec-nr $package $test $port"
+    echo "$hash noir-projects/labs/scripts/run_test.sh aztec-nr $package $test $txe_port"
   done
 
   # Oracle roundtrip tests run against a dedicated resolver instead of TXE