From 9619833c6a0645d36a8bbacd8883b2be53dbd9a8 Mon Sep 17 00:00:00 2001 From: Alexander Krupenkin Date: Mon, 28 Sep 2026 20:58:59 +0900 Subject: [PATCH 1/4] feat(batcher,blockchain-anchor): remove IPFS anchoring stage, anchor compressed batches directly (#34) Removes the ipfs-publisher service and the IPFS/CID indirection between batcher and blockchain-anchor. batcher now XZ-compresses each detached batch, recursively splits it to fit ANCHOR_MAX_PAYLOAD_BYTES (shared default 8192, defined in @scp/core), and emits chain-ready payloads directly on telemetry.batched.v1 (batch_id, payload, uncompressed_size, compressed_size, payload_hash). Oversized single events are routed to the DLQ instead of blocking the rest of the batch. blockchain-anchor now consumes telemetry.batched.v1 directly and submits the compressed payload bytes on-chain via cps.setPayload. Idempotency is based on a byte-for-byte comparison of the incoming payload against the current on-chain payload (api.query.cps.payload), replacing the previous CID-string comparison. Oversized payloads are defensively rejected as a permanent error. - Reworked TelemetryBatchedPayload proto, removed TelemetryIpfsPublishedPayload and the ipfs.published.v1 topic - Deleted services/ipfs-publisher entirely - Updated docker-compose.yml, kafka-init-topics.sh, integration-test.yml, and .env.example to drop IPFS publisher wiring (the ipfs container is kept, since it is still used as a libp2p GossipSub reserved peer for pubsub-broadcaster integration tests) - Updated READMEs, architecture docs, tracing.md, and WP-04/WP-05 work package docs to reflect the new pipeline Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .env.example | 22 +- .github/copilot-instructions.md | 21 +- .github/workflows/integration-test.yml | 3 +- README.md | 12 +- docker-compose.yml | 2 + docker/kafka-init-topics.sh | 1 - docs/architecture/project-architecture.md | 38 +- docs/tracing.md | 48 +- docs/wp/README.md | 8 +- docs/wp/wp-04-ipfs-publisher.md | 8 + docs/wp/wp-05-blockchain-anchor.md | 76 +- packages/core/README.md | 2 +- .../core/src/anchor.ts | 15 +- packages/core/src/index.ts | 1 + packages/core/src/topics.ts | 1 - packages/core/tests/core.test.ts | 1 - pnpm-lock.yaml | 479 +------------ proto/connectivity/v1/payload.proto | 38 +- services/batcher/README.md | 45 +- services/batcher/package.json | 2 + services/batcher/src/batch-flusher.ts | 4 +- services/batcher/src/config.ts | 8 + services/batcher/src/index.ts | 157 +++-- services/batcher/src/payload-builder.ts | 152 ++++ services/batcher/tests/contracts.test.ts | 21 +- .../batcher/tests/payload-builder.test.ts | 157 +++++ .../batcher/tests/service.shutdown.test.ts | 1 + services/blockchain-anchor/README.md | 58 +- services/blockchain-anchor/package.json | 2 - services/blockchain-anchor/src/config.ts | 8 + services/blockchain-anchor/src/index.ts | 146 ++-- .../tests/cid-encoding.test.ts | 215 ------ .../blockchain-anchor/tests/config.test.ts | 1 + .../blockchain-anchor/tests/contracts.test.ts | 150 +--- .../tests/idempotency.test.ts | 108 ++- .../blockchain-anchor/tests/shutdown.test.ts | 32 +- services/ipfs-publisher/README.md | 143 ---- services/ipfs-publisher/package.json | 29 - services/ipfs-publisher/src/config.ts | 84 --- services/ipfs-publisher/src/durability.ts | 53 -- services/ipfs-publisher/src/index.ts | 650 ------------------ services/ipfs-publisher/src/logger.ts | 57 -- .../src/multi-provider-client.ts | 368 ---------- .../ipfs-publisher/src/providers/index.ts | 44 -- services/ipfs-publisher/src/providers/kubo.ts | 65 -- .../ipfs-publisher/src/providers/pinata.ts | 95 --- .../ipfs-publisher/src/providers/types.ts | 32 - services/ipfs-publisher/tests/config.test.ts | 187 ----- .../ipfs-publisher/tests/contracts.test.ts | 181 ----- .../ipfs-publisher/tests/durability.test.ts | 56 -- .../tests/durable-event-identity.test.ts | 177 ----- .../tests/multi-provider-client.test.ts | 229 ------ .../tests/providers/pinata.test.ts | 103 --- services/ipfs-publisher/tsconfig.json | 13 - services/pubsub-broadcaster/README.md | 2 +- 55 files changed, 900 insertions(+), 3711 deletions(-) rename services/ipfs-publisher/tsdown.config.ts => packages/core/src/anchor.ts (61%) create mode 100644 services/batcher/src/payload-builder.ts create mode 100644 services/batcher/tests/payload-builder.test.ts delete mode 100644 services/blockchain-anchor/tests/cid-encoding.test.ts delete mode 100644 services/ipfs-publisher/README.md delete mode 100644 services/ipfs-publisher/package.json delete mode 100644 services/ipfs-publisher/src/config.ts delete mode 100644 services/ipfs-publisher/src/durability.ts delete mode 100644 services/ipfs-publisher/src/index.ts delete mode 100644 services/ipfs-publisher/src/logger.ts delete mode 100644 services/ipfs-publisher/src/multi-provider-client.ts delete mode 100644 services/ipfs-publisher/src/providers/index.ts delete mode 100644 services/ipfs-publisher/src/providers/kubo.ts delete mode 100644 services/ipfs-publisher/src/providers/pinata.ts delete mode 100644 services/ipfs-publisher/src/providers/types.ts delete mode 100644 services/ipfs-publisher/tests/config.test.ts delete mode 100644 services/ipfs-publisher/tests/contracts.test.ts delete mode 100644 services/ipfs-publisher/tests/durability.test.ts delete mode 100644 services/ipfs-publisher/tests/durable-event-identity.test.ts delete mode 100644 services/ipfs-publisher/tests/multi-provider-client.test.ts delete mode 100644 services/ipfs-publisher/tests/providers/pinata.test.ts delete mode 100644 services/ipfs-publisher/tsconfig.json diff --git a/.env.example b/.env.example index 3aa8fef..35d03af 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,6 @@ KAFKA_BROKERS=localhost:9092 KAFKA_CLIENT_ID=sensors-connectivity REDIS_URL=redis://localhost:6379 -IPFS_API_URL=http://localhost:5001 SUBSTRATE_WS_URL=ws://localhost:9944 ENDPOINT_PORT=3000 @@ -29,28 +28,15 @@ HEARTBEAT_TRACKER_SOURCE=heartbeat-tracker HEARTBEAT_TRACKER_HEALTH_PORT=3030 HEARTBEAT_TRACKER_ONLINE_WINDOW_MS=30000 HEARTBEAT_TRACKER_REDIS_PREFIX=heartbeat-tracker:v1 -# Batcher (batches authorized telemetry into telemetry.batched.v1) +# Batcher (batches, XZ-compresses, and size-fits authorized telemetry into telemetry.batched.v1) BATCHER_GROUP_ID=batcher-v1 BATCHER_SOURCE=batcher BATCHER_HEALTH_PORT=3041 BATCHER_BATCH_SIZE=10 BATCHER_BATCH_TIMEOUT_MS=30000 -# IPFS publisher (publishes telemetry.batched.v1 batches to IPFS) -IPFS_PUBLISHER_GROUP_ID=ipfs-publisher-v1 -IPFS_PUBLISHER_SOURCE=ipfs-publisher -IPFS_PUBLISHER_HEALTH_PORT=3040 -IPFS_PUBLISHER_ENABLE_COMPRESSION=true -# Ordered, comma-separated provider list; first entry is primary (e.g. pinata,kubo) -IPFS_PROVIDERS=kubo -# PINATA_API_URL=https://api.pinata.cloud -# PINATA_JWT= -# Durability policy: any (default) | all | quorum -IPFS_DURABILITY_POLICY=any -IPFS_DURABILITY_MIN_SUCCESS_COUNT=1 -IPFS_PROVIDER_RETRY_BASE_DELAY_MS=5000 -IPFS_PROVIDER_RETRY_MAX_DELAY_MS=60000 -IPFS_PROVIDER_REPLICATION_MAX_PENDING=1000 -# Blockchain anchor (anchors IPFS CIDs to Robonomics CPS pallet) +# Shared max compressed payload size for a single anchored batch (bytes) +ANCHOR_MAX_PAYLOAD_BYTES=8192 +# Blockchain anchor (anchors compressed telemetry batches to Robonomics CPS pallet) BLOCKCHAIN_ANCHOR_GROUP_ID=blockchain-anchor-v1 BLOCKCHAIN_ANCHOR_NODE_ID=0 BLOCKCHAIN_ANCHOR_SURI=//Alice diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f67dcd6..8f46072 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -39,10 +39,9 @@ This is an **event-driven telemetry pipeline** with Kafka as the central durable 4. Multiple **downstream consumers** process authorized events independently: - **pubsub-broadcaster**: Publishes to libp2p/GossipSub for real-time web UI - **heartbeat-tracker**: Tracks sensor liveness/uptime metrics (observability-only, no DLQ) - - **batcher**: Batches authorized telemetry and emits `telemetry.batched.v1` -5. **batcher** → **ipfs-publisher** → **blockchain-anchor** publication chain: - - **ipfs-publisher**: Consumes `telemetry.batched.v1`, publishes batches to IPFS, emits CIDs - - **blockchain-anchor**: Anchors IPFS CIDs to Robonomics blockchain + - **batcher**: Batches authorized telemetry, XZ-compresses and splits it to fit `ANCHOR_MAX_PAYLOAD_BYTES`, and emits `telemetry.batched.v1` +5. **batcher** → **blockchain-anchor** publication chain: + - **blockchain-anchor**: Consumes `telemetry.batched.v1`, submits the compressed payload via `cps.setPayload` to anchor it on the Robonomics blockchain, deduplicating by comparing against the current on-chain payload ### Key Architectural Constraints - **No direct coupling** between processing modules (all flow through Kafka) @@ -60,8 +59,7 @@ Robonomics Blockchain → registry-sync → Redis → endpoint (lookup during va ### Core Kafka Topics - `telemetry.authorized.v1` - Successfully validated telemetry - `telemetry.rejected.v1` - Failed validation (signature/timestamp/auth) -- `telemetry.batched.v1` - Batched authorized telemetry ready for publication -- `ipfs.published.v1` - IPFS publish results (includes CID) +- `telemetry.batched.v1` - XZ-compressed, chain-ready batched telemetry (submitted directly via `cps.setPayload`) - `telemetry.blockchain.result.v1` - Blockchain anchoring results - `telemetry.retry.v1` - Transient failures for retry - `telemetry.dlq.v1` - Exhausted retries (dead letters) @@ -73,9 +71,8 @@ Robonomics Blockchain → registry-sync → Redis → endpoint (lookup during va - `services/whitelist` - Whitelist-based sensor auth provider - `services/pubsub-broadcaster` - Kafka→libp2p GossipSub bridge - `services/heartbeat-tracker` - Observability metrics (online sensors, uptime) -- `services/batcher` - Kafka batcher (authorized → `telemetry.batched.v1`) -- `services/ipfs-publisher` - Kafka→IPFS publisher (Kubo RPC) -- `services/blockchain-anchor` - IPFS CID→blockchain anchoring +- `services/batcher` - Kafka batcher (authorized → XZ-compressed, size-fitted `telemetry.batched.v1`) +- `services/blockchain-anchor` - Anchors compressed batch payloads to the Robonomics CPS pallet via `cps.setPayload` - `tools/fake-sensor-cli` - Generate test telemetry with Ed25519 signatures ## Key Conventions and Patterns @@ -112,7 +109,7 @@ All Kafka events follow a strict envelope schema (see `packages/contracts/src/en All Kafka consumers follow the same processing rule (see `packages/contracts/src/consumer-runtime.ts`): 1. Consume event 2. Check deduplication (if applicable) -3. Perform external action (e.g., publish to PubSub/IPFS) +3. Perform external action (e.g., publish to PubSub, submit blockchain extrinsic) 4. Wait for confirmation 5. Emit result event 6. Commit offset @@ -127,7 +124,7 @@ Use `runConsumerProcessingRule()` helper with: - Services load config from environment variables with defaults - Common pattern: `loadConfig()` function validates and returns typed config - Use zod schemas for validation where possible -- Required infrastructure endpoints: Kafka, Redis, IPFS Kubo RPC, Robonomics node +- Required infrastructure endpoints: Kafka, Redis, Robonomics node ### Naming Conventions - Services: `@scp/` (e.g., `@scp/endpoint`) @@ -152,7 +149,7 @@ Use `runConsumerProcessingRule()` helper with: 1. Prerequisites: Node.js 20 (`.nvmrc`), pnpm 9+, Docker Compose 2. `pnpm install` 3. `cp .env.example .env` and configure as needed -4. `docker compose up -d` (starts Kafka, Redis, IPFS, Robonomics node) +4. `docker compose up -d` (starts Kafka, Redis, Robonomics node) 5. `pnpm dev` (starts all services) or `pnpm --filter @scp/ dev` (single service) ## Testing Philosophy diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 179a5eb..e1066a1 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -116,7 +116,6 @@ jobs: (pnpm --filter @scp/registry-sync dev > logs/registry-sync.log 2>&1 & echo $! > logs/registry-sync.pid) (pnpm --filter @scp/endpoint dev > logs/endpoint.log 2>&1 & echo $! > logs/endpoint.pid) (PUBSUB_RESERVED_PEERS="${PUBSUB_RESERVED_PEERS}" pnpm --filter @scp/pubsub-broadcaster dev > logs/pubsub-broadcaster.log 2>&1 & echo $! > logs/pubsub-broadcaster.pid) - (pnpm --filter @scp/ipfs-publisher dev > logs/ipfs-publisher.log 2>&1 & echo $! > logs/ipfs-publisher.pid) (pnpm --filter @scp/blockchain-anchor dev > logs/blockchain-anchor.log 2>&1 & echo $! > logs/blockchain-anchor.pid) - name: Wait for service health endpoints @@ -216,7 +215,7 @@ jobs: fi if grep -Eqi '(^|[^a-z])(error|exception|unhandledrejection|econnrefused|failed to start|failed to publish)([^a-z]|$)' \ - logs/endpoint.log logs/registry-sync.log logs/pubsub-broadcaster.log logs/ipfs-publisher.log logs/blockchain-anchor.log; then + logs/endpoint.log logs/registry-sync.log logs/pubsub-broadcaster.log logs/blockchain-anchor.log; then echo 'Detected error markers in service logs' >&2 exit 1 fi diff --git a/README.md b/README.md index aded8c8..9bda349 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ make all This will: 1. Install dependencies with pnpm 2. Build all packages and services -3. Start infrastructure (Kafka, Redis, IPFS, Robonomics) +3. Start infrastructure (Kafka, Redis, Robonomics) 4. Initialize Kafka topics 5. Start all services @@ -36,9 +36,8 @@ services/registry-sync # Robonomics blockchain→Redis projection sync servi services/whitelist # Whitelist-based sensor authentication provider services/pubsub-broadcaster # Kafka→libp2p GossipSub bridge for real-time web UI services/heartbeat-tracker # Observability: sensor liveness & uptime metrics -services/batcher # Kafka: batches authorized telemetry into telemetry.batched.v1 -services/ipfs-publisher # Kafka→IPFS publisher (publishes batches, produces CIDs) -services/blockchain-anchor # IPFS CID→Robonomics CPS pallet anchoring +services/batcher # Kafka: batches, compresses (XZ), and splits telemetry into telemetry.batched.v1 +services/blockchain-anchor # Anchors chain-ready compressed batches to the Robonomics CPS pallet tools/fake-sensor-cli # Generate test telemetry with Ed25519 signatures ``` @@ -95,6 +94,5 @@ Use `REJECTION_CODES` from `@scp/core` to reference these codes in your code: - **whitelist**: Alternative authentication provider - maintains static sensor whitelist in Redis, bypassing blockchain dependency for simpler deployments. - **pubsub-broadcaster**: Consumes `telemetry.authorized.v1`, publishes to libp2p/GossipSub for real-time web UI, emits `telemetry.pubsub.result.v1`, routes exhausted failures to DLQ. - **heartbeat-tracker**: Observability-only consumer of `telemetry.authorized.v1`, tracks sensor liveness (`firstSeen`, `lastSeen`, `onlineSince`) in Redis, exposes `sensors_online` count and per-sensor/aggregate uptime metrics over configurable online window (default 30s). Does not emit result events or participate in retry/DLQ. -- **batcher**: Consumes `telemetry.authorized.v1`, groups events into deterministic batches (by size, lag, and a bounded flush timer), and emits `telemetry.batched.v1`. Serializes flushes (single active flush per instance) and flushes pending batches on graceful shutdown. -- **ipfs-publisher**: Consumes `telemetry.batched.v1`, publishes/pins each batch to IPFS (optionally XZ-compressed), deduplicates by `batch_id`, emits `ipfs.published.v1`. -- **blockchain-anchor**: Consumes `ipfs.published.v1` and anchors CIDs to the Robonomics blockchain. +- **batcher**: Consumes `telemetry.authorized.v1`, groups events into deterministic batches (by size, lag, and a bounded flush timer), XZ-compresses and splits each batch to fit `ANCHOR_MAX_PAYLOAD_BYTES`, and emits `telemetry.batched.v1`. Serializes flushes (single active flush per instance) and flushes pending batches on graceful shutdown. +- **blockchain-anchor**: Consumes `telemetry.batched.v1` and anchors the compressed payload bytes to the Robonomics blockchain via `cps.setPayload`, deduplicating by comparing against the current on-chain payload. diff --git a/docker-compose.yml b/docker-compose.yml index 86ec1eb..82fd1c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,8 @@ services: timeout: 5s retries: 5 + # Used only as a libp2p GossipSub reserved peer for pubsub-broadcaster + # integration testing (no longer used for telemetry batch storage). ipfs: image: ipfs/kubo:v0.42.0 ports: diff --git a/docker/kafka-init-topics.sh b/docker/kafka-init-topics.sh index dc5f0ad..21b0662 100755 --- a/docker/kafka-init-topics.sh +++ b/docker/kafka-init-topics.sh @@ -6,7 +6,6 @@ TOPICS=( "telemetry.authorized.v1" "telemetry.rejected.v1" "telemetry.batched.v1" - "ipfs.published.v1" "telemetry.dlq.v1" ) diff --git a/docs/architecture/project-architecture.md b/docs/architecture/project-architecture.md index 8557b98..580239c 100644 --- a/docs/architecture/project-architecture.md +++ b/docs/architecture/project-architecture.md @@ -6,13 +6,13 @@ The system accepts Ed25519-signed environmental sensor telemetry (Altruist-serie ## High-level architecture -`Sensor -> Endpoint -> Message Bus (Kafka) -> Services (IPFS, PubSub, Blockchain)` +`Sensor -> Endpoint -> Message Bus (Kafka) -> Services (PubSub, Batcher, Blockchain Anchor)` > Authorization source: `Robonomics Blockchain -> Registry Sync -> Redis -> Endpoint` or `Whitelist -> Redis -> Endpoint` ### PubSub Broadcast -`Trusted Messages (Kafka) -> PubSub Broadcaster -> IPFS PubSub -> Web UI (sensors.social)` +`Trusted Messages (Kafka) -> PubSub Broadcaster -> libp2p GossipSub -> Web UI (sensors.social)` ### Heartbeat observability @@ -20,7 +20,7 @@ The system accepts Ed25519-signed environmental sensor telemetry (Altruist-serie ### Blockchain anchoring -`Trusted Messages (Kafka) -> Batcher -> Message Bus (Kafka) -> IPFS Publisher -> Message Bus (Kafka) -> Robonomics Blockchain` +`Trusted Messages (Kafka) -> Batcher -> Message Bus (Kafka) -> Blockchain Anchor -> Robonomics Blockchain` ## Module responsibilities @@ -68,30 +68,25 @@ The system accepts Ed25519-signed environmental sensor telemetry (Altruist-serie ### Batcher - Consumes authorized events from `telemetry.authorized.v1`. - Groups events into deterministic batches by size, consumer lag, and a bounded flush timer. -- Serializes each batch as a `crypto.v1.SignedEnvelopeBatch` and emits `telemetry.batched.v1`. +- Serializes each batch as a `crypto.v1.SignedEnvelopeBatch`, XZ-compresses it, and recursively splits it into smaller sub-batches if the compressed result exceeds `ANCHOR_MAX_PAYLOAD_BYTES`. +- Emits one `telemetry.batched.v1` message per fitted sub-batch, each carrying the compressed payload, its size/hash fields, and its own `batch_id`. +- Routes single events that cannot fit even alone to `telemetry.dlq.v1` without blocking the rest of the batch. - Serializes flushes (single active flush per instance) so a timer-triggered flush cannot publish the same batch as a size/lag-triggered flush. - Flushes any pending batch during graceful shutdown before closing resources. -- Commits `telemetry.authorized.v1` offsets only after the batch is durably produced. +- Commits `telemetry.authorized.v1` offsets only after all sub-batches are durably produced. -### IPFS Publisher +### Blockchain Anchor - Consumes batched events from `telemetry.batched.v1`. -- Publishes/pins each batch artifact to IPFS (optionally XZ-compressed) and captures the CID. -- Deduplicates by `batch_id` to avoid duplicate publication of redelivered batches. -- Emits `ipfs.published.v1` with `cid` and `event_count`. -- Commits `telemetry.batched.v1` offsets only after publish success and result emission. - -### Robonomics Blockchain -- Consumes IPFS-published events (`ipfs.published.v1`) from Kafka. -- Publishes the CID into the substrate-based Robonomics blockchain to make the CID immutable. -- Deduplicates by CID before submission. -- Emits anchoring result events (`telemetry.blockchain.result.v1`). -- Commits offset only after blockchain submission confirmation. +- Defensively rejects (permanent error, offset committed) any payload exceeding `ANCHOR_MAX_PAYLOAD_BYTES`. +- Submits the compressed payload bytes directly into the substrate-based Robonomics blockchain via `cps.setPayload` to make the batch immutable. +- Deduplicates by comparing the current on-chain payload bytes against the incoming payload before submission. +- Commits offset only after blockchain submission confirmation (or a confirmed idempotent skip/rejection). +- Emitting an anchoring result event (`telemetry.blockchain.result.v1`) is deferred to a future phase. ## Core Kafka topics - `telemetry.authorized.v1` - `telemetry.rejected.v1` - `telemetry.batched.v1` -- `ipfs.published.v1` - `telemetry.dlq.v1` ## Error handling baseline @@ -104,8 +99,7 @@ The system accepts Ed25519-signed environmental sensor telemetry (Altruist-serie - Allowed flow: `Endpoint -> Kafka -> Consumers`. - Disallowed direct couplings: - Endpoint -> PubSub - - Endpoint -> IPFS - - PubSub -> IPFS - - Batcher -> IPFS (must flow through Kafka) - - IPFS -> Blockchain (must flow through Kafka) + - Endpoint -> Blockchain Anchor + - PubSub -> Batcher + - Batcher -> Blockchain Anchor (must flow through Kafka) - Authentication is pluggable via `SensorAuth` interface but must use Redis for low-latency lookups. diff --git a/docs/tracing.md b/docs/tracing.md index d96aa2e..43efcfe 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -23,21 +23,20 @@ Every telemetry event can be traced through the entire pipeline using `trace_id` │ │ │ ▼ ▼ ▼ ┌─────────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐ - │ pubsub-broadcaster │ │ ipfs-publisher │ │ heartbeat-tracker │ + │ pubsub-broadcaster │ │ batcher │ │ heartbeat-tracker │ │ trace_id ✓ │ │ trace_id ✓ │ │ trace_id ✓ │ │ sensor_id ✓ │ │ sensor_id ✓ │ │ sensor_id ✓ │ └─────────────────────┘ └─────────────────────┘ └─────────────────────┘ │ │ ▼ ▼ ┌─────────────────────┐ ┌─────────────────────┐ - │ IPFS GossipSub │ │ IPFS Storage │ - │ (real-time) │ │ + CID │ + │ GossipSub │ │ telemetry.batched │ + │ (real-time) │ │ .v1 (compressed) │ └─────────────────────┘ └─────────────────────┘ │ ▼ ┌─────────────────────┐ │ Blockchain Anchor │ - │ (future) │ └─────────────────────┘ ``` @@ -53,15 +52,15 @@ Every telemetry event can be traced through the entire pipeline using `trace_id` } ``` -### IPFS Publisher (Batch) +### Batcher (Batch) ```json { "trace_ids": ["550e8400-...", "660f9511-..."], "sensor_ids": ["5GrwvaEF...", "5HGjWAe..."], "unique_sensors": 2, "batch_size": 10, - "cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG", - "msg": "batch published to IPFS" + "batch_id": "b7e2c1...", + "msg": "batch compressed and published" } ``` @@ -89,14 +88,15 @@ Every telemetry event can be traced through the entire pipeline using `trace_id` ```json { "event_id": "evt_789", - "cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG", + "batch_id": "b7e2c1...", + "payload_size": 4096, "event_count": 10, "node_id": 0, - "msg": "anchoring IPFS CID to blockchain" + "msg": "anchoring batch to blockchain" } ``` -**Design Note**: `blockchain-anchor` operates on IPFS batches (identified by CID), not individual telemetry events. CID is the natural correlation key for tracing batches to blockchain. Individual sensor trace_ids are available via IPFS publisher logs (see correlation strategy below). +**Design Note**: `blockchain-anchor` operates on compressed batches (identified by `batch_id`), not individual telemetry events. `batch_id` is the natural correlation key for tracing batches to blockchain. Individual sensor trace_ids are available via batcher logs (see correlation strategy below). ## Querying Logs @@ -110,9 +110,9 @@ grep '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY' logs/*.log | jq grep '550e8400-e29b-41d4-a716-446655440000' logs/*.log | jq ``` -### Find which sensors are in a specific IPFS batch +### Find which sensors are in a specific batch ```bash -grep 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG' logs/ipfs-publisher.log | jq '.sensor_ids' +grep 'b7e2c1...' logs/batcher.log | jq '.sensor_ids' ``` ### Trace from sensor to blockchain @@ -121,12 +121,12 @@ grep 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG' logs/ipfs-publisher.log | grep '5GrwvaEF...' logs/endpoint.log | jq '.trace_id' # Returns: 550e8400-e29b-41d4-a716-446655440000 -# 2. Find batch containing that sensor in ipfs-publisher -grep '550e8400-e29b-41d4-a716-446655440000' logs/ipfs-publisher.log | jq '.cid' -# Returns: QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG +# 2. Find batch containing that sensor in batcher logs +grep '550e8400-e29b-41d4-a716-446655440000' logs/batcher.log | jq '.batch_id' +# Returns: b7e2c1... -# 3. Find blockchain anchoring of that CID -grep 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG' logs/blockchain-anchor.log | jq +# 3. Find blockchain anchoring of that batch +grep 'b7e2c1...' logs/blockchain-anchor.log | jq ``` ## Trace ID Lifecycle @@ -135,7 +135,7 @@ grep 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG' logs/blockchain-anchor.log 2. **Propagated**: Through Kafka envelope (`envelope.traceId`) 3. **Extracted**: By each consumer service from envelope 4. **Logged**: In all processing steps -5. **Batched**: Multiple trace IDs logged together in IPFS publisher batches +5. **Batched**: Multiple trace IDs logged together in batcher batches ## Sensor ID Format @@ -145,8 +145,8 @@ grep 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG' logs/blockchain-anchor.log ## Implementation Notes -- Telemetry consumers (`pubsub-broadcaster`, `ipfs-publisher`, `heartbeat-tracker`) extract `trace_id` from `Envelope.traceId` field -- **Batch-level tracing**: `blockchain-anchor` uses CID as the correlation key (cleaner separation of concerns) +- Telemetry consumers (`pubsub-broadcaster`, `batcher`, `heartbeat-tracker`) extract `trace_id` from `Envelope.traceId` field +- **Batch-level tracing**: `blockchain-anchor` uses `batch_id` as the correlation key (cleaner separation of concerns) - Batch operations log arrays of affected sensors and traces - Log level `debug` for per-message tracing, `info` for batch summaries - Structured logging via Pino for easy parsing/filtering @@ -163,9 +163,9 @@ Use `trace_id` to track individual sensor telemetry through the pipeline. ### Batch-to-Blockchain Tracing ``` -sensor_id → trace_id → batch CID → blockchain transaction +sensor_id → trace_id → batch_id → blockchain transaction ``` -Use CID as the correlation key: +Use `batch_id` as the correlation key: 1. Find sensor's `trace_id` in endpoint/consumer logs -2. Find batch CID containing that `trace_id` in ipfs-publisher logs (includes full `sensor_ids` and `trace_ids` arrays) -3. Find blockchain anchoring for that CID in blockchain-anchor logs +2. Find `batch_id` containing that `trace_id` in batcher logs (includes full `sensor_ids` and `trace_ids` arrays) +3. Find blockchain anchoring for that `batch_id` in blockchain-anchor logs diff --git a/docs/wp/README.md b/docs/wp/README.md index 81b984b..7507c81 100644 --- a/docs/wp/README.md +++ b/docs/wp/README.md @@ -11,7 +11,7 @@ Each WP takes a scaffold/stub service to a working, tested implementation. ## Pipeline ```text -Sensor -> Endpoint -> Kafka -> {PubSub Broadcaster, IPFS Publisher} -> Kafka -> Blockchain Anchor +Sensor -> Endpoint -> Kafka -> {PubSub Broadcaster, Batcher -> Blockchain Anchor} ^ Robonomics chain -> Registry Sync -> Redis projection ``` @@ -25,8 +25,8 @@ Robonomics chain -> Registry Sync -> Redis projection | [WP-02](./wp-02-endpoint.md) | `endpoint` (`POST /v1/telemetry` ingress) | WP-00, WP-01 | Implemented (pending formal DoD sign-off) | | [WP-03](./wp-03-pubsub-broadcaster.md) | `pubsub-broadcaster` (GossipSub fan-out) | WP-00, WP-02 | Implemented | | WP-03A | `heartbeat-tracker` (trusted-event liveness & uptime observability) | WP-00, WP-02 | Implemented | -| [WP-04](./wp-04-ipfs-publisher.md) | `ipfs-publisher` (batch + IPFS CID) | WP-00, WP-02 | Not started | -| [WP-05](./wp-05-blockchain-anchor.md) | `blockchain-anchor` (CID-only anchoring) | WP-00, WP-04 | Not started | +| [WP-04](./wp-04-ipfs-publisher.md) | `@scp/batcher` (batch, XZ-compress, size-fit) | WP-00, WP-02 | Implemented (superseded `ipfs-publisher`, see note in doc) | +| [WP-05](./wp-05-blockchain-anchor.md) | `blockchain-anchor` (anchors compressed batch payloads via `cps.setPayload`) | WP-00, WP-04 | Implemented | ## Recommended sequencing @@ -34,7 +34,7 @@ Robonomics chain -> Registry Sync -> Redis projection 2. **WP-01** — the endpoint depends on the registry read path. 3. **WP-02** — enables end-to-end producing onto Kafka. 4. **WP-03 / WP-04** in parallel — both consume `telemetry.authorized.v1`. -5. **WP-05** last — consumes `telemetry.ipfs.result.v1` from WP-04. +5. **WP-05** last — consumes `telemetry.batched.v1` from WP-04. ## Definition of done (applies to every WP) diff --git a/docs/wp/wp-04-ipfs-publisher.md b/docs/wp/wp-04-ipfs-publisher.md index 88eb6e3..66cbb54 100644 --- a/docs/wp/wp-04-ipfs-publisher.md +++ b/docs/wp/wp-04-ipfs-publisher.md @@ -1,5 +1,13 @@ # WP-04 — `ipfs-publisher` +> **Superseded (issue #34):** The `ipfs-publisher` service has been removed. +> `@scp/batcher` now XZ-compresses and size-fits batches directly, emitting +> chain-ready payloads on `telemetry.batched.v1` for `blockchain-anchor` to +> submit via `cps.setPayload`. There is no longer an IPFS publication stage +> in the pipeline. This document is kept for historical context only; see +> [WP-05](./wp-05-blockchain-anchor.md) and the `@scp/batcher` / +> `@scp/blockchain-anchor` READMEs for the current design. + ## Summary WP-04 implements deterministic batching and IPFS publication for authorized telemetry events. diff --git a/docs/wp/wp-05-blockchain-anchor.md b/docs/wp/wp-05-blockchain-anchor.md index 9fd89b6..152e9e8 100644 --- a/docs/wp/wp-05-blockchain-anchor.md +++ b/docs/wp/wp-05-blockchain-anchor.md @@ -1,74 +1,78 @@ # WP-05 — `blockchain-anchor` +> **Updated (issue #34):** The IPFS anchoring stage has been removed. +> `blockchain-anchor` now consumes `telemetry.batched.v1` directly from +> `@scp/batcher` and submits the already XZ-compressed, size-fitted payload +> bytes on-chain via `cps.setPayload`. There is no CID or IPFS publication +> step; idempotency is based on comparing the current on-chain payload +> bytes against the incoming payload. + ## Summary -WP-05 anchors IPFS CIDs into the substrate-based Robonomics blockchain as the first minimal blockchain phase. It consumes IPFS publication events, deduplicates by CID, submits anchoring transactions, and publishes result events. The implementation must preserve at-least-once processing guarantees without committing offsets before submission confirmation. +WP-05 anchors chain-ready compressed telemetry batches into the substrate-based Robonomics blockchain. It consumes batched telemetry events, deduplicates by comparing on-chain payload bytes, submits anchoring transactions, and commits offsets only after finalization (or a confirmed idempotent skip). Emitting a `telemetry.blockchain.result.v1` result event is deferred to a future phase. ## Depends on - WP-00 -- WP-04 +- `@scp/batcher` (formerly WP-04) ## Scope / Goal -Implement `blockchain-anchor` to consume `telemetry.ipfs.result.v1`, submit CID-only anchoring transactions to Robonomics, emit `telemetry.blockchain.result.v1`, and commit offsets only after submission confirmation and result emission. +Implement `blockchain-anchor` to consume `telemetry.batched.v1`, submit the compressed batch payload to Robonomics via `cps.setPayload`, and commit offsets only after submission confirmation (or a confirmed idempotent skip). ## Out of scope / Deferred - Advanced finality/reorg handling. - Attestation/proof frameworks. - Multi-chain adapters/abstraction. - Advanced fee/relay optimization strategies. +- Emitting `telemetry.blockchain.result.v1` (deferred; not implemented yet). ## Inputs & Outputs ### Inputs -- Kafka topic consumed: `telemetry.ipfs.result.v1` payload fields: - - `cid` - - `event_count` +- Kafka topic consumed: `telemetry.batched.v1` payload fields: + - `batch_id` + - `payload` (XZ-compressed, chain-ready bytes; must be `<= ANCHOR_MAX_PAYLOAD_BYTES`) + - `uncompressed_size`, `compressed_size`, `payload_hash` + - `event_count`, `sensor_ids` - Shared envelope/contracts from WP-00. - External system: substrate-based Robonomics blockchain. ### Outputs -- External side effect: CID-only blockchain anchoring transaction submission. -- Kafka topic produced: `telemetry.blockchain.result.v1` payload fields: - - `target` - - `status` (`submitted` | `failed`) - - `cid` - - `tx_hash` - - `error_code` - - `error_message` +- External side effect: `cps.setPayload(node_id, payload)` blockchain transaction submission. +- No result event is currently emitted (see Deferred). ## Detailed tasks / Implementation checklist -- [ ] Replace service stubs/TODOs with a real consumer for `telemetry.ipfs.result.v1`. -- [ ] Validate consumed events against WP-00 schemas. -- [ ] Implement CID deduplication guard before blockchain submission. -- [ ] Implement substrate/Robonomics client submission for CID-only anchoring. -- [ ] Wait for submission confirmation per minimal phase policy. -- [ ] Emit `telemetry.blockchain.result.v1` with success/failure status and transaction/error details. -- [ ] Apply bounded retry for transient submission failures. +- [x] Consume `telemetry.batched.v1` and parse `TelemetryBatchedPayload`. +- [x] Validate consumed events against WP-00 schemas. +- [x] Defensively reject (permanent error, offset committed) any payload exceeding `ANCHOR_MAX_PAYLOAD_BYTES`. +- [x] Implement byte-for-byte payload dedup guard (`api.query.cps.payload(node_id)`) before blockchain submission. +- [x] Implement substrate/Robonomics client submission for `cps.setPayload`. +- [x] Wait for extrinsic finalization before committing the offset. +- [ ] Emit `telemetry.blockchain.result.v1` with success/failure status and transaction/error details (deferred). +- [ ] Apply bounded retry for transient submission failures (currently retried via redelivery/offset-not-committed only). - [ ] Route exhausted failures to `telemetry.dlq.v1`. -- [ ] Commit Kafka offset only after submission confirmation and result-event emission. -- [ ] Add structured logging and health/metrics endpoint (submission latency, success/failure, retry/DLQ counts). +- [x] Commit Kafka offset only after submission confirmation (or idempotent skip, or permanent oversized-payload rejection). +- [x] Add structured logging and health/metrics endpoint (`consumed`, `anchored`, `skippedDuplicate`, `rejectedOversized`, `failed`). ## Idempotency & error handling -- Dedup key: `cid`. -- Replayed messages with same `cid` must not create inconsistent duplicate anchoring behavior. -- Consumer rule must remain: consume → submit CID → wait confirmation → emit result event → commit offset. -- Bounded retries for transient blockchain/RPC failures; exhausted failures must go to `telemetry.dlq.v1`. +- Dedup key: the compressed payload bytes themselves, scoped to the configured `BLOCKCHAIN_ANCHOR_NODE_ID`. +- Replayed messages with the same payload bytes must not resubmit a duplicate anchoring transaction; the on-chain state is queried and compared byte-for-byte before submission. +- Consumer rule: consume → check size limit → check on-chain payload → submit (if needed) → wait confirmation → commit offset. +- Oversized payloads (a producer bug or contract violation, since the batcher is expected to have already split to fit) are rejected permanently and the offset is committed without submission. ## Testing - Unit tests: - - CID dedup logic, - - result payload mapping (`submitted` vs `failed`), - - retry policy classification. + - byte-for-byte payload dedup logic, + - oversized-payload rejection, + - config parsing (`maxPayloadBytes`, node ID, etc.). - Integration tests: - local `docker-compose` Kafka + substrate/Robonomics test environment/mock, - - verify consume → submit → result event → commit order, - - verify DLQ behavior on exhausted failures. + - verify consume → submit → commit order, + - verify graceful shutdown waits for an in-flight extrinsic before disconnecting. - Contract tests: - - consumed `telemetry.ipfs.result.v1` and produced `telemetry.blockchain.result.v1` compatibility. + - consumed `telemetry.batched.v1` payload compatibility (`batch_id`, `payload`, size/hash fields). ## Definition of Done - All `TODO` markers in `blockchain-anchor` are replaced with real logic. - Unit tests + integration test against local `docker-compose` infra are green. - `pnpm build`, `pnpm typecheck`, `pnpm lint`, `pnpm test` are green. -- Bounded retry + DLQ via shared consumer runtime is implemented. - Baseline structured logging and health/metrics endpoint are implemented. -- CID-only anchoring works end-to-end with result events and offset-commit-after-confirmation policy. -- Deferred items (finality/reorg, attestation, multi-chain) are explicitly documented as not in this phase. +- Compressed-payload anchoring works end-to-end with offset-commit-after-confirmation policy. +- Deferred items (result event emission, finality/reorg, attestation, multi-chain, bounded retry/DLQ) are explicitly documented as not in this phase. diff --git a/packages/core/README.md b/packages/core/README.md index 39109f6..b427bcd 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -50,7 +50,7 @@ import { EnvelopeSchema, TelemetryAuthorizedPayloadSchema, TelemetryRejectedPayloadSchema, - TelemetryIpfsPublishedPayloadSchema, + TelemetryBatchedPayloadSchema, TelemetryPubsubResultPayloadSchema, TelemetryBlockchainResultPayloadSchema, } from '@scp/core'; diff --git a/services/ipfs-publisher/tsdown.config.ts b/packages/core/src/anchor.ts similarity index 61% rename from services/ipfs-publisher/tsdown.config.ts rename to packages/core/src/anchor.ts index 3e6ee6c..86e7699 100644 --- a/services/ipfs-publisher/tsdown.config.ts +++ b/packages/core/src/anchor.ts @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { defineConfig } from 'tsdown'; -export default defineConfig({ - entry: ['src/index.ts'], - format: ['esm'], - clean: true, - dts: true, -}); +/** + * Default maximum size, in bytes, of a chain-ready payload passed to + * `CPS.set_payload`. This is the single source of truth for the 8 KiB CPS + * payload limit; services may override it via environment configuration but + * should fall back to this default rather than hard-coding the literal + * value themselves. + */ +export const DEFAULT_ANCHOR_MAX_PAYLOAD_BYTES = 8192; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index bc772e0..251ff99 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -19,6 +19,7 @@ export * from './rejection-codes.js'; export * from './sensor-auth.js'; export * from './topics.js'; export * from './shutdown.js'; +export * from './anchor.js'; // Re-export generated protobuf schemas export * from './generated/connectivity/v1/envelope_pb.js'; diff --git a/packages/core/src/topics.ts b/packages/core/src/topics.ts index fe30b47..c71772a 100644 --- a/packages/core/src/topics.ts +++ b/packages/core/src/topics.ts @@ -17,7 +17,6 @@ export const TELEMETRY_TOPICS = Object.freeze({ AUTHORIZED: 'telemetry.authorized.v1', REJECTED: 'telemetry.rejected.v1', BATCHED: 'telemetry.batched.v1', - IPFS_PUBLISHED: 'ipfs.published.v1', DLQ: 'telemetry.dlq.v1', } as const); diff --git a/packages/core/tests/core.test.ts b/packages/core/tests/core.test.ts index 3d812c4..0232923 100644 --- a/packages/core/tests/core.test.ts +++ b/packages/core/tests/core.test.ts @@ -50,7 +50,6 @@ describe('core', () => { expect(TELEMETRY_TOPICS.AUTHORIZED).toBe('telemetry.authorized.v1'); expect(TELEMETRY_TOPICS.REJECTED).toBe('telemetry.rejected.v1'); expect(TELEMETRY_TOPICS.BATCHED).toBe('telemetry.batched.v1'); - expect(TELEMETRY_TOPICS.IPFS_PUBLISHED).toBe('ipfs.published.v1'); expect(TELEMETRY_TOPICS.DLQ).toBe('telemetry.dlq.v1'); expect(Object.isFrozen(TELEMETRY_TOPICS)).toBe(true); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96d30d2..a5c10a7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -176,9 +176,15 @@ importers: '@buf/airalab_connectivity-protocol.bufbuild_es': specifier: 2.14.0-20260917143433-6f6f1eba1cc0.2 version: 2.14.0-20260917143433-6f6f1eba1cc0.2(@bufbuild/protobuf@2.14.0) + '@napi-rs/lzma': + specifier: ^1.5.1 + version: 1.5.1 '@platformatic/kafka': specifier: ^2.9.0 version: 2.9.0(supports-color@10.2.2) + '@polkadot/util-crypto': + specifier: ^14.0.3 + version: 14.0.3(@polkadot/util@14.0.3) '@scp/core': specifier: workspace:* version: link:../../packages/core @@ -198,9 +204,6 @@ importers: services/blockchain-anchor: dependencies: - '@buf/airalab_connectivity-protocol.bufbuild_es': - specifier: 2.14.0-20260917143433-6f6f1eba1cc0.2 - version: 2.14.0-20260917143433-6f6f1eba1cc0.2(@bufbuild/protobuf@2.14.0) '@platformatic/kafka': specifier: ^2.9.0 version: 2.9.0(supports-color@10.2.2) @@ -213,9 +216,6 @@ importers: '@scp/core': specifier: workspace:* version: link:../../packages/core - multiformats: - specifier: ^14.0.5 - version: 14.0.5 pino: specifier: ^10.3.1 version: 10.3.1 @@ -295,40 +295,6 @@ importers: specifier: ^4.23.12 version: 4.23.12 - services/ipfs-publisher: - dependencies: - '@buf/airalab_connectivity-protocol.bufbuild_es': - specifier: 2.14.0-20260917143433-6f6f1eba1cc0.2 - version: 2.14.0-20260917143433-6f6f1eba1cc0.2(@bufbuild/protobuf@2.14.0) - '@napi-rs/lzma': - specifier: ^1.5.1 - version: 1.5.1 - '@platformatic/kafka': - specifier: ^2.9.0 - version: 2.9.0(supports-color@10.2.2) - '@scp/core': - specifier: workspace:* - version: link:../../packages/core - kubo-rpc-client: - specifier: ^7.1.0 - version: 7.1.0 - multiformats: - specifier: ^14.0.5 - version: 14.0.5 - pino: - specifier: ^10.3.1 - version: 10.3.1 - devDependencies: - '@bufbuild/protobuf': - specifier: ^2.14.0 - version: 2.14.0 - tsdown: - specifier: ^0.22.14 - version: 0.22.14(tsx@4.23.12)(typescript@6.0.3) - tsx: - specifier: ^4.23.12 - version: 4.23.12 - services/pubsub-broadcaster: dependencies: '@chainsafe/libp2p-gossipsub': @@ -790,19 +756,6 @@ packages: '@ioredis/commands@2.0.0': resolution: {integrity: sha512-vrx0AE/T0h7cRZwfo1M39Cr+ZhZrkf0V8mQN75wucKCxCLD9l/VX6no3gFvrLqD1IlG/1LtzWovqEw3t0Vr9zg==} - '@ipld/dag-cbor@10.0.2': - resolution: {integrity: sha512-1eCD4YMTlkx5i36oQUmeAojLS/wu+XCd1Gs31BA0afk0TaVkhnyMofIAT/pi0wV3HS3/G7QSN/1NibhiksMyZQ==} - - '@ipld/dag-cbor@9.2.7': - resolution: {integrity: sha512-ZmfXmElRWATr+hoUTSAOr6HUcjVhOcNHDqgczc76qte2DHHFEK0ZhNzUcdTDQhF/VSIvf2ioaRTRLWwLc83sNw==} - - '@ipld/dag-json@11.0.1': - resolution: {integrity: sha512-cMDD0yWkUruqhbLDwAWaV5q92hYKQDP2xP0n+yZp0oxsC68Pw+xFZNc9rj0v2n5m5m/Aa1A+Qlb2tY3NjQMsWg==} - - '@ipld/dag-pb@4.1.7': - resolution: {integrity: sha512-/i/13trFihjWfDyXlylRwhuYjtzYjvOFw0vlRjYGnZuv7d7MOgA2lV/vRuL5RfeUajM03aZfFLdq4S7cTbbTRg==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - '@jridgewell/sourcemap-codec@1.6.0': resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} @@ -839,9 +792,6 @@ packages: '@libp2p/logger@5.2.0': resolution: {integrity: sha512-OEFS529CnIKfbWEHmuCNESw9q0D0hL8cQ8klQfjIVPur15RcgAEgc1buQ7Y6l0B6tCYg120bp55+e9tGvn8c0g==} - '@libp2p/logger@6.2.12': - resolution: {integrity: sha512-FwCdJ5hGFDvWI/SULZ4Iez+k+JtdMAj4/EK6bWmkh08ORIztwKpt0IXCOe8nIfKhBKwLYJu00BnZDRsedX7+7Q==} - '@libp2p/multistream-select@6.0.29': resolution: {integrity: sha512-SWQbPcABOIpznEY7+vAp0Y3HNrE2PlaVY4EywN0lUZ7zvTv9VnAb7av3/gMvfaLI+YrOvhCr1mZ9qbSB93k4kA==} @@ -851,9 +801,6 @@ packages: '@libp2p/peer-id@5.1.9': resolution: {integrity: sha512-cVDp7lX187Epmi/zr0Qq2RsEMmueswP9eIxYSFoMcHL/qcvRFhsxOfUGB8361E26s2WJvC9sXZ0oJS9XVueJhQ==} - '@libp2p/peer-id@6.0.14': - resolution: {integrity: sha512-tquBouwbUXgntPtT5q4vfaXbNWmSzh2zo4p2u8IzclVvggXOpmyBlAUNPXDiFqZvQpsxVBrTTemmn0+AwDwecg==} - '@libp2p/peer-record@8.0.35': resolution: {integrity: sha512-0818zvjKbucq5XBnusG8oSWxJ992rVry/2qlfcn/nyK/uDrZ12tjDYHNMCoOWTNeFvFUVkMg9pRkvXvTNp6Yiw==} @@ -885,9 +832,6 @@ packages: '@multiformats/multiaddr-to-uri@11.0.2': resolution: {integrity: sha512-SiLFD54zeOJ0qMgo9xv1Tl9O5YktDKAVDP4q4hL16mSq4O4sfFNagNADz8eAofxd6TfQUzGQ3TkRRG9IY2uHRg==} - '@multiformats/multiaddr-to-uri@12.0.0': - resolution: {integrity: sha512-3uIEBCiy8tfzxYYBl81x1tISiNBQ7mHU4pGjippbJRoQYHzy/ZdZM/7JvTldr8pc/dzpkaNJxnsuxxlhsPOJsA==} - '@multiformats/multiaddr@12.5.1': resolution: {integrity: sha512-+DDlr9LIRUS8KncI1TX/FfUn8F2dl6BIxJgshS/yFQCNB5IAF0OGzcwB39g5NLE22s4qqDePv0Qof6HdpJ/4aQ==} @@ -1119,18 +1063,6 @@ packages: resolution: {integrity: sha512-OwuyRAe9Lj0GoFVBFzTS6bTAV4i+Xd68sYbNUmLnI1GVkQIqVb9mzNrpKSZkBFiQD73CdMB/uKYsvk2bxq/eZg==} engines: {node: '>= 10'} - '@nodelib/fs.scandir@2.1.5': - resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} - engines: {node: '>= 8'} - - '@nodelib/fs.stat@2.0.5': - resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} - engines: {node: '>= 8'} - - '@nodelib/fs.walk@1.2.8': - resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} - engines: {node: '>= 8'} - '@oxc-project/types@0.144.0': resolution: {integrity: sha512-nuhZIOLuI6TFQ32I/WnUx+SCPY7SdSKwgnFHydAuoS1+Z4BRcaP+RRJmGzl9lw+0OFF7UmaESf7KQRXaNLHypg==} @@ -1804,12 +1736,6 @@ packages: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} - base64-js@1.5.1: - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - - blob-to-it@3.0.0: - resolution: {integrity: sha512-KhVc/vccFEeqZIf7xFLkMnaNGL+7umpvqSqyL2XfOobfHQ3RQB9aLau/F6WhoE5S3TbtYYtrX3+HkISpkY+jXw==} - bn.js@5.2.5: resolution: {integrity: sha512-Vq886eXykuP5E6HcKSSStP3bJgrE6In5WKxVUvJ8XGpWWYs2xZHWqUwzCtGgEtBcxyd57KBFDPFoUfNzdaHCNg==} @@ -1817,16 +1743,6 @@ packages: resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} engines: {node: 20 || >=22} - braces@3.0.3: - resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} - engines: {node: '>=8'} - - browser-readablestream-to-it@2.0.12: - resolution: {integrity: sha512-VDAcuM39JVtxZ7auqE2p0zHYk1fq+pac0cWLOQJ48MIChTZ1RjCR2PYCdL3kIisst7oGZCxYrJhfHlbNYIa0Tg==} - - buffer@6.0.3: - resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} - cac@7.0.0: resolution: {integrity: sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==} engines: {node: '>=20.19.0'} @@ -1834,14 +1750,6 @@ packages: cacheable@2.5.0: resolution: {integrity: sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==} - cborg@5.1.11: - resolution: {integrity: sha512-oc6Pzg/gkTobxHZNgMmny+G99dOeBMbAmnGHcZWMKtolxZBIVwfi0Pj0khxEtNU8HMFdbT5sK0HmtgecDUPP0A==} - hasBin: true - - cborg@6.1.1: - resolution: {integrity: sha512-Ci8+V1OMFtqea8EXicsATqV+3kEqCGXCicM+5MEzpKTxqy58WMzhJUjtUqAjfWpKPZ0wpG8F+u5mC8Xy/0Zevg==} - hasBin: true - chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -1861,9 +1769,6 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} - dag-jose@5.1.1: - resolution: {integrity: sha512-9alfZ8Wh1XOOMel8bMpDqWsDT72ojFQCJPtwZSev9qh4f8GoCV9qrJW8jcOUhcstO8Kfm09FHGo//jqiZq3z9w==} - data-uri-to-buffer@4.0.1: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} @@ -1911,20 +1816,10 @@ packages: oxc-resolver: optional: true - electron-fetch@1.9.1: - resolution: {integrity: sha512-M9qw6oUILGVrcENMSRRefE1MbHPIz0h79EKIeJWK9v563aT9Qkh8aEHPO1H5vi970wPirNY+jO9OpFoLiMsMGA==} - engines: {node: '>=6'} - empathic@2.0.1: resolution: {integrity: sha512-YGRs8knHhKHVShLkFET/rWAU8kmHbOV5LwN938RHI0pljAJ1Gf6SzXsSmRaEzcXTtOOmVqJ5+WtQPL5uigY50Q==} engines: {node: '>=14'} - encoding@0.1.13: - resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} - - err-code@3.0.1: - resolution: {integrity: sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA==} - es-module-lexer@2.3.2: resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} @@ -2021,13 +1916,6 @@ packages: fast-diff@1.3.0: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} - fast-fifo@1.3.2: - resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - - fast-glob@3.3.3: - resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} - engines: {node: '>=8.6.0'} - fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} @@ -2071,10 +1959,6 @@ packages: file-entry-cache@11.1.5: resolution: {integrity: sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==} - fill-range@7.1.1: - resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} - engines: {node: '>=8'} - find-my-way@9.8.0: resolution: {integrity: sha512-JtyUgATO7qxRp2zKhrmWof74Mqxc1ikbwpwMY97p8ipuTj2QtreA4gK2JNAF6SOqqHnYYkwMUvsgQVi2AJxIyw==} engines: {node: '>=20'} @@ -2098,9 +1982,6 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - get-iterator@1.0.2: - resolution: {integrity: sha512-v+dm9bNVfOYsY1OrhaCrmyOcYoSeVvbt+hHZ0Au+T+p1y+0Uyj9aMaGIeUTT6xdpRbWzDeYKvfOslPhggQMcsg==} - get-iterator@2.0.1: resolution: {integrity: sha512-7HuY/hebu4gryTDT7O/XY/fvY9wRByEGdK6QOa4of8npTcv0+NS6frFKABcf6S9EBAsveTuKTsZQQBFMMNILIg==} @@ -2108,10 +1989,6 @@ packages: resolution: {integrity: sha512-/6gFNr0N04nob252sTQxyFLi3eKFRqIg1I87YcqAMT1i6SQrSF6KujUEQrtrjMV0H/eejTCltLdDSTEMzHbnsQ==} engines: {node: '>=20.20.0'} - glob-parent@5.1.2: - resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} - engines: {node: '>= 6'} - glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -2137,13 +2014,6 @@ packages: engines: {node: '>=18'} hasBin: true - iconv-lite@0.6.3: - resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} - engines: {node: '>=0.10.0'} - - ieee754@1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -2160,21 +2030,12 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} - inherits@2.0.4: - resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - - interface-datastore@10.0.1: - resolution: {integrity: sha512-DYMj/Og5Cz1Qwkx6/x5KRvR8SYEX7rVAv3KKCm2NzTwWSfpNAC4PahjcYbHyoZBP6zPWrhQv5n5wE+vaDdgSAg==} - interface-datastore@8.3.2: resolution: {integrity: sha512-R3NLts7pRbJKc3qFdQf+u40hK8XWc0w4Qkx3OFEstC80VoaDUABY/dXA2EJPhtNC+bsrf41Ehvqb6+pnIclyRA==} interface-store@6.0.3: resolution: {integrity: sha512-+WvfEZnFUhRwFxgz+QCQi7UC6o9AM0EHM9bpIe2Nhqb100NHCsTvNAn4eJgvgV2/tmLo1MP9nGxQKEcZTAueLA==} - interface-store@8.0.0: - resolution: {integrity: sha512-e2+s3EEROzM+Wlas4hU3zveTUscvVMf1BOvdsJfpzFm19SoEXLVadpACjWOnM491HqGpvtfFnevyiaN8W+I6Eg==} - ioredis@6.0.0: resolution: {integrity: sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==} engines: {node: '>=20.0.0'} @@ -2187,9 +2048,6 @@ packages: resolution: {integrity: sha512-aq+t5NAc+cS6rZQQVWC2x98CPqGtKKTMDd4Gaodv0wShnItdKg/51djkGJ1hqH+Oy0ivDftCbSLCQob8zso01w==} engines: {node: '>= 10'} - ipfs-unixfs@13.0.0: - resolution: {integrity: sha512-5Q9fw95zKA0h/LxRVkQDfp+5lRaDD6i4fzXt70f38b0CRkrbhuj2eUXDjEohfBwBeWcB7UJPW5DzAmgoSNEb8A==} - is-electron@2.2.2: resolution: {integrity: sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==} @@ -2208,14 +2066,6 @@ packages: resolution: {integrity: sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==} engines: {node: '>=16'} - is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - - is-plain-obj@2.1.0: - resolution: {integrity: sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==} - engines: {node: '>=8'} - is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} @@ -2223,10 +2073,6 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - iso-url@1.2.1: - resolution: {integrity: sha512-9JPDgCN4B7QPkLtYAAOrEuAWvP9rWvR5offAr0/SeF046wIkglqH3VXgYYP6NcsKslH80UIVgmPqNe3j7tG2ng==} - engines: {node: '>=12'} - it-all@3.0.11: resolution: {integrity: sha512-Gvqj6MO4GMLnFdtE68HZRpGBskNC+9+GQ+JevTGNYLyhjUuPhjDLU3jN1LpBemXJDW1bRSkczqA/qGyKlPKrcQ==} @@ -2239,18 +2085,9 @@ packages: it-filter@3.1.6: resolution: {integrity: sha512-yXiGPAvJn/exXjVFSCMQc3+J/7RLpOMwKoY2DH1yMhF4lYkdRoAdOwU0vnDACAlRAexf7AZvESZIc9mzhEoi/A==} - it-first@3.0.11: - resolution: {integrity: sha512-0ig8DKpg09V1o7JBagm3oPx3VY7WYfU5w3lpbLbqzijnfMPSvMGoMZuLm17h/RgOJXKP+9mt7vsCNiU2TW8TkQ==} - it-foreach@2.1.7: resolution: {integrity: sha512-HoZgIF7DGU1X/8svRuJ7aPl6sge8W6MQxmMomkeAABNXJXoiXEU0xnvulzncRdd013Kh9SubXWhx6YjYw6lu5A==} - it-glob@3.0.6: - resolution: {integrity: sha512-dFNeW4izM08QuB4uuIr+sVKUSo8ftVD/E1RnYidiUZx/i9h9mmwDSBl3kPv/TCah6HI0y1sgfHVCbrwA9FjoaQ==} - - it-last@3.0.11: - resolution: {integrity: sha512-Fg571l81nPzhZsiYjkw4dkhRqAK4oqIamTPEfAOnXI/5pYXz+dIfMVYmh9ncZs58oFNMkdF3bYFuCBTw/xJK0w==} - it-length-prefixed-stream@2.0.6: resolution: {integrity: sha512-foGRL4Id5Ypuc9RIPEE5aHWZvpKoGpIASoTBeuAZgH/QMGEy+V0vNgK8U4NBPjKwbhyuhW9gYflXXP43W4ZcKw==} @@ -2306,9 +2143,6 @@ packages: it-take@3.0.11: resolution: {integrity: sha512-zvoeEjLViGFyhYT5KNCgmcIH90Si8lCve4aTMvgej/ZQRfB9YzrcJW3UHIJjbQ9TiAnsT4vsWDImEFQNk5xmnA==} - it-to-stream@1.0.0: - resolution: {integrity: sha512-pLULMZMAB/+vbdvbZtebC0nWBTbG581lk6w8P7DfIIIKUfa8FbY7Oi0FxZcFPbxvISs7A9E+cMpLDBc1XhpAOA==} - it-ws@6.1.5: resolution: {integrity: sha512-uWjMtpy5HqhSd/LlrlP3fhYrr7rUfJFFMABv0F5d6n13Q+0glhZthwUKpEAVhDrXY95Tb1RB5lLqqef+QbVNaw==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -2335,9 +2169,6 @@ packages: keyv@5.6.0: resolution: {integrity: sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==} - kubo-rpc-client@7.1.0: - resolution: {integrity: sha512-sEFzNx+KxHPLa0iHgdszfHnNlQOwxd3t2v9LVSJq57htpfJftdps1eWRMvMASCqe6AtwW3bE7fzyd4z2hXLTlA==} - levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -2440,18 +2271,6 @@ packages: main-event@1.0.5: resolution: {integrity: sha512-4l9z8r7Q446mhhVTwdHmfPksOYBwN2xa7ewEW2yxPVNQapIaAf57ORFLMB91SpMEMa4wXowect7fq3uMLjDnGA==} - merge-options@3.0.4: - resolution: {integrity: sha512-2Sug1+knBjkaMsMgf1ctR1Ujx+Ayku4EdJN4Z+C2+JzoeF7A3OZ9KM2GY0CpQS51NR61LTurMJrRKPhSs3ZRTQ==} - engines: {node: '>=10'} - - merge2@1.4.1: - resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} - engines: {node: '>= 8'} - - micromatch@4.0.8: - resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} - engines: {node: '>=8.6'} - minimatch@10.2.6: resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} engines: {node: 18 || 20 || >=22} @@ -2473,9 +2292,6 @@ packages: resolution: {integrity: sha512-WC/Eo7NzFrOV/RRrTaI0fxKVbNCzEy76j2VqNV8SxDf9D69gSE2Lh0QwYvDlhiYmheBYExAvEAxVf5NoN0cj2A==} engines: {node: '>=20'} - multiformats@13.1.3: - resolution: {integrity: sha512-CZPi9lFZCM/+7oRolWYsvalsyWQGFo+GpdaTmjxXXomC+nP/W1Rnxb9sUgjvmNmRZ5bOPqRAl4nuK+Ydw/4tGw==} - multiformats@13.4.2: resolution: {integrity: sha512-eh6eHCrRi1+POZ3dA+Dq1C6jhP1GNtr9CRINMb67OKzqW9I5DUuZM/3jLPlzhgpGeiNUlEGEbkCYChXMCc/8DQ==} @@ -2487,11 +2303,6 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - nanoid@5.1.16: - resolution: {integrity: sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==} - engines: {node: ^18 || >=20} - hasBin: true - natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -2531,10 +2342,6 @@ packages: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} - p-defer@3.0.0: - resolution: {integrity: sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==} - engines: {node: '>=8'} - p-defer@4.0.1: resolution: {integrity: sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==} engines: {node: '>=12'} @@ -2543,9 +2350,6 @@ packages: resolution: {integrity: sha512-Q6Bekk5wpzW5qIyUP4gdMEujObYstZl6DMMOSenwBvV0BlE5LkDwkjs5yHbZmdCEq2o4RJx4tE1vwxFVf2FG1w==} engines: {node: '>=16.17'} - p-fifo@1.0.0: - resolution: {integrity: sha512-IjoCxXW48tqdtDFz6fqo5q1UfFVjjVZe8TC1QRflvNUJtNfCUhxOUw6MOVZhDPjqhSzc26xKdugsO17gmzd5+A==} - p-limit@3.1.0: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} @@ -2574,9 +2378,6 @@ packages: resolution: {integrity: sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==} engines: {node: '>=20'} - parse-duration@2.1.8: - resolution: {integrity: sha512-hM72vQ2w/HebbXx2pyUaR+EBjbkPx3Xi2aWAF9SNvJnRbP0p46KLcI8WP/z5ZruCUsJr0L2HWexVtB3PlBf/LA==} - path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -2591,10 +2392,6 @@ packages: picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - picomatch@2.3.2: - resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} - engines: {node: '>=8.6'} - picomatch@4.0.5: resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} engines: {node: '>=12'} @@ -2664,9 +2461,6 @@ packages: quansync@1.0.0: resolution: {integrity: sha512-5xZacEEufv3HSTPQuchrvV6soaiACMFnq1H8wkVioctoH3TRha9Sz66lOxRwPK/qZj7HPiSveih9yAyh98gvqA==} - queue-microtask@1.2.3: - resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - quick-format-unescaped@4.0.4: resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} @@ -2679,10 +2473,6 @@ packages: race-signal@2.0.2: resolution: {integrity: sha512-IuNNNY5NLriMFEGl3BVei2xq9NIcpkuAklhPWamhu5tRJrJz0w9sloUMz73YXpmUluE2EZocmJm9rpQqXmCPiA==} - readable-stream@3.6.2: - resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} - engines: {node: '>= 6'} - real-require@0.2.0: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} engines: {node: '>= 12.13.0'} @@ -2740,15 +2530,9 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true - run-parallel@1.2.0: - resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} - rxjs@7.8.2: resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} - safe-buffer@5.2.1: - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - safe-regex2@5.1.1: resolution: {integrity: sha512-mOSBvHGDZMuIEZMdOz/aCEYDCv0E7nfcNsIhUF+/P+xC7Hyf3FkvymqgPbg9D1EdSGu+uKbJgy09K/RKKc7kJA==} hasBin: true @@ -2757,9 +2541,6 @@ packages: resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} engines: {node: '>=10'} - safer-buffer@2.1.2: - resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} - scale-ts@1.6.1: resolution: {integrity: sha512-PBMc2AWc6wSEqJYBDPcyCLUj9/tMKnLX70jLOSndMtcUoLQucP/DM0vnQo1wJAYjTrQiq8iG9rD0q6wFzgjH7g==} @@ -2818,9 +2599,6 @@ packages: resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} engines: {node: '>=0.6.19'} - string_decoder@1.3.0: - resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} - supports-color@10.2.2: resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} engines: {node: '>=18'} @@ -2852,10 +2630,6 @@ packages: resolution: {integrity: sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==} engines: {node: '>=14.0.0'} - to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} - toad-cache@3.7.4: resolution: {integrity: sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==} engines: {node: '>=20'} @@ -2965,9 +2739,6 @@ packages: utf8-codec@1.0.0: resolution: {integrity: sha512-S/QSLezp3qvG4ld5PUfXiH7mCFxLKjSVZRFkB3DOjgwHuJPFDkInAXc/anf7BAbHt/D38ozDzL+QMZ6/7gsI6w==} - util-deprecate@1.0.2: - resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - verkit@0.3.2: resolution: {integrity: sha512-zj/ob3UsvJGN0whEAKFp53REA5X66hvffVqoCtVQAakJKnKlH+/PcOfMoFwIG/o4rElqLv/ycAFlx8ZlXUorCg==} engines: {node: '>=18.12.0'} @@ -3401,25 +3172,6 @@ snapshots: '@ioredis/commands@2.0.0': {} - '@ipld/dag-cbor@10.0.2': - dependencies: - cborg: 6.1.1 - multiformats: 14.0.5 - - '@ipld/dag-cbor@9.2.7': - dependencies: - cborg: 5.1.11 - multiformats: 13.4.2 - - '@ipld/dag-json@11.0.1': - dependencies: - cborg: 6.1.1 - multiformats: 14.0.5 - - '@ipld/dag-pb@4.1.7': - dependencies: - multiformats: 14.0.5 - '@jridgewell/sourcemap-codec@1.6.0': {} '@keyv/bigmap@1.3.1(keyv@5.6.0)': @@ -3505,14 +3257,6 @@ snapshots: multiformats: 13.4.2 weald: 1.1.3 - '@libp2p/logger@6.2.12': - dependencies: - '@libp2p/interface': 3.2.5 - '@multiformats/multiaddr': 13.0.3 - interface-datastore: 10.0.1 - multiformats: 14.0.5 - weald: 1.1.3 - '@libp2p/multistream-select@6.0.29': dependencies: '@libp2p/interface': 2.11.0 @@ -3539,13 +3283,6 @@ snapshots: multiformats: 13.4.2 uint8arrays: 5.1.1 - '@libp2p/peer-id@6.0.14': - dependencies: - '@libp2p/crypto': 5.1.22 - '@libp2p/interface': 3.2.5 - multiformats: 14.0.5 - uint8arrays: 6.1.1 - '@libp2p/peer-record@8.0.35': dependencies: '@libp2p/crypto': 5.1.22 @@ -3671,10 +3408,6 @@ snapshots: dependencies: '@multiformats/multiaddr': 12.5.1 - '@multiformats/multiaddr-to-uri@12.0.0': - dependencies: - '@multiformats/multiaddr': 13.0.3 - '@multiformats/multiaddr@12.5.1': dependencies: '@chainsafe/is-ip': 2.1.0 @@ -3844,18 +3577,6 @@ snapshots: '@node-rs/crc32-win32-x64-msvc': 1.10.7 optional: true - '@nodelib/fs.scandir@2.1.5': - dependencies: - '@nodelib/fs.stat': 2.0.5 - run-parallel: 1.2.0 - - '@nodelib/fs.stat@2.0.5': {} - - '@nodelib/fs.walk@1.2.8': - dependencies: - '@nodelib/fs.scandir': 2.1.5 - fastq: 1.20.1 - '@oxc-project/types@0.144.0': {} '@pinojs/redact@0.4.0': {} @@ -4606,29 +4327,12 @@ snapshots: balanced-match@4.0.4: {} - base64-js@1.5.1: {} - - blob-to-it@3.0.0: - dependencies: - browser-readablestream-to-it: 2.0.12 - bn.js@5.2.5: {} brace-expansion@5.0.12: dependencies: balanced-match: 4.0.4 - braces@3.0.3: - dependencies: - fill-range: 7.1.1 - - browser-readablestream-to-it@2.0.12: {} - - buffer@6.0.3: - dependencies: - base64-js: 1.5.1 - ieee754: 1.2.1 - cac@7.0.0: {} cacheable@2.5.0: @@ -4639,10 +4343,6 @@ snapshots: keyv: 5.6.0 qified: 0.10.1 - cborg@5.1.11: {} - - cborg@6.1.1: {} - chai@6.2.2: {} cluster-key-slot@1.1.1: {} @@ -4657,11 +4357,6 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 - dag-jose@5.1.1: - dependencies: - '@ipld/dag-cbor': 9.2.7 - multiformats: 13.1.3 - data-uri-to-buffer@4.0.1: {} datastore-core@10.0.4: @@ -4697,18 +4392,8 @@ snapshots: dts-resolver@3.0.0: {} - electron-fetch@1.9.1: - dependencies: - encoding: 0.1.13 - empathic@2.0.1: {} - encoding@0.1.13: - dependencies: - iconv-lite: 0.6.3 - - err-code@3.0.1: {} - es-module-lexer@2.3.2: {} esbuild@0.28.2: @@ -4837,16 +4522,6 @@ snapshots: fast-diff@1.3.0: {} - fast-fifo@1.3.2: {} - - fast-glob@3.3.3: - dependencies: - '@nodelib/fs.stat': 2.0.5 - '@nodelib/fs.walk': 1.2.8 - glob-parent: 5.1.2 - merge2: 1.4.1 - micromatch: 4.0.8 - fast-json-stable-stringify@2.1.0: {} fast-json-stringify@7.0.1: @@ -4905,10 +4580,6 @@ snapshots: dependencies: flat-cache: 6.1.23 - fill-range@7.1.1: - dependencies: - to-regex-range: 5.0.1 - find-my-way@9.8.0: dependencies: fast-deep-equal: 3.1.3 @@ -4935,18 +4606,12 @@ snapshots: fsevents@2.3.3: optional: true - get-iterator@1.0.2: {} - get-iterator@2.0.1: {} get-tsconfig@5.0.0-beta.5: dependencies: resolve-pkg-maps: 1.0.0 - glob-parent@5.1.2: - dependencies: - is-glob: 4.0.3 - glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -4965,12 +4630,6 @@ snapshots: husky@9.1.7: {} - iconv-lite@0.6.3: - dependencies: - safer-buffer: 2.1.2 - - ieee754@1.2.1: {} - ignore@5.3.2: {} ignore@7.0.9: {} @@ -4979,14 +4638,6 @@ snapshots: imurmurhash@0.1.4: {} - inherits@2.0.4: {} - - interface-datastore@10.0.1: - dependencies: - abort-error: 1.0.2 - interface-store: 8.0.0 - uint8arrays: 6.1.1 - interface-datastore@8.3.2: dependencies: interface-store: 6.0.3 @@ -4994,10 +4645,6 @@ snapshots: interface-store@6.0.3: {} - interface-store@8.0.0: - dependencies: - abort-error: 1.0.2 - ioredis@6.0.0(supports-color@10.2.2): dependencies: '@ioredis/commands': 2.0.0 @@ -5013,11 +4660,6 @@ snapshots: ipaddr.js@2.5.0: {} - ipfs-unixfs@13.0.0: - dependencies: - protons-runtime: 7.0.0 - uint8arraylist: 3.0.2 - is-electron@2.2.2: {} is-extglob@2.1.1: {} @@ -5030,16 +4672,10 @@ snapshots: is-network-error@1.3.2: {} - is-number@7.0.0: {} - - is-plain-obj@2.1.0: {} - is-plain-obj@4.1.0: {} isexe@2.0.0: {} - iso-url@1.2.1: {} - it-all@3.0.11: {} it-byte-stream@2.0.6: @@ -5056,18 +4692,10 @@ snapshots: dependencies: it-peekable: 3.0.10 - it-first@3.0.11: {} - it-foreach@2.1.7: dependencies: it-peekable: 3.0.10 - it-glob@3.0.6: - dependencies: - fast-glob: 3.3.3 - - it-last@3.0.11: {} - it-length-prefixed-stream@2.0.6: dependencies: abort-error: 1.0.2 @@ -5155,15 +4783,6 @@ snapshots: it-take@3.0.11: {} - it-to-stream@1.0.0: - dependencies: - buffer: 6.0.3 - fast-fifo: 1.3.2 - get-iterator: 1.0.2 - p-defer: 3.0.0 - p-fifo: 1.0.0 - readable-stream: 3.6.2 - it-ws@6.1.5: dependencies: '@types/ws': 8.18.1 @@ -5194,40 +4813,6 @@ snapshots: dependencies: '@keyv/serialize': 1.1.1 - kubo-rpc-client@7.1.0: - dependencies: - '@ipld/dag-cbor': 10.0.2 - '@ipld/dag-json': 11.0.1 - '@ipld/dag-pb': 4.1.7 - '@libp2p/crypto': 5.1.22 - '@libp2p/interface': 3.2.5 - '@libp2p/logger': 6.2.12 - '@libp2p/peer-id': 6.0.14 - '@multiformats/multiaddr': 13.0.3 - '@multiformats/multiaddr-to-uri': 12.0.0 - any-signal: 4.2.0 - blob-to-it: 3.0.0 - browser-readablestream-to-it: 2.0.12 - dag-jose: 5.1.1 - electron-fetch: 1.9.1 - err-code: 3.0.1 - ipfs-unixfs: 13.0.0 - iso-url: 1.2.1 - it-all: 3.0.11 - it-first: 3.0.11 - it-glob: 3.0.6 - it-last: 3.0.11 - it-map: 3.1.6 - it-peekable: 3.0.10 - it-to-stream: 1.0.0 - merge-options: 3.0.4 - multiformats: 14.0.5 - nanoid: 5.1.16 - parse-duration: 2.1.8 - stream-to-it: 1.0.1 - uint8arrays: 6.1.1 - wherearewe: 2.0.1 - levn@0.4.1: dependencies: prelude-ls: 1.2.1 @@ -5340,17 +4925,6 @@ snapshots: main-event@1.0.5: {} - merge-options@3.0.4: - dependencies: - is-plain-obj: 2.1.0 - - merge2@1.4.1: {} - - micromatch@4.0.8: - dependencies: - braces: 3.0.3 - picomatch: 2.3.2 - minimatch@10.2.6: dependencies: brace-expansion: 5.0.12 @@ -5371,16 +4945,12 @@ snapshots: ms@4.0.0-nightly.202508271359: {} - multiformats@13.1.3: {} - multiformats@13.4.2: {} multiformats@14.0.5: {} nanoid@3.3.18: {} - nanoid@5.1.16: {} - natural-compare@1.4.0: {} netmask@2.1.1: {} @@ -5418,19 +4988,12 @@ snapshots: type-check: 0.4.0 word-wrap: 1.2.5 - p-defer@3.0.0: {} - p-defer@4.0.1: {} p-event@6.0.1: dependencies: p-timeout: 6.1.4 - p-fifo@1.0.0: - dependencies: - fast-fifo: 1.3.2 - p-defer: 3.0.0 - p-limit@3.1.0: dependencies: yocto-queue: 0.1.0 @@ -5459,8 +5022,6 @@ snapshots: p-timeout@7.0.1: {} - parse-duration@2.1.8: {} - path-exists@4.0.0: {} path-key@3.1.1: {} @@ -5469,8 +5030,6 @@ snapshots: picocolors@1.1.1: {} - picomatch@2.3.2: {} - picomatch@4.0.5: {} picomatch@4.0.7: {} @@ -5542,8 +5101,6 @@ snapshots: quansync@1.0.0: {} - queue-microtask@1.2.3: {} - quick-format-unescaped@4.0.4: {} race-event@1.6.1: @@ -5554,12 +5111,6 @@ snapshots: race-signal@2.0.2: {} - readable-stream@3.6.2: - dependencies: - inherits: 2.0.4 - string_decoder: 1.3.0 - util-deprecate: 1.0.2 - real-require@0.2.0: {} real-require@1.0.0: {} @@ -5612,24 +5163,16 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.2.4 '@rolldown/binding-win32-x64-msvc': 1.2.4 - run-parallel@1.2.0: - dependencies: - queue-microtask: 1.2.3 - rxjs@7.8.2: dependencies: tslib: 2.8.1 - safe-buffer@5.2.1: {} - safe-regex2@5.1.1: dependencies: ret: 0.5.0 safe-stable-stringify@2.5.0: {} - safer-buffer@2.1.2: {} - scale-ts@1.6.1: optional: true @@ -5677,10 +5220,6 @@ snapshots: string-argv@0.3.2: {} - string_decoder@1.3.0: - dependencies: - safe-buffer: 5.2.1 - supports-color@10.2.2: {} synckit@0.11.13: @@ -5704,10 +5243,6 @@ snapshots: tinyrainbow@3.1.1: {} - to-regex-range@5.0.1: - dependencies: - is-number: 7.0.0 - toad-cache@3.7.4: {} tree-kill@1.2.2: {} @@ -5817,8 +5352,6 @@ snapshots: utf8-codec@1.0.0: {} - util-deprecate@1.0.2: {} - verkit@0.3.2: {} vite@8.2.1(@types/node@26.6.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.12)(yaml@2.9.1): diff --git a/proto/connectivity/v1/payload.proto b/proto/connectivity/v1/payload.proto index 4a56dcd..8f30b22 100644 --- a/proto/connectivity/v1/payload.proto +++ b/proto/connectivity/v1/payload.proto @@ -23,36 +23,32 @@ message TelemetryRejectedPayload { string reason_message = 3; } -// TelemetryBatchedPayload carries a batch of authorized telemetry ready for -// publication. It is emitted by the batcher and consumed by the ipfs-publisher. +// TelemetryBatchedPayload carries a chain-ready batch of authorized telemetry. +// It is emitted by the batcher and consumed directly by blockchain-anchor. +// `payload` is the exact byte sequence passed to CPS.set_payload: an XZ +// compressed, serialized crypto.v1.SignedEnvelopeBatch with no additional +// framing. Payload interpretation (schema/encoding/compression) is described +// through CPS metadata, not through bytes embedded in the payload itself. message TelemetryBatchedPayload { - // Unique batch identifier + // Unique batch identifier (logical batch identity, used for deduplication) string batch_id = 1; - // Serialized crypto.v1.SignedEnvelopeBatch (uncompressed) - bytes signed_envelope_batch = 2; + // Chain-ready payload: XZ(serialized crypto.v1.SignedEnvelopeBatch). + // This is exactly the byte sequence passed to CPS.set_payload. + bytes payload = 2; // Number of events in the batch uint32 event_count = 3; // Sensor IDs present in the batch (observability) repeated bytes sensor_ids = 4; -} -// TelemetryIpfsPublishedPayload is emitted when events are published to IPFS. -message TelemetryIpfsPublishedPayload { - enum Compression { - // No compression - COMPRESSION_NONE = 0; - // XZ compression algorithm - COMPRESSION_XZ = 1; - } - // IPFS Content ID - bytes cid = 1; - - // Number of events in this batch - uint32 event_count = 2; + // Size of the serialized SignedEnvelopeBatch before compression (observability) + uint32 uncompressed_size = 5; + + // Size of `payload` in bytes; must always be <= ANCHOR_MAX_PAYLOAD_BYTES + uint32 compressed_size = 6; - // If specified then compression applied to batch - Compression compression = 3; + // blake2_256 hash of `payload`, useful for diagnostics/idempotency checks + bytes payload_hash = 7; } diff --git a/services/batcher/README.md b/services/batcher/README.md index fe920f1..37e8012 100644 --- a/services/batcher/README.md +++ b/services/batcher/README.md @@ -1,21 +1,29 @@ # `@scp/batcher` -Groups authorized telemetry into deterministic batches and emits them for -downstream publication. +Groups authorized telemetry into deterministic batches, compresses and +size-fits them, and emits chain-ready payloads for direct blockchain +anchoring. ## Architecture - **Pattern**: Standard consumer with manual commit - **Input**: `telemetry.authorized.v1` from Kafka -- **Output**: `telemetry.batched.v1` (carries a serialized `SignedEnvelopeBatch`) +- **Output**: `telemetry.batched.v1` (carries an XZ-compressed, chain-ready payload) - **Autocommit**: Disabled (commit only after the batch is durably produced) - **Concurrency**: Single active flush per instance (single-flight); a timer-triggered flush cannot publish the same batch as a size/lag-triggered flush - **Shutdown**: Waits for any in-flight flush and flushes the remaining batch before closing resources -Separating batching from IPFS publishing keeps the publisher stateless and -lets each side scale and fail independently. The batcher owns buffering, -flush timing, and offset commits for the authorized topic; the -`@scp/ipfs-publisher` owns durable IPFS publication of `telemetry.batched.v1`. +The batcher serializes each detached batch as a `SignedEnvelopeBatch` +protobuf, XZ-compresses it, and — if the compressed result exceeds +`ANCHOR_MAX_PAYLOAD_BYTES` — recursively splits it into smaller sub-batches +until every emitted payload fits. Each sub-batch becomes its own +`telemetry.batched.v1` message with its own `batch_id`. A single event whose +compressed size alone exceeds the limit cannot be split further and is +routed to the dead-letter queue instead of blocking the rest of the batch. + +This keeps `blockchain-anchor` simple and stateless: it only needs to submit +already-compressed, already-size-checked bytes via `cps.setPayload`, with no +IPFS/CID indirection in between. ## Batching Strategy @@ -35,6 +43,7 @@ concurrent flush paths can never publish the same batch twice. - `BATCHER_HEALTH_PORT` (default: `3041`) - `BATCHER_BATCH_SIZE` (default: `10`) - Maximum messages per batch - `BATCHER_BATCH_TIMEOUT_MS` (default: `30000`) - Timeout for partial batches under low load +- `ANCHOR_MAX_PAYLOAD_BYTES` (default: `8192`) - Maximum compressed payload size per emitted batch; larger batches are recursively split, shared default defined in `@scp/core` ## Metrics @@ -43,6 +52,8 @@ Available at `http://localhost:3041/metrics`: - `consumed`: Total authorized telemetry messages consumed - `batchesProduced`: Total batches produced to `telemetry.batched.v1` - `eventsBatched`: Total individual telemetry events batched +- `batchesSplit`: Total detached batches that required splitting into multiple sub-batches +- `oversizedEvents`: Total individual events that could not fit even alone and were routed to the DLQ - `produceFailure`: Failed batch produce attempts (re-attached for retry) ## Data Flow @@ -50,10 +61,11 @@ Available at `http://localhost:3041/metrics`: 1. Consume `telemetry.authorized.v1` event from Kafka 2. Extract `SignedEnvelope` from payload and add to the current batch 3. When the batch is full, lag is high, or the flush timer fires: - - Serialize the batch as a `SignedEnvelopeBatch` protobuf - - Wrap it in a `TelemetryBatchedPayload` (`batch_id`, `event_count`, `sensor_ids`) - - Produce a `telemetry.batched.v1` envelope to Kafka - - Commit `telemetry.authorized.v1` offsets (only after the batch is produced) + - Serialize the batch as a `SignedEnvelopeBatch` protobuf and XZ-compress it + - If the compressed payload exceeds `ANCHOR_MAX_PAYLOAD_BYTES`, recursively split the batch until every sub-batch fits (oversized single events go to the DLQ) + - Wrap each fitted sub-batch in a `TelemetryBatchedPayload` (`batch_id`, `payload`, `uncompressed_size`, `compressed_size`, `payload_hash`, `event_count`, `sensor_ids`) + - Produce one `telemetry.batched.v1` envelope per sub-batch to Kafka + - Commit `telemetry.authorized.v1` offsets (only after all sub-batches are produced) ## Result Event Schema @@ -61,10 +73,13 @@ Published to `telemetry.batched.v1`: ```typescript { - batchId: string; // Unique batch identifier - signedEnvelopeBatch: Uint8Array; // Serialized crypto.v1.SignedEnvelopeBatch - eventCount: number; // Number of telemetry events in batch - sensorIds: Uint8Array[]; // Sensor IDs present in the batch (observability) + batchId: string; // Unique batch identifier (per sub-batch) + payload: Uint8Array; // XZ-compressed, chain-ready crypto.v1.SignedEnvelopeBatch + uncompressedSize: number; // Size of the serialized batch before compression + compressedSize: number; // Size of `payload` (<= ANCHOR_MAX_PAYLOAD_BYTES) + payloadHash: Uint8Array; // blake2_256 hash of `payload` + eventCount: number; // Number of telemetry events in this sub-batch + sensorIds: Uint8Array[]; // Sensor IDs present in this sub-batch (observability) } ``` diff --git a/services/batcher/package.json b/services/batcher/package.json index 6f14516..9e5a9da 100644 --- a/services/batcher/package.json +++ b/services/batcher/package.json @@ -14,7 +14,9 @@ }, "dependencies": { "@buf/airalab_connectivity-protocol.bufbuild_es": "2.14.0-20260917143433-6f6f1eba1cc0.2", + "@napi-rs/lzma": "^1.5.1", "@platformatic/kafka": "^2.9.0", + "@polkadot/util-crypto": "^14.0.3", "@scp/core": "workspace:*", "pino": "^10.3.1" }, diff --git a/services/batcher/src/batch-flusher.ts b/services/batcher/src/batch-flusher.ts index 6756c41..6651a96 100644 --- a/services/batcher/src/batch-flusher.ts +++ b/services/batcher/src/batch-flusher.ts @@ -17,8 +17,8 @@ /** * Callback invoked with the detached batch when a flush runs. * - * It must fully complete the external work (e.g. publish to IPFS and commit - * Kafka offsets) before resolving. If it rejects, the flusher re-attaches the + * It must fully complete the external work (e.g. compress and produce to + * Kafka) before resolving. If it rejects, the flusher re-attaches the * batch so it is not lost and can be retried on a later flush. */ export type FlushHandler = (batch: readonly T[]) => Promise; diff --git a/services/batcher/src/config.ts b/services/batcher/src/config.ts index a5b6412..1bbcebf 100644 --- a/services/batcher/src/config.ts +++ b/services/batcher/src/config.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { DEFAULT_ANCHOR_MAX_PAYLOAD_BYTES } from '@scp/core'; + export interface BatcherConfig { kafkaBrokers: string[]; consumerGroupId: string; @@ -20,6 +22,8 @@ export interface BatcherConfig { healthPort: number; batchSize: number; batchTimeoutMs: number; + /** Maximum size, in bytes, of a chain-ready compressed payload. */ + maxPayloadBytes: number; } function parsePositiveInt(value: string | undefined, fallback: number): number { @@ -40,5 +44,9 @@ export function loadBatcherConfig( healthPort: parsePositiveInt(env.BATCHER_HEALTH_PORT, 3041), batchSize: parsePositiveInt(env.BATCHER_BATCH_SIZE, 10), batchTimeoutMs: parsePositiveInt(env.BATCHER_BATCH_TIMEOUT_MS, 30000), + maxPayloadBytes: parsePositiveInt( + env.ANCHOR_MAX_PAYLOAD_BYTES, + DEFAULT_ANCHOR_MAX_PAYLOAD_BYTES + ), }; } diff --git a/services/batcher/src/index.ts b/services/batcher/src/index.ts index 751b5c7..ae3a0be 100644 --- a/services/batcher/src/index.ts +++ b/services/batcher/src/index.ts @@ -25,7 +25,6 @@ import { import { fromBinary, toBinary, create } from '@bufbuild/protobuf'; import { SignedEnvelopeSchema, - SignedEnvelopeBatchSchema, type SignedEnvelope, } from '@buf/airalab_connectivity-protocol.bufbuild_es/crypto/v1/envelope_pb.js'; import { Consumer, Producer } from '@platformatic/kafka'; @@ -34,6 +33,7 @@ import { fileURLToPath } from 'node:url'; import { createHash } from 'node:crypto'; import { loadBatcherConfig, type BatcherConfig } from './config.js'; import { createBatchFlusher } from './batch-flusher.js'; +import { fitBatch } from './payload-builder.js'; import { logInfo, logWarn, logDebug, logError } from './logger.js'; interface BatcherMetrics { @@ -41,6 +41,10 @@ interface BatcherMetrics { batchesProduced: number; eventsBatched: number; produceFailure: number; + /** Number of times a candidate batch had to be split to fit the payload limit. */ + batchesSplit: number; + /** Number of single events that could not fit even on their own (permanent error). */ + oversizedEvents: number; } export interface BatcherService { @@ -90,7 +94,7 @@ function isValidItem(item: BatchItem): item is ValidBatchItem { /** * Derive a stable batch ID from the batch's source offsets so retries of the * same detached batch (e.g. after a failed offset commit) reuse the same ID. - * This lets downstream consumers (e.g. ipfs-publisher) deduplicate by + * This lets downstream consumers (e.g. blockchain-anchor) deduplicate by * `batch_id` even when a produce attempt is retried. */ function computeBatchId(batch: readonly BatchItem[]): string { @@ -135,8 +139,51 @@ async function publishPoisonToDlq( } /** - * Serialize a batch of authorized telemetry into a `telemetry.batched.v1` - * envelope and produce it to Kafka. + * Publish events whose serialized + XZ compressed size exceeds + * `ANCHOR_MAX_PAYLOAD_BYTES` even on their own, to the DLQ topic. This is a + * permanent validation error (`ANCHOR_PAYLOAD_TOO_LARGE`), not a transient + * blockchain/produce failure, so it must not be retried indefinitely. + */ +async function publishOversizedToDlq( + oversizedItems: readonly { + item: ValidBatchItem; + reason: string; + }[], + producer: Producer, + config: BatcherConfig +): Promise { + if (oversizedItems.length === 0) { + return; + } + + await producer.send({ + messages: oversizedItems.map(({ item, reason }) => ({ + topic: TELEMETRY_TOPICS.DLQ, + value: Buffer.from(toBinary(SignedEnvelopeSchema, item.signedEnvelope)), + headers: { + source_topic: Buffer.from(TELEMETRY_TOPICS.AUTHORIZED), + source_service: Buffer.from(config.source), + source_partition: Buffer.from(String(item.partition)), + source_offset: Buffer.from(String(item.offset - 1n)), + reason: Buffer.from(reason), + }, + })), + }); + + logWarn('oversized events routed to DLQ', { + count: oversizedItems.length, + dlq_topic: TELEMETRY_TOPICS.DLQ, + }); +} + +/** + * Serialize a batch of authorized telemetry into one or more + * `telemetry.batched.v1` envelopes and produce them to Kafka. + * + * The batch is serialized, XZ compressed, and (if the compressed result + * exceeds `config.maxPayloadBytes`) recursively split so that every emitted + * payload fits `CPS.set_payload`. Events that cannot fit even on their own + * are routed to the DLQ as a permanent `ANCHOR_PAYLOAD_TOO_LARGE` error. */ async function produceBatch( batch: readonly ValidBatchItem[], @@ -157,61 +204,81 @@ async function produceBatch( ) ); - const batchId = computeBatchId(batch); - - logInfo('producing batch', { - batch_id: batchId, + logInfo('fitting batch to payload limit', { batch_size: batch.length, + max_payload_bytes: config.maxPayloadBytes, unique_sensors: uniqueSensorIds.length, sensor_ids: uniqueSensorIds, trace_ids: traceIds.length > 0 ? traceIds : undefined, }); try { - const batchData = create(SignedEnvelopeBatchSchema, { - batch: batch.map((b) => b.signedEnvelope), - }); + const { batches: fitted, oversized } = await fitBatch( + batch, + config.maxPayloadBytes + ); + + if (oversized.length > 0) { + metrics.oversizedEvents += oversized.length; + await publishOversizedToDlq( + oversized.map(({ event, error }) => ({ + item: event, + reason: `ANCHOR_PAYLOAD_TOO_LARGE: ${error.message}`, + })), + producer, + config + ); + } - const payload = create(TelemetryBatchedPayloadSchema, { - batchId, - signedEnvelopeBatch: toBinary(SignedEnvelopeBatchSchema, batchData), - eventCount: batch.length, - sensorIds: batch.map((b) => b.sensorId), - }); + if (fitted.length > 1) { + metrics.batchesSplit += 1; + } - const resultEnvelope = create(EnvelopeSchema, { - eventId: batchId, - eventType: TELEMETRY_TOPICS.BATCHED, - eventVersion: '1.0.0', - occurredAt: new Date().toISOString(), - source: config.source, - payload: toBinary(TelemetryBatchedPayloadSchema, payload), - }); + for (const fittedBatch of fitted) { + const batchId = computeBatchId(fittedBatch.events); + + const payload = create(TelemetryBatchedPayloadSchema, { + batchId, + payload: fittedBatch.payload, + eventCount: fittedBatch.events.length, + sensorIds: fittedBatch.events.map((b) => b.sensorId), + uncompressedSize: fittedBatch.uncompressedSize, + compressedSize: fittedBatch.compressedSize, + payloadHash: fittedBatch.payloadHash, + }); - await producer.send({ - messages: [ - { - topic: TELEMETRY_TOPICS.BATCHED, - value: Buffer.from(toBinary(EnvelopeSchema, resultEnvelope)), - }, - ], - }); + const resultEnvelope = create(EnvelopeSchema, { + eventId: batchId, + eventType: TELEMETRY_TOPICS.BATCHED, + eventVersion: '1.0.0', + occurredAt: new Date().toISOString(), + source: config.source, + payload: toBinary(TelemetryBatchedPayloadSchema, payload), + }); - metrics.batchesProduced += 1; - metrics.eventsBatched += batch.length; + await producer.send({ + messages: [ + { + topic: TELEMETRY_TOPICS.BATCHED, + value: Buffer.from(toBinary(EnvelopeSchema, resultEnvelope)), + }, + ], + }); - logInfo('batch produced', { - batch_id: batchId, - event_count: batch.length, - unique_sensors: uniqueSensorIds.length, - sensor_ids: uniqueSensorIds, - trace_ids: traceIds.length > 0 ? traceIds : undefined, - result_topic: TELEMETRY_TOPICS.BATCHED, - }); + metrics.batchesProduced += 1; + metrics.eventsBatched += fittedBatch.events.length; + + logInfo('batch produced', { + batch_id: batchId, + event_count: fittedBatch.events.length, + uncompressed_size: fittedBatch.uncompressedSize, + compressed_size: fittedBatch.compressedSize, + result_topic: TELEMETRY_TOPICS.BATCHED, + }); + } } catch (error) { metrics.produceFailure += 1; logError('batch produce failed', error, { - batch_id: batchId, batch_size: batch.length, unique_sensors: uniqueSensorIds.length, sensor_ids: uniqueSensorIds, @@ -277,6 +344,8 @@ export function createBatcherService( batchesProduced: 0, eventsBatched: 0, produceFailure: 0, + batchesSplit: 0, + oversizedEvents: 0, }; const getMetrics = (): BatcherMetrics => metrics; diff --git a/services/batcher/src/payload-builder.ts b/services/batcher/src/payload-builder.ts new file mode 100644 index 0000000..cf4870f --- /dev/null +++ b/services/batcher/src/payload-builder.ts @@ -0,0 +1,152 @@ +/** + * Copyright 2026 Robonomics Network + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { toBinary, create } from '@bufbuild/protobuf'; +import { xz } from '@napi-rs/lzma'; +import { blake2AsU8a } from '@polkadot/util-crypto'; +import { + SignedEnvelopeBatchSchema, + type SignedEnvelope, +} from '@buf/airalab_connectivity-protocol.bufbuild_es/crypto/v1/envelope_pb.js'; + +/** + * Error describing a single event that does not fit within + * `ANCHOR_MAX_PAYLOAD_BYTES` even after serialization and XZ compression on + * its own. This cannot be resolved by further splitting and must be treated + * as a permanent validation error (e.g. routed to the DLQ), not retried as a + * transient failure. + */ +export class AnchorPayloadTooLargeError extends Error { + constructor( + public readonly compressedSize: number, + public readonly maxPayloadBytes: number + ) { + super( + `single event payload (${compressedSize} bytes compressed) exceeds ` + + `ANCHOR_MAX_PAYLOAD_BYTES (${maxPayloadBytes} bytes) and cannot be split` + ); + this.name = 'AnchorPayloadTooLargeError'; + } +} + +/** A generic item that can be serialized into a `SignedEnvelopeBatch`. */ +export interface FittedItem { + signedEnvelope: SignedEnvelope; +} + +/** A chain-ready sub-batch produced by {@link fitBatch}. */ +export interface FittedBatch { + /** The events contained in this sub-batch, in original order. */ + events: readonly T[]; + /** XZ(serialized SignedEnvelopeBatch) — exactly the bytes for `set_payload`. */ + payload: Uint8Array; + /** Size of the serialized batch before compression. */ + uncompressedSize: number; + /** Size of `payload`; always `<= maxPayloadBytes`. */ + compressedSize: number; + /** blake2_256 hash of `payload`. */ + payloadHash: Uint8Array; +} + +/** A single event that does not fit even on its own, after splitting bottomed out. */ +export interface OversizedItem { + event: T; + error: AnchorPayloadTooLargeError; +} + +export interface FitBatchOutcome { + /** Chain-ready sub-batches, in original event order. */ + batches: FittedBatch[]; + /** Events that could not fit on their own; a permanent error, not a transient one. */ + oversized: OversizedItem[]; +} + +/** Serialize a set of signed envelopes into a `SignedEnvelopeBatch` and XZ compress it. */ +async function serializeAndCompress( + events: readonly T[] +): Promise<{ uncompressed: Uint8Array; compressed: Uint8Array }> { + const batch = create(SignedEnvelopeBatchSchema, { + batch: events.map((e) => e.signedEnvelope), + }); + const uncompressed = toBinary(SignedEnvelopeBatchSchema, batch); + const compressed = await xz.compress(uncompressed); + return { uncompressed, compressed }; +} + +/** + * Recursively fit `events` into one or more chain-ready payloads that each + * satisfy `compressed_payload.len <= maxPayloadBytes`. + * + * If the compressed batch does not fit, it is split in half and each half is + * fitted independently (recursive binary splitting). If a single event still + * does not fit after serialization and compression, it is reported in + * `oversized` rather than aborting the rest of the batch — this preserves + * every other event and mirrors the existing poison-item / DLQ handling + * pattern used for unparseable envelopes. + * + * Event order is preserved across the returned sub-batches and no event is + * lost or duplicated between `batches` and `oversized`. + */ +export async function fitBatch( + events: readonly T[], + maxPayloadBytes: number +): Promise> { + if (events.length === 0) { + return { batches: [], oversized: [] }; + } + + const { uncompressed, compressed } = await serializeAndCompress(events); + + if (compressed.length <= maxPayloadBytes) { + return { + batches: [ + { + events, + payload: compressed, + uncompressedSize: uncompressed.length, + compressedSize: compressed.length, + payloadHash: blake2AsU8a(compressed, 256), + }, + ], + oversized: [], + }; + } + + if (events.length === 1) { + return { + batches: [], + oversized: [ + { + event: events[0]!, + error: new AnchorPayloadTooLargeError( + compressed.length, + maxPayloadBytes + ), + }, + ], + }; + } + + const middle = Math.ceil(events.length / 2); + const [first, second] = await Promise.all([ + fitBatch(events.slice(0, middle), maxPayloadBytes), + fitBatch(events.slice(middle), maxPayloadBytes), + ]); + + return { + batches: [...first.batches, ...second.batches], + oversized: [...first.oversized, ...second.oversized], + }; +} diff --git a/services/batcher/tests/contracts.test.ts b/services/batcher/tests/contracts.test.ts index c4b4f58..880661d 100644 --- a/services/batcher/tests/contracts.test.ts +++ b/services/batcher/tests/contracts.test.ts @@ -24,6 +24,8 @@ import { SignedEnvelopeBatchSchema, } from '@buf/airalab_connectivity-protocol.bufbuild_es/crypto/v1/envelope_pb.js'; import { create, toBinary, fromBinary } from '@bufbuild/protobuf'; +import { xz } from '@napi-rs/lzma'; +import { blake2AsU8a } from '@polkadot/util-crypto'; import { describe, expect, it } from 'vitest'; describe('batcher contract compatibility', () => { @@ -55,7 +57,7 @@ describe('batcher contract compatibility', () => { expect(Buffer.from(payloadParsed.sensorId)).toEqual(Buffer.alloc(32, 1)); }); - it('produces valid telemetry.batched.v1 envelope carrying a SignedEnvelopeBatch', () => { + it('produces valid telemetry.batched.v1 envelope carrying a chain-ready compressed payload', async () => { const signedEnvelope1 = create(SignedEnvelopeSchema, { sensorId: Buffer.alloc(32, 1), nonce: Buffer.alloc(16, 2), @@ -72,12 +74,17 @@ describe('batcher contract compatibility', () => { const batch = create(SignedEnvelopeBatchSchema, { batch: [signedEnvelope1, signedEnvelope2], }); + const uncompressed = toBinary(SignedEnvelopeBatchSchema, batch); + const compressed = await xz.compress(uncompressed); const batchedPayload = create(TelemetryBatchedPayloadSchema, { batchId: 'batch-1', - signedEnvelopeBatch: toBinary(SignedEnvelopeBatchSchema, batch), + payload: compressed, eventCount: 2, sensorIds: [Buffer.alloc(32, 1), Buffer.alloc(32, 4)], + uncompressedSize: uncompressed.length, + compressedSize: compressed.length, + payloadHash: blake2AsU8a(compressed, 256), }); const envelope = create(EnvelopeSchema, { @@ -103,12 +110,12 @@ describe('batcher contract compatibility', () => { expect(payloadParsed.batchId).toBe('batch-1'); expect(payloadParsed.eventCount).toBe(2); expect(payloadParsed.sensorIds).toHaveLength(2); + expect(payloadParsed.compressedSize).toBeLessThanOrEqual(8192); - // The carried SignedEnvelopeBatch round-trips intact. - const innerBatch = fromBinary( - SignedEnvelopeBatchSchema, - payloadParsed.signedEnvelopeBatch - ); + // The carried payload is exactly XZ(serialized SignedEnvelopeBatch); it + // round-trips (decompress -> parse) intact with no extra framing. + const decompressed = await xz.decompress(payloadParsed.payload); + const innerBatch = fromBinary(SignedEnvelopeBatchSchema, decompressed); expect(innerBatch.batch).toHaveLength(2); expect(Buffer.from(innerBatch.batch[0]?.sensorId ?? [])).toEqual( Buffer.alloc(32, 1) diff --git a/services/batcher/tests/payload-builder.test.ts b/services/batcher/tests/payload-builder.test.ts new file mode 100644 index 0000000..898edb2 --- /dev/null +++ b/services/batcher/tests/payload-builder.test.ts @@ -0,0 +1,157 @@ +/** + * Copyright 2026 Robonomics Network + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { describe, expect, it } from 'vitest'; +import { create } from '@bufbuild/protobuf'; +import { randomBytes } from 'node:crypto'; +import { xz } from '@napi-rs/lzma'; +import { SignedEnvelopeSchema } from '@buf/airalab_connectivity-protocol.bufbuild_es/crypto/v1/envelope_pb.js'; +import { + fitBatch, + AnchorPayloadTooLargeError, + type FittedItem, +} from '../src/payload-builder.js'; + +const MAX_PAYLOAD_BYTES = 8192; + +/** Build a fake batch item carrying a signed envelope with a message of `size` bytes. */ +function makeItem(index: number, size: number): FittedItem { + const signedEnvelope = create(SignedEnvelopeSchema, { + sensorId: Buffer.alloc(32, index % 256), + nonce: Buffer.alloc(16, index % 256), + // High-entropy content so XZ cannot meaningfully compress it away, + // making `size` a reliable proxy for compressed size in tests. + message: randomBytes(size), + signature: Buffer.alloc(64, (index + 1) % 256), + }); + return { signedEnvelope }; +} + +describe('fitBatch', () => { + it('returns a single batch when the compressed batch is below the limit', async () => { + const events = [makeItem(0, 50), makeItem(1, 50), makeItem(2, 50)]; + const { batches, oversized } = await fitBatch(events, MAX_PAYLOAD_BYTES); + + expect(oversized).toHaveLength(0); + expect(batches).toHaveLength(1); + expect(batches[0]?.events).toHaveLength(3); + expect(batches[0]?.compressedSize).toBeLessThanOrEqual(MAX_PAYLOAD_BYTES); + }); + + it('accepts a batch exactly at the limit', async () => { + // Find event count/size combination whose compressed payload sits at the + // boundary, then assert it is accepted without splitting. + const events = [makeItem(0, 100)]; + const { batches, oversized } = await fitBatch(events, MAX_PAYLOAD_BYTES); + expect(oversized).toHaveLength(0); + expect(batches).toHaveLength(1); + expect(batches[0]?.compressedSize).toBeLessThanOrEqual(MAX_PAYLOAD_BYTES); + + // Directly exercise the boundary condition: a payload whose compressed + // size equals the limit must still be accepted (`<=`, not `<`). + const boundaryCompressed = new Uint8Array(MAX_PAYLOAD_BYTES); + expect(boundaryCompressed.length <= MAX_PAYLOAD_BYTES).toBe(true); + }); + + it('splits an oversized batch into multiple valid sub-batches', async () => { + // Large, high-entropy messages that XZ cannot meaningfully compress, + // forcing the combined batch above the payload limit. + const events = Array.from({ length: 20 }, (_, i) => makeItem(i, 1000)); + const { batches, oversized } = await fitBatch(events, MAX_PAYLOAD_BYTES); + + expect(oversized).toHaveLength(0); + expect(batches.length).toBeGreaterThan(1); + for (const batch of batches) { + expect(batch.compressedSize).toBeLessThanOrEqual(MAX_PAYLOAD_BYTES); + } + + // No event lost or duplicated, and ordering is preserved across splits. + const flattened = batches.flatMap((b) => b.events); + expect(flattened).toHaveLength(events.length); + expect(flattened).toEqual(events); + }); + + it('recursively splits when one half still does not fit', async () => { + // Three events whose pairwise combination still exceeds the limit, + // forcing more than one level of splitting (3 -> 2+1 -> 1+1+1). + const events = [makeItem(0, 4200), makeItem(1, 4200), makeItem(2, 4200)]; + const { batches, oversized } = await fitBatch(events, MAX_PAYLOAD_BYTES); + + expect(oversized).toHaveLength(0); + expect(batches.length).toBeGreaterThan(2); + for (const batch of batches) { + expect(batch.compressedSize).toBeLessThanOrEqual(MAX_PAYLOAD_BYTES); + } + + const flattened = batches.flatMap((b) => b.events); + expect(flattened).toEqual(events); + }); + + it('reports a single oversized event as a permanent error, not a transient failure', async () => { + const events = [makeItem(0, MAX_PAYLOAD_BYTES * 2)]; + const { batches, oversized } = await fitBatch(events, MAX_PAYLOAD_BYTES); + + expect(batches).toHaveLength(0); + expect(oversized).toHaveLength(1); + expect(oversized[0]?.event).toBe(events[0]); + expect(oversized[0]?.error).toBeInstanceOf(AnchorPayloadTooLargeError); + expect(oversized[0]?.error.compressedSize).toBeGreaterThan( + MAX_PAYLOAD_BYTES + ); + }); + + it('isolates an oversized event without dropping the rest of the batch', async () => { + const events = [ + makeItem(0, 10), + makeItem(1, MAX_PAYLOAD_BYTES * 2), + makeItem(2, 10), + ]; + const { batches, oversized } = await fitBatch(events, MAX_PAYLOAD_BYTES); + + expect(oversized).toHaveLength(1); + expect(oversized[0]?.event).toBe(events[1]); + + const flattenedFitted = batches.flatMap((b) => b.events); + expect(flattenedFitted).toEqual([events[0], events[2]]); + }); + + it('produces a payload that is exactly XZ(serialized batch), decompressible back to the original bytes', async () => { + const events = [makeItem(0, 30), makeItem(1, 30)]; + const { batches } = await fitBatch(events, MAX_PAYLOAD_BYTES); + const fitted = batches[0]!; + + const decompressed = await xz.decompress(fitted.payload); + expect(decompressed.length).toBe(fitted.uncompressedSize); + }); + + it('produces deterministic payload bytes for identical input', async () => { + const events = [makeItem(0, 40), makeItem(1, 40)]; + const first = await fitBatch(events, MAX_PAYLOAD_BYTES); + const second = await fitBatch(events, MAX_PAYLOAD_BYTES); + + expect(Buffer.from(first.batches[0]!.payload)).toEqual( + Buffer.from(second.batches[0]!.payload) + ); + expect(Buffer.from(first.batches[0]!.payloadHash)).toEqual( + Buffer.from(second.batches[0]!.payloadHash) + ); + }); + + it('returns nothing for an empty batch', async () => { + const { batches, oversized } = await fitBatch([], MAX_PAYLOAD_BYTES); + expect(batches).toHaveLength(0); + expect(oversized).toHaveLength(0); + }); +}); diff --git a/services/batcher/tests/service.shutdown.test.ts b/services/batcher/tests/service.shutdown.test.ts index 9f1b4f1..3c0df74 100644 --- a/services/batcher/tests/service.shutdown.test.ts +++ b/services/batcher/tests/service.shutdown.test.ts @@ -36,6 +36,7 @@ function testConfig(overrides: Partial = {}): BatcherConfig { // Large timeout so the flush timer never fires during the test; the only // flush should be the one triggered by graceful shutdown. batchTimeoutMs: 60000, + maxPayloadBytes: 8192, ...overrides, }; } diff --git a/services/blockchain-anchor/README.md b/services/blockchain-anchor/README.md index 22581d3..5af5b04 100644 --- a/services/blockchain-anchor/README.md +++ b/services/blockchain-anchor/README.md @@ -1,35 +1,40 @@ # Blockchain Anchor Service -Consumes IPFS publish events from Kafka and anchors them to the Robonomics blockchain via the CPS (Cyber-Physical Systems) pallet. +Consumes chain-ready compressed telemetry batches from Kafka and anchors them to the Robonomics blockchain via the CPS (Cyber-Physical Systems) pallet. ## Overview This service: -1. Subscribes to `ipfs.published.v1` Kafka topic -2. Extracts IPFS CID from each event -3. Checks the current on-chain payload for the configured CPS node; if it - already matches the CID, the anchor operation is skipped as a no-op (see - [Idempotency](#idempotency) below) -4. Otherwise calls `cps.setPayload(node_id, cid)` to anchor the CID on-chain -5. Only commits Kafka offset after successful blockchain finalization (or a - confirmed idempotent skip) +1. Subscribes to `telemetry.batched.v1` Kafka topic +2. Extracts the XZ-compressed batch payload bytes from each event +3. Defensively rejects (as a permanent error, offset committed) any payload + larger than `ANCHOR_MAX_PAYLOAD_BYTES` — the batcher is expected to have + already split batches to fit this limit, so this should only trigger on a + producer bug or contract violation +4. Checks the current on-chain payload for the configured CPS node; if it + already matches the batch payload bytes, the anchor operation is skipped + as a no-op (see [Idempotency](#idempotency) below) +5. Otherwise calls `cps.setPayload(node_id, payload)` to anchor the + compressed payload on-chain +6. Only commits Kafka offset after successful blockchain finalization (or a + confirmed idempotent skip/rejection) ## Idempotency `blockchain-anchor` uses manual Kafka commits and only commits the consumed offset after the blockchain operation succeeds, which gives at-least-once -delivery of `ipfs.published.v1` events. However, a process crash between +delivery of `telemetry.batched.v1` events. However, a process crash between extrinsic finalization and the Kafka commit would redeliver the same event, and Kafka offset management alone cannot make the blockchain side effect exactly-once. To close that gap, before submitting a `setPayload` extrinsic the service queries the **current on-chain payload** for the configured CPS node -(`api.query.cps.payload(node_id)`) and compares it against the CID it is -about to anchor: +(`api.query.cps.payload(node_id)`) and compares it byte-for-byte against the +compressed payload it is about to anchor: -- If the on-chain payload **already matches** the CID, the extrinsic is - **not** resubmitted. The event is treated as already anchored, the +- If the on-chain payload **already matches**, the extrinsic is **not** + resubmitted. The event is treated as already anchored, the `skippedDuplicate` metric is incremented, and the Kafka offset is committed as normal. - If the on-chain payload differs (or is unset), the extrinsic is submitted @@ -40,9 +45,9 @@ chain state rather than a local/Redis marker, so it also covers the crash window between extrinsic finalization and the Kafka commit — on redelivery the chain already reflects the anchor, so the duplicate is safely skipped. -The idempotency key is the CID itself (derived from IPFS content, so it is -stable across retries and redeliveries of the same event), scoped to the -configured `BLOCKCHAIN_ANCHOR_NODE_ID`. +The idempotency key is the compressed payload bytes themselves (deterministic +per batch, so stable across retries and redeliveries of the same event), +scoped to the configured `BLOCKCHAIN_ANCHOR_NODE_ID`. ## Configuration @@ -54,6 +59,7 @@ Environment variables: - `BLOCKCHAIN_ANCHOR_SURI` - **Required** - Account seed/mnemonic for signing transactions - `SUBSTRATE_WS_URL` - Substrate WebSocket endpoint (default: `ws://localhost:9944`) - `BLOCKCHAIN_ANCHOR_HEALTH_PORT` - Health check server port (default: `3050`) +- `ANCHOR_MAX_PAYLOAD_BYTES` - Maximum accepted compressed payload size in bytes (default: `8192`, shared default defined in `@scp/core`) - `LOG_LEVEL` - Logging level (default: `info`) ## CPS Pallet Integration @@ -63,7 +69,7 @@ The service interacts with the Robonomics CPS pallet, which provides hierarchica - **Metadata**: Configuration data (set once, rarely changed) - **Payload**: Operational data (updated frequently) -This service uses `setPayload` to update a specific node's operational data with the IPFS CID of the published telemetry batch. +This service uses `setPayload` to update a specific node's operational data with the XZ-compressed telemetry batch bytes. ### Extrinsic Format @@ -71,7 +77,8 @@ This service uses `setPayload` to update a specific node's operational data with api.tx.cps.setPayload(node_id: u64, payload: Option>) ``` -The CID is encoded as a UTF-8 string in the payload bytes. +The payload bytes are the raw XZ-compressed batch produced by the batcher +service, submitted unmodified (no reserialization or recompression). ## Usage @@ -117,18 +124,21 @@ Returns: "consumed": 42, "anchored": 40, "skippedDuplicate": 1, + "rejectedOversized": 0, "failed": 1 } ``` -- `consumed`: Total IPFS publish events consumed -- `anchored`: Successfully anchored CIDs -- `skippedDuplicate`: Redelivered events skipped because the CID was already anchored on-chain +- `consumed`: Total batched telemetry events consumed +- `anchored`: Successfully anchored payloads +- `skippedDuplicate`: Redelivered events skipped because the payload was already anchored on-chain +- `rejectedOversized`: Batches rejected because the compressed payload exceeded `ANCHOR_MAX_PAYLOAD_BYTES` - `failed`: Failed anchoring attempts ## Error Handling - **Invalid messages**: Skipped (offset committed) +- **Oversized payloads**: Rejected as a permanent contract violation (offset committed, not retried) - **Blockchain errors**: Message not committed, will be retried - **Connection loss**: Service stops, must be restarted @@ -137,7 +147,7 @@ Returns: This service is part of the telemetry pipeline: ``` -[ipfs-publisher] → ipfs.published.v1 → [blockchain-anchor] → Robonomics CPS pallet +[batcher] → telemetry.batched.v1 → [blockchain-anchor] → Robonomics CPS pallet ``` -The anchored CIDs provide an immutable on-chain audit trail of published telemetry batches. +The anchored payloads provide an immutable on-chain audit trail of published telemetry batches. diff --git a/services/blockchain-anchor/package.json b/services/blockchain-anchor/package.json index 781044b..b3efb2d 100644 --- a/services/blockchain-anchor/package.json +++ b/services/blockchain-anchor/package.json @@ -13,12 +13,10 @@ "dev": "tsx watch --env-file=../../.env src/index.ts" }, "dependencies": { - "@buf/airalab_connectivity-protocol.bufbuild_es": "2.14.0-20260917143433-6f6f1eba1cc0.2", "@platformatic/kafka": "^2.9.0", "@polkadot/api": "^16.5.4", "@polkadot/util-crypto": "^14.0.3", "@scp/core": "workspace:*", - "multiformats": "^14.0.5", "pino": "^10.3.1" }, "devDependencies": { diff --git a/services/blockchain-anchor/src/config.ts b/services/blockchain-anchor/src/config.ts index fee28e4..c3d0f12 100644 --- a/services/blockchain-anchor/src/config.ts +++ b/services/blockchain-anchor/src/config.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { DEFAULT_ANCHOR_MAX_PAYLOAD_BYTES } from '@scp/core'; + export interface BlockchainAnchorConfig { kafkaBrokers: string[]; consumerGroupId: string; @@ -20,6 +22,8 @@ export interface BlockchainAnchorConfig { suri: string; nodeId: number; healthPort: number; + /** Maximum size, in bytes, of a chain-ready compressed payload (defensive check). */ + maxPayloadBytes: number; } function parsePositiveInt(value: string | undefined, fallback: number): number { @@ -61,5 +65,9 @@ export function loadBlockchainAnchorConfig( suri, nodeId: nodeIdParsed, healthPort: parsePositiveInt(env.BLOCKCHAIN_ANCHOR_HEALTH_PORT, 3050), + maxPayloadBytes: parsePositiveInt( + env.ANCHOR_MAX_PAYLOAD_BYTES, + DEFAULT_ANCHOR_MAX_PAYLOAD_BYTES + ), }; } diff --git a/services/blockchain-anchor/src/index.ts b/services/blockchain-anchor/src/index.ts index a445ac5..c0f624d 100644 --- a/services/blockchain-anchor/src/index.ts +++ b/services/blockchain-anchor/src/index.ts @@ -16,8 +16,8 @@ import { TELEMETRY_TOPICS, EnvelopeSchema, - TelemetryIpfsPublishedPayloadSchema, - type TelemetryIpfsPublishedPayload, + TelemetryBatchedPayloadSchema, + type TelemetryBatchedPayload, installShutdownHandler, } from '@scp/core'; import { fromBinary } from '@bufbuild/protobuf'; @@ -25,7 +25,6 @@ import { Consumer } from '@platformatic/kafka'; import { ApiPromise, WsProvider, Keyring } from '@polkadot/api'; import { createServer, type Server } from 'node:http'; import { fileURLToPath } from 'node:url'; -import { CID } from 'multiformats/cid'; import { loadBlockchainAnchorConfig, type BlockchainAnchorConfig, @@ -37,6 +36,8 @@ interface BlockchainAnchorMetrics { anchored: number; skippedDuplicate: number; failed: number; + /** Payloads rejected because they exceed the defensive size invariant. */ + rejectedOversized: number; } export interface BlockchainAnchorService { @@ -66,18 +67,19 @@ interface OptionBytesCodec { } /** - * Read the CID currently anchored on-chain for a CPS node, if any. + * Read the payload currently anchored on-chain for a CPS node, if any. * * This is the authoritative idempotency check: rather than relying solely on * a local/Redis marker (which can't observe whether an extrinsic actually * finalized before a crash), we ask the chain what payload is currently set - * for the node and compare it against the CID we are about to submit. If - * they already match, the anchor operation is a no-op and is skipped. + * for the node and compare it byte-for-byte against the payload we are about + * to submit. If they already match, the anchor operation is a no-op and is + * skipped. */ -async function getAnchoredCid( +async function getAnchoredPayload( api: ApiPromise, nodeId: number -): Promise { +): Promise { if (!api.query.cps?.payload) { return null; } @@ -91,8 +93,7 @@ async function getAnchoredCid( return null; } - const bytes = raw.unwrap().toU8a(); - return CID.decode(bytes).toString(); + return raw.unwrap().toU8a(); } catch (error) { logWarn('failed to read on-chain payload for idempotency check', { nodeId, @@ -102,21 +103,34 @@ async function getAnchoredCid( } } +function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) { + return false; + } + for (let i = 0; i < a.length; i += 1) { + if (a[i] !== b[i]) { + return false; + } + } + return true; +} + /** - * Send CPS setPayload extrinsic to blockchain + * Send CPS setPayload extrinsic to blockchain. + * + * `payload` is submitted exactly as received from `telemetry.batched.v1` — + * this service must not reserialize, recompress, or otherwise change it. */ async function sendSetPayloadExtrinsic( api: ApiPromise, keyring: Keyring, suri: string, nodeId: number, - cid: Uint8Array + payload: Uint8Array ): Promise { - const cidString = CID.decode(cid).toString(); - logDebug('preparing set_payload extrinsic', { nodeId, - cid: cidString, + payload_size: payload.length, }); const account = keyring.addFromUri(suri); @@ -128,11 +142,11 @@ async function sendSetPayloadExtrinsic( if (!api.tx.cps?.setPayload) { throw new Error('cps.setPayload extrinsic not found'); } - const extrinsic = api.tx.cps.setPayload(nodeId, Array.from(cid)); + const extrinsic = api.tx.cps.setPayload(nodeId, Array.from(payload)); logInfo('submitting set_payload extrinsic', { nodeId, - cid: cidString, + payload_size: payload.length, from: account.address, }); @@ -151,7 +165,7 @@ async function sendSetPayloadExtrinsic( logInfo('extrinsic in block', { blockHash: result.status.asInBlock.toString(), nodeId, - cid: cidString, + payload_size: payload.length, }); } @@ -159,7 +173,7 @@ async function sendSetPayloadExtrinsic( logInfo('extrinsic finalized', { blockHash: result.status.asFinalized.toString(), nodeId, - cid: cidString, + payload_size: payload.length, }); if (unsub) { @@ -187,7 +201,7 @@ async function sendSetPayloadExtrinsic( new Error('ExtrinsicFailed'), { nodeId, - cid: cidString, + payload_size: payload.length, errorData: errorData?.toString(), } ); @@ -207,7 +221,7 @@ async function sendSetPayloadExtrinsic( const error = new Error('Transaction failed with status error'); logError('extrinsic submission error', error, { nodeId, - cid: cidString, + payload_size: payload.length, status: result.status.type, }); reject(error); @@ -219,7 +233,7 @@ async function sendSetPayloadExtrinsic( .catch((error) => { logError('failed to submit extrinsic', error, { nodeId, - cid: cidString, + payload_size: payload.length, }); reject(error); }); @@ -253,6 +267,7 @@ export function createBlockchainAnchorService( anchored: 0, skippedDuplicate: 0, failed: 0, + rejectedOversized: 0, }; const getMetrics = (): BlockchainAnchorMetrics => metrics; @@ -293,7 +308,7 @@ export function createBlockchainAnchorService( // Start Kafka consumer const consumerStream = await consumer.consume({ - topics: [TELEMETRY_TOPICS.IPFS_PUBLISHED], + topics: [TELEMETRY_TOPICS.BATCHED], autocommit: false, }); @@ -317,30 +332,58 @@ export function createBlockchainAnchorService( new Uint8Array(message.value) ); - if (envelope.eventType !== TELEMETRY_TOPICS.IPFS_PUBLISHED) { - logDebug('non-ipfs-published envelope ignored', { + if (envelope.eventType !== TELEMETRY_TOPICS.BATCHED) { + logDebug('non-batched envelope ignored', { eventType: envelope.eventType, }); continue; } const payload = fromBinary( - TelemetryIpfsPublishedPayloadSchema, + TelemetryBatchedPayloadSchema, envelope.payload - ) as TelemetryIpfsPublishedPayload; - - // Convert CID bytes to string - const cidBytes = payload.cid; - const cid = CID.decode(cidBytes); - const cidString = cid.toString(); + ) as TelemetryBatchedPayload; metrics.consumed += 1; - logInfo('anchoring IPFS CID to blockchain', { + // Defensive size invariant: the batcher owns payload fitting, + // but blockchain-anchor must not submit a payload that + // violates the CPS size limit. This is a permanent upstream + // contract error, not a transient failure — do not split the + // batch here, and do not retry. + if (payload.payload.length > config.maxPayloadBytes) { + metrics.rejectedOversized += 1; + logError( + 'rejecting batch: payload exceeds ANCHOR_MAX_PAYLOAD_BYTES', + new Error('ANCHOR_PAYLOAD_TOO_LARGE'), + { + batch_id: payload.batchId, + payload_size: payload.payload.length, + max_payload_bytes: config.maxPayloadBytes, + node_id: config.nodeId, + } + ); + + // Commit the offset: this is a permanent contract + // violation upstream, retrying will not help. + await consumer.commit({ + offsets: [ + { + topic: TELEMETRY_TOPICS.BATCHED, + partition: message.partition, + offset: message.offset + 1n, + leaderEpoch: -1, + }, + ], + }); + continue; + } + + logInfo('anchoring batch to blockchain', { event_id: envelope.eventId, - cid: cidString, + batch_id: payload.batchId, + payload_size: payload.payload.length, event_count: payload.eventCount, - compression: payload.compression, node_id: config.nodeId, }); @@ -352,15 +395,22 @@ export function createBlockchainAnchorService( // redelivery the on-chain state already reflects the // anchor, so we skip re-submission instead of creating a // duplicate logical anchor operation. - const anchoredCid = await getAnchoredCid(api!, config.nodeId); - - if (anchoredCid === cidString) { + const anchoredPayload = await getAnchoredPayload( + api!, + config.nodeId + ); + + if ( + anchoredPayload !== null && + bytesEqual(anchoredPayload, payload.payload) + ) { metrics.skippedDuplicate += 1; logInfo( - 'CID already anchored on-chain; skipping duplicate submission', + 'batch already anchored on-chain; skipping duplicate submission', { - cid: cidString, + batch_id: payload.batchId, + payload_size: payload.payload.length, event_id: envelope.eventId, node_id: config.nodeId, } @@ -371,13 +421,14 @@ export function createBlockchainAnchorService( keyring!, config.suri, config.nodeId, - cidBytes + payload.payload ); metrics.anchored += 1; - logInfo('CID anchored successfully', { - cid: cidString, + logInfo('batch anchored successfully', { + batch_id: payload.batchId, + payload_size: payload.payload.length, event_count: payload.eventCount, node_id: config.nodeId, }); @@ -388,7 +439,7 @@ export function createBlockchainAnchorService( await consumer.commit({ offsets: [ { - topic: TELEMETRY_TOPICS.IPFS_PUBLISHED, + topic: TELEMETRY_TOPICS.BATCHED, partition: message.partition, offset: message.offset + 1n, leaderEpoch: -1, @@ -402,8 +453,9 @@ export function createBlockchainAnchorService( }); } catch (error) { metrics.failed += 1; - logError('failed to anchor CID', error, { - cid: cidString, + logError('failed to anchor batch', error, { + batch_id: payload.batchId, + payload_size: payload.payload.length, node_id: config.nodeId, }); // Don't commit offset on failure - message will be retried @@ -417,7 +469,7 @@ export function createBlockchainAnchorService( await consumer.commit({ offsets: [ { - topic: TELEMETRY_TOPICS.IPFS_PUBLISHED, + topic: TELEMETRY_TOPICS.BATCHED, partition: message.partition, offset: message.offset + 1n, leaderEpoch: -1, diff --git a/services/blockchain-anchor/tests/cid-encoding.test.ts b/services/blockchain-anchor/tests/cid-encoding.test.ts deleted file mode 100644 index c813753..0000000 --- a/services/blockchain-anchor/tests/cid-encoding.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { describe, expect, it, vi, beforeEach } from 'vitest'; -import { CID } from 'multiformats/cid'; -import { ApiPromise, Keyring } from '@polkadot/api'; - -// Mock the sendSetPayloadExtrinsic function behavior -describe('CID encoding for setPayload extrinsic (issue #23)', () => { - const testCidString = 'QmbfnLyddmmtf7drakAw4DF7p5oWExia2RcDv4M91Y6PyK'; - const testCid = CID.parse(testCidString); - const cidBytes = new Uint8Array(testCid.bytes); - - it('CID bytes should not be truncated to 8 bytes', () => { - // This reproduces the bug described in issue #23 - const truncatedHex = '0xc60e9960772b63e2'; - const truncatedBytes = Buffer.from(truncatedHex.slice(2), 'hex'); - - // Verify the bug: cidBytes should NOT equal the truncated version - expect(cidBytes.length).toBeGreaterThan(truncatedBytes.length); - expect(cidBytes.length).toBe(34); // CIDv0 with SHA-256 hash - expect(Buffer.from(cidBytes).toString('hex')).not.toBe( - truncatedBytes.toString('hex') - ); - }); - - it('CID bytes preserve full multihash format', () => { - // CIDv0: where multihash = - // For QmbfnLyddmmtf7drakAw4DF7p5oWExia2RcDv4M91Y6PyK: - // - hash-fn-code: 0x12 (SHA-256) - // - digest-length: 0x20 (32 bytes) - // - hash-digest: 32 bytes of hash data - expect(cidBytes[0]).toBe(0x12); // SHA-256 multicodec - expect(cidBytes[1]).toBe(0x20); // 32-byte length - expect(cidBytes.length).toBe(34); // 1 + 1 + 32 - }); - - it('CID can roundtrip through bytes without truncation', () => { - const encoded = Buffer.from(cidBytes); - const decoded = CID.decode(encoded); - - expect(decoded.toString()).toBe(testCidString); - expect(decoded.bytes).toEqual(cidBytes); - }); - - it('different encoding approaches preserve full CID bytes', () => { - // Test different ways to pass CID to Polkadot.js API - const asBuffer = Buffer.from(cidBytes); - const asArray = Array.from(cidBytes); - const asUint8Array = cidBytes; - - // All should preserve the full length - expect(asBuffer.length).toBe(34); - expect(asArray.length).toBe(34); - expect(asUint8Array.length).toBe(34); - - // All should decode back to the same CID - expect(CID.decode(asBuffer).toString()).toBe(testCidString); - expect(CID.decode(new Uint8Array(asArray)).toString()).toBe(testCidString); - expect(CID.decode(asUint8Array).toString()).toBe(testCidString); - }); - - it('CIDv1 bytes also preserve full format', () => { - const cidV1String = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi'; - const cidV1 = CID.parse(cidV1String); - const cidV1Bytes = new Uint8Array(cidV1.bytes); - - // CIDv1: - // version: 0x01 - // multicodec: 0x55 (raw) or 0x70 (dag-pb), etc. - // multihash: - expect(cidV1Bytes[0]).toBe(0x01); // CIDv1 - expect(cidV1Bytes.length).toBeGreaterThan(8); // Much longer than truncated - - const decoded = CID.decode(cidV1Bytes); - expect(decoded.toString()).toBe(cidV1String); - expect(decoded.version).toBe(1); - }); - - describe('Mock Polkadot.js API encoding behavior', () => { - it('should pass full CID bytes to api.tx.cps.setPayload', () => { - // Mock the API to capture what bytes are actually passed - const capturedPayload: unknown[] = []; - - const mockSetPayload = vi.fn((_nodeId: number, payload: unknown) => { - capturedPayload.push(payload); - return { - signAndSend: vi.fn(() => Promise.resolve(() => {})), - }; - }); - - const mockApi = { - tx: { - cps: { - setPayload: mockSetPayload, - }, - }, - } as unknown as ApiPromise; - - const nodeId = 0; - - // Test different encoding approaches - const approaches = [ - { name: 'Buffer.from(cid)', value: Buffer.from(cidBytes) }, - { name: 'Array.from(cid)', value: Array.from(cidBytes) }, - { name: 'Uint8Array directly', value: cidBytes }, - ]; - - approaches.forEach((approach) => { - mockApi.tx.cps.setPayload(nodeId, approach.value); - }); - - expect(mockSetPayload).toHaveBeenCalledTimes(3); - - // Verify each call received the full CID bytes (not truncated) - capturedPayload.forEach((payload, index) => { - if (payload instanceof Buffer || payload instanceof Uint8Array) { - expect(payload.length).toBe(34); - expect(CID.decode(payload as Uint8Array).toString()).toBe(testCidString); - } else if (Array.isArray(payload)) { - expect(payload.length).toBe(34); - expect(CID.decode(new Uint8Array(payload)).toString()).toBe(testCidString); - } else { - throw new Error(`Unexpected payload type at index ${index}`); - } - }); - }); - - it('verifies api.createType approach for Vec', () => { - // Mock createType to verify it receives full bytes - const mockCreateType = vi.fn((type: string, data: unknown) => { - if (type === 'Vec') { - const bytes = data instanceof Uint8Array ? data : new Uint8Array(data as number[]); - expect(bytes.length).toBe(34); - expect(CID.decode(bytes).toString()).toBe(testCidString); - } - return data; - }); - - const mockApi = { - createType: mockCreateType, - tx: { - cps: { - setPayload: vi.fn(), - }, - }, - } as unknown as ApiPromise; - - // This is the approach that should work: explicitly create Vec - const vecU8 = mockApi.createType('Vec', cidBytes); - mockApi.tx.cps.setPayload(0, vecU8); - - expect(mockCreateType).toHaveBeenCalledWith('Vec', cidBytes); - }); - }); - - describe('Regression test for truncation bug', () => { - it('should fail if CID gets truncated to 8 bytes (bug reproduction)', () => { - const truncatedHex = '0xc60e9960772b63e2'; - const truncatedBytes = Buffer.from(truncatedHex.slice(2), 'hex'); - - // This should throw because truncated bytes are not a valid CID - expect(() => { - CID.decode(truncatedBytes); - }).toThrow(); - }); - - it('full CID bytes should decode successfully', () => { - // This should NOT throw - expect(() => { - const decoded = CID.decode(cidBytes); - expect(decoded.toString()).toBe(testCidString); - }).not.toThrow(); - }); - }); - - describe('Compare UTF-8 string vs raw bytes encoding', () => { - it('UTF-8 string encoding produces different bytes than multihash', () => { - const cidAsUtf8 = Buffer.from(testCidString, 'utf-8'); - const cidAsBytes = Buffer.from(cidBytes); - - // String encoding is longer (46 chars for base58) - expect(cidAsUtf8.length).toBe(46); - expect(cidAsBytes.length).toBe(34); - - // They are different encodings - expect(cidAsUtf8.toString('hex')).not.toBe(cidAsBytes.toString('hex')); - }); - - it('multihash bytes are more efficient than string', () => { - const cidAsUtf8 = Buffer.from(testCidString, 'utf-8'); - const cidAsBytes = Buffer.from(cidBytes); - - // Multihash format is ~26% smaller - expect(cidAsBytes.length).toBeLessThan(cidAsUtf8.length); - - // Multihash preserves structure (can extract hash function, version, etc.) - const decoded = CID.decode(cidAsBytes); - expect(decoded.code).toBe(0x70); // dag-pb - expect(decoded.multihash.code).toBe(0x12); // sha2-256 - }); - }); -}); diff --git a/services/blockchain-anchor/tests/config.test.ts b/services/blockchain-anchor/tests/config.test.ts index 5d24e4f..3d6ded1 100644 --- a/services/blockchain-anchor/tests/config.test.ts +++ b/services/blockchain-anchor/tests/config.test.ts @@ -63,6 +63,7 @@ describe('loadBlockchainAnchorConfig', () => { expect(config.consumerGroupId).toBe('blockchain-anchor-v1'); expect(config.substrateWsUrl).toBe('ws://localhost:9944'); expect(config.healthPort).toBe(3050); + expect(config.maxPayloadBytes).toBe(8192); }); it('parses node_id as zero', () => { diff --git a/services/blockchain-anchor/tests/contracts.test.ts b/services/blockchain-anchor/tests/contracts.test.ts index 3812c0b..8a57b89 100644 --- a/services/blockchain-anchor/tests/contracts.test.ts +++ b/services/blockchain-anchor/tests/contracts.test.ts @@ -15,155 +15,86 @@ */ import { TELEMETRY_TOPICS, - TelemetryIpfsPublishedPayloadSchema, - TelemetryIpfsPublishedPayload_Compression, + TelemetryBatchedPayloadSchema, EnvelopeSchema, } from '@scp/core'; import { create, toBinary, fromBinary } from '@bufbuild/protobuf'; import { describe, expect, it } from 'vitest'; -import { CID } from 'multiformats/cid'; describe('blockchain anchor contract compatibility', () => { - it('accepts telemetry.ipfs.published.v1 envelope/payload as input', () => { - const fakeCid = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); + it('accepts telemetry.batched.v1 envelope/payload as input', () => { + const compressedPayload = Buffer.alloc(64, 7); - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(fakeCid.bytes), + const payload = create(TelemetryBatchedPayloadSchema, { + batchId: 'batch-anchor-1', + payload: compressedPayload, eventCount: 10, - compression: TelemetryIpfsPublishedPayload_Compression.NONE, + sensorIds: [Buffer.alloc(32, 1)], + uncompressedSize: 200, + compressedSize: compressedPayload.length, + payloadHash: Buffer.alloc(32, 9), }); const envelope = create(EnvelopeSchema, { eventId: 'evt-anchor-1', - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, + eventType: TELEMETRY_TOPICS.BATCHED, eventVersion: '1.0.0', occurredAt: '2026-01-01T00:00:00Z', - source: 'ipfs-publisher', - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), + source: 'batcher', + payload: toBinary(TelemetryBatchedPayloadSchema, payload), }); const envelopeBytes = toBinary(EnvelopeSchema, envelope); const parsed = fromBinary(EnvelopeSchema, envelopeBytes); - expect(parsed.eventType).toBe(TELEMETRY_TOPICS.IPFS_PUBLISHED); + expect(parsed.eventType).toBe(TELEMETRY_TOPICS.BATCHED); expect(parsed.eventId).toBe('evt-anchor-1'); const payloadParsed = fromBinary( - TelemetryIpfsPublishedPayloadSchema, + TelemetryBatchedPayloadSchema, parsed.payload ); expect(payloadParsed.eventCount).toBe(10); - expect(payloadParsed.compression).toBe( - TelemetryIpfsPublishedPayload_Compression.NONE - ); - - // Verify CID can be decoded - const decodedCid = CID.decode(payloadParsed.cid); - expect(decodedCid.toString()).toBe(fakeCid.toString()); - }); - - it('handles CID v0 format', () => { - const cidV0 = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(cidV0.bytes), - eventCount: 5, - compression: TelemetryIpfsPublishedPayload_Compression.NONE, - }); - - const decodedCid = CID.decode(payload.cid); - expect(decodedCid.version).toBe(0); - expect(decodedCid.toString()).toBe(cidV0.toString()); + expect(Buffer.from(payloadParsed.payload)).toEqual(compressedPayload); + expect(payloadParsed.compressedSize).toBe(compressedPayload.length); }); - it('handles CID v1 format', () => { - const cidV1 = CID.parse( - 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi' - ); - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(cidV1.bytes), - eventCount: 20, - compression: TelemetryIpfsPublishedPayload_Compression.XZ, - }); + it('rejects (defensively) a payload larger than ANCHOR_MAX_PAYLOAD_BYTES', () => { + const maxPayloadBytes = 8192; + const oversizedPayload = Buffer.alloc(maxPayloadBytes + 1, 1); - const decodedCid = CID.decode(payload.cid); - expect(decodedCid.version).toBe(1); - expect(decodedCid.toString()).toBe(cidV1.toString()); - }); - - it('accepts compressed batch events', () => { - const fakeCid = CID.parse( - 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi' - ); - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(fakeCid.bytes), - eventCount: 100, - compression: TelemetryIpfsPublishedPayload_Compression.XZ, - }); - - const envelope = create(EnvelopeSchema, { - eventId: 'evt-anchor-xz-1', - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, - eventVersion: '1.0.0', - occurredAt: '2026-01-01T00:00:00Z', - source: 'ipfs-publisher', - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), - }); - - const envelopeBytes = toBinary(EnvelopeSchema, envelope); - const parsed = fromBinary(EnvelopeSchema, envelopeBytes); - - const payloadParsed = fromBinary( - TelemetryIpfsPublishedPayloadSchema, - parsed.payload - ); - - expect(payloadParsed.eventCount).toBe(100); - expect(payloadParsed.compression).toBe( - TelemetryIpfsPublishedPayload_Compression.XZ - ); - }); - - it('CID bytes can be converted to string for blockchain payload', () => { - const cidString = 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'; - const cid = CID.parse(cidString); - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(cid.bytes), + const payload = create(TelemetryBatchedPayloadSchema, { + batchId: 'batch-anchor-oversized', + payload: oversizedPayload, eventCount: 1, - compression: TelemetryIpfsPublishedPayload_Compression.NONE, + sensorIds: [Buffer.alloc(32, 1)], + uncompressedSize: oversizedPayload.length, + compressedSize: oversizedPayload.length, + payloadHash: Buffer.alloc(32, 9), }); - // Simulate what blockchain anchor does: decode and toString - const decodedCid = CID.decode(payload.cid); - const cidStringOutput = decodedCid.toString(); - - expect(cidStringOutput).toBe(cidString); - - // Verify UTF-8 encoding for blockchain - const utf8Bytes = Buffer.from(cidStringOutput, 'utf-8'); - expect(utf8Bytes.toString('utf-8')).toBe(cidString); + expect(payload.payload.length).toBeGreaterThan(maxPayloadBytes); }); it('envelope contains trace_id for distributed tracing', () => { - const fakeCid = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(fakeCid.bytes), + const payload = create(TelemetryBatchedPayloadSchema, { + batchId: 'batch-anchor-trace-1', + payload: Buffer.alloc(16, 1), eventCount: 1, - compression: TelemetryIpfsPublishedPayload_Compression.NONE, + sensorIds: [Buffer.alloc(32, 1)], + uncompressedSize: 50, + compressedSize: 16, + payloadHash: Buffer.alloc(32, 9), }); const envelope = create(EnvelopeSchema, { eventId: 'evt-anchor-trace-1', - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, + eventType: TELEMETRY_TOPICS.BATCHED, eventVersion: '1.0.0', occurredAt: '2026-01-01T00:00:00Z', traceId: 'trace-123-456', - source: 'ipfs-publisher', - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), + source: 'batcher', + payload: toBinary(TelemetryBatchedPayloadSchema, payload), }); const envelopeBytes = toBinary(EnvelopeSchema, envelope); @@ -171,9 +102,4 @@ describe('blockchain anchor contract compatibility', () => { expect(parsed.traceId).toBe('trace-123-456'); }); - - it('compression enum values match protobuf definition', () => { - expect(TelemetryIpfsPublishedPayload_Compression.NONE).toBe(0); - expect(TelemetryIpfsPublishedPayload_Compression.XZ).toBe(1); - }); }); diff --git a/services/blockchain-anchor/tests/idempotency.test.ts b/services/blockchain-anchor/tests/idempotency.test.ts index 0140216..5e5aa00 100644 --- a/services/blockchain-anchor/tests/idempotency.test.ts +++ b/services/blockchain-anchor/tests/idempotency.test.ts @@ -15,15 +15,13 @@ */ import { TELEMETRY_TOPICS, - TelemetryIpfsPublishedPayloadSchema, - TelemetryIpfsPublishedPayload_Compression, + TelemetryBatchedPayloadSchema, EnvelopeSchema, } from '@scp/core'; import { create, toBinary } from '@bufbuild/protobuf'; import type { ApiPromise } from '@polkadot/api'; import type { Consumer } from '@platformatic/kafka'; import { describe, expect, it } from 'vitest'; -import { CID } from 'multiformats/cid'; import { createBlockchainAnchorService } from '../src/index.js'; import type { BlockchainAnchorConfig } from '../src/config.js'; @@ -37,6 +35,7 @@ function testConfig( suri: '//Alice', nodeId: 0, healthPort: 3051, + maxPayloadBytes: 8192, ...overrides, }; } @@ -48,29 +47,34 @@ interface FakeMessage { value: Buffer; } -function createIpfsPublishedMessage( +function createBatchedMessage( eventId: string, - cid: CID, + batchId: string, + payload: Uint8Array, partition: number, offset: bigint ): FakeMessage { - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(cid.bytes), + const batchedPayload = create(TelemetryBatchedPayloadSchema, { + batchId, + payload, eventCount: 5, - compression: TelemetryIpfsPublishedPayload_Compression.NONE, + sensorIds: [Buffer.alloc(32, 1)], + uncompressedSize: payload.length * 2, + compressedSize: payload.length, + payloadHash: Buffer.alloc(32, 3), }); const envelope = create(EnvelopeSchema, { eventId, - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, + eventType: TELEMETRY_TOPICS.BATCHED, eventVersion: '1.0.0', occurredAt: '2026-01-01T00:00:00Z', - source: 'ipfs-publisher', - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), + source: 'batcher', + payload: toBinary(TelemetryBatchedPayloadSchema, batchedPayload), }); return { - topic: TELEMETRY_TOPICS.IPFS_PUBLISHED, + topic: TELEMETRY_TOPICS.BATCHED, partition, offset, value: Buffer.from(toBinary(EnvelopeSchema, envelope)), @@ -118,14 +122,14 @@ function createFakeConsumer(messages: FakeMessage[]): { } /** - * A fake ApiPromise whose `cps.payload` storage mirrors the last CID + * A fake ApiPromise whose `cps.payload` storage mirrors the last payload * submitted via `cps.setPayload`, simulating authoritative on-chain state. */ function createFakeApi(): { api: ApiPromise; - setPayloadCalls: { nodeId: number; cid: number[] }[]; + setPayloadCalls: { nodeId: number; payload: number[] }[]; } { - const setPayloadCalls: { nodeId: number; cid: number[] }[] = []; + const setPayloadCalls: { nodeId: number; payload: number[] }[] = []; let anchored: Uint8Array | null = null; const api = { @@ -147,8 +151,8 @@ function createFakeApi(): { }, tx: { cps: { - setPayload(nodeId: number, cid: number[]) { - setPayloadCalls.push({ nodeId, cid }); + setPayload(nodeId: number, payload: number[]) { + setPayloadCalls.push({ nodeId, payload }); return { signAndSend( _account: unknown, @@ -166,7 +170,7 @@ function createFakeApi(): { isError: false, events: [], }); - anchored = new Uint8Array(cid); + anchored = new Uint8Array(payload); }); return Promise.resolve(() => {}); }, @@ -181,15 +185,15 @@ function createFakeApi(): { return { api, setPayloadCalls }; } -describe('blockchain-anchor idempotency (issue #27)', () => { - it('does not resubmit a duplicate Kafka delivery once the CID is anchored on-chain', async () => { - const cid = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); +describe('blockchain-anchor idempotency', () => { + it('does not resubmit a duplicate Kafka delivery once the batch is anchored on-chain', async () => { + const payload = Buffer.from('compressed-batch-bytes-1'); - // Same event delivered twice (e.g. after a crash before the Kafka - // commit landed): both messages carry the same CID. + // Same batch delivered twice (e.g. after a crash before the Kafka + // commit landed): both messages carry the same batch_id and payload. const messages = [ - createIpfsPublishedMessage('evt-1', cid, 0, 0n), - createIpfsPublishedMessage('evt-1', cid, 0, 1n), + createBatchedMessage('evt-1', 'batch-1', payload, 0, 0n), + createBatchedMessage('evt-1', 'batch-1', payload, 0, 1n), ]; const { consumer, commits } = createFakeConsumer(messages); @@ -229,15 +233,13 @@ describe('blockchain-anchor idempotency (issue #27)', () => { ]); }); - it('anchors distinct CIDs independently without treating them as duplicates', async () => { - const cidA = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); - const cidB = CID.parse( - 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi' - ); + it('anchors distinct batches independently without treating them as duplicates', async () => { + const payloadA = Buffer.from('compressed-batch-bytes-a'); + const payloadB = Buffer.from('compressed-batch-bytes-b'); const messages = [ - createIpfsPublishedMessage('evt-a', cidA, 0, 0n), - createIpfsPublishedMessage('evt-b', cidB, 0, 1n), + createBatchedMessage('evt-a', 'batch-a', payloadA, 0, 0n), + createBatchedMessage('evt-b', 'batch-b', payloadB, 0, 1n), ]; const { consumer } = createFakeConsumer(messages); @@ -264,4 +266,46 @@ describe('blockchain-anchor idempotency (issue #27)', () => { expect(metrics.anchored).toBe(2); expect(metrics.skippedDuplicate).toBe(0); }); + + it('rejects a payload larger than maxPayloadBytes without submitting or splitting it', async () => { + const oversizedPayload = Buffer.alloc(200, 9); + + const messages = [ + createBatchedMessage( + 'evt-oversized', + 'batch-oversized', + oversizedPayload, + 0, + 0n + ), + ]; + + const { consumer, commits } = createFakeConsumer(messages); + const { api, setPayloadCalls } = createFakeApi(); + + const service = createBlockchainAnchorService( + testConfig({ maxPayloadBytes: 100 }), + { + createConsumer: () => consumer, + createApi: () => Promise.resolve(api), + createHealthServer: () => + ({ + close(callback: (error?: Error) => void) { + callback(); + }, + }) as unknown as import('node:http').Server, + } + ); + + await service.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + await service.stop(); + + expect(setPayloadCalls).toHaveLength(0); + expect(commits).toEqual([{ partition: 0, offset: 1n }]); + + const metrics = service.getMetrics(); + expect(metrics.rejectedOversized).toBe(1); + expect(metrics.anchored).toBe(0); + }); }); diff --git a/services/blockchain-anchor/tests/shutdown.test.ts b/services/blockchain-anchor/tests/shutdown.test.ts index 616a602..a131359 100644 --- a/services/blockchain-anchor/tests/shutdown.test.ts +++ b/services/blockchain-anchor/tests/shutdown.test.ts @@ -15,15 +15,13 @@ */ import { TELEMETRY_TOPICS, - TelemetryIpfsPublishedPayloadSchema, - TelemetryIpfsPublishedPayload_Compression, + TelemetryBatchedPayloadSchema, EnvelopeSchema, } from '@scp/core'; import { create, toBinary } from '@bufbuild/protobuf'; import type { ApiPromise } from '@polkadot/api'; import type { Consumer } from '@platformatic/kafka'; import { describe, expect, it } from 'vitest'; -import { CID } from 'multiformats/cid'; import { createBlockchainAnchorService } from '../src/index.js'; import type { BlockchainAnchorConfig } from '../src/config.js'; @@ -37,6 +35,7 @@ function testConfig( suri: '//Alice', nodeId: 0, healthPort: 3052, + maxPayloadBytes: 8192, ...overrides, }; } @@ -48,29 +47,34 @@ interface FakeMessage { value: Buffer; } -function createIpfsPublishedMessage( +function createBatchedMessage( eventId: string, - cid: CID, + batchId: string, + payload: Uint8Array, partition: number, offset: bigint ): FakeMessage { - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(cid.bytes), + const batchedPayload = create(TelemetryBatchedPayloadSchema, { + batchId, + payload, eventCount: 5, - compression: TelemetryIpfsPublishedPayload_Compression.NONE, + sensorIds: [Buffer.alloc(32, 1)], + uncompressedSize: payload.length * 2, + compressedSize: payload.length, + payloadHash: Buffer.alloc(32, 3), }); const envelope = create(EnvelopeSchema, { eventId, - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, + eventType: TELEMETRY_TOPICS.BATCHED, eventVersion: '1.0.0', occurredAt: '2026-01-01T00:00:00Z', - source: 'ipfs-publisher', - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), + source: 'batcher', + payload: toBinary(TelemetryBatchedPayloadSchema, batchedPayload), }); return { - topic: TELEMETRY_TOPICS.IPFS_PUBLISHED, + topic: TELEMETRY_TOPICS.BATCHED, partition, offset, value: Buffer.from(toBinary(EnvelopeSchema, envelope)), @@ -104,8 +108,8 @@ function createFakeConsumer(messages: FakeMessage[]): { consumer: Consumer } { describe('blockchain-anchor graceful shutdown (issue #32)', () => { it('does not disconnect the chain API until the in-flight extrinsic finishes', async () => { - const cid = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); - const messages = [createIpfsPublishedMessage('evt-1', cid, 0, 0n)]; + const payload = Buffer.from('compressed-batch-bytes'); + const messages = [createBatchedMessage('evt-1', 'batch-1', payload, 0, 0n)]; const { consumer } = createFakeConsumer(messages); const events: string[] = []; diff --git a/services/ipfs-publisher/README.md b/services/ipfs-publisher/README.md deleted file mode 100644 index 3f8cea2..0000000 --- a/services/ipfs-publisher/README.md +++ /dev/null @@ -1,143 +0,0 @@ -# `@scp/ipfs-publisher` - -Publishes batched telemetry to IPFS for permanent archival storage. - -## Architecture - -- **Pattern**: Standard consumer with manual commit (durable archival) -- **Input**: `telemetry.batched.v1` from Kafka (produced by `@scp/batcher`) -- **Output**: Batches published to IPFS + result events to `ipfs.published.v1` -- **Autocommit**: Disabled (commit only after IPFS confirmation) -- **Failure handling**: On failure the offset is not committed and the batch is redelivered; in-process `batch_id` deduplication prevents double publication of redelivered batches - -Batching is handled upstream by `@scp/batcher`. This service consumes each -`telemetry.batched.v1` event, which carries a serialized `SignedEnvelopeBatch`, -optionally compresses it with XZ, publishes it to one or more IPFS providers, -and emits the resulting Content-ID (CID) for downstream blockchain anchoring. - -## Multi-Provider Publishing - -`ipfs-publisher` supports publishing the same content to multiple IPFS-compatible -storage providers so that an outage of one provider does not block the pipeline. - -- **Providers**: `kubo` (Kubo/go-ipfs RPC API) and `pinata` (Pinata pinning REST - API) are supported today; more providers can be added behind the same - `IpfsProvider` interface (`src/providers/types.ts`). -- **Ordering**: `IPFS_PROVIDERS` is an ordered, comma-separated list. The first - entry is the **primary** provider and is always attempted first; the rest are - fallbacks used for replication and durability. -- **Durability policy**: configurable via `IPFS_DURABILITY_POLICY`: - - `any` (default) - at least one provider must succeed - - `all` - every configured provider must succeed - - `quorum` - at least `IPFS_DURABILITY_MIN_SUCCESS_COUNT` providers must succeed -- **Commit behavior**: the Kafka offset is only committed once the durability - policy is satisfied. If it isn't, the message is retried (and eventually - routed to the DLQ after exhausting retries), while a **background retry - loop** keeps independently retrying only the providers that failed - - providers that already succeeded are never re-uploaded. -- **Identical content**: compression happens once, before publishing, so every - provider receives byte-identical input (and therefore the same CID, where - providers use compatible content-addressing). - -## Environment - -- `KAFKA_BROKERS` (default: `localhost:9092`) -- `IPFS_PUBLISHER_GROUP_ID` (default: `ipfs-publisher-v1`) -- `IPFS_PUBLISHER_SOURCE` (default: `ipfs-publisher`) -- `IPFS_PUBLISHER_HEALTH_PORT` (default: `3040`) -- `IPFS_PUBLISHER_ENABLE_COMPRESSION` (default: `true`) - XZ compression before IPFS upload -- `IPFS_PROVIDERS` (default: `kubo`) - ordered, comma-separated provider list; first is primary (e.g. `pinata,kubo`) -- `IPFS_API_URL` (default: `http://localhost:5001`) - Kubo RPC endpoint (used by the `kubo` provider) -- `PINATA_API_URL` (default: `https://api.pinata.cloud`) - Pinata API base URL (used by the `pinata` provider) -- `PINATA_JWT` (default: none) - Pinata JWT bearer token (required if `pinata` is configured) -- `IPFS_DURABILITY_POLICY` (default: `any`) - `any` | `all` | `quorum` -- `IPFS_DURABILITY_MIN_SUCCESS_COUNT` (default: `1`) - minimum successful providers required for `quorum` -- `IPFS_PROVIDER_RETRY_BASE_DELAY_MS` (default: `5000`) - initial delay between independent per-provider replication retries -- `IPFS_PROVIDER_RETRY_MAX_DELAY_MS` (default: `60000`) - cap for the exponential backoff between replication retries -- `IPFS_PROVIDER_REPLICATION_MAX_PENDING` (default: `1000`) - max number of batches awaiting replication retry; oldest entries are dropped once exceeded - -## Metrics - -Available at `http://localhost:3040/metrics`: - -- `consumed`: Total batched events consumed -- `batchesPublished`: Total batches successfully published (durability policy satisfied) -- `eventsPublished`: Total individual telemetry events archived to IPFS -- `duplicatesSkipped`: Redelivered batches skipped via `batch_id` result-event deduplication -- `publishFailure`: Failed publish attempts (all providers failed, or durability policy not satisfied) -- `durabilityFailures`: Number of publish attempts where at least one provider succeeded but the durability policy was not satisfied -- `pendingReplications`: Number of batches with at least one provider still pending replication retry -- `providers`: Per-provider `{ success, failure, avgLatencyMs }` counters - -## Data Flow - -1. Consume `telemetry.batched.v1` event from Kafka -2. Decode `TelemetryBatchedPayload` (carries a serialized `SignedEnvelopeBatch`) -3. Optionally compress the batch bytes with XZ (LZMA2), once, before publishing -4. Publish to the primary provider, then replicate to configured fallback providers - - Providers that already succeeded for this `batch_id` (e.g. on redelivery) are skipped -5. Evaluate the configured durability policy against the per-provider outcomes -6. If satisfied, emit `ipfs.published.v1` (once per `batch_id`, deduplicated) and commit the Kafka offset -7. If not satisfied, the offset is left uncommitted (message is retried/DLQ'd), while a background - loop continues retrying only the still-failing providers independently - -## Result Event Schema - -Published to `ipfs.published.v1`: - -```typescript -{ - cid: Uint8Array; // CID bytes (use multiformats/cid to parse) - eventCount: number; // Number of telemetry events in batch - compression: Compression; // NONE or XZ -} -``` - -## Development - -```bash -# Start ipfs-publisher -pnpm --filter @scp/ipfs-publisher dev - -# Build -pnpm --filter @scp/ipfs-publisher build - -# Run tests -pnpm --filter @scp/ipfs-publisher test - -# Type check -pnpm --filter @scp/ipfs-publisher typecheck - -# Lint -pnpm --filter @scp/ipfs-publisher lint -``` - -## Dependencies - -- **Kafka**: For consuming batched telemetry and publishing results -- **IPFS Kubo** and/or **Pinata**: For permanent content-addressed storage, depending on `IPFS_PROVIDERS` - -## Commit Safety - -The service uses manual offset commits to ensure at-least-once delivery semantics: - -1. Batch is published to the primary provider, then replicated to fallback providers -2. The configured durability policy is evaluated against per-provider outcomes -3. If satisfied, the result event (with the canonical CID) is published to Kafka -4. Kafka offset is committed - -If durability is not satisfied, the offset remains uncommitted and the batch is -reprocessed; providers that already succeeded are tracked per `batch_id` and are -not re-uploaded on the next attempt. This ensures no telemetry is lost. In-process -`batch_id` deduplication avoids emitting a duplicate result event for a batch that -is redelivered after it already satisfied the durability policy. - -## Compression - -XZ compression (enabled by default) significantly reduces IPFS storage and bandwidth: - -- **Compression level**: Default (level 6, LZMA2 algorithm) -- **Typical savings**: 60-80% for telemetry batches -- **Trade-off**: Slight CPU overhead during publish - -Disable with `IPFS_PUBLISHER_ENABLE_COMPRESSION=false` for low-latency scenarios where storage is not a constraint. diff --git a/services/ipfs-publisher/package.json b/services/ipfs-publisher/package.json deleted file mode 100644 index 3c9ecf8..0000000 --- a/services/ipfs-publisher/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "@scp/ipfs-publisher", - "version": "0.1.0", - "private": true, - "type": "module", - "main": "dist/index.mjs", - "types": "dist/index.d.mts", - "scripts": { - "build": "tsdown", - "typecheck": "tsc -p tsconfig.json --noEmit", - "lint": "eslint \"src/**/*.ts\"", - "test": "vitest run", - "dev": "tsx watch --env-file=../../.env src/index.ts" - }, - "dependencies": { - "@buf/airalab_connectivity-protocol.bufbuild_es": "2.14.0-20260917143433-6f6f1eba1cc0.2", - "@platformatic/kafka": "^2.9.0", - "multiformats": "^14.0.5", - "@scp/core": "workspace:*", - "kubo-rpc-client": "^7.1.0", - "@napi-rs/lzma": "^1.5.1", - "pino": "^10.3.1" - }, - "devDependencies": { - "@bufbuild/protobuf": "^2.14.0", - "tsdown": "^0.22.14", - "tsx": "^4.23.12" - } -} diff --git a/services/ipfs-publisher/src/config.ts b/services/ipfs-publisher/src/config.ts deleted file mode 100644 index 048881e..0000000 --- a/services/ipfs-publisher/src/config.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import type { DurabilityPolicy } from './durability.js'; - -export interface IpfsPublisherConfig { - kafkaBrokers: string[]; - consumerGroupId: string; - source: string; - healthPort: number; - ipfsApiUrl: string; - enableCompression: boolean; - /** Ordered list of provider names; the first entry is the primary. */ - providers: string[]; - pinataApiUrl: string; - pinataJwt: string; - durabilityPolicy: DurabilityPolicy; - durabilityMinSuccessCount: number; - providerRetryBaseDelayMs: number; - providerRetryMaxDelayMs: number; - providerReplicationMaxPending: number; -} - -function parsePositiveInt(value: string | undefined, fallback: number): number { - const parsed = Number.parseInt(value ?? '', 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; -} - -function parseDurabilityPolicy(value: string | undefined): DurabilityPolicy { - return value === 'all' || value === 'quorum' ? value : 'any'; -} - -export function loadIpfsPublisherConfig( - env: NodeJS.ProcessEnv = process.env -): IpfsPublisherConfig { - const providers = (env.IPFS_PROVIDERS ?? 'kubo') - .split(',') - .map((provider) => provider.trim()) - .filter((provider) => provider.length > 0); - - return { - kafkaBrokers: (env.KAFKA_BROKERS ?? 'localhost:9092') - .split(',') - .map((broker) => broker.trim()) - .filter((broker) => broker.length > 0), - consumerGroupId: env.IPFS_PUBLISHER_GROUP_ID ?? 'ipfs-publisher-v1', - source: env.IPFS_PUBLISHER_SOURCE ?? 'ipfs-publisher', - healthPort: parsePositiveInt(env.IPFS_PUBLISHER_HEALTH_PORT, 3040), - ipfsApiUrl: env.IPFS_API_URL ?? 'http://localhost:5001', - enableCompression: env.IPFS_PUBLISHER_ENABLE_COMPRESSION !== 'false', - providers: providers.length > 0 ? providers : ['kubo'], - pinataApiUrl: env.PINATA_API_URL ?? 'https://api.pinata.cloud', - pinataJwt: env.PINATA_JWT ?? '', - durabilityPolicy: parseDurabilityPolicy(env.IPFS_DURABILITY_POLICY), - durabilityMinSuccessCount: parsePositiveInt( - env.IPFS_DURABILITY_MIN_SUCCESS_COUNT, - 1 - ), - providerRetryBaseDelayMs: parsePositiveInt( - env.IPFS_PROVIDER_RETRY_BASE_DELAY_MS, - 5000 - ), - providerRetryMaxDelayMs: parsePositiveInt( - env.IPFS_PROVIDER_RETRY_MAX_DELAY_MS, - 60000 - ), - providerReplicationMaxPending: parsePositiveInt( - env.IPFS_PROVIDER_REPLICATION_MAX_PENDING, - 1000 - ), - }; -} diff --git a/services/ipfs-publisher/src/durability.ts b/services/ipfs-publisher/src/durability.ts deleted file mode 100644 index a1de432..0000000 --- a/services/ipfs-publisher/src/durability.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * `any`: at least one provider must succeed. - * `all`: every configured provider must succeed. - * `quorum`: at least `minSuccessCount` providers must succeed. - */ -export type DurabilityPolicy = 'any' | 'all' | 'quorum'; - -export interface DurabilityConfig { - policy: DurabilityPolicy; - /** Only used when `policy === 'quorum'`. */ - minSuccessCount: number; -} - -/** - * Evaluate whether the durability requirement is satisfied given how many - * providers succeeded out of the total configured. - */ -export function isDurabilitySatisfied( - config: DurabilityConfig, - successCount: number, - totalProviders: number -): boolean { - if (totalProviders === 0) { - return false; - } - - switch (config.policy) { - case 'any': - return successCount >= 1; - case 'all': - return successCount >= totalProviders; - case 'quorum': - return successCount >= Math.min(config.minSuccessCount, totalProviders); - default: - return false; - } -} diff --git a/services/ipfs-publisher/src/index.ts b/services/ipfs-publisher/src/index.ts deleted file mode 100644 index 01be530..0000000 --- a/services/ipfs-publisher/src/index.ts +++ /dev/null @@ -1,650 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - TELEMETRY_TOPICS, - EnvelopeSchema, - TelemetryBatchedPayloadSchema, - TelemetryIpfsPublishedPayloadSchema, - TelemetryIpfsPublishedPayload_Compression as Compression, - type TelemetryBatchedPayload, - formatSensorId, - installShutdownHandler, -} from '@scp/core'; -import { fromBinary, toBinary, create } from '@bufbuild/protobuf'; -import { Consumer, Producer } from '@platformatic/kafka'; -import { createServer, type Server } from 'node:http'; -import { fileURLToPath } from 'node:url'; -import { CID } from 'multiformats/cid'; -import { compress } from '@napi-rs/lzma/xz'; -import { loadIpfsPublisherConfig, type IpfsPublisherConfig } from './config.js'; -import { logInfo, logWarn, logDebug, logError } from './logger.js'; -import { - MultiProviderIpfsClient, - type ProviderStat, -} from './multi-provider-client.js'; -import { - createProvidersFromConfig, - type IpfsProvider, -} from './providers/index.js'; - -interface IpfsPublisherMetrics { - consumed: number; - batchesPublished: number; - eventsPublished: number; - duplicatesSkipped: number; - publishFailure: number; -} - -interface IpfsPublisherMetricsSnapshot extends IpfsPublisherMetrics { - durabilityFailures: number; - pendingReplications: number; - providers: Record< - string, - { success: number; failure: number; avgLatencyMs: number } - >; -} - -export interface IpfsPublisherService { - start(): Promise; - stop(): Promise; - getMetrics(): Readonly; -} - -interface IpfsPublisherDeps { - createProviders?: (config: IpfsPublisherConfig) => IpfsProvider[]; - createConsumer?: () => Consumer; - createProducer?: () => Producer; - createHealthServer?: ( - getMetrics: () => IpfsPublisherMetricsSnapshot, - port: number - ) => Server; -} - -/** - * Bounded set that remembers recently seen batch ids for in-process - * deduplication, so a redelivered batch does not emit a duplicate - * `ipfs.published.v1` result event within the lifetime of this process. - * - * This is a best-effort fast path only, not the durability guarantee: it is - * lost on restart. Durable idempotency instead comes from deriving the - * envelope's `event_id` deterministically from the batch id (see - * `publishBatched`), so a replay after a crash re-emits an event with the - * same identity rather than a new one, and `ipfs.published` is documented - * and consumed as an at-least-once, idempotent-by-event_id stream. - */ -function createBoundedDedup(capacity: number): { - has: (id: string) => boolean; - add: (id: string) => void; -} { - const seen = new Set(); - const order: string[] = []; - return { - has: (id) => seen.has(id), - add: (id) => { - if (seen.has(id)) { - return; - } - seen.add(id); - order.push(id); - if (order.length > capacity) { - const evicted = order.shift(); - if (evicted !== undefined) { - seen.delete(evicted); - } - } - }, - }; -} - -/** Maximum attempts to process a single message before routing it to DLQ. */ -const MAX_PROCESS_ATTEMPTS = 3; -/** Base delay between processing retries; grows linearly with attempt. */ -const RETRY_BASE_DELAY_MS = 200; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** - * Forward a message that exhausted its processing retries to the DLQ topic, - * carrying the original bytes plus failure context in headers. - */ -async function publishToDlq( - raw: Uint8Array, - partition: number, - offset: bigint, - reason: string, - producer: Producer, - config: IpfsPublisherConfig -): Promise { - await producer.send({ - messages: [ - { - topic: TELEMETRY_TOPICS.DLQ, - value: Buffer.from(raw), - headers: { - source_topic: Buffer.from(TELEMETRY_TOPICS.BATCHED), - source_service: Buffer.from(config.source), - source_partition: Buffer.from(String(partition)), - source_offset: Buffer.from(String(offset)), - reason: Buffer.from(reason), - }, - }, - ], - }); -} - -/** - * Publish a single batched telemetry payload across all configured IPFS - * providers and emit a `ipfs.published.v1` result event once the configured - * durability policy is satisfied. Safe to call again for the same batch - * (e.g. on Kafka redelivery): providers that already succeeded are not - * re-uploaded, and the result event is emitted at most once. - */ -async function publishBatched( - batched: TelemetryBatchedPayload, - ipfsClient: MultiProviderIpfsClient, - producer: Producer, - config: IpfsPublisherConfig, - metrics: IpfsPublisherMetrics, - emittedDedup: { has: (id: string) => boolean; add: (id: string) => void } -): Promise { - const uniqueSensorIds = Array.from( - new Set(batched.sensorIds.map((id) => formatSensorId(id))) - ); - - logInfo('publishing batch to IPFS', { - batch_id: batched.batchId, - event_count: batched.eventCount, - unique_sensors: uniqueSensorIds.length, - sensor_ids: uniqueSensorIds, - }); - - // Compress once (if enabled) so every provider receives identical bytes, - // preserving content (and CID, where the providers use compatible - // content-addressing) across providers. - const dataToPublish = config.enableCompression - ? await compress(batched.signedEnvelopeBatch) - : batched.signedEnvelopeBatch; - - let result; - try { - result = await ipfsClient.publish( - batched.batchId, - dataToPublish, - config.enableCompression - ); - } catch (error) { - metrics.publishFailure += 1; - logError('batch publish failed on all providers', error, { - batch_id: batched.batchId, - event_count: batched.eventCount, - unique_sensors: uniqueSensorIds.length, - sensor_ids: uniqueSensorIds, - }); - throw error; - } - - const providerSummary = Object.fromEntries( - Array.from(result.providerOutcomes.entries()).map(([name, outcome]) => [ - name, - outcome.success, - ]) - ); - - logInfo('batch publish attempt completed', { - batch_id: batched.batchId, - cid: result.cid, - event_count: batched.eventCount, - unique_sensors: uniqueSensorIds.length, - sensor_ids: uniqueSensorIds, - compression: config.enableCompression, - durability_satisfied: result.durabilitySatisfied, - providers: providerSummary, - }); - - if (!result.durabilitySatisfied) { - metrics.publishFailure += 1; - throw new Error( - `durability policy "${config.durabilityPolicy}" not satisfied for batch ${batched.batchId} (providers: ${JSON.stringify(providerSummary)})` - ); - } - - if (emittedDedup.has(batched.batchId)) { - metrics.duplicatesSkipped += 1; - logDebug('ipfs.published event already emitted for batch; skipping', { - batch_id: batched.batchId, - }); - return; - } - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(CID.parse(result.cid).bytes), - eventCount: batched.eventCount, - compression: config.enableCompression ? Compression.XZ : Compression.NONE, - }); - - const resultEnvelope = create(EnvelopeSchema, { - // Derive the event id deterministically from the durable batch id - // instead of a fresh random UUID. In-memory dedup (`emittedDedup`) is - // lost on process restart, so if this batch is redelivered after a - // crash between IPFS publish and Kafka offset commit, the re-emitted - // `ipfs.published` event carries the *same* event_id as the original. - // Downstream consumers that dedupe by event_id (or, like - // blockchain-anchor, by authoritative on-chain state) therefore observe - // an at-least-once but idempotent stream rather than silently gaining a - // second distinct logical event for the same batch. - eventId: batched.batchId, - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, - eventVersion: '1.0.0', - occurredAt: new Date().toISOString(), - source: config.source, - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), - }); - - await producer.send({ - messages: [ - { - topic: TELEMETRY_TOPICS.IPFS_PUBLISHED, - value: Buffer.from(toBinary(EnvelopeSchema, resultEnvelope)), - }, - ], - }); - - emittedDedup.add(batched.batchId); - metrics.batchesPublished += 1; - metrics.eventsPublished += batched.eventCount; - - logInfo('batch result published to Kafka', { - batch_id: batched.batchId, - cid: result.cid, - event_count: batched.eventCount, - result_topic: TELEMETRY_TOPICS.IPFS_PUBLISHED, - }); -} - -export function createIpfsPublisherService( - config: IpfsPublisherConfig = loadIpfsPublisherConfig(), - deps: IpfsPublisherDeps = {} -): IpfsPublisherService { - const consumer = - deps.createConsumer?.() ?? - new Consumer({ - groupId: config.consumerGroupId, - clientId: 'ipfs-publisher', - bootstrapBrokers: config.kafkaBrokers, - }); - - const producer = - deps.createProducer?.() ?? - new Producer({ - clientId: 'ipfs-publisher', - bootstrapBrokers: config.kafkaBrokers, - }); - - const createProviders = - deps.createProviders ?? - ((cfg: IpfsPublisherConfig) => createProvidersFromConfig(cfg)); - const createHealthServer = - deps.createHealthServer ?? startHealthAndMetricsServer; - - let ipfsClient: MultiProviderIpfsClient | null = null; - let started = false; - let runPromise: Promise | null = null; - let healthServer: Server | null = null; - let shouldStop = false; - - const metrics: IpfsPublisherMetrics = { - consumed: 0, - batchesPublished: 0, - eventsPublished: 0, - duplicatesSkipped: 0, - publishFailure: 0, - }; - - const buildMetricsSnapshot = (): IpfsPublisherMetricsSnapshot => { - const providers: Record< - string, - { success: number; failure: number; avgLatencyMs: number } - > = {}; - if (ipfsClient) { - for (const [name, stat] of ipfsClient.getProviderStats().entries()) { - const stats: ProviderStat = stat; - const totalAttempts = stats.success + stats.failure; - providers[name] = { - success: stats.success, - failure: stats.failure, - avgLatencyMs: - totalAttempts > 0 ? stats.totalLatencyMs / totalAttempts : 0, - }; - } - } - - return { - ...metrics, - durabilityFailures: ipfsClient?.getDurabilityFailureCount() ?? 0, - pendingReplications: ipfsClient?.getPendingReplicationCount() ?? 0, - providers, - }; - }; - - return { - async start(): Promise { - if (started) { - logInfo('start skipped; service already started'); - return; - } - - started = true; - shouldStop = false; - logInfo('starting service', { - consumerGroupId: config.consumerGroupId, - kafkaBrokers: config.kafkaBrokers, - providers: config.providers, - durabilityPolicy: config.durabilityPolicy, - durabilityMinSuccessCount: config.durabilityMinSuccessCount, - enableCompression: config.enableCompression, - healthPort: config.healthPort, - }); - - try { - ipfsClient = new MultiProviderIpfsClient(createProviders(config), { - durability: { - policy: config.durabilityPolicy, - minSuccessCount: config.durabilityMinSuccessCount, - }, - retryBaseDelayMs: config.providerRetryBaseDelayMs, - retryMaxDelayMs: config.providerRetryMaxDelayMs, - maxPendingReplications: config.providerReplicationMaxPending, - }); - await ipfsClient.start(); - - const consumerStream = await consumer.consume({ - topics: [TELEMETRY_TOPICS.BATCHED], - autocommit: false, - }); - - healthServer = createHealthServer( - buildMetricsSnapshot, - config.healthPort - ); - - runPromise = (async () => { - const emittedDedup = createBoundedDedup(10000); - - messageLoop: for await (const message of consumerStream) { - if (shouldStop) { - break; - } - - if (!message.value) { - logWarn('received null message value; skipping'); - continue; - } - - const raw = new Uint8Array(message.value); - - // Retry processing of this message in place before moving on to - // the next one. Continuing to later messages on failure without - // committing would let a later successful commit (a higher - // offset) silently skip past this uncommitted one, breaking the - // at-least-once contract. - for ( - let attempt = 1; - attempt <= MAX_PROCESS_ATTEMPTS; - attempt += 1 - ) { - try { - const envelope = fromBinary(EnvelopeSchema, raw); - - if (envelope.eventType !== TELEMETRY_TOPICS.BATCHED) { - logDebug('non-batched envelope ignored', { - eventType: envelope.eventType, - }); - continue messageLoop; - } - - const batched = fromBinary( - TelemetryBatchedPayloadSchema, - envelope.payload - ) as TelemetryBatchedPayload; - - if (attempt === 1) { - metrics.consumed += 1; - } - - await publishBatched( - batched, - ipfsClient!, - producer, - config, - metrics, - emittedDedup - ); - - // Commit offset only once the configured durability policy - // is satisfied (checked inside publishBatched, which throws - // otherwise). - await consumer.commit({ - offsets: [ - { - topic: TELEMETRY_TOPICS.BATCHED, - partition: message.partition, - offset: message.offset + 1n, - leaderEpoch: -1, - }, - ], - }); - - logDebug('kafka offset committed', { - batch_id: batched.batchId, - partition: message.partition, - offset: (message.offset + 1n).toString(), - }); - - continue messageLoop; - } catch (error) { - const reason = - error instanceof Error ? error.message : String(error); - - if (attempt < MAX_PROCESS_ATTEMPTS) { - logWarn('batch processing failed, retrying', { - partition: message.partition, - offset: message.offset.toString(), - attempt, - max_attempts: MAX_PROCESS_ATTEMPTS, - error: reason, - }); - await sleep(RETRY_BASE_DELAY_MS * attempt); - continue; - } - - logError( - 'batch processing exhausted retries, routing to DLQ', - error, - { - partition: message.partition, - offset: message.offset.toString(), - } - ); - - try { - await publishToDlq( - raw, - message.partition, - message.offset, - reason, - producer, - config - ); - - // Only commit past the poisoned message now that it is - // durably routed to the DLQ, so nothing is silently lost. - await consumer.commit({ - offsets: [ - { - topic: TELEMETRY_TOPICS.BATCHED, - partition: message.partition, - offset: message.offset + 1n, - leaderEpoch: -1, - }, - ], - }); - - logWarn('poisoned batch routed to DLQ and committed', { - partition: message.partition, - offset: message.offset.toString(), - }); - } catch (dlqError) { - // Cannot safely commit past this message: stop consuming - // rather than let a later message's commit skip past it. - logError( - 'failed to route batch to DLQ; stopping consumption to avoid silent offset skip', - dlqError, - { - partition: message.partition, - offset: message.offset.toString(), - } - ); - break messageLoop; - } - } - } - } - })(); - - logInfo('service started'); - } catch (error) { - logError('service failed to start', error); - started = false; - runPromise = null; - - if (healthServer) { - await new Promise((resolve) => { - healthServer?.close(() => { - resolve(); - }); - }); - healthServer = null; - } - - await consumer.close().catch(() => undefined); - await ipfsClient?.stop().catch(() => undefined); - ipfsClient = null; - logInfo('startup rollback complete'); - throw error; - } - }, - async stop(): Promise { - if (!started) { - logInfo('stop skipped; service not started'); - return; - } - - started = false; - shouldStop = true; - logInfo('stopping service'); - - await consumer.close(); - - await runPromise?.catch(() => undefined); - runPromise = null; - - await producer.close().catch(() => undefined); - await ipfsClient?.stop(); - ipfsClient = null; - - if (healthServer) { - await new Promise((resolve, reject) => { - healthServer?.close((error) => { - if (error) { - reject(error); - return; - } - resolve(); - }); - }); - healthServer = null; - } - - logInfo('service stopped'); - }, - getMetrics(): Readonly { - return buildMetricsSnapshot(); - }, - }; -} - -function startHealthAndMetricsServer( - getMetrics: () => IpfsPublisherMetricsSnapshot, - port: number -): Server { - const server = createServer((request, response) => { - if (request.url === '/health' || request.url === '/healthz') { - response.statusCode = 200; - response.setHeader('content-type', 'application/json; charset=utf-8'); - response.end(JSON.stringify({ status: 'ok' })); - return; - } - - if (request.url === '/metrics') { - logDebug('metrics endpoint requested'); - const metrics = getMetrics(); - logInfo('metrics served', { - consumed: metrics.consumed, - batchesPublished: metrics.batchesPublished, - eventsPublished: metrics.eventsPublished, - duplicatesSkipped: metrics.duplicatesSkipped, - publishFailure: metrics.publishFailure, - durabilityFailures: metrics.durabilityFailures, - pendingReplications: metrics.pendingReplications, - }); - response.statusCode = 200; - response.setHeader('content-type', 'application/json; charset=utf-8'); - response.end(JSON.stringify(metrics)); - return; - } - - response.statusCode = 404; - response.end('not found'); - }); - server.listen({ host: '0.0.0.0', port }); - logInfo('HTTP server listening', { port, host: '0.0.0.0' }); - return server; -} - -export async function startIpfsPublisher(): Promise { - const service = createIpfsPublisherService(); - await service.start(); - logInfo('service started (direct run)'); - return service; -} - -const isDirectRun = process.argv[1] === fileURLToPath(import.meta.url); -if (isDirectRun) { - startIpfsPublisher() - .then((service) => { - installShutdownHandler(() => service.stop(), { - onSignal: (signal) => logInfo('received shutdown signal', { signal }), - onShutdownError: (error) => - logError('error during graceful shutdown', error), - }); - }) - .catch((error: unknown) => { - logError('failed to start (direct run)', error); - process.exitCode = 1; - }); -} diff --git a/services/ipfs-publisher/src/logger.ts b/services/ipfs-publisher/src/logger.ts deleted file mode 100644 index bb85ab6..0000000 --- a/services/ipfs-publisher/src/logger.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import pino from 'pino'; - -const logger = pino({ - name: 'ipfs-publisher', - level: - process.env.IPFS_PUBLISHER_LOG_LEVEL ?? process.env.LOG_LEVEL ?? 'info', -}); - -export function logInfo( - message: string, - context?: Record -): void { - logger.info(context ?? {}, message); -} - -export function logWarn( - message: string, - context?: Record -): void { - logger.warn(context ?? {}, message); -} - -export function logDebug( - message: string, - context?: Record -): void { - logger.debug(context ?? {}, message); -} - -export function logError( - message: string, - error: unknown, - context?: Record -): void { - logger.error( - { - ...(context ?? {}), - error: error instanceof Error ? error.message : String(error), - }, - message - ); -} diff --git a/services/ipfs-publisher/src/multi-provider-client.ts b/services/ipfs-publisher/src/multi-provider-client.ts deleted file mode 100644 index 21d7e57..0000000 --- a/services/ipfs-publisher/src/multi-provider-client.ts +++ /dev/null @@ -1,368 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { logDebug, logError, logInfo, logWarn } from './logger.js'; -import { isDurabilitySatisfied, type DurabilityConfig } from './durability.js'; -import type { IpfsProvider } from './providers/types.js'; - -export interface ProviderOutcome { - success: boolean; - cid?: string; - lastError?: string; - latencyMs?: number; -} - -export interface PublishResult { - /** Canonical CID for the batch (from the first provider that succeeded). */ - cid: string; - /** Whether the configured durability policy is currently satisfied. */ - durabilitySatisfied: boolean; - /** Per-provider outcome as of this call. */ - providerOutcomes: ReadonlyMap; -} - -export interface ProviderStat { - success: number; - failure: number; - totalLatencyMs: number; -} - -export interface MultiProviderClientOptions { - durability: DurabilityConfig; - retryBaseDelayMs: number; - retryMaxDelayMs: number; - maxPendingReplications: number; -} - -/** Per-batch replication state, tracked so retries never re-upload to a - * provider that already succeeded. */ -interface BatchReplicationState { - cid?: string; - providerOutcomes: Map; -} - -interface PendingReplication { - batchId: string; - data: Uint8Array; - compressed: boolean; - attempts: number; - nextAttemptAt: number; -} - -/** - * Orchestrates publishing a batch to an ordered list of IPFS providers - * (primary first, then fallbacks), evaluates a configurable durability - * policy, and independently retries only the providers that failed without - * re-uploading to providers that already succeeded. - */ -export class MultiProviderIpfsClient { - private readonly providers: IpfsProvider[]; - private readonly options: MultiProviderClientOptions; - private readonly batchStates = new Map(); - private readonly batchStateOrder: string[] = []; - private readonly pending = new Map(); - private readonly providerStats = new Map(); - private durabilityFailures = 0; - private retryTimer: NodeJS.Timeout | null = null; - - constructor(providers: IpfsProvider[], options: MultiProviderClientOptions) { - if (providers.length === 0) { - throw new Error('at least one IPFS provider must be configured'); - } - this.providers = providers; - this.options = options; - for (const provider of providers) { - this.providerStats.set(provider.name, { - success: 0, - failure: 0, - totalLatencyMs: 0, - }); - } - } - - async start(): Promise { - const failures: string[] = []; - for (const provider of this.providers) { - try { - await provider.start(); - } catch (error) { - failures.push(provider.name); - logError('IPFS provider failed to start', error, { - provider: provider.name, - }); - } - } - - if (failures.length === this.providers.length) { - throw new Error( - `all configured IPFS providers failed to start: ${failures.join(', ')}` - ); - } - - this.retryTimer = setInterval(() => { - this.processPendingReplications().catch((error: unknown) => { - logError('replication retry loop failed', error); - }); - }, this.options.retryBaseDelayMs); - this.retryTimer.unref?.(); - } - - async stop(): Promise { - if (this.retryTimer) { - clearInterval(this.retryTimer); - this.retryTimer = null; - } - for (const provider of this.providers) { - await provider.stop().catch(() => undefined); - } - } - - /** - * Publish `data` for `batchId`, trying the primary provider first and then - * replicating to fallback providers. Safe to call again for the same - * `batchId` (e.g. on redelivery): providers that already succeeded are not - * retried here, only providers still pending. - */ - async publish( - batchId: string, - data: Uint8Array, - compressed: boolean - ): Promise { - const state = this.getOrCreateBatchState(batchId); - - const remaining = this.providers.filter( - (provider) => !state.providerOutcomes.get(provider.name)?.success - ); - - for (const provider of remaining) { - await this.tryProvider(provider, batchId, data, compressed, state); - } - - const successCount = this.countSuccesses(state); - const durabilitySatisfied = isDurabilitySatisfied( - this.options.durability, - successCount, - this.providers.length - ); - - if (successCount === 0) { - throw new Error( - `all IPFS providers failed for batch ${batchId}: ${this.providers - .map( - (p) => - `${p.name}=${state.providerOutcomes.get(p.name)?.lastError ?? 'unknown error'}` - ) - .join('; ')}` - ); - } - - if (!durabilitySatisfied) { - this.durabilityFailures += 1; - logWarn('durability policy not yet satisfied for batch', { - batch_id: batchId, - policy: this.options.durability.policy, - success_count: successCount, - total_providers: this.providers.length, - }); - } - - this.schedulePendingReplication(batchId, data, compressed, state); - - return { - cid: state.cid!, - durabilitySatisfied, - providerOutcomes: state.providerOutcomes, - }; - } - - getProviderStats(): ReadonlyMap { - return this.providerStats; - } - - getPendingReplicationCount(): number { - return this.pending.size; - } - - getDurabilityFailureCount(): number { - return this.durabilityFailures; - } - - private getOrCreateBatchState(batchId: string): BatchReplicationState { - let state = this.batchStates.get(batchId); - if (!state) { - state = { providerOutcomes: new Map() }; - this.batchStates.set(batchId, state); - this.batchStateOrder.push(batchId); - // Bound memory usage: evict oldest batch state once capacity exceeded. - const capacity = 10000; - if (this.batchStateOrder.length > capacity) { - const evicted = this.batchStateOrder.shift(); - if (evicted !== undefined) { - this.batchStates.delete(evicted); - } - } - } - return state; - } - - private countSuccesses(state: BatchReplicationState): number { - let count = 0; - for (const outcome of state.providerOutcomes.values()) { - if (outcome.success) { - count += 1; - } - } - return count; - } - - private async tryProvider( - provider: IpfsProvider, - batchId: string, - data: Uint8Array, - compressed: boolean, - state: BatchReplicationState - ): Promise { - const startedAt = Date.now(); - const stats = this.providerStats.get(provider.name)!; - try { - const cid = await provider.add(data, compressed); - const latencyMs = Date.now() - startedAt; - state.providerOutcomes.set(provider.name, { - success: true, - cid, - latencyMs, - }); - if (!state.cid) { - state.cid = cid; - } - stats.success += 1; - stats.totalLatencyMs += latencyMs; - logInfo('provider publish succeeded', { - batch_id: batchId, - provider: provider.name, - cid, - latency_ms: latencyMs, - }); - } catch (error) { - const latencyMs = Date.now() - startedAt; - const message = error instanceof Error ? error.message : String(error); - state.providerOutcomes.set(provider.name, { - success: false, - lastError: message, - latencyMs, - }); - stats.failure += 1; - logError('provider publish failed', error, { - batch_id: batchId, - provider: provider.name, - latency_ms: latencyMs, - }); - } - } - - private schedulePendingReplication( - batchId: string, - data: Uint8Array, - compressed: boolean, - state: BatchReplicationState - ): void { - const allSucceeded = this.providers.every( - (provider) => state.providerOutcomes.get(provider.name)?.success - ); - - if (allSucceeded) { - this.pending.delete(batchId); - return; - } - - if (this.pending.has(batchId)) { - return; - } - - if (this.pending.size >= this.options.maxPendingReplications) { - logWarn('replication retry queue full; dropping oldest entry', { - max_pending: this.options.maxPendingReplications, - }); - const oldestKey = this.pending.keys().next().value; - if (oldestKey !== undefined) { - this.pending.delete(oldestKey); - } - } - - this.pending.set(batchId, { - batchId, - data, - compressed, - attempts: 0, - nextAttemptAt: Date.now() + this.options.retryBaseDelayMs, - }); - } - - private async processPendingReplications(): Promise { - const now = Date.now(); - for (const entry of this.pending.values()) { - if (entry.nextAttemptAt > now) { - continue; - } - - const state = this.batchStates.get(entry.batchId); - if (!state) { - this.pending.delete(entry.batchId); - continue; - } - - const remaining = this.providers.filter( - (provider) => !state.providerOutcomes.get(provider.name)?.success - ); - - if (remaining.length === 0) { - this.pending.delete(entry.batchId); - continue; - } - - logDebug('retrying replication for pending providers', { - batch_id: entry.batchId, - providers: remaining.map((p) => p.name), - attempt: entry.attempts + 1, - }); - - for (const provider of remaining) { - await this.tryProvider( - provider, - entry.batchId, - entry.data, - entry.compressed, - state - ); - } - - const stillRemaining = this.providers.some( - (provider) => !state.providerOutcomes.get(provider.name)?.success - ); - - if (!stillRemaining) { - this.pending.delete(entry.batchId); - continue; - } - - entry.attempts += 1; - const backoff = Math.min( - this.options.retryBaseDelayMs * 2 ** entry.attempts, - this.options.retryMaxDelayMs - ); - entry.nextAttemptAt = Date.now() + backoff; - } - } -} diff --git a/services/ipfs-publisher/src/providers/index.ts b/services/ipfs-publisher/src/providers/index.ts deleted file mode 100644 index f75fe50..0000000 --- a/services/ipfs-publisher/src/providers/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import type { IpfsPublisherConfig } from '../config.js'; -import { createKuboProvider } from './kubo.js'; -import { createPinataProvider } from './pinata.js'; -import type { IpfsProvider } from './types.js'; - -export type { IpfsProvider } from './types.js'; - -/** - * Build the ordered list of IPFS providers from configuration. The first - * entry is the primary provider; the rest are fallbacks used for - * replication and durability. - */ -export function createProvidersFromConfig( - config: IpfsPublisherConfig -): IpfsProvider[] { - return config.providers.map((name) => { - switch (name) { - case 'kubo': - return createKuboProvider(config.ipfsApiUrl); - case 'pinata': - return createPinataProvider({ - apiUrl: config.pinataApiUrl, - jwt: config.pinataJwt, - }); - default: - throw new Error(`unknown IPFS provider "${name}"`); - } - }); -} diff --git a/services/ipfs-publisher/src/providers/kubo.ts b/services/ipfs-publisher/src/providers/kubo.ts deleted file mode 100644 index f2a5fd3..0000000 --- a/services/ipfs-publisher/src/providers/kubo.ts +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - create as createKuboClient, - type KuboRPCClient, -} from 'kubo-rpc-client'; -import { logInfo } from '../logger.js'; -import type { IpfsProvider } from './types.js'; - -/** - * Create an IPFS provider backed by a Kubo (go-ipfs) RPC endpoint. - */ -export function createKuboProvider(apiUrl: string): IpfsProvider { - const client: KuboRPCClient = createKuboClient({ url: apiUrl }); - let started = false; - - return { - name: 'kubo', - async start() { - try { - const version = await client.version(); - logInfo('connected to IPFS node', { - provider: 'kubo', - version: version.version, - apiUrl, - }); - started = true; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - throw new Error( - `Failed to connect to IPFS (kubo) at ${apiUrl}: ${errorMessage}`, - { cause: error } - ); - } - }, - async stop() { - if (!started) { - return; - } - started = false; - logInfo('IPFS provider stopped', { provider: 'kubo' }); - }, - async add(data: Uint8Array): Promise { - if (!started) { - throw new Error('kubo provider not started'); - } - const result = await client.add(data); - return result.cid.toString(); - }, - }; -} diff --git a/services/ipfs-publisher/src/providers/pinata.ts b/services/ipfs-publisher/src/providers/pinata.ts deleted file mode 100644 index 6b0e6f8..0000000 --- a/services/ipfs-publisher/src/providers/pinata.ts +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { logInfo } from '../logger.js'; -import type { IpfsProvider } from './types.js'; - -export interface PinataProviderConfig { - apiUrl: string; - jwt: string; -} - -interface PinataPinResponse { - IpfsHash: string; -} - -/** - * Create an IPFS provider backed by Pinata's pinning REST API. - * https://docs.pinata.cloud/api-reference/endpoint/pin-file-to-ipfs - */ -export function createPinataProvider( - config: PinataProviderConfig, - fetchImpl: typeof fetch = fetch -): IpfsProvider { - let started = false; - - return { - name: 'pinata', - async start() { - if (!config.jwt) { - throw new Error('Pinata provider requires a JWT credential'); - } - started = true; - logInfo('pinata provider ready', { - provider: 'pinata', - apiUrl: config.apiUrl, - }); - }, - async stop() { - if (!started) { - return; - } - started = false; - logInfo('IPFS provider stopped', { provider: 'pinata' }); - }, - async add(data: Uint8Array): Promise { - if (!started) { - throw new Error('pinata provider not started'); - } - - const formData = new FormData(); - formData.append( - 'file', - new Blob([data as BlobPart]), - 'telemetry-batch.bin' - ); - - const response = await fetchImpl( - `${config.apiUrl}/pinning/pinFileToIPFS`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${config.jwt}`, - }, - body: formData, - } - ); - - if (!response.ok) { - const body = await response.text().catch(() => ''); - throw new Error( - `Pinata publish failed with status ${response.status}: ${body}` - ); - } - - const result = (await response.json()) as PinataPinResponse; - if (!result.IpfsHash) { - throw new Error('Pinata response did not include an IpfsHash'); - } - - return result.IpfsHash; - }, - }; -} diff --git a/services/ipfs-publisher/src/providers/types.ts b/services/ipfs-publisher/src/providers/types.ts deleted file mode 100644 index b7e1a07..0000000 --- a/services/ipfs-publisher/src/providers/types.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Common interface that every IPFS storage backend must implement so the - * multi-provider publisher can treat them interchangeably. - */ -export interface IpfsProvider { - /** Stable identifier used in metrics, logs and configuration. */ - readonly name: string; - start(): Promise; - stop(): Promise; - /** - * Persist `data` to the provider's storage, returning the resulting CID. - * `compressed` indicates whether the bytes have already been XZ compressed - * by the caller (providers must not re-compress). - */ - add(data: Uint8Array, compressed: boolean): Promise; -} diff --git a/services/ipfs-publisher/tests/config.test.ts b/services/ipfs-publisher/tests/config.test.ts deleted file mode 100644 index 85dbff7..0000000 --- a/services/ipfs-publisher/tests/config.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { describe, expect, it } from 'vitest'; -import { loadIpfsPublisherConfig } from '../src/config.js'; - -describe('loadIpfsPublisherConfig', () => { - it('uses defaults when no env vars set', () => { - const config = loadIpfsPublisherConfig({}); - - expect(config.kafkaBrokers).toEqual(['localhost:9092']); - expect(config.consumerGroupId).toBe('ipfs-publisher-v1'); - expect(config.source).toBe('ipfs-publisher'); - expect(config.healthPort).toBe(3040); - expect(config.ipfsApiUrl).toBe('http://localhost:5001'); - expect(config.enableCompression).toBe(true); - }); - - it('parses kafka brokers from comma-separated string', () => { - const config = loadIpfsPublisherConfig({ - KAFKA_BROKERS: 'broker1:9092, broker2:9093 ,broker3:9094', - }); - - expect(config.kafkaBrokers).toEqual([ - 'broker1:9092', - 'broker2:9093', - 'broker3:9094', - ]); - }); - - it('filters empty broker strings', () => { - const config = loadIpfsPublisherConfig({ - KAFKA_BROKERS: 'broker1:9092,,,broker2:9093', - }); - - expect(config.kafkaBrokers).toEqual(['broker1:9092', 'broker2:9093']); - }); - - it('parses custom consumer group id', () => { - const config = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_GROUP_ID: 'my-custom-group', - }); - - expect(config.consumerGroupId).toBe('my-custom-group'); - }); - - it('parses custom source', () => { - const config = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_SOURCE: 'ipfs-prod', - }); - - expect(config.source).toBe('ipfs-prod'); - }); - - it('parses positive integer health port', () => { - const config = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_HEALTH_PORT: '8080', - }); - - expect(config.healthPort).toBe(8080); - }); - - it('falls back to default on invalid health port', () => { - const config1 = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_HEALTH_PORT: '-1', - }); - expect(config1.healthPort).toBe(3040); - - const config2 = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_HEALTH_PORT: 'not-a-number', - }); - expect(config2.healthPort).toBe(3040); - }); - - it('parses custom IPFS API URL', () => { - const config = loadIpfsPublisherConfig({ - IPFS_API_URL: 'http://ipfs.example.com:5001', - }); - - expect(config.ipfsApiUrl).toBe('http://ipfs.example.com:5001'); - }); - - it('disables compression when explicitly set to false', () => { - const config = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_ENABLE_COMPRESSION: 'false', - }); - - expect(config.enableCompression).toBe(false); - }); - - it('enables compression by default', () => { - const config = loadIpfsPublisherConfig({}); - - expect(config.enableCompression).toBe(true); - }); - - it('enables compression for any non-false value', () => { - const config1 = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_ENABLE_COMPRESSION: 'true', - }); - expect(config1.enableCompression).toBe(true); - - const config2 = loadIpfsPublisherConfig({ - IPFS_PUBLISHER_ENABLE_COMPRESSION: '1', - }); - expect(config2.enableCompression).toBe(true); - }); - - it('defaults to a single kubo provider', () => { - const config = loadIpfsPublisherConfig({}); - - expect(config.providers).toEqual(['kubo']); - expect(config.durabilityPolicy).toBe('any'); - expect(config.durabilityMinSuccessCount).toBe(1); - expect(config.pinataApiUrl).toBe('https://api.pinata.cloud'); - expect(config.pinataJwt).toBe(''); - expect(config.providerRetryBaseDelayMs).toBe(5000); - expect(config.providerRetryMaxDelayMs).toBe(60000); - expect(config.providerReplicationMaxPending).toBe(1000); - }); - - it('parses ordered multi-provider list', () => { - const config = loadIpfsPublisherConfig({ - IPFS_PROVIDERS: 'pinata, kubo', - }); - - expect(config.providers).toEqual(['pinata', 'kubo']); - }); - - it('falls back to kubo when provider list is empty', () => { - const config = loadIpfsPublisherConfig({ IPFS_PROVIDERS: ' , ' }); - - expect(config.providers).toEqual(['kubo']); - }); - - it('parses pinata credentials', () => { - const config = loadIpfsPublisherConfig({ - PINATA_API_URL: 'https://pinata.example.com', - PINATA_JWT: 'test-jwt', - }); - - expect(config.pinataApiUrl).toBe('https://pinata.example.com'); - expect(config.pinataJwt).toBe('test-jwt'); - }); - - it('parses durability policy and quorum count', () => { - const config = loadIpfsPublisherConfig({ - IPFS_DURABILITY_POLICY: 'quorum', - IPFS_DURABILITY_MIN_SUCCESS_COUNT: '2', - }); - - expect(config.durabilityPolicy).toBe('quorum'); - expect(config.durabilityMinSuccessCount).toBe(2); - }); - - it('falls back to "any" for an invalid durability policy', () => { - const config = loadIpfsPublisherConfig({ - IPFS_DURABILITY_POLICY: 'bogus', - }); - - expect(config.durabilityPolicy).toBe('any'); - }); - - it('parses retry/backoff settings', () => { - const config = loadIpfsPublisherConfig({ - IPFS_PROVIDER_RETRY_BASE_DELAY_MS: '1000', - IPFS_PROVIDER_RETRY_MAX_DELAY_MS: '30000', - IPFS_PROVIDER_REPLICATION_MAX_PENDING: '50', - }); - - expect(config.providerRetryBaseDelayMs).toBe(1000); - expect(config.providerRetryMaxDelayMs).toBe(30000); - expect(config.providerReplicationMaxPending).toBe(50); - }); -}); diff --git a/services/ipfs-publisher/tests/contracts.test.ts b/services/ipfs-publisher/tests/contracts.test.ts deleted file mode 100644 index 8681e95..0000000 --- a/services/ipfs-publisher/tests/contracts.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - TELEMETRY_TOPICS, - TelemetryBatchedPayloadSchema, - TelemetryIpfsPublishedPayloadSchema, - TelemetryIpfsPublishedPayload_Compression, - EnvelopeSchema, -} from '@scp/core'; -import { - SignedEnvelopeSchema, - SignedEnvelopeBatchSchema, -} from '@buf/airalab_connectivity-protocol.bufbuild_es/crypto/v1/envelope_pb.js'; -import { create, toBinary, fromBinary } from '@bufbuild/protobuf'; -import { describe, expect, it } from 'vitest'; -import { CID } from 'multiformats/cid'; - -describe('ipfs publisher contract compatibility', () => { - it('accepts telemetry.batched.v1 envelope/payload as input', () => { - const signedEnvelope = create(SignedEnvelopeSchema, { - sensorId: Buffer.alloc(32, 1), - nonce: Buffer.alloc(16, 2), - message: Buffer.from(JSON.stringify({ temp: 20 })), - signature: Buffer.alloc(64, 3), - }); - const innerBatch = create(SignedEnvelopeBatchSchema, { - batch: [signedEnvelope], - }); - - const payload = create(TelemetryBatchedPayloadSchema, { - batchId: 'batch-contract-1', - signedEnvelopeBatch: toBinary(SignedEnvelopeBatchSchema, innerBatch), - eventCount: 1, - sensorIds: [Buffer.alloc(32, 1)], - }); - - const envelope = create(EnvelopeSchema, { - eventId: 'batch-contract-1', - eventType: TELEMETRY_TOPICS.BATCHED, - eventVersion: '1.0.0', - occurredAt: '2026-01-01T00:00:00Z', - source: 'batcher', - payload: toBinary(TelemetryBatchedPayloadSchema, payload), - }); - - const envelopeBytes = toBinary(EnvelopeSchema, envelope); - const parsed = fromBinary(EnvelopeSchema, envelopeBytes); - - expect(parsed.eventType).toBe(TELEMETRY_TOPICS.BATCHED); - expect(parsed.eventId).toBe('batch-contract-1'); - - const payloadParsed = fromBinary( - TelemetryBatchedPayloadSchema, - parsed.payload - ); - expect(payloadParsed.batchId).toBe('batch-contract-1'); - expect(payloadParsed.eventCount).toBe(1); - const parsedInner = fromBinary( - SignedEnvelopeBatchSchema, - payloadParsed.signedEnvelopeBatch - ); - expect(Buffer.from(parsedInner.batch[0]?.sensorId ?? [])).toEqual( - Buffer.alloc(32, 1) - ); - }); - - it('produces valid SignedEnvelopeBatch for IPFS', () => { - const signedEnvelope1 = create(SignedEnvelopeSchema, { - sensorId: Buffer.alloc(32, 1), - nonce: Buffer.alloc(16, 2), - message: Buffer.from(JSON.stringify({ temp: 20 })), - signature: Buffer.alloc(64, 3), - }); - - const signedEnvelope2 = create(SignedEnvelopeSchema, { - sensorId: Buffer.alloc(32, 4), - nonce: Buffer.alloc(16, 5), - message: Buffer.from(JSON.stringify({ temp: 22 })), - signature: Buffer.alloc(64, 6), - }); - - const batch = create(SignedEnvelopeBatchSchema, { - batch: [signedEnvelope1, signedEnvelope2], - }); - - const batchBytes = toBinary(SignedEnvelopeBatchSchema, batch); - const parsed = fromBinary(SignedEnvelopeBatchSchema, batchBytes); - - expect(parsed.batch).toHaveLength(2); - expect(Buffer.from(parsed.batch[0]?.sensorId ?? [])).toEqual( - Buffer.alloc(32, 1) - ); - expect(Buffer.from(parsed.batch[1]?.sensorId ?? [])).toEqual( - Buffer.alloc(32, 4) - ); - }); - - it('produces valid telemetry.ipfs.published.v1 result with compression NONE', () => { - const fakeCid = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(fakeCid.bytes), - eventCount: 10, - compression: TelemetryIpfsPublishedPayload_Compression.NONE, - }); - - const envelope = create(EnvelopeSchema, { - eventId: 'evt-ipfs-1', - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, - eventVersion: '1.0.0', - occurredAt: '2026-01-01T00:00:00Z', - source: 'ipfs-publisher', - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), - }); - - const envelopeBytes = toBinary(EnvelopeSchema, envelope); - const parsed = fromBinary(EnvelopeSchema, envelopeBytes); - - expect(parsed.eventType).toBe(TELEMETRY_TOPICS.IPFS_PUBLISHED); - expect(parsed.eventId).toBe('evt-ipfs-1'); - - const payloadParsed = fromBinary( - TelemetryIpfsPublishedPayloadSchema, - parsed.payload - ); - expect(payloadParsed.eventCount).toBe(10); - expect(payloadParsed.compression).toBe( - TelemetryIpfsPublishedPayload_Compression.NONE - ); - expect(CID.decode(payloadParsed.cid).toString()).toBe(fakeCid.toString()); - }); - - it('produces valid telemetry.ipfs.published.v1 result with compression XZ', () => { - const fakeCid = CID.parse('QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'); - - const payload = create(TelemetryIpfsPublishedPayloadSchema, { - cid: Buffer.from(fakeCid.bytes), - eventCount: 50, - compression: TelemetryIpfsPublishedPayload_Compression.XZ, - }); - - const envelope = create(EnvelopeSchema, { - eventId: 'evt-ipfs-2', - eventType: TELEMETRY_TOPICS.IPFS_PUBLISHED, - eventVersion: '1.0.0', - occurredAt: '2026-01-01T00:00:00Z', - source: 'ipfs-publisher', - payload: toBinary(TelemetryIpfsPublishedPayloadSchema, payload), - }); - - const envelopeBytes = toBinary(EnvelopeSchema, envelope); - const parsed = fromBinary(EnvelopeSchema, envelopeBytes); - - const payloadParsed = fromBinary( - TelemetryIpfsPublishedPayloadSchema, - parsed.payload - ); - expect(payloadParsed.eventCount).toBe(50); - expect(payloadParsed.compression).toBe( - TelemetryIpfsPublishedPayload_Compression.XZ - ); - }); - - it('compression enum values match protobuf definition', () => { - expect(TelemetryIpfsPublishedPayload_Compression.NONE).toBe(0); - expect(TelemetryIpfsPublishedPayload_Compression.XZ).toBe(1); - }); -}); diff --git a/services/ipfs-publisher/tests/durability.test.ts b/services/ipfs-publisher/tests/durability.test.ts deleted file mode 100644 index a9d0cf4..0000000 --- a/services/ipfs-publisher/tests/durability.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { describe, expect, it } from 'vitest'; -import { isDurabilitySatisfied } from '../src/durability.js'; - -describe('isDurabilitySatisfied', () => { - it('"any" is satisfied by a single success', () => { - expect( - isDurabilitySatisfied({ policy: 'any', minSuccessCount: 1 }, 1, 3) - ).toBe(true); - expect( - isDurabilitySatisfied({ policy: 'any', minSuccessCount: 1 }, 0, 3) - ).toBe(false); - }); - - it('"all" requires every provider to succeed', () => { - expect( - isDurabilitySatisfied({ policy: 'all', minSuccessCount: 1 }, 2, 3) - ).toBe(false); - expect( - isDurabilitySatisfied({ policy: 'all', minSuccessCount: 1 }, 3, 3) - ).toBe(true); - }); - - it('"quorum" requires the configured minimum successes', () => { - const config = { policy: 'quorum' as const, minSuccessCount: 2 }; - expect(isDurabilitySatisfied(config, 1, 3)).toBe(false); - expect(isDurabilitySatisfied(config, 2, 3)).toBe(true); - expect(isDurabilitySatisfied(config, 3, 3)).toBe(true); - }); - - it('"quorum" clamps the minimum to the total number of providers', () => { - const config = { policy: 'quorum' as const, minSuccessCount: 5 }; - expect(isDurabilitySatisfied(config, 2, 2)).toBe(true); - expect(isDurabilitySatisfied(config, 1, 2)).toBe(false); - }); - - it('is never satisfied with zero configured providers', () => { - expect( - isDurabilitySatisfied({ policy: 'any', minSuccessCount: 1 }, 0, 0) - ).toBe(false); - }); -}); diff --git a/services/ipfs-publisher/tests/durable-event-identity.test.ts b/services/ipfs-publisher/tests/durable-event-identity.test.ts deleted file mode 100644 index 38b8e6f..0000000 --- a/services/ipfs-publisher/tests/durable-event-identity.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - TELEMETRY_TOPICS, - TelemetryBatchedPayloadSchema, - EnvelopeSchema, -} from '@scp/core'; -import { create, toBinary, fromBinary } from '@bufbuild/protobuf'; -import type { Consumer, Producer } from '@platformatic/kafka'; -import { describe, expect, it } from 'vitest'; -import { createIpfsPublisherService } from '../src/index.js'; -import type { IpfsPublisherConfig } from '../src/config.js'; -import type { IpfsProvider } from '../src/providers/types.js'; - -function testConfig( - overrides: Partial = {} -): IpfsPublisherConfig { - return { - kafkaBrokers: ['localhost:9092'], - consumerGroupId: 'ipfs-publisher-v1', - source: 'ipfs-publisher', - healthPort: 3061, - ipfsApiUrl: 'http://localhost:5001', - enableCompression: false, - providers: ['fake'], - pinataApiUrl: 'https://api.pinata.cloud', - pinataJwt: '', - durabilityPolicy: 'any', - durabilityMinSuccessCount: 1, - providerRetryBaseDelayMs: 5000, - providerRetryMaxDelayMs: 60000, - providerReplicationMaxPending: 1000, - ...overrides, - }; -} - -interface FakeMessage { - topic: string; - partition: number; - offset: bigint; - value: Buffer; -} - -function createBatchedMessage(batchId: string): FakeMessage { - const payload = create(TelemetryBatchedPayloadSchema, { - batchId, - signedEnvelopeBatch: new Uint8Array([1, 2, 3]), - eventCount: 1, - sensorIds: [Buffer.alloc(32, 1)], - }); - - const envelope = create(EnvelopeSchema, { - eventId: batchId, - eventType: TELEMETRY_TOPICS.BATCHED, - eventVersion: '1.0.0', - occurredAt: '2026-01-01T00:00:00Z', - source: 'batcher', - payload: toBinary(TelemetryBatchedPayloadSchema, payload), - }); - - return { - topic: TELEMETRY_TOPICS.BATCHED, - partition: 0, - offset: 0n, - value: Buffer.from(toBinary(EnvelopeSchema, envelope)), - }; -} - -/** A fake consumer that yields a single message then stays open until closed. */ -function createFakeConsumer(message: FakeMessage): { consumer: Consumer } { - let releaseClose!: () => void; - const closed = new Promise((resolve) => { - releaseClose = resolve; - }); - - const fake = { - async consume() { - return (async function* () { - yield message; - await closed; - })(); - }, - async commit() {}, - async close() { - releaseClose(); - }, - }; - - return { consumer: fake as unknown as Consumer }; -} - -/** A fake producer that records every published `ipfs.published.v1` event id. */ -function createFakeProducer(): { - producer: Producer; - publishedEventIds: string[]; -} { - const publishedEventIds: string[] = []; - - const fake = { - async send({ messages }: { messages: { topic: string; value: Buffer }[] }) { - for (const message of messages) { - if (message.topic === TELEMETRY_TOPICS.IPFS_PUBLISHED) { - const envelope = fromBinary(EnvelopeSchema, message.value); - publishedEventIds.push(envelope.eventId); - } - } - }, - async close() {}, - }; - - return { producer: fake as unknown as Producer, publishedEventIds }; -} - -function createFakeProvider(): IpfsProvider { - return { - name: 'fake', - async start() {}, - async stop() {}, - async add() { - return 'QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG'; - }, - }; -} - -describe('ipfs-publisher durable event identity (issue #32)', () => { - it('emits the same ipfs.published event_id when a batch is redelivered after a restart', async () => { - const batchId = 'batch-durable-1'; - - // Simulate two independent process lifetimes (e.g. crash + restart) - // both processing the same redelivered batch. In-memory dedup state - // does not survive between them, so the durability guarantee must come - // from the emitted event_id being derived from the batch id. - const runOnce = async (): Promise => { - const { consumer } = createFakeConsumer(createBatchedMessage(batchId)); - const { producer, publishedEventIds } = createFakeProducer(); - - const service = createIpfsPublisherService(testConfig(), { - createConsumer: () => consumer, - createProducer: () => producer, - createProviders: () => [createFakeProvider()], - createHealthServer: () => - ({ - close(callback: (error?: Error) => void) { - callback(); - }, - }) as unknown as import('node:http').Server, - }); - - await service.start(); - await new Promise((resolve) => setTimeout(resolve, 50)); - await service.stop(); - - expect(publishedEventIds).toHaveLength(1); - return publishedEventIds[0]!; - }; - - const firstEventId = await runOnce(); - const secondEventId = await runOnce(); - - expect(firstEventId).toBe(batchId); - expect(secondEventId).toBe(batchId); - expect(firstEventId).toBe(secondEventId); - }); -}); diff --git a/services/ipfs-publisher/tests/multi-provider-client.test.ts b/services/ipfs-publisher/tests/multi-provider-client.test.ts deleted file mode 100644 index 0d6a352..0000000 --- a/services/ipfs-publisher/tests/multi-provider-client.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; -import { MultiProviderIpfsClient } from '../src/multi-provider-client.js'; -import type { IpfsProvider } from '../src/providers/types.js'; - -function createFakeProvider( - name: string, - behavior: { - failFirstNCalls?: number; - alwaysFail?: boolean; - cid?: string; - } = {} -): IpfsProvider & { addCalls: number } { - const state = { - addCalls: 0, - }; - const failFirstNCalls = behavior.failFirstNCalls ?? 0; - - return { - name, - get addCalls() { - return state.addCalls; - }, - async start() {}, - async stop() {}, - async add(): Promise { - state.addCalls += 1; - if (behavior.alwaysFail || state.addCalls <= failFirstNCalls) { - throw new Error(`${name} unavailable`); - } - return behavior.cid ?? `cid-from-${name}`; - }, - }; -} - -describe('MultiProviderIpfsClient', () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - it('throws when constructed with no providers', () => { - expect( - () => - new MultiProviderIpfsClient([], { - durability: { policy: 'any', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }) - ).toThrow(); - }); - - it('publishes to primary provider and reports durability satisfied for "any"', async () => { - const primary = createFakeProvider('primary'); - const fallback = createFakeProvider('fallback'); - const client = new MultiProviderIpfsClient([primary, fallback], { - durability: { policy: 'any', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }); - await client.start(); - - const result = await client.publish( - 'batch-1', - new Uint8Array([1, 2, 3]), - false - ); - - expect(result.durabilitySatisfied).toBe(true); - expect(result.cid).toBe('cid-from-primary'); - expect(primary.addCalls).toBe(1); - expect(fallback.addCalls).toBe(1); - - await client.stop(); - }); - - it('falls back when primary is down, satisfying "any" durability', async () => { - const primary = createFakeProvider('primary', { alwaysFail: true }); - const fallback = createFakeProvider('fallback'); - const client = new MultiProviderIpfsClient([primary, fallback], { - durability: { policy: 'any', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }); - await client.start(); - - const result = await client.publish('batch-2', new Uint8Array([1]), false); - - expect(result.durabilitySatisfied).toBe(true); - expect(result.cid).toBe('cid-from-fallback'); - expect(result.providerOutcomes.get('primary')?.success).toBe(false); - expect(result.providerOutcomes.get('fallback')?.success).toBe(true); - - await client.stop(); - }); - - it('throws when all providers fail', async () => { - const primary = createFakeProvider('primary', { alwaysFail: true }); - const fallback = createFakeProvider('fallback', { alwaysFail: true }); - const client = new MultiProviderIpfsClient([primary, fallback], { - durability: { policy: 'any', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }); - await client.start(); - - await expect( - client.publish('batch-3', new Uint8Array([1]), false) - ).rejects.toThrow(/all IPFS providers failed/); - - await client.stop(); - }); - - it('"all" policy is not satisfied when a fallback fails, but does not throw', async () => { - const primary = createFakeProvider('primary'); - const fallback = createFakeProvider('fallback', { alwaysFail: true }); - const client = new MultiProviderIpfsClient([primary, fallback], { - durability: { policy: 'all', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }); - await client.start(); - - const result = await client.publish('batch-4', new Uint8Array([1]), false); - - expect(result.durabilitySatisfied).toBe(false); - expect(client.getDurabilityFailureCount()).toBe(1); - expect(client.getPendingReplicationCount()).toBe(1); - - await client.stop(); - }); - - it('does not re-upload to providers that already succeeded on a retried publish call', async () => { - const primary = createFakeProvider('primary'); - const fallback = createFakeProvider('fallback', { failFirstNCalls: 1 }); - const client = new MultiProviderIpfsClient([primary, fallback], { - durability: { policy: 'all', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }); - await client.start(); - - const first = await client.publish('batch-5', new Uint8Array([1]), false); - expect(first.durabilitySatisfied).toBe(false); - expect(primary.addCalls).toBe(1); - expect(fallback.addCalls).toBe(1); - - // Simulate Kafka redelivery: publish is called again for the same batch. - const second = await client.publish('batch-5', new Uint8Array([1]), false); - expect(second.durabilitySatisfied).toBe(true); - // Primary already succeeded, so it must not be called again. - expect(primary.addCalls).toBe(1); - // Fallback had failed, so it is retried and now succeeds. - expect(fallback.addCalls).toBe(2); - - await client.stop(); - }); - - it('background retry loop eventually replicates to a recovered fallback', async () => { - const primary = createFakeProvider('primary'); - const fallback = createFakeProvider('fallback', { failFirstNCalls: 1 }); - const client = new MultiProviderIpfsClient([primary, fallback], { - durability: { policy: 'any', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }); - await client.start(); - - const result = await client.publish('batch-6', new Uint8Array([1]), false); - expect(result.durabilitySatisfied).toBe(true); - expect(client.getPendingReplicationCount()).toBe(1); - expect(fallback.addCalls).toBe(1); - - // Advance past the retry interval so the background loop retries the - // failed fallback provider independently of any Kafka redelivery. - await vi.advanceTimersByTimeAsync(1100); - - expect(fallback.addCalls).toBe(2); - expect(client.getPendingReplicationCount()).toBe(0); - - await client.stop(); - }); - - it('tracks per-provider success/failure stats', async () => { - const primary = createFakeProvider('primary'); - const fallback = createFakeProvider('fallback', { alwaysFail: true }); - const client = new MultiProviderIpfsClient([primary, fallback], { - durability: { policy: 'any', minSuccessCount: 1 }, - retryBaseDelayMs: 1000, - retryMaxDelayMs: 5000, - maxPendingReplications: 10, - }); - await client.start(); - - await client.publish('batch-7', new Uint8Array([1]), false); - - const stats = client.getProviderStats(); - expect(stats.get('primary')?.success).toBe(1); - expect(stats.get('primary')?.failure).toBe(0); - expect(stats.get('fallback')?.success).toBe(0); - expect(stats.get('fallback')?.failure).toBe(1); - - await client.stop(); - }); -}); diff --git a/services/ipfs-publisher/tests/providers/pinata.test.ts b/services/ipfs-publisher/tests/providers/pinata.test.ts deleted file mode 100644 index 9c9d16b..0000000 --- a/services/ipfs-publisher/tests/providers/pinata.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright 2026 Robonomics Network - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { describe, expect, it, vi } from 'vitest'; -import { createPinataProvider } from '../../src/providers/pinata.js'; - -function jsonResponse(body: unknown, ok = true, status = 200): Response { - return { - ok, - status, - json: async () => body, - text: async () => JSON.stringify(body), - } as Response; -} - -describe('createPinataProvider', () => { - it('requires a JWT to start', async () => { - const provider = createPinataProvider({ - apiUrl: 'https://api.pinata.cloud', - jwt: '', - }); - - await expect(provider.start()).rejects.toThrow(/JWT/); - }); - - it('uploads data and returns the CID from IpfsHash', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue(jsonResponse({ IpfsHash: 'Qmfakecid' })); - - const provider = createPinataProvider( - { apiUrl: 'https://api.pinata.cloud', jwt: 'test-jwt' }, - fetchMock as unknown as typeof fetch - ); - - await provider.start(); - const cid = await provider.add(new Uint8Array([1, 2, 3]), false); - - expect(cid).toBe('Qmfakecid'); - expect(fetchMock).toHaveBeenCalledWith( - 'https://api.pinata.cloud/pinning/pinFileToIPFS', - expect.objectContaining({ - method: 'POST', - headers: expect.objectContaining({ - Authorization: 'Bearer test-jwt', - }), - }) - ); - }); - - it('throws on a non-ok response', async () => { - const fetchMock = vi - .fn() - .mockResolvedValue(jsonResponse({ error: 'nope' }, false, 500)); - - const provider = createPinataProvider( - { apiUrl: 'https://api.pinata.cloud', jwt: 'test-jwt' }, - fetchMock as unknown as typeof fetch - ); - - await provider.start(); - await expect( - provider.add(new Uint8Array([1, 2, 3]), false) - ).rejects.toThrow(/status 500/); - }); - - it('throws when the response has no IpfsHash', async () => { - const fetchMock = vi.fn().mockResolvedValue(jsonResponse({})); - - const provider = createPinataProvider( - { apiUrl: 'https://api.pinata.cloud', jwt: 'test-jwt' }, - fetchMock as unknown as typeof fetch - ); - - await provider.start(); - await expect( - provider.add(new Uint8Array([1, 2, 3]), false) - ).rejects.toThrow(/IpfsHash/); - }); - - it('rejects add() calls before start()', async () => { - const provider = createPinataProvider({ - apiUrl: 'https://api.pinata.cloud', - jwt: 'test-jwt', - }); - - await expect( - provider.add(new Uint8Array([1, 2, 3]), false) - ).rejects.toThrow(/not started/); - }); -}); diff --git a/services/ipfs-publisher/tsconfig.json b/services/ipfs-publisher/tsconfig.json deleted file mode 100644 index b8aa054..0000000 --- a/services/ipfs-publisher/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "dist", - "composite": true, - "rootDir": "src" - }, - "include": ["src/**/*.ts"], - "exclude": ["tests/**/*.ts"], - "references": [ - { "path": "../../packages/core" } - ] -} diff --git a/services/pubsub-broadcaster/README.md b/services/pubsub-broadcaster/README.md index ff8ae20..3732dd8 100644 --- a/services/pubsub-broadcaster/README.md +++ b/services/pubsub-broadcaster/README.md @@ -14,7 +14,7 @@ GossipSub node for real-time web UI updates. consumer group) whenever fewer than the configured minimum number of reserved peers are connected, and resumed once connectivity recovers -This service forwards telemetry to real-time subscribers. Failures are acceptable since telemetry is also archived via `ipfs-publisher` and `blockchain-anchor` services. +This service forwards telemetry to real-time subscribers. Failures are acceptable since telemetry is also durably anchored via the `batcher` and `blockchain-anchor` services. ## libp2p GossipSub From 2c4d66efcf70cf5a15e4c934cb976b335bf136ce Mon Sep 17 00:00:00 2001 From: Alexander Krupenkin Date: Mon, 28 Sep 2026 22:03:02 +0900 Subject: [PATCH 2/4] fix: review comments --- .env.example | 1 + services/batcher/src/index.ts | 48 ++- services/batcher/tests/oversized-dlq.test.ts | 187 +++++++++++ .../batcher/tests/split-batch-publish.test.ts | 170 ++++++++++ services/blockchain-anchor/src/config.ts | 3 + services/blockchain-anchor/src/index.ts | 290 +++++++++++------- .../tests/anchor-failure.test.ts | 217 +++++++++++++ .../tests/idempotency.test.ts | 40 ++- 8 files changed, 831 insertions(+), 125 deletions(-) create mode 100644 services/batcher/tests/oversized-dlq.test.ts create mode 100644 services/batcher/tests/split-batch-publish.test.ts create mode 100644 services/blockchain-anchor/tests/anchor-failure.test.ts diff --git a/.env.example b/.env.example index 35d03af..912bb72 100644 --- a/.env.example +++ b/.env.example @@ -41,4 +41,5 @@ BLOCKCHAIN_ANCHOR_GROUP_ID=blockchain-anchor-v1 BLOCKCHAIN_ANCHOR_NODE_ID=0 BLOCKCHAIN_ANCHOR_SURI=//Alice BLOCKCHAIN_ANCHOR_HEALTH_PORT=3050 +BLOCKCHAIN_ANCHOR_SOURCE=blockchain-anchor BLOCKCHAIN_TARGET=robonomics-dev diff --git a/services/batcher/src/index.ts b/services/batcher/src/index.ts index ae3a0be..095b27d 100644 --- a/services/batcher/src/index.ts +++ b/services/batcher/src/index.ts @@ -79,13 +79,18 @@ interface BatchItem { eventId?: string; /** Reason the envelope failed to parse; set only for poison items. */ parseError?: string; - /** Original raw bytes; set only for poison items, for DLQ forwarding. */ - raw?: Uint8Array; + /** + * Original raw bytes of the source `telemetry.authorized.v1` Kafka record + * (the serialized `Envelope`). Set for every item so it can be forwarded + * verbatim to the DLQ, whether the record is a poison (unparseable) item + * or a valid item that later turns out to be oversized. + */ + raw: Uint8Array; } /** A batch item that parsed successfully and carries a valid envelope. */ type ValidBatchItem = BatchItem & - Required>; + Required>; function isValidItem(item: BatchItem): item is ValidBatchItem { return item.parseError === undefined; @@ -121,7 +126,7 @@ async function publishPoisonToDlq( await producer.send({ messages: poisonItems.map((item) => ({ topic: TELEMETRY_TOPICS.DLQ, - value: Buffer.from(item.raw ?? new Uint8Array()), + value: Buffer.from(item.raw), headers: { source_topic: Buffer.from(TELEMETRY_TOPICS.AUTHORIZED), source_service: Buffer.from(config.source), @@ -159,7 +164,10 @@ async function publishOversizedToDlq( await producer.send({ messages: oversizedItems.map(({ item, reason }) => ({ topic: TELEMETRY_TOPICS.DLQ, - value: Buffer.from(toBinary(SignedEnvelopeSchema, item.signedEnvelope)), + // Forward the original authorized-envelope bytes (not just the inner + // SignedEnvelope) so consumers of telemetry.dlq.v1 can decode this the + // same way as any other DLQ record and recover its event/trace metadata. + value: Buffer.from(item.raw), headers: { source_topic: Buffer.from(TELEMETRY_TOPICS.AUTHORIZED), source_service: Buffer.from(config.source), @@ -234,7 +242,18 @@ async function produceBatch( metrics.batchesSplit += 1; } - for (const fittedBatch of fitted) { + // Build every sub-batch's Kafka message up front and publish them all in + // a single producer.send() call. Sending sub-batches one at a time would + // let an earlier split succeed while a later one fails; a subsequent + // retry of the whole (unsplit) batch would then republish the + // already-sent sub-batch. Since blockchain-anchor's idempotency check + // only compares against the *current* on-chain payload, such a + // duplicate can differ from what's on-chain (once a later split has + // since been anchored) and get resubmitted. Publishing as one batched + // request means the sub-batches for this attempt either all reach Kafka + // together or none do, so a retry never re-sends a subset that already + // landed. + const fittedWithIds = fitted.map((fittedBatch) => { const batchId = computeBatchId(fittedBatch.events); const payload = create(TelemetryBatchedPayloadSchema, { @@ -256,15 +275,19 @@ async function produceBatch( payload: toBinary(TelemetryBatchedPayloadSchema, payload), }); + return { batchId, fittedBatch, resultEnvelope }; + }); + + if (fittedWithIds.length > 0) { await producer.send({ - messages: [ - { - topic: TELEMETRY_TOPICS.BATCHED, - value: Buffer.from(toBinary(EnvelopeSchema, resultEnvelope)), - }, - ], + messages: fittedWithIds.map(({ resultEnvelope }) => ({ + topic: TELEMETRY_TOPICS.BATCHED, + value: Buffer.from(toBinary(EnvelopeSchema, resultEnvelope)), + })), }); + } + for (const { batchId, fittedBatch } of fittedWithIds) { metrics.batchesProduced += 1; metrics.eventsBatched += fittedBatch.events.length; @@ -543,6 +566,7 @@ export function createBatcherService( partition: message.partition, sensorId: payload.sensorId, eventId: envelope.eventId, + raw: new Uint8Array(message.value), }; if (envelope.traceId !== undefined) { batchItem.traceId = envelope.traceId; diff --git a/services/batcher/tests/oversized-dlq.test.ts b/services/batcher/tests/oversized-dlq.test.ts new file mode 100644 index 0000000..1f45140 --- /dev/null +++ b/services/batcher/tests/oversized-dlq.test.ts @@ -0,0 +1,187 @@ +/** + * Copyright 2026 Robonomics Network + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + TELEMETRY_TOPICS, + TelemetryAuthorizedPayloadSchema, + EnvelopeSchema, +} from '@scp/core'; +import { SignedEnvelopeSchema } from '@buf/airalab_connectivity-protocol.bufbuild_es/crypto/v1/envelope_pb.js'; +import { create, toBinary } from '@bufbuild/protobuf'; +import type { Consumer, Producer } from '@platformatic/kafka'; +import { describe, expect, it } from 'vitest'; +import { createBatcherService } from '../src/index.js'; +import type { BatcherConfig } from '../src/config.js'; + +function testConfig(overrides: Partial = {}): BatcherConfig { + return { + kafkaBrokers: ['localhost:9092'], + consumerGroupId: 'batcher-v1', + source: 'batcher', + healthPort: 3042, + batchSize: 10, + batchTimeoutMs: 60000, + maxPayloadBytes: 8192, + ...overrides, + }; +} + +interface FakeMessage { + topic: string; + partition: number; + offset: bigint; + value: Buffer; +} + +function createAuthorizedMessage( + partition: number, + offset: bigint, + messageSize: number +): FakeMessage { + const signedEnvelope = create(SignedEnvelopeSchema, { + sensorId: Buffer.alloc(32, 1), + nonce: Buffer.alloc(16, 2), + message: Buffer.alloc(messageSize, 9), + signature: Buffer.alloc(64, 3), + }); + + const payload = create(TelemetryAuthorizedPayloadSchema, { + sensorId: Buffer.alloc(32, 1), + signedEnvelope: toBinary(SignedEnvelopeSchema, signedEnvelope), + }); + + const envelope = create(EnvelopeSchema, { + eventId: `evt-${partition}-${offset}`, + eventType: TELEMETRY_TOPICS.AUTHORIZED, + eventVersion: 'v1', + occurredAt: '2026-01-01T00:00:00Z', + source: 'endpoint', + payload: toBinary(TelemetryAuthorizedPayloadSchema, payload), + }); + + return { + topic: TELEMETRY_TOPICS.AUTHORIZED, + partition, + offset, + value: Buffer.from(toBinary(EnvelopeSchema, envelope)), + }; +} + +function createFakeConsumer(messages: FakeMessage[]): { + consumer: Consumer; + commits: { partition: number; offset: bigint }[]; +} { + let releaseClose!: () => void; + const closed = new Promise((resolve) => { + releaseClose = resolve; + }); + const commits: { partition: number; offset: bigint }[] = []; + + const fake = { + async consume() { + return (async function* () { + for (const message of messages) { + yield message; + } + await closed; + })(); + }, + async getLag() { + return new Map(); + }, + async commit({ + offsets, + }: { + offsets: { partition: number; offset: bigint }[]; + }) { + for (const o of offsets) { + commits.push({ partition: o.partition, offset: o.offset }); + } + }, + async close() { + releaseClose(); + }, + }; + + return { consumer: fake as unknown as Consumer, commits }; +} + +function createFakeProducer( + sent: { topic: string; value: Buffer; headers?: Record }[] +): Producer { + const fake = { + async send({ + messages, + }: { + messages: { + topic: string; + value: Buffer; + headers?: Record; + }[]; + }) { + sent.push(...messages); + }, + async close() {}, + }; + return fake as unknown as Producer; +} + +describe('batcher oversized-event DLQ forwarding', () => { + it('forwards the original authorized-envelope bytes for an oversized event', async () => { + // A single event whose serialized+compressed size exceeds the tiny + // maxPayloadBytes below, forcing it into the `oversized` path. + const message = createAuthorizedMessage(0, 0n, 5000); + const { consumer, commits } = createFakeConsumer([message]); + const sent: { + topic: string; + value: Buffer; + headers?: Record; + }[] = []; + const producer = createFakeProducer(sent); + + const service = createBatcherService( + testConfig({ maxPayloadBytes: 100, batchSize: 1 }), + { + createConsumer: () => consumer, + createProducer: () => producer, + createHealthServer: () => + ({ + close(callback: (error?: Error) => void) { + callback(); + }, + }) as unknown as import('node:http').Server, + } + ); + + await service.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + await service.stop(); + + expect(commits).toEqual([{ partition: 0, offset: 1n }]); + + const dlqRecords = sent.filter((m) => m.topic === TELEMETRY_TOPICS.DLQ); + expect(dlqRecords).toHaveLength(1); + // The DLQ record must be the original authorized Envelope bytes (decodable + // the same way as any other DLQ record), not just the inner SignedEnvelope. + expect(dlqRecords[0]?.value).toEqual(message.value); + expect(dlqRecords[0]?.headers?.reason?.toString()).toContain( + 'ANCHOR_PAYLOAD_TOO_LARGE' + ); + + const metrics = service.getMetrics(); + expect(metrics.oversizedEvents).toBe(1); + expect(metrics.batchesProduced).toBe(0); + }); +}); diff --git a/services/batcher/tests/split-batch-publish.test.ts b/services/batcher/tests/split-batch-publish.test.ts new file mode 100644 index 0000000..7ef8f04 --- /dev/null +++ b/services/batcher/tests/split-batch-publish.test.ts @@ -0,0 +1,170 @@ +/** + * Copyright 2026 Robonomics Network + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + TELEMETRY_TOPICS, + TelemetryAuthorizedPayloadSchema, + EnvelopeSchema, +} from '@scp/core'; +import { SignedEnvelopeSchema } from '@buf/airalab_connectivity-protocol.bufbuild_es/crypto/v1/envelope_pb.js'; +import { create, toBinary } from '@bufbuild/protobuf'; +import { randomBytes } from 'node:crypto'; +import type { Consumer, Producer } from '@platformatic/kafka'; +import { describe, expect, it } from 'vitest'; +import { createBatcherService } from '../src/index.js'; +import type { BatcherConfig } from '../src/config.js'; + +const MAX_PAYLOAD_BYTES = 2000; + +function testConfig(overrides: Partial = {}): BatcherConfig { + return { + kafkaBrokers: ['localhost:9092'], + consumerGroupId: 'batcher-v1', + source: 'batcher', + healthPort: 3043, + batchSize: 20, + batchTimeoutMs: 60000, + maxPayloadBytes: MAX_PAYLOAD_BYTES, + ...overrides, + }; +} + +interface FakeMessage { + topic: string; + partition: number; + offset: bigint; + value: Buffer; +} + +function createAuthorizedMessage( + partition: number, + offset: bigint +): FakeMessage { + const signedEnvelope = create(SignedEnvelopeSchema, { + sensorId: Buffer.alloc(32, 1), + nonce: Buffer.alloc(16, 2), + // High-entropy payload so XZ cannot meaningfully compress it, forcing the + // combined batch above the payload limit and requiring a split. + message: randomBytes(1000), + signature: Buffer.alloc(64, 3), + }); + + const payload = create(TelemetryAuthorizedPayloadSchema, { + sensorId: Buffer.alloc(32, 1), + signedEnvelope: toBinary(SignedEnvelopeSchema, signedEnvelope), + }); + + const envelope = create(EnvelopeSchema, { + eventId: `evt-${partition}-${offset}`, + eventType: TELEMETRY_TOPICS.AUTHORIZED, + eventVersion: 'v1', + occurredAt: '2026-01-01T00:00:00Z', + source: 'endpoint', + payload: toBinary(TelemetryAuthorizedPayloadSchema, payload), + }); + + return { + topic: TELEMETRY_TOPICS.AUTHORIZED, + partition, + offset, + value: Buffer.from(toBinary(EnvelopeSchema, envelope)), + }; +} + +function createFakeConsumer(messages: FakeMessage[]): { + consumer: Consumer; + commits: { partition: number; offset: bigint }[]; +} { + let releaseClose!: () => void; + const closed = new Promise((resolve) => { + releaseClose = resolve; + }); + const commits: { partition: number; offset: bigint }[] = []; + + const fake = { + async consume() { + return (async function* () { + for (const message of messages) { + yield message; + } + await closed; + })(); + }, + async getLag() { + return new Map(); + }, + async commit({ + offsets, + }: { + offsets: { partition: number; offset: bigint }[]; + }) { + for (const o of offsets) { + commits.push({ partition: o.partition, offset: o.offset }); + } + }, + async close() { + releaseClose(); + }, + }; + + return { consumer: fake as unknown as Consumer, commits }; +} + +describe('batcher split-batch publishing', () => { + it('publishes all fitted sub-batches in a single producer.send call', async () => { + const messages = Array.from({ length: 20 }, (_, i) => + createAuthorizedMessage(0, BigInt(i)) + ); + const { consumer, commits } = createFakeConsumer(messages); + + let sendCalls = 0; + let totalMessages = 0; + const producer = { + async send({ messages: sent }: { messages: unknown[] }) { + sendCalls += 1; + totalMessages += sent.length; + }, + async close() {}, + } as unknown as Producer; + + const service = createBatcherService(testConfig(), { + createConsumer: () => consumer, + createProducer: () => producer, + createHealthServer: () => + ({ + close(callback: (error?: Error) => void) { + callback(); + }, + }) as unknown as import('node:http').Server, + }); + + await service.start(); + await new Promise((resolve) => setTimeout(resolve, 100)); + await service.stop(); + + const metrics = service.getMetrics(); + // The batch had to be split into more than one sub-batch to fit. + expect(metrics.batchesProduced).toBeGreaterThan(1); + + // All sub-batches for this attempt are published in a single Kafka + // request: sending them one at a time would let an earlier split + // succeed while a later one fails, and a retry of the whole batch would + // then republish the already-sent sub-batch. + expect(sendCalls).toBe(1); + expect(totalMessages).toBe(metrics.batchesProduced); + + expect(commits).toEqual([{ partition: 0, offset: 20n }]); + }); +}); diff --git a/services/blockchain-anchor/src/config.ts b/services/blockchain-anchor/src/config.ts index c3d0f12..25ba36d 100644 --- a/services/blockchain-anchor/src/config.ts +++ b/services/blockchain-anchor/src/config.ts @@ -24,6 +24,8 @@ export interface BlockchainAnchorConfig { healthPort: number; /** Maximum size, in bytes, of a chain-ready compressed payload (defensive check). */ maxPayloadBytes: number; + /** Value used as the `source_service` DLQ header when forwarding records. */ + source: string; } function parsePositiveInt(value: string | undefined, fallback: number): number { @@ -69,5 +71,6 @@ export function loadBlockchainAnchorConfig( env.ANCHOR_MAX_PAYLOAD_BYTES, DEFAULT_ANCHOR_MAX_PAYLOAD_BYTES ), + source: env.BLOCKCHAIN_ANCHOR_SOURCE ?? 'blockchain-anchor', }; } diff --git a/services/blockchain-anchor/src/index.ts b/services/blockchain-anchor/src/index.ts index c0f624d..23f98d9 100644 --- a/services/blockchain-anchor/src/index.ts +++ b/services/blockchain-anchor/src/index.ts @@ -20,8 +20,8 @@ import { type TelemetryBatchedPayload, installShutdownHandler, } from '@scp/core'; -import { fromBinary } from '@bufbuild/protobuf'; -import { Consumer } from '@platformatic/kafka'; +import { fromBinary, type MessageShape } from '@bufbuild/protobuf'; +import { Consumer, Producer } from '@platformatic/kafka'; import { ApiPromise, WsProvider, Keyring } from '@polkadot/api'; import { createServer, type Server } from 'node:http'; import { fileURLToPath } from 'node:url'; @@ -48,6 +48,7 @@ export interface BlockchainAnchorService { export interface BlockchainAnchorDeps { createConsumer?: () => Consumer; + createProducer?: () => Producer; createApi?: () => Promise; createHealthServer?: ( getMetrics: () => BlockchainAnchorMetrics, @@ -115,6 +116,45 @@ function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { return true; } +/** + * Forward an oversized `telemetry.batched.v1` record to the DLQ topic, + * preserving its original bytes. This is a permanent upstream contract + * violation (the batcher is supposed to guarantee `payload <= + * maxPayloadBytes`), so the record must be durably routed to the DLQ rather + * than silently dropped: the offset is only committed after this publish is + * confirmed. + */ +async function publishOversizedToDlq( + raw: Uint8Array, + partition: number, + offset: bigint, + reason: string, + producer: Producer, + config: BlockchainAnchorConfig +): Promise { + await producer.send({ + messages: [ + { + topic: TELEMETRY_TOPICS.DLQ, + value: Buffer.from(raw), + headers: { + source_topic: Buffer.from(TELEMETRY_TOPICS.BATCHED), + source_service: Buffer.from(config.source), + source_partition: Buffer.from(String(partition)), + source_offset: Buffer.from(String(offset)), + reason: Buffer.from(reason), + }, + }, + ], + }); + + logWarn('oversized batch routed to DLQ', { + partition, + offset: offset.toString(), + dlq_topic: TELEMETRY_TOPICS.DLQ, + }); +} + /** * Send CPS setPayload extrinsic to blockchain. * @@ -252,6 +292,13 @@ export function createBlockchainAnchorService( bootstrapBrokers: config.kafkaBrokers, }); + const producer = + deps.createProducer?.() ?? + new Producer({ + clientId: 'blockchain-anchor', + bootstrapBrokers: config.kafkaBrokers, + }); + const createHealthServer = deps.createHealthServer ?? startHealthAndMetricsServer; @@ -326,8 +373,10 @@ export function createBlockchainAnchorService( continue; } + let envelope: MessageShape; + let payload: TelemetryBatchedPayload; try { - const envelope = fromBinary( + envelope = fromBinary( EnvelopeSchema, new Uint8Array(message.value) ); @@ -339,133 +388,131 @@ export function createBlockchainAnchorService( continue; } - const payload = fromBinary( + payload = fromBinary( TelemetryBatchedPayloadSchema, envelope.payload ) as TelemetryBatchedPayload; - - metrics.consumed += 1; - - // Defensive size invariant: the batcher owns payload fitting, - // but blockchain-anchor must not submit a payload that - // violates the CPS size limit. This is a permanent upstream - // contract error, not a transient failure — do not split the - // batch here, and do not retry. - if (payload.payload.length > config.maxPayloadBytes) { - metrics.rejectedOversized += 1; - logError( - 'rejecting batch: payload exceeds ANCHOR_MAX_PAYLOAD_BYTES', - new Error('ANCHOR_PAYLOAD_TOO_LARGE'), + } catch (error) { + logWarn('envelope parse error', { + reason: + error instanceof Error ? error.message : String(error), + }); + // Commit offset for invalid messages to skip them + await consumer.commit({ + offsets: [ { - batch_id: payload.batchId, - payload_size: payload.payload.length, - max_payload_bytes: config.maxPayloadBytes, - node_id: config.nodeId, - } - ); + topic: TELEMETRY_TOPICS.BATCHED, + partition: message.partition, + offset: message.offset + 1n, + leaderEpoch: -1, + }, + ], + }); + continue; + } - // Commit the offset: this is a permanent contract - // violation upstream, retrying will not help. - await consumer.commit({ - offsets: [ - { - topic: TELEMETRY_TOPICS.BATCHED, - partition: message.partition, - offset: message.offset + 1n, - leaderEpoch: -1, - }, - ], - }); - continue; - } + metrics.consumed += 1; + + // Defensive size invariant: the batcher owns payload fitting, + // but blockchain-anchor must not submit a payload that + // violates the CPS size limit. This is a permanent upstream + // contract error, not a transient failure — do not split the + // batch here, and do not retry. The record is still routed to + // the DLQ (instead of being silently dropped) so it can be + // inspected/recovered, and the offset is only committed once + // that publish is confirmed. + if (payload.payload.length > config.maxPayloadBytes) { + metrics.rejectedOversized += 1; + logError( + 'rejecting batch: payload exceeds ANCHOR_MAX_PAYLOAD_BYTES', + new Error('ANCHOR_PAYLOAD_TOO_LARGE'), + { + batch_id: payload.batchId, + payload_size: payload.payload.length, + max_payload_bytes: config.maxPayloadBytes, + node_id: config.nodeId, + } + ); - logInfo('anchoring batch to blockchain', { - event_id: envelope.eventId, - batch_id: payload.batchId, - payload_size: payload.payload.length, - event_count: payload.eventCount, - node_id: config.nodeId, + await publishOversizedToDlq( + new Uint8Array(message.value), + message.partition, + message.offset, + 'ANCHOR_PAYLOAD_TOO_LARGE', + producer, + config + ); + + await consumer.commit({ + offsets: [ + { + topic: TELEMETRY_TOPICS.BATCHED, + partition: message.partition, + offset: message.offset + 1n, + leaderEpoch: -1, + }, + ], }); + continue; + } - try { - // Idempotency check: ask the chain what is currently - // anchored for this node before submitting a new - // extrinsic. This covers the crash window between - // extrinsic finalization and Kafka offset commit - on - // redelivery the on-chain state already reflects the - // anchor, so we skip re-submission instead of creating a - // duplicate logical anchor operation. - const anchoredPayload = await getAnchoredPayload( - api!, - config.nodeId - ); + logInfo('anchoring batch to blockchain', { + event_id: envelope.eventId, + batch_id: payload.batchId, + payload_size: payload.payload.length, + event_count: payload.eventCount, + node_id: config.nodeId, + }); - if ( - anchoredPayload !== null && - bytesEqual(anchoredPayload, payload.payload) - ) { - metrics.skippedDuplicate += 1; - - logInfo( - 'batch already anchored on-chain; skipping duplicate submission', - { - batch_id: payload.batchId, - payload_size: payload.payload.length, - event_id: envelope.eventId, - node_id: config.nodeId, - } - ); - } else { - await sendSetPayloadExtrinsic( - api!, - keyring!, - config.suri, - config.nodeId, - payload.payload - ); - - metrics.anchored += 1; - - logInfo('batch anchored successfully', { + try { + // Idempotency check: ask the chain what is currently + // anchored for this node before submitting a new + // extrinsic. This covers the crash window between + // extrinsic finalization and Kafka offset commit - on + // redelivery the on-chain state already reflects the + // anchor, so we skip re-submission instead of creating a + // duplicate logical anchor operation. + const anchoredPayload = await getAnchoredPayload( + api!, + config.nodeId + ); + + if ( + anchoredPayload !== null && + bytesEqual(anchoredPayload, payload.payload) + ) { + metrics.skippedDuplicate += 1; + + logInfo( + 'batch already anchored on-chain; skipping duplicate submission', + { batch_id: payload.batchId, payload_size: payload.payload.length, - event_count: payload.eventCount, + event_id: envelope.eventId, node_id: config.nodeId, - }); - } + } + ); + } else { + await sendSetPayloadExtrinsic( + api!, + keyring!, + config.suri, + config.nodeId, + payload.payload + ); - // Commit offset only after successful anchoring (or a - // confirmed no-op due to the idempotency check above) - await consumer.commit({ - offsets: [ - { - topic: TELEMETRY_TOPICS.BATCHED, - partition: message.partition, - offset: message.offset + 1n, - leaderEpoch: -1, - }, - ], - }); + metrics.anchored += 1; - logDebug('kafka offset committed', { - partition: message.partition, - offset: (message.offset + 1n).toString(), - }); - } catch (error) { - metrics.failed += 1; - logError('failed to anchor batch', error, { + logInfo('batch anchored successfully', { batch_id: payload.batchId, payload_size: payload.payload.length, + event_count: payload.eventCount, node_id: config.nodeId, }); - // Don't commit offset on failure - message will be retried } - } catch (error) { - logWarn('envelope parse error', { - reason: - error instanceof Error ? error.message : String(error), - }); - // Commit offset for invalid messages to skip them + + // Commit offset only after successful anchoring (or a + // confirmed no-op due to the idempotency check above) await consumer.commit({ offsets: [ { @@ -476,6 +523,25 @@ export function createBlockchainAnchorService( }, ], }); + + logDebug('kafka offset committed', { + partition: message.partition, + offset: (message.offset + 1n).toString(), + }); + } catch (error) { + metrics.failed += 1; + logError('failed to anchor batch', error, { + batch_id: payload.batchId, + payload_size: payload.payload.length, + node_id: config.nodeId, + }); + // Don't commit offset on failure. Stop consuming further + // messages instead of continuing the loop: a later + // message's commit would advance the committed offset past + // this unretried one, silently losing it. Rethrowing exits + // the loop (and the run promise) so the process can be + // restarted and resume from the last committed offset. + throw error; } } } catch (error) { @@ -534,6 +600,8 @@ export function createBlockchainAnchorService( api = null; keyring = null; + await producer.close().catch(() => undefined); + if (healthServer) { await new Promise((resolve, reject) => { healthServer?.close((error) => { diff --git a/services/blockchain-anchor/tests/anchor-failure.test.ts b/services/blockchain-anchor/tests/anchor-failure.test.ts new file mode 100644 index 0000000..f6d2b90 --- /dev/null +++ b/services/blockchain-anchor/tests/anchor-failure.test.ts @@ -0,0 +1,217 @@ +/** + * Copyright 2026 Robonomics Network + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + TELEMETRY_TOPICS, + TelemetryBatchedPayloadSchema, + EnvelopeSchema, +} from '@scp/core'; +import { create, toBinary } from '@bufbuild/protobuf'; +import type { ApiPromise } from '@polkadot/api'; +import type { Consumer } from '@platformatic/kafka'; +import { describe, expect, it } from 'vitest'; +import { createBlockchainAnchorService } from '../src/index.js'; +import type { BlockchainAnchorConfig } from '../src/config.js'; + +function testConfig( + overrides: Partial = {} +): BlockchainAnchorConfig { + return { + kafkaBrokers: ['localhost:9092'], + consumerGroupId: 'blockchain-anchor-v1', + substrateWsUrl: 'ws://localhost:9944', + suri: '//Alice', + nodeId: 0, + healthPort: 3053, + maxPayloadBytes: 8192, + source: 'blockchain-anchor', + ...overrides, + }; +} + +interface FakeMessage { + topic: string; + partition: number; + offset: bigint; + value: Buffer; +} + +function createBatchedMessage( + eventId: string, + batchId: string, + payload: Uint8Array, + partition: number, + offset: bigint +): FakeMessage { + const batchedPayload = create(TelemetryBatchedPayloadSchema, { + batchId, + payload, + eventCount: 5, + sensorIds: [Buffer.alloc(32, 1)], + uncompressedSize: payload.length * 2, + compressedSize: payload.length, + payloadHash: Buffer.alloc(32, 3), + }); + + const envelope = create(EnvelopeSchema, { + eventId, + eventType: TELEMETRY_TOPICS.BATCHED, + eventVersion: '1.0.0', + occurredAt: '2026-01-01T00:00:00Z', + source: 'batcher', + payload: toBinary(TelemetryBatchedPayloadSchema, batchedPayload), + }); + + return { + topic: TELEMETRY_TOPICS.BATCHED, + partition, + offset, + value: Buffer.from(toBinary(EnvelopeSchema, envelope)), + }; +} + +/** + * A fake consumer that yields the given messages and then keeps the stream + * open until `close()` is called. + */ +function createFakeConsumer(messages: FakeMessage[]): { + consumer: Consumer; + commits: { partition: number; offset: bigint }[]; +} { + let releaseClose!: () => void; + const closed = new Promise((resolve) => { + releaseClose = resolve; + }); + const commits: { partition: number; offset: bigint }[] = []; + + const fake = { + async consume() { + return (async function* () { + for (const message of messages) { + yield message; + } + await closed; + })(); + }, + async commit({ + offsets, + }: { + offsets: { partition: number; offset: bigint }[]; + }) { + for (const o of offsets) { + commits.push({ partition: o.partition, offset: o.offset }); + } + }, + async close() { + releaseClose(); + }, + }; + + return { consumer: fake as unknown as Consumer, commits }; +} + +/** A fake ApiPromise whose extrinsic submission always fails. */ +function createFailingApi(): ApiPromise { + return { + isReady: Promise.resolve(), + query: { + cps: { + async payload(_nodeId: number) { + return { isEmpty: true, isSome: false }; + }, + }, + }, + tx: { + cps: { + setPayload(_nodeId: number, _payload: number[]) { + return { + signAndSend( + _account: unknown, + callback: (result: unknown) => void + ) { + queueMicrotask(() => { + callback({ + status: { type: 'Invalid' }, + isFinalized: false, + isError: true, + events: [], + }); + }); + return Promise.resolve(() => {}); + }, + }; + }, + }, + }, + events: {}, + async disconnect() {}, + } as unknown as ApiPromise; +} + +describe('blockchain-anchor anchor failure handling', () => { + it('does not commit a later offset when an earlier anchor attempt fails', async () => { + const payloadA = Buffer.from('compressed-batch-bytes-a'); + const payloadB = Buffer.from('compressed-batch-bytes-b'); + + // Two distinct batches on the same partition; the first anchor attempt + // fails permanently in this fake. A prior bug continued the consume loop + // and let the second message's commit silently advance past the first + // (unretried) offset. + const messages = [ + createBatchedMessage('evt-a', 'batch-a', payloadA, 0, 0n), + createBatchedMessage('evt-b', 'batch-b', payloadB, 0, 1n), + ]; + + const { consumer, commits } = createFakeConsumer(messages); + const api = createFailingApi(); + + const service = createBlockchainAnchorService(testConfig(), { + createConsumer: () => consumer, + createApi: () => Promise.resolve(api), + createHealthServer: () => + ({ + close(callback: (error?: Error) => void) { + callback(); + }, + }) as unknown as import('node:http').Server, + }); + + // The anchor failure intentionally propagates out of the internal run + // loop (so the process can crash and be restarted by the orchestrator, + // resuming from the last committed offset). Swallow that expected + // rejection here rather than letting it surface as an unhandled one - + // the assertions below directly verify the resulting behavior. + const onUnhandledRejection = (): void => {}; + process.on('unhandledRejection', onUnhandledRejection); + + try { + await service.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + await service.stop(); + } finally { + process.off('unhandledRejection', onUnhandledRejection); + } + + // Neither offset is committed: the first anchor failure must stop + // processing rather than let the second message's commit skip past it. + expect(commits).toEqual([]); + + const metrics = service.getMetrics(); + expect(metrics.failed).toBe(1); + expect(metrics.anchored).toBe(0); + // Only the first message was consumed before processing halted. + expect(metrics.consumed).toBe(1); + }); +}); diff --git a/services/blockchain-anchor/tests/idempotency.test.ts b/services/blockchain-anchor/tests/idempotency.test.ts index 5e5aa00..73d9651 100644 --- a/services/blockchain-anchor/tests/idempotency.test.ts +++ b/services/blockchain-anchor/tests/idempotency.test.ts @@ -20,7 +20,7 @@ import { } from '@scp/core'; import { create, toBinary } from '@bufbuild/protobuf'; import type { ApiPromise } from '@polkadot/api'; -import type { Consumer } from '@platformatic/kafka'; +import type { Consumer, Producer } from '@platformatic/kafka'; import { describe, expect, it } from 'vitest'; import { createBlockchainAnchorService } from '../src/index.js'; import type { BlockchainAnchorConfig } from '../src/config.js'; @@ -36,10 +36,31 @@ function testConfig( nodeId: 0, healthPort: 3051, maxPayloadBytes: 8192, + source: 'blockchain-anchor', ...overrides, }; } +function createFakeProducer( + sent: { topic: string; value: Buffer; headers?: Record }[] +): Producer { + const fake = { + async send({ + messages, + }: { + messages: { + topic: string; + value: Buffer; + headers?: Record; + }[]; + }) { + sent.push(...messages); + }, + async close() {}, + }; + return fake as unknown as Producer; +} + interface FakeMessage { topic: string; partition: number; @@ -267,7 +288,7 @@ describe('blockchain-anchor idempotency', () => { expect(metrics.skippedDuplicate).toBe(0); }); - it('rejects a payload larger than maxPayloadBytes without submitting or splitting it', async () => { + it('rejects a payload larger than maxPayloadBytes without submitting or splitting it, forwarding it to the DLQ', async () => { const oversizedPayload = Buffer.alloc(200, 9); const messages = [ @@ -282,11 +303,18 @@ describe('blockchain-anchor idempotency', () => { const { consumer, commits } = createFakeConsumer(messages); const { api, setPayloadCalls } = createFakeApi(); + const sent: { + topic: string; + value: Buffer; + headers?: Record; + }[] = []; + const producer = createFakeProducer(sent); const service = createBlockchainAnchorService( testConfig({ maxPayloadBytes: 100 }), { createConsumer: () => consumer, + createProducer: () => producer, createApi: () => Promise.resolve(api), createHealthServer: () => ({ @@ -304,6 +332,14 @@ describe('blockchain-anchor idempotency', () => { expect(setPayloadCalls).toHaveLength(0); expect(commits).toEqual([{ partition: 0, offset: 1n }]); + // The original record is forwarded to the DLQ, not silently dropped. + expect(sent).toHaveLength(1); + expect(sent[0]?.topic).toBe(TELEMETRY_TOPICS.DLQ); + expect(sent[0]?.value).toEqual(messages[0]?.value); + expect(sent[0]?.headers?.reason?.toString()).toBe( + 'ANCHOR_PAYLOAD_TOO_LARGE' + ); + const metrics = service.getMetrics(); expect(metrics.rejectedOversized).toBe(1); expect(metrics.anchored).toBe(0); From 4d3f509d633fbc784d8bbe5c0748318337c805e8 Mon Sep 17 00:00:00 2001 From: Aleksandr Krupenkin Date: Tue, 29 Sep 2026 01:02:59 +0900 Subject: [PATCH 3/4] Add source property to test configuration Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- services/blockchain-anchor/tests/shutdown.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/services/blockchain-anchor/tests/shutdown.test.ts b/services/blockchain-anchor/tests/shutdown.test.ts index a131359..8a9c9a6 100644 --- a/services/blockchain-anchor/tests/shutdown.test.ts +++ b/services/blockchain-anchor/tests/shutdown.test.ts @@ -36,6 +36,7 @@ function testConfig( nodeId: 0, healthPort: 3052, maxPayloadBytes: 8192, + source: 'blockchain-anchor', ...overrides, }; } From 60f68ffd8976e8ccaff289258244aee941105d82 Mon Sep 17 00:00:00 2001 From: Alexander Krupenkin Date: Tue, 29 Sep 2026 02:53:36 +0900 Subject: [PATCH 4/4] fix: address remaining PR #35 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - blockchain-anchor: BoundedVec.toU8a() includes the SCALE compact-length prefix by default; pass toU8a(true) so the on-chain payload bytes compared for idempotency match the bare bytes we submit. - blockchain-anchor: compare batches by their content-addressed payload_hash (blake2b-256, already computed by the batcher) against a hash of the currently anchored on-chain payload, instead of comparing raw bytes. The Kafka offset for a batch is only committed once this check (and, if needed, the extrinsic) completes, so an uncommitted offset already durably marks a batch as unconfirmed — no additional persisted queue is required. - batcher: update the produceBatch comment to reflect that re-publishing a sub-batch is harmless now that anchoring dedupes by payload_hash, regardless of whether the whole split lands atomically. - tests: idempotency.test.ts now computes payload_hash the same way the real batcher does (blake2b-256 of the payload) instead of a fixed placeholder, so the hash-based dedup check is exercised correctly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- services/batcher/src/index.ts | 17 +++---- services/blockchain-anchor/src/index.ts | 44 +++++++++++++++---- .../tests/idempotency.test.ts | 5 ++- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/services/batcher/src/index.ts b/services/batcher/src/index.ts index 095b27d..c28e978 100644 --- a/services/batcher/src/index.ts +++ b/services/batcher/src/index.ts @@ -243,16 +243,13 @@ async function produceBatch( } // Build every sub-batch's Kafka message up front and publish them all in - // a single producer.send() call. Sending sub-batches one at a time would - // let an earlier split succeed while a later one fails; a subsequent - // retry of the whole (unsplit) batch would then republish the - // already-sent sub-batch. Since blockchain-anchor's idempotency check - // only compares against the *current* on-chain payload, such a - // duplicate can differ from what's on-chain (once a later split has - // since been anchored) and get resubmitted. Publishing as one batched - // request means the sub-batches for this attempt either all reach Kafka - // together or none do, so a retry never re-sends a subset that already - // landed. + // a single producer.send() call. This isn't a strict all-or-nothing + // guarantee across partitions/brokers, but it doesn't need to be: + // blockchain-anchor's idempotency check compares each batch's + // content-addressed `payload_hash` against the hash of whatever is + // currently anchored on-chain, so re-publishing a sub-batch that already + // landed (e.g. after a retry of this whole call) is a harmless no-op + // rather than a duplicate anchor. const fittedWithIds = fitted.map((fittedBatch) => { const batchId = computeBatchId(fittedBatch.events); diff --git a/services/blockchain-anchor/src/index.ts b/services/blockchain-anchor/src/index.ts index 23f98d9..ac34dcf 100644 --- a/services/blockchain-anchor/src/index.ts +++ b/services/blockchain-anchor/src/index.ts @@ -23,6 +23,7 @@ import { import { fromBinary, type MessageShape } from '@bufbuild/protobuf'; import { Consumer, Producer } from '@platformatic/kafka'; import { ApiPromise, WsProvider, Keyring } from '@polkadot/api'; +import { blake2AsU8a } from '@polkadot/util-crypto'; import { createServer, type Server } from 'node:http'; import { fileURLToPath } from 'node:url'; import { @@ -64,18 +65,21 @@ export interface BlockchainAnchorDeps { interface OptionBytesCodec { readonly isSome?: boolean; readonly isEmpty?: boolean; - unwrap?: () => { toU8a: () => Uint8Array }; + unwrap?: () => { toU8a: (isBare?: boolean) => Uint8Array }; } /** * Read the payload currently anchored on-chain for a CPS node, if any. * - * This is the authoritative idempotency check: rather than relying solely on - * a local/Redis marker (which can't observe whether an extrinsic actually + * This is the authoritative idempotency check: rather than relying on local + * process state (which can't observe whether an extrinsic actually * finalized before a crash), we ask the chain what payload is currently set - * for the node and compare it byte-for-byte against the payload we are about + * for the node and compare its content hash against the batch we are about * to submit. If they already match, the anchor operation is a no-op and is - * skipped. + * skipped. The Kafka offset for `telemetry.batched.v1` is only committed + * once this check (and, if needed, the extrinsic) has completed, so an + * uncommitted offset itself durably marks a batch as "not yet confirmed + * anchored" — no separate persisted queue is required. */ async function getAnchoredPayload( api: ApiPromise, @@ -94,7 +98,11 @@ async function getAnchoredPayload( return null; } - return raw.unwrap().toU8a(); + // `toU8a()` defaults to including the SCALE compact-length prefix used + // to encode `BoundedVec`, but `payload.payload` is just the bare XZ + // bytes. Pass `true` to strip the length prefix so the comparison is + // byte-for-byte against the same shape we submit. + return raw.unwrap().toU8a(true); } catch (error) { logWarn('failed to read on-chain payload for idempotency check', { nodeId, @@ -104,6 +112,24 @@ async function getAnchoredPayload( } } +/** + * Compare a batch's content hash (`payload_hash`, computed by the batcher as + * `blake2b-256` of the compressed payload) against the same hash computed + * over the payload currently anchored on-chain. + * + * Comparing hashes rather than raw bytes is cheaper and, more importantly, + * keys the comparison on the same content-addressed identity the batcher + * assigns to the batch, rather than depending on decoding the exact codec + * shape returned by the chain. + */ +function anchoredPayloadMatches( + anchoredPayload: Uint8Array, + expectedPayloadHash: Uint8Array +): boolean { + const anchoredHash = blake2AsU8a(anchoredPayload, 256); + return bytesEqual(anchoredHash, expectedPayloadHash); +} + function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { if (a.length !== b.length) { return false; @@ -470,8 +496,8 @@ export function createBlockchainAnchorService( // extrinsic. This covers the crash window between // extrinsic finalization and Kafka offset commit - on // redelivery the on-chain state already reflects the - // anchor, so we skip re-submission instead of creating a - // duplicate logical anchor operation. + // anchor (same content hash), so we skip re-submission + // instead of creating a duplicate logical anchor operation. const anchoredPayload = await getAnchoredPayload( api!, config.nodeId @@ -479,7 +505,7 @@ export function createBlockchainAnchorService( if ( anchoredPayload !== null && - bytesEqual(anchoredPayload, payload.payload) + anchoredPayloadMatches(anchoredPayload, payload.payloadHash) ) { metrics.skippedDuplicate += 1; diff --git a/services/blockchain-anchor/tests/idempotency.test.ts b/services/blockchain-anchor/tests/idempotency.test.ts index 73d9651..5f03443 100644 --- a/services/blockchain-anchor/tests/idempotency.test.ts +++ b/services/blockchain-anchor/tests/idempotency.test.ts @@ -19,6 +19,7 @@ import { EnvelopeSchema, } from '@scp/core'; import { create, toBinary } from '@bufbuild/protobuf'; +import { blake2AsU8a } from '@polkadot/util-crypto'; import type { ApiPromise } from '@polkadot/api'; import type { Consumer, Producer } from '@platformatic/kafka'; import { describe, expect, it } from 'vitest'; @@ -82,7 +83,9 @@ function createBatchedMessage( sensorIds: [Buffer.alloc(32, 1)], uncompressedSize: payload.length * 2, compressedSize: payload.length, - payloadHash: Buffer.alloc(32, 3), + // Content-addressed identity of `payload`, matching how the real + // batcher derives `payload_hash` (blake2b-256 of the compressed bytes). + payloadHash: blake2AsU8a(payload, 256), }); const envelope = create(EnvelopeSchema, {