feat: self-contained identity webhook + collector price oracle and metering fixes

.env.example

@@ -0,0 +1,81 @@
+# Clearinghouse stack — copy to .env before starting.
+#
+#   cp .env.example .env
+#   $EDITOR .env
+#
+# Sections list which Compose service reads each variable. Unused vars in a container are ignored.
+
+# --- Shared (identity-webhook + remote-signer) ---
+# Local dev only — change before any shared or production use.
+WEBHOOK_SECRET=dev-webhook-secret-change-me
+
+# --- Shared (remote-signer + openmeter-collector) ---
+KAFKA_BROKERS=kafka:9092
+KAFKA_GATEWAY_TOPIC=livepeer-gateway-events
+
+# --- identity-webhook ---
+IDENTITY_ISSUER=http://identity-webhook:8090
+# Demo API key — must match the Bearer token end users send to remote-signer.
+DEMO_API_KEY=sk_demo_local_key
+DEMO_CLIENT_ID=demo-client
+DEMO_USER_ID=demo-user
+
+# Optional
+USAGE_SUBJECT_TYPE=api_key_user
+# API_KEY_PREFIX=sk_
+# DEMO_API_KEYS={"sk_other":{"clientId":"app-b","userId":"user-b"}}
+
+# OAuth / OIDC — set OIDC_ISSUER to enable JWT auth (tried before sk_ keys).
+# Clearinghouse / Auth0 signer JWTs (M2M mint or Builder API exchange):
+# OIDC_ISSUER=https://pymthouse.us.auth0.com/
+# OIDC_AUDIENCE=livepeer-clearinghouse
+# OIDC_CLIENT_CLAIM=app_client_id
+# OIDC_SUBJECT_CLAIM=external_user_id
+# OIDC_SUBJECT_TYPE=external_user_id
+# OIDC_REQUIRED_SCOPES=sign:job
+#
+# Generic OIDC (raw IdP access tokens with azp/sub):
+# OIDC_ISSUER=https://your-tenant.auth0.com/
+# OIDC_AUDIENCE=https://api.your-platform.com
+# OIDC_JWKS_URI=                       # defaults to ${OIDC_ISSUER}/.well-known/jwks.json
+# OIDC_CLIENT_CLAIM=azp
+# OIDC_SUBJECT_CLAIM=sub
+# OIDC_SUBJECT_TYPE=oidc_user
+# OIDC_REQUIRED_SCOPES=
+
+# Reference only — not read by Compose services; for minting test JWTs.
+# Canonical copy: auth0-provisioner/provision/.env.livepeer
+# DEMO_APP_AUTH0_AUDIENCE=livepeer-clearinghouse
+# DEMO_APP_AUTH0_PUBLIC_CLIENT_ID=
+# DEMO_APP_AUTH0_M2M_CLIENT_ID=
+# DEMO_APP_AUTH0_M2M_CLIENT_SECRET=
+
+# --- kafka (Redpanda) ---
+KAFKA_ADVERTISED_ADDR=kafka:9092
+
+# --- remote-signer ---
+# Leave REMOTE_SIGNER_WEBHOOK_URL empty to skip auth (dev only; loopback-only in Compose).
+# To expose the signer on all host interfaces, use a Compose override — see README.
+REMOTE_SIGNER_WEBHOOK_URL=http://identity-webhook:8090/authorize
+SIGNER_NETWORK=arbitrum-one-mainnet
+SIGNER_PORT=8081
+ETH_RPC_URL=https://arb1.arbitrum.io/rpc
+SIGNER_ETH_ADDR=
+SIGNER_ETH_KEYSTORE_PATH=/data/keystore
+
+# Optional — host keystore/password under remote-signer/data/ (mounted at /data).
+# SIGNER_ETH_KEYSTORE_PATH defaults to /data/keystore when that directory exists.
+# ORCH_WEBHOOK_URL and LIVE_AI_CAP_REPORT_INTERVAL apply when SIGNER_REMOTE_DISCOVERY=1.
+SIGNER_REMOTE_DISCOVERY=0
+# ORCH_WEBHOOK_URL=
+# LIVE_AI_CAP_REPORT_INTERVAL=
+
+# --- openmeter-collector ---
+# Konnect: https://<region>.api.konghq.com/v3/openmeter/events
+# Self-hosted: https://<host>/api/v1/events
+OPENMETER_INGEST_URL=https://us.api.konghq.com/v3/openmeter/events
+OPENMETER_API_KEY=
+
+# Optional (collector defaults shown)
+PRICE_ORACLE_URL=https://api.coinbase.com/v2/prices/ETH-USD/spot
+PRICE_ORACLE_REFRESH=5m

.gitignore

@@ -1,4 +1,7 @@
-kafka/.env
-remote-signer/.env
-openmeter-collector/.env
+.env
 remote-signer/data/
+node_modules/
+
+# Editor / OS
+.vscode/
+.DS_Store

README.md

@@ -1,12 +1,13 @@
 # clearinghouse
 
 Docker Compose stack for the clearinghouse runtime:
-**Redpanda → go-livepeer remote signer → OpenMeter/Benthos collector → Konnect metering**.
+**identity-webhook → Redpanda → go-livepeer remote signer → OpenMeter/Benthos collector → Konnect metering**.
 
 ## Components
 
 | Service | Role | Docs |
 | --- | --- | --- |
+| **identity-webhook** (`identity-webhook`) | Resolves end-user credentials (API keys and/or OAuth/OIDC JWTs) to `auth_id` for go-livepeer's `/authorize` hook. Self-contained: implements the go-livepeer webhook wire protocol in-repo, verifying JWTs with `jose`. | [jose](https://github.com/panva/jose) |
 | **Redpanda** (`kafka`) | Kafka-compatible event bus. The signer publishes gateway events; the collector consumes them. | [Redpanda docs](https://docs.redpanda.com/) |
 | **go-livepeer remote signer** (`remote-signer`) | Signs Livepeer payment tickets and emits `create_signed_ticket` events to Kafka. | [go-livepeer](https://github.com/livepeer/go-livepeer) |
 | **OpenMeter collector** (`openmeter-collector`) | Benthos pipeline: filters Kafka events, converts fees to USD micros, POSTs CloudEvents to OpenMeter ingest. | [OpenMeter collector](https://openmeter.io/docs/collectors) |
@@ -26,39 +27,59 @@ Signer HTTP request
 
 **Redpanda over Apache Kafka.** The stack uses Redpanda as the Kafka-compatible broker. Redpanda runs as a single-binary dev container with no ZooKeeper dependency and faster local startup.
 
-**Identity & auth.** The signer container runs `go-livepeer` directly. In the normal path, every signing request is authorized by go-livepeer's `-remoteSignerWebhookUrl` hook, which calls your `/authorize` endpoint with `Authorization: Bearer <WEBHOOK_SECRET>` — no reverse proxy or gateway in front of the signer. For local alive checks only, leave `REMOTE_SIGNER_WEBHOOK_URL` empty to omit the webhook hook.
+**Identity & auth.** The in-compose **identity-webhook** is self-contained: it implements go-livepeer's remote-signer webhook wire protocol in-repo (`identity-webhook/protocol.mjs`) and pluggable end-user verifiers (`identity-webhook/verifiers.mjs`) — an API-key verifier and an OAuth/OIDC verifier built on [`jose`](https://github.com/panva/jose). The signer container runs `go-livepeer` directly; every signing request is authorized by go-livepeer's `-remoteSignerWebhookUrl` hook, which calls `/authorize` with `Authorization: Bearer <WEBHOOK_SECRET>`. End users present their credential to the signer — `Authorization: Bearer sk_…` (API key) or `Authorization: Bearer <jwt>` (OIDC) — and the webhook resolves it to `auth_id = "{client_id}:{usage_subject}"`. Set `OIDC_ISSUER`/`OIDC_AUDIENCE` to enable bring-your-own-OAuth (JWTs verified against your IdP's JWKS); configure both API keys and OIDC to accept either. For local alive checks only, leave `REMOTE_SIGNER_WEBHOOK_URL` empty to omit the webhook hook.
 
 **CLI port not exposed.** go-livepeer's `-cliAddr` (admin/RPC) is bound to `127.0.0.1:4935` inside the container and is never published or mapped to the host.
 
-**Signing port is loopback-only by default.** `8081` is published as `127.0.0.1:8081:8081` — host-only, not the network. `/generate-live-payment` mints PM tickets from the operator's on-chain deposit, and with `REMOTE_SIGNER_WEBHOOK_URL` unset it has **no auth** — anyone who reaches it can drain the deposit. Expose it (`0.0.0.0:8081:8081` or a reverse proxy) **only with the webhook set**.
+**Signing port loopback-only by default.** Compose publishes the signing HTTP port as `127.0.0.1:8081` so an accidentally unauthenticated signer (when `REMOTE_SIGNER_WEBHOOK_URL` is empty) is not reachable from the LAN. To expose on all interfaces — e.g. for a gateway on another host — add a Compose override:
 
-**Per-service configuration.** Each service reads a local `.env` file mounted at `/service/.env` and sourced by its entrypoint. Copy the `.env.example` in each service directory before starting the stack.
+```yaml
+# docker-compose.override.yml
+services:
+  remote-signer:
+    ports:
+      - "8081:8081"
+```
+
+Only bind `0.0.0.0` when `REMOTE_SIGNER_WEBHOOK_URL` and `WEBHOOK_SECRET` are set; an open signer can drain deposits.
+
+**Stack configuration.** Copy [`.env.example`](.env.example) to `.env` at the repo root before starting. All Compose services read from that file — kafka, remote-signer, and openmeter-collector mount it at `/service/.env` and source it in the entrypoint; identity-webhook reads it via Compose `env_file`.
 
 ## Local stack
 
-### 1. Quick check — Kafka + signer
+### 1. Quick check — Kafka + identity webhook + signer
 
-Start here before wiring identity or metering. This runs only the Kafka broker and remote signer so you can confirm the core path is alive.
+Start here before wiring metering. This runs the broker, identity webhook, and remote signer.
 
 ```bash
-cp kafka/.env.example kafka/.env
-cp remote-signer/.env.example remote-signer/.env
-$EDITOR remote-signer/.env
+cp .env.example .env
+$EDITOR .env
 # For a local alive check without an identity webhook:
 #   REMOTE_SIGNER_WEBHOOK_URL=
 #   WEBHOOK_SECRET=
 
-docker compose up -d --build kafka remote-signer
+docker compose up -d --build kafka identity-webhook remote-signer
 docker compose logs -f remote-signer
 ```
 
+Verify the identity webhook (simulates go-livepeer calling `/authorize`; secret matches `.env.example`):
+
+```bash
+docker compose exec identity-webhook \
+  curl -sS -X POST http://localhost:8090/authorize \
+    -H "Authorization: Bearer dev-webhook-secret-change-me" \
+    -H "Content-Type: application/json" \
+    -d '{"headers":{"Authorization":["Bearer sk_demo_local_key"]}}'
+# expected: "status":200, "auth_id":"demo-client:demo-user"
+```
+
 Expected result: `remote-signer` starts cleanly, connects to Kafka, and serves the signing HTTP port.
 
 Smoketests:
 
 ```bash
 docker compose ps
-# kafka "healthy" (signer only starts once Kafka is healthy), remote-signer "Up"
+# kafka "healthy", identity-webhook "healthy" (signer waits for both), remote-signer "Up"
 
 curl -fsS -X POST http://localhost:8081/sign-orchestrator-info
 # {"address":"0x…","signature":"0x…"} — keystore unlocked, signer can sign
@@ -75,11 +96,10 @@ docker compose port remote-signer 8081
 
 ### 2. Full stack — add metering
 
-After the quick check passes, add the OpenMeter collector and hosted metering configuration. Provision OpenMeter meters/features (see [OpenMeter/Konnect bootstrap](#openmeterkonnect-bootstrap)), then configure the collector:
+After the quick check passes, add the OpenMeter collector and hosted metering configuration. Provision OpenMeter meters/features (see [OpenMeter/Konnect bootstrap](#openmeterkonnect-bootstrap)), then set `OPENMETER_API_KEY` in `.env`:
 
 ```bash
-cp openmeter-collector/.env.example openmeter-collector/.env
-$EDITOR openmeter-collector/.env
+$EDITOR .env
 
 docker compose up -d --build
 docker compose logs -f
@@ -104,13 +124,16 @@ Re-runs are dedup-safe (OpenMeter deduplicates by event id). A real signer-emitt
 
 ## Environment variables
 
-Each service documents its variables in its own `.env.example`:
+All variables are documented in [`.env.example`](.env.example), grouped by service:
 
-| Service | Config file | Key variables |
-| --- | --- | --- |
-| `kafka` | [`kafka/.env.example`](kafka/.env.example) | `KAFKA_ADVERTISED_ADDR` |
-| `remote-signer` | [`remote-signer/.env.example`](remote-signer/.env.example) | `REMOTE_SIGNER_WEBHOOK_URL`, `WEBHOOK_SECRET`, `SIGNER_*`, `KAFKA_BROKERS`, `KAFKA_GATEWAY_TOPIC` |
-| `openmeter-collector` | [`openmeter-collector/.env.example`](openmeter-collector/.env.example) | `KAFKA_BROKERS`, `KAFKA_GATEWAY_TOPIC`, `OPENMETER_INGEST_URL`, `OPENMETER_API_KEY`, `ETH_USD_PRICE` |
+| Service | Key variables |
+| --- | --- |
+| `identity-webhook` | `WEBHOOK_SECRET`, `IDENTITY_ISSUER`, `DEMO_API_KEY`, `DEMO_CLIENT_ID`, `DEMO_USER_ID`, `API_KEY_PREFIX` (optional, default `sk_`), `OIDC_*` (optional) |
+| `kafka` | `KAFKA_ADVERTISED_ADDR` |
+| `remote-signer` | `REMOTE_SIGNER_WEBHOOK_URL`, `WEBHOOK_SECRET`, `SIGNER_*`, `KAFKA_BROKERS`, `KAFKA_GATEWAY_TOPIC` |
+| `openmeter-collector` | `KAFKA_BROKERS`, `KAFKA_GATEWAY_TOPIC`, `OPENMETER_INGEST_URL`, `OPENMETER_API_KEY`, `PRICE_ORACLE_URL`, `PRICE_ORACLE_REFRESH` |
+
+Shared keys (`WEBHOOK_SECRET`, `KAFKA_BROKERS`, `KAFKA_GATEWAY_TOPIC`) are listed once at the top of `.env.example`.
 
 Signer state (keystore, `.eth-password`, chain DB) is stored under [`remote-signer/data/`](remote-signer/data/), bind-mounted to `/data` in the container.
 
@@ -119,13 +142,12 @@ mkdir -p remote-signer/data/keystore
 cp /path/to/your/keystore/* remote-signer/data/keystore/
 cp /path/to/your/.eth-password remote-signer/data/.eth-password
 
-cp remote-signer/.env.example remote-signer/.env
-$EDITOR remote-signer/.env
+$EDITOR .env
 ```
 
 Set `SIGNER_ETH_KEYSTORE_PATH=/data/keystore` (container path) and `SIGNER_ETH_ADDR` to your funded signer address. If `SIGNER_ETH_KEYSTORE_PATH` is unset, the entrypoint uses `/data/keystore` when that directory exists.
 
-To change the host signing port from `8081`, use a Compose override file.
+To change the host signing port or bind on all interfaces, use a Compose override file (see **Signing port loopback-only by default** under Design decisions).
 
 ## OpenMeter/Konnect bootstrap
 
@@ -155,12 +177,21 @@ Signer computed_fee (wei)
             → clearinghouse_default_ppu subscription per customer
 ```
 
-Markup rules are defined in the bootstrap CLI catalog. Collector
-pipeline config: [`openmeter-collector/collector.yaml`](openmeter-collector/collector.yaml).
-The collector does not yet emit `billable_usd_micros` (phase 2); until then the billable meter
-stays empty while the catalog is ready.
+Collector pipeline config: [`openmeter-collector/collector.yaml`](openmeter-collector/collector.yaml).
+The collector emits `billable_usd_micros` as an interim passthrough equal to
+`network_fee_usd_micros` so the billable meter validates and accumulates. Phase-2
+markup rules (network × pipeline/model multiplier) are not applied yet — until then
+`billable_usd_micros == network_fee_usd_micros`.
 
 ### Identity contract (collector)
 
-The collector expects Kafka `auth_id` as `client_id:external_user_id` (first-colon split).
-Konnect customer key matches that compound id (e.g. `demo-client:demo-user`).
+Upstream Kafka events carry `auth_id` as `client_id:usage_subject` (webhook → go-livepeer state → Kafka; unchanged).
+
+The collector parses `auth_id` once (first-colon split) and emits normalized CloudEvents to Konnect/OpenMeter:
+
+- `subject` = end user (`usage_subject`)
+- `data.client_id` = tenant
+- `data.usage_subject` = end user
+- `data.auth_id` retained for compatibility; `data.external_user_id` mirrors `usage_subject` for meter `groupBy`
+
+Demo API key defaults: `sk_demo_local_key` → `demo-client:demo-user` (configured in `.env`).

docker-compose.yml

@@ -1,25 +1,45 @@
-# Clearinghouse stack: Redpanda + go-livepeer remote signer + OpenMeter collector.
+# Clearinghouse stack: identity webhook + Redpanda + remote signer + OpenMeter collector.
 #
 # Redpanda is the Kafka-compatible broker. The signer container runs go-livepeer directly —
-# only its signing HTTP port (8081) is published to the host. The CLI/admin port (-cliAddr)
-# is bound to loopback inside the container and is never exposed or published.
+# only its signing HTTP port (8081) is published to the host on loopback (127.0.0.1).
+# The CLI/admin port (-cliAddr) is bound to loopback inside the container and is never
+# published. To expose the signing port on all interfaces, add a Compose override:
+#   services.remote-signer.ports: ["8081:8081"]
+# Only do this when REMOTE_SIGNER_WEBHOOK_URL is set — an unauthenticated signer on the
+# LAN can drain deposits.
 #
-# Per-service config: copy each service's .env.example to .env before starting.
+# Stack config: copy .env.example to .env at the repo root before starting.
 #
 # Full stack:
 #   docker compose up -d --build
 #
-# Kafka + signer only (no metering):
-#   docker compose up -d --build kafka remote-signer
+# Kafka + identity webhook + signer only (no metering):
+#   docker compose up -d --build kafka identity-webhook remote-signer
 
 services:
+  identity-webhook:
+    build:
+      context: .
+      dockerfile: identity-webhook/Dockerfile
+    restart: unless-stopped
+    env_file:
+      - ./.env
+    environment:
+      PORT: "8090"
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://localhost:8090/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 10s
+
   kafka:
     build:
       context: .
       dockerfile: kafka/Dockerfile
     restart: unless-stopped
     volumes:
-      - ./kafka/.env:/service/.env:ro
+      - ./.env:/service/.env:ro
     healthcheck:
       test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy'"]
       interval: 10s
@@ -38,11 +58,9 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
     ports:
-      # Loopback only — the signer mints payments with no auth when the webhook is unset.
-      # Use "0.0.0.0:8081:8081" to expose remotely ONLY with REMOTE_SIGNER_WEBHOOK_URL set.
       - "127.0.0.1:8081:8081"
     volumes:
-      - ./remote-signer/.env:/service/.env:ro
+      - ./.env:/service/.env:ro
       - ./remote-signer/data:/data
 
   openmeter-collector:
@@ -54,7 +72,7 @@ services:
       kafka:
         condition: service_healthy
     volumes:
-      - ./openmeter-collector/.env:/service/.env:ro
+      - ./.env:/service/.env:ro
 
 # Signer datadir is bind-mounted from remote-signer/data.
 # Copy keystore + .eth-password into remote-signer/data before first run.

identity-webhook/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:22-alpine
+
+RUN apk add --no-cache curl
+
+WORKDIR /app
+
+COPY identity-webhook/package.json identity-webhook/package-lock.json ./
+RUN npm ci --omit=dev
+
+COPY identity-webhook/keys.mjs identity-webhook/protocol.mjs identity-webhook/verifiers.mjs identity-webhook/server.mjs ./
+
+ENV PORT=8090
+
+EXPOSE 8090
+
+CMD ["node", "server.mjs"]