From f176e2ea999f3a340d313b3cbd408dc8e6d9d886 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Thu, 18 Jun 2026 18:16:14 -0400 Subject: [PATCH 01/17] feat(deploy): add Docker Compose stack for runtime services Kafka, remote signer, and OpenMeter collector with local compose targets. --- deploy/.env.example | 29 ++++++ deploy/README.md | 106 ++++++++++++++++++++++ deploy/docker-compose.yml | 72 +++++++++++++++ deploy/kafka/Dockerfile | 13 +++ deploy/kafka/entrypoint.sh | 15 +++ deploy/openmeter-collector/Dockerfile | 6 ++ deploy/openmeter-collector/collector.yaml | 74 +++++++++++++++ deploy/remote-signer/Dockerfile | 25 +++++ deploy/remote-signer/entrypoint.sh | 55 +++++++++++ 9 files changed, 395 insertions(+) create mode 100644 deploy/.env.example create mode 100644 deploy/README.md create mode 100644 deploy/docker-compose.yml create mode 100644 deploy/kafka/Dockerfile create mode 100644 deploy/kafka/entrypoint.sh create mode 100644 deploy/openmeter-collector/Dockerfile create mode 100644 deploy/openmeter-collector/collector.yaml create mode 100644 deploy/remote-signer/Dockerfile create mode 100755 deploy/remote-signer/entrypoint.sh diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..90ef863 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,29 @@ +# Runtime stack env for deploy/docker-compose.yml +# +# cp deploy/.env.example deploy/.env +# $EDITOR deploy/.env + +# --- Identity webhook (required) --- +REMOTE_SIGNER_WEBHOOK_URL=https://your-platform.vercel.app/api/signer/authorize +WEBHOOK_SECRET= + +# --- Signer --- +SIGNER_NETWORK=arbitrum-one-mainnet +ETH_RPC_URL=https://arb1.arbitrum.io/rpc +SIGNER_ETH_ADDR= +SIGNER_HOST_PORT=8081 +SIGNER_REMOTE_DISCOVERY=0 +# ORCH_WEBHOOK_URL= +# LIVE_AI_CAP_REPORT_INTERVAL= + +# --- Kafka --- +KAFKA_GATEWAY_TOPIC=livepeer-gateway-events + +# --- OpenMeter / Konnect (required for collector) --- +OPENMETER_URL= +OPENMETER_INGEST_URL= +OPENMETER_API_KEY= +ETH_USD_PRICE=3500 + +# --- Optional (collector identity metadata) --- +# AUTH0_PUBLIC_CLIENT_ID= diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..71ed8de --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,106 @@ +# clearinghouse deploy/ + +Docker Compose stack for the clearinghouse runtime: +**Kafka/Redpanda → go-livepeer remote signer → OpenMeter/Benthos collector → Konnect metering**. + +## Design decisions + +**No Apache DMZ.** The remote signer container runs `go-livepeer` directly — +there is no Apache reverse proxy or `mod_authnz_jwt` layer in front of it. +Identity validation is handled by the `-remoteSignerWebhookUrl` hook (your +`/authorize` endpoint) and the shared `WEBHOOK_SECRET`. + +**CLI port not exposed.** go-livepeer's `-cliAddr` (admin/RPC) is bound to +`127.0.0.1:4935` inside the container and is never published or mapped to the +host — not in `docker-compose.yml`. Only the signing HTTP port +(`8081`) is exposed. + +## Local stack + +```bash +cp deploy/.env.example deploy/.env +$EDITOR deploy/.env + +# If using clearinghouse-bootstrap (feat/go-bootstrap-cli), merge WEBHOOK_SECRET +# and Konnect vars from .env.livepeer into deploy/.env. + +docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build +docker compose -f deploy/docker-compose.yml --env-file deploy/.env logs -f +docker compose -f deploy/docker-compose.yml --env-file deploy/.env down +``` + +Kafka + signer only (no metering): + +```bash +docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build kafka remote-signer +``` + +Verify CLI port is not published: + +```bash +docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-signer 4935 +# expected: no output / error (port is not mapped) +docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-signer 8081 +# expected: 0.0.0.0:8081 +``` + +## Environment variables + +| Variable | Required | Default | Description | +|---|---|---|---| +| `REMOTE_SIGNER_WEBHOOK_URL` | yes | — | Identity webhook URL (`/authorize` endpoint) | +| `WEBHOOK_SECRET` | yes | — | Shared secret passed as `Authorization: Bearer` to the webhook (from bootstrap) | +| `SIGNER_NETWORK` | no | `arbitrum-one-mainnet` | go-livepeer `-network` | +| `ETH_RPC_URL` | no | public arb1 endpoint | Arbitrum RPC | +| `SIGNER_ETH_ADDR` | no | — | Funded signer Ethereum address | +| `SIGNER_HOST_PORT` | no | `8081` | Host port for the signing HTTP endpoint | +| `KAFKA_GATEWAY_TOPIC` | no | `livepeer-gateway-events` | Kafka topic | +| `OPENMETER_URL` | yes | — | OpenMeter / Konnect base URL (from bootstrap) | +| `OPENMETER_INGEST_URL` | yes | — | Ingest endpoint (`${OPENMETER_URL}/events` for Konnect) | +| `OPENMETER_API_KEY` | yes | — | Konnect PAT (`kpat_…`) (from bootstrap) | +| `ETH_USD_PRICE` | no | `3500` | ETH/USD rate for Wei→USD micros conversion | +| `AUTH0_PUBLIC_CLIENT_ID` | no | — | Auth0 public client id (from bootstrap) | + +## OpenMeter/Konnect bootstrap + +Provision meters, features, and the default pay-per-use plan before starting the collector. +Use the Go `clearinghouse-bootstrap` CLI (`feat/go-bootstrap-cli`) or your existing Konnect setup. + +Creates (additive — existing pymthouse objects are untouched): + +| Object | Key | Purpose | +|--------|-----|---------| +| Meter | `network_fee_usd_micros` | Raw network cost from signer | +| Meter | `billable_usd_micros` | Post-markup billable amount (collector phase 2) | +| Meter | `signed_ticket_count` | Request counts | +| Feature | `network_spend` | Trial/network spend feature | +| Feature | `billable_spend` | Billable usage feature | +| Plan | `clearinghouse_default_ppu` | Pay-per-use rate card | + +Idempotent — safe to re-run. + +### Two-meter billing model + +```text +Signer computed_fee (wei) + → collector: network_fee_usd_micros (raw network cost — observability) + → collector: billable_usd_micros (network × pipeline/model markup — billing) + → billable_spend feature + → clearinghouse_default_ppu subscription per customer +``` + +Markup rules are defined in the bootstrap CLI catalog. Collector +pipeline config: [`deploy/openmeter-collector/collector.yaml`](openmeter-collector/collector.yaml). +The collector does not yet emit `billable_usd_micros` (phase 2); until then the billable meter +stays empty while the catalog is ready. + +### Auth0 identity contract + +- Webhook returns `auth_id = "{azp}:{sub}"` (`CLAIM_CLIENT_ID=azp`, `USAGE_SUBJECT_TYPE=auth0_user_id`) +- Collector splits on first colon → `client_id` / `external_user_id` +- Konnect customer key: `{AUTH0_PUBLIC_CLIENT_ID}:{auth0|sub}` + +Example customer key: `abc123xyz:auth0|user456` + +Per-customer provisioning (Konnect customer + subscription) is a follow-up — not +yet implemented in the Go CLI. diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..7bbfe29 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,72 @@ +# Clearinghouse deploy stack: Kafka + go-livepeer remote signer + OpenMeter collector. +# +# No Apache DMZ — the signer container runs go-livepeer directly. Only its signing +# HTTP port (8081) is published to the host. The CLI/admin port (-cliAddr) is bound +# to loopback inside the container and is never exposed or published. +# +# Full stack: +# docker compose -f deploy/docker-compose.yml --env-file .env up -d --build +# +# Kafka + signer only (no metering): +# docker compose -f deploy/docker-compose.yml --env-file .env up -d --build kafka remote-signer + +services: + kafka: + build: + context: .. + dockerfile: deploy/kafka/Dockerfile + restart: unless-stopped + environment: + KAFKA_ADVERTISED_ADDR: kafka:9092 + healthcheck: + test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy'"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 15s + + remote-signer: + build: + context: .. + dockerfile: deploy/remote-signer/Dockerfile + restart: unless-stopped + depends_on: + kafka: + condition: service_healthy + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "${SIGNER_HOST_PORT:-8081}:8081" + volumes: + - signer_data:/data + environment: + SIGNER_NETWORK: ${SIGNER_NETWORK:-arbitrum-one-mainnet} + SIGNER_PORT: "8081" + ETH_RPC_URL: ${ETH_RPC_URL:-https://arb1.arbitrum.io/rpc} + SIGNER_ETH_ADDR: ${SIGNER_ETH_ADDR:-} + SIGNER_REMOTE_DISCOVERY: ${SIGNER_REMOTE_DISCOVERY:-0} + ORCH_WEBHOOK_URL: ${ORCH_WEBHOOK_URL:-} + LIVE_AI_CAP_REPORT_INTERVAL: ${LIVE_AI_CAP_REPORT_INTERVAL:-} + KAFKA_BROKERS: kafka:9092 + KAFKA_GATEWAY_TOPIC: ${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events} + REMOTE_SIGNER_WEBHOOK_URL: ${REMOTE_SIGNER_WEBHOOK_URL} + WEBHOOK_SECRET: ${WEBHOOK_SECRET} + + openmeter-collector: + build: + context: .. + dockerfile: deploy/openmeter-collector/Dockerfile + restart: unless-stopped + depends_on: + kafka: + condition: service_healthy + environment: + KAFKA_BROKERS: kafka:9092 + KAFKA_GATEWAY_TOPIC: ${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events} + OPENMETER_URL: ${OPENMETER_URL} + OPENMETER_INGEST_URL: ${OPENMETER_INGEST_URL} + OPENMETER_API_KEY: ${OPENMETER_API_KEY} + ETH_USD_PRICE: ${ETH_USD_PRICE:-3500} + +volumes: + signer_data: diff --git a/deploy/kafka/Dockerfile b/deploy/kafka/Dockerfile new file mode 100644 index 0000000..1c353f9 --- /dev/null +++ b/deploy/kafka/Dockerfile @@ -0,0 +1,13 @@ +# Clearinghouse Kafka bus (Redpanda dev-container mode). +# KAFKA_ADVERTISED_ADDR defaults to kafka:9092 for local docker-compose. +FROM docker.redpanda.com/redpandadata/redpanda:v24.2.4 + +ENV CLUSTER_ID=6ac74f21-e6d3-453b-b5d6-de4fb811e8bb +ENV KAFKA_ADVERTISED_ADDR=kafka:9092 + +USER root +COPY deploy/kafka/entrypoint.sh /clearinghouse-entrypoint.sh +RUN chmod +x /clearinghouse-entrypoint.sh +USER redpanda + +ENTRYPOINT ["/clearinghouse-entrypoint.sh"] diff --git a/deploy/kafka/entrypoint.sh b/deploy/kafka/entrypoint.sh new file mode 100644 index 0000000..2ec60bb --- /dev/null +++ b/deploy/kafka/entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu + +ADVERTISED="${KAFKA_ADVERTISED_ADDR:-kafka:9092}" + +# Base image ENTRYPOINT is /entrypoint.sh -> exec /usr/bin/rpk "$@". +# Shell-form CMD becomes /bin/sh -c ..., which rpk rejects (-c flag). +# Exec-form CMD works; entrypoint allows runtime KAFKA_ADVERTISED_ADDR override. +exec /usr/bin/rpk redpanda start \ + --kafka-addr "internal://0.0.0.0:9092" \ + --advertise-kafka-addr "internal://${ADVERTISED}" \ + --mode dev-container \ + --smp 1 \ + --memory 512M \ + --overprovisioned diff --git a/deploy/openmeter-collector/Dockerfile b/deploy/openmeter-collector/Dockerfile new file mode 100644 index 0000000..1a79f70 --- /dev/null +++ b/deploy/openmeter-collector/Dockerfile @@ -0,0 +1,6 @@ +# OpenMeter Benthos collector: Kafka create_signed_ticket events -> Konnect/OpenMeter ingest. +FROM ghcr.io/openmeterio/benthos-collector:latest + +COPY deploy/openmeter-collector/collector.yaml /config.yaml + +CMD ["-c", "/config.yaml"] diff --git a/deploy/openmeter-collector/collector.yaml b/deploy/openmeter-collector/collector.yaml new file mode 100644 index 0000000..d520b94 --- /dev/null +++ b/deploy/openmeter-collector/collector.yaml @@ -0,0 +1,74 @@ +# OpenMeter Collector: go-livepeer remote signer Kafka -> hosted OpenMeter/Konnect. +# +# Notes: +# - Topic carries multiple event types; filter to create_signed_ticket only. +# - auth_id is expected as "client_id:external_user_id" from the signer webhook response. +# - Uses first-colon split semantics to preserve compatibility with builder-sdk. +# - Konnect ingest: POST {OPENMETER_INGEST_URL} (set to ${OPENMETER_URL}/events). +# For self-hosted OpenMeter set OPENMETER_INGEST_URL to ${OPENMETER_URL}/api/v1/events. + +input: + kafka: + addresses: + - ${KAFKA_BROKERS} + topics: + - ${KAFKA_GATEWAY_TOPIC} + consumer_group: openmeter-collector + start_from_oldest: true + +pipeline: + processors: + - mapping: | + root = if this.type != "create_signed_ticket" { deleted() } + + - mapping: | + let data = this.data.or({}) + let auth_id = $data.auth_id.or("").string().trim() + root = if $auth_id == "" { deleted() } + + let colon = $auth_id.index_of(":") + let is_compound = $colon > 0 && $colon < $auth_id.length() - 1 + let client_id = if $is_compound { $auth_id.slice(0, $colon) } else { $auth_id } + let external_user_id = if $is_compound { $auth_id.slice($colon + 1) } else { $auth_id } + + let eth_usd = env("ETH_USD_PRICE").number().or(3500) + let fee_wei = $data.computed_fee.number().or(0) + let fee_usd_micros = ($fee_wei * $eth_usd / 1000000000000).round() + + root = { + "specversion": "1.0", + "type": "create_signed_ticket", + "id": $data.request_id.or(uuid_v4()), + "source": "go-livepeer-remote-signer", + "subject": $auth_id, + "time": $data.current_time.or(now()), + "data": { + "client_id": $client_id, + "external_user_id": $external_user_id, + "network_fee_usd_micros": $fee_usd_micros, + "pipeline": if $data.pipeline != "" && $data.pipeline != null { $data.pipeline } else { "unknown" }, + "model_id": if $data.model_id != "" && $data.model_id != null { $data.model_id } else { "unknown" }, + "pixels": $data.pixels.string().or("0"), + "fee_wei": $data.computed_fee.or("0"), + "gateway_request_id": $data.request_id, + "auth_id": $auth_id, + } + } + + - catch: + - log: + level: ERROR + message: "signed_ticket mapping failed: ${! error() }" + - mapping: root = deleted() + +output: + http_client: + url: ${OPENMETER_INGEST_URL} + verb: POST + headers: + Authorization: "Bearer ${OPENMETER_API_KEY}" + Content-Type: application/cloudevents+json + successful_on: + - 200 + - 202 + - 204 diff --git a/deploy/remote-signer/Dockerfile b/deploy/remote-signer/Dockerfile new file mode 100644 index 0000000..ec62004 --- /dev/null +++ b/deploy/remote-signer/Dockerfile @@ -0,0 +1,25 @@ +# go-livepeer remote signer — extends livepeer/go-livepeer:latest (CUDA libs required by binary). +# Do not copy livepeer into debian:slim; the binary links libnppig.so and other CUDA NPP libs. +# +# No Apache DMZ: go-livepeer runs directly in this container. Only the signing HTTP port +# is EXPOSE'd and published. The CLI/admin port (-cliAddr) is bound to loopback inside +# the container and is never exposed or published — not here, not in docker-compose. +FROM livepeer/go-livepeer:latest + +RUN mkdir -p /data + +COPY deploy/remote-signer/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENV SIGNER_NETWORK=arbitrum-one-mainnet +ENV SIGNER_PORT=8081 +ENV ETH_RPC_URL=https://arb1.arbitrum.io/rpc +ENV SIGNER_ETH_ADDR="" +ENV SIGNER_REMOTE_DISCOVERY=0 +ENV KAFKA_BROKERS=kafka:9092 +ENV KAFKA_GATEWAY_TOPIC=livepeer-gateway-events + +EXPOSE 8081 +VOLUME ["/data"] + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/deploy/remote-signer/entrypoint.sh b/deploy/remote-signer/entrypoint.sh new file mode 100755 index 0000000..fae66a4 --- /dev/null +++ b/deploy/remote-signer/entrypoint.sh @@ -0,0 +1,55 @@ +#!/bin/sh +set -eu + +SIGNER_NETWORK="${SIGNER_NETWORK:-arbitrum-one-mainnet}" +SIGNER_PORT="${SIGNER_PORT:-8081}" +ETH_RPC_URL="${ETH_RPC_URL:-https://arb1.arbitrum.io/rpc}" +KAFKA_BROKERS="${KAFKA_BROKERS:-kafka:9092}" +KAFKA_GATEWAY_TOPIC="${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events}" + +if [ -z "${REMOTE_SIGNER_WEBHOOK_URL:-}" ]; then + echo "entrypoint: REMOTE_SIGNER_WEBHOOK_URL is required (identity webhook URL)" >&2 + exit 1 +fi +if [ -z "${WEBHOOK_SECRET:-}" ]; then + echo "entrypoint: WEBHOOK_SECRET is required" >&2 + exit 1 +fi + +if [ ! -f /data/.eth-password ]; then + echo "" >/data/.eth-password +fi + +ARGS="-remoteSigner" +ARGS="$ARGS -network=${SIGNER_NETWORK}" +ARGS="$ARGS -httpAddr=0.0.0.0:${SIGNER_PORT}" +# CLI/admin port: loopback only — never published, no Apache DMZ in front of this container. +ARGS="$ARGS -cliAddr=127.0.0.1:4935" +ARGS="$ARGS -ethUrl=${ETH_RPC_URL}" +ARGS="$ARGS -ethPassword=/data/.eth-password" +ARGS="$ARGS -datadir=/data" +ARGS="$ARGS -v=99" + +if [ -n "${SIGNER_ETH_ADDR:-}" ]; then + ARGS="$ARGS -ethAcctAddr=${SIGNER_ETH_ADDR}" +fi + +ARGS="$ARGS -remoteSignerWebhookUrl=${REMOTE_SIGNER_WEBHOOK_URL}" +ARGS="$ARGS -remoteSignerWebhookHeaders=Authorization:Bearer ${WEBHOOK_SECRET}" + +ARGS="$ARGS -monitor" +ARGS="$ARGS -kafkaBootstrapServers=${KAFKA_BROKERS}" +ARGS="$ARGS -kafkaGatewayTopic=${KAFKA_GATEWAY_TOPIC}" + +if [ "${SIGNER_REMOTE_DISCOVERY:-0}" = "1" ] || [ "${SIGNER_REMOTE_DISCOVERY:-0}" = "true" ]; then + ARGS="$ARGS -remoteDiscovery=true" + if [ -n "${ORCH_WEBHOOK_URL:-}" ]; then + ARGS="$ARGS -orchWebhookUrl=${ORCH_WEBHOOK_URL}" + fi + if [ -n "${LIVE_AI_CAP_REPORT_INTERVAL:-}" ]; then + ARGS="$ARGS -liveAICapReportInterval=${LIVE_AI_CAP_REPORT_INTERVAL}" + fi +fi + +echo "entrypoint: starting livepeer $ARGS" >&2 +exec /usr/local/bin/livepeer $ARGS From 116502e0f3df513f00b4299b81c43409a3603a94 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Thu, 18 Jun 2026 19:43:53 -0400 Subject: [PATCH 02/17] fix(deploy): update Docker Compose configuration and entrypoint script Changed the volume binding for the signer service to use a configurable data directory. Updated the entrypoint script to correct the remote signer webhook header format for compatibility. --- deploy/docker-compose.yml | 6 +++--- deploy/remote-signer/entrypoint.sh | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 7bbfe29..e718acf 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -38,7 +38,7 @@ services: ports: - "${SIGNER_HOST_PORT:-8081}:8081" volumes: - - signer_data:/data + - ${SIGNER_DATA_DIR:-./data}:/data environment: SIGNER_NETWORK: ${SIGNER_NETWORK:-arbitrum-one-mainnet} SIGNER_PORT: "8081" @@ -68,5 +68,5 @@ services: OPENMETER_API_KEY: ${OPENMETER_API_KEY} ETH_USD_PRICE: ${ETH_USD_PRICE:-3500} -volumes: - signer_data: +# Signer datadir is bind-mounted from deploy/data (see SIGNER_DATA_DIR). +# Copy keystore + .eth-password from pymthouse/data before first run. diff --git a/deploy/remote-signer/entrypoint.sh b/deploy/remote-signer/entrypoint.sh index fae66a4..e32b035 100755 --- a/deploy/remote-signer/entrypoint.sh +++ b/deploy/remote-signer/entrypoint.sh @@ -35,7 +35,9 @@ if [ -n "${SIGNER_ETH_ADDR:-}" ]; then fi ARGS="$ARGS -remoteSignerWebhookUrl=${REMOTE_SIGNER_WEBHOOK_URL}" -ARGS="$ARGS -remoteSignerWebhookHeaders=Authorization:Bearer ${WEBHOOK_SECRET}" +# X-Api-Key has no spaces — Authorization:Bearer ${WEBHOOK_SECRET} breaks when +# /usr/local/bin/livepeer $ARGS word-splits the secret into a stray argv token. +ARGS="$ARGS -remoteSignerWebhookHeaders=X-Api-Key:${WEBHOOK_SECRET}" ARGS="$ARGS -monitor" ARGS="$ARGS -kafkaBootstrapServers=${KAFKA_BROKERS}" From 53b18655f9285ae9c382a15a3149f3b09ce5c03c Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Mon, 22 Jun 2026 11:50:34 -0400 Subject: [PATCH 03/17] chore(deploy): remove unused Auth0 env var and tidy identity docs Drop AUTH0_PUBLIC_CLIENT_ID from deploy env/README now that the stacked identity-webhook uses API-key auth. Remove PR delivery notes from README. --- deploy/.env.example | 7 ++----- deploy/README.md | 13 +++---------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/deploy/.env.example b/deploy/.env.example index 90ef863..ed906e9 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -20,10 +20,7 @@ SIGNER_REMOTE_DISCOVERY=0 KAFKA_GATEWAY_TOPIC=livepeer-gateway-events # --- OpenMeter / Konnect (required for collector) --- -OPENMETER_URL= -OPENMETER_INGEST_URL= +OPENMETER_URL=https://us.api.konghq.com/v3/openmeter +OPENMETER_INGEST_URL=https://us.api.konghq.com/v3/openmeter/events OPENMETER_API_KEY= ETH_USD_PRICE=3500 - -# --- Optional (collector identity metadata) --- -# AUTH0_PUBLIC_CLIENT_ID= diff --git a/deploy/README.md b/deploy/README.md index 71ed8de..8095b80 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -59,7 +59,6 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s | `OPENMETER_INGEST_URL` | yes | — | Ingest endpoint (`${OPENMETER_URL}/events` for Konnect) | | `OPENMETER_API_KEY` | yes | — | Konnect PAT (`kpat_…`) (from bootstrap) | | `ETH_USD_PRICE` | no | `3500` | ETH/USD rate for Wei→USD micros conversion | -| `AUTH0_PUBLIC_CLIENT_ID` | no | — | Auth0 public client id (from bootstrap) | ## OpenMeter/Konnect bootstrap @@ -94,13 +93,7 @@ pipeline config: [`deploy/openmeter-collector/collector.yaml`](openmeter-collect The collector does not yet emit `billable_usd_micros` (phase 2); until then the billable meter stays empty while the catalog is ready. -### Auth0 identity contract +### Identity contract (collector) -- Webhook returns `auth_id = "{azp}:{sub}"` (`CLAIM_CLIENT_ID=azp`, `USAGE_SUBJECT_TYPE=auth0_user_id`) -- Collector splits on first colon → `client_id` / `external_user_id` -- Konnect customer key: `{AUTH0_PUBLIC_CLIENT_ID}:{auth0|sub}` - -Example customer key: `abc123xyz:auth0|user456` - -Per-customer provisioning (Konnect customer + subscription) is a follow-up — not -yet implemented in the Go CLI. +The collector expects Kafka `auth_id` as `client_id:external_user_id` (first-colon split). +Konnect customer key matches that compound id (e.g. `demo-client:demo-user`). From e11ec1e3e036ec859ea52404b1e55583204cccf1 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Mon, 22 Jun 2026 12:24:35 -0400 Subject: [PATCH 04/17] fix(deploy): update remote signer entrypoint script and README documentation Refactor the entrypoint script to use a more concise argument passing method and correct the format of the `WEBHOOK_SECRET` in the README to include the required syntax for the Authorization header. --- deploy/README.md | 2 +- deploy/remote-signer/entrypoint.sh | 44 ++++++++++++++---------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 8095b80..667656f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -49,7 +49,7 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s | Variable | Required | Default | Description | |---|---|---|---| | `REMOTE_SIGNER_WEBHOOK_URL` | yes | — | Identity webhook URL (`/authorize` endpoint) | -| `WEBHOOK_SECRET` | yes | — | Shared secret passed as `Authorization: Bearer` to the webhook (from bootstrap) | +| `WEBHOOK_SECRET` | yes | — | Shared secret passed as `Authorization:Bearer ` to the webhook (from bootstrap) | | `SIGNER_NETWORK` | no | `arbitrum-one-mainnet` | go-livepeer `-network` | | `ETH_RPC_URL` | no | public arb1 endpoint | Arbitrum RPC | | `SIGNER_ETH_ADDR` | no | — | Funded signer Ethereum address | diff --git a/deploy/remote-signer/entrypoint.sh b/deploy/remote-signer/entrypoint.sh index e32b035..bdf5aba 100755 --- a/deploy/remote-signer/entrypoint.sh +++ b/deploy/remote-signer/entrypoint.sh @@ -20,38 +20,34 @@ if [ ! -f /data/.eth-password ]; then echo "" >/data/.eth-password fi -ARGS="-remoteSigner" -ARGS="$ARGS -network=${SIGNER_NETWORK}" -ARGS="$ARGS -httpAddr=0.0.0.0:${SIGNER_PORT}" -# CLI/admin port: loopback only — never published, no Apache DMZ in front of this container. -ARGS="$ARGS -cliAddr=127.0.0.1:4935" -ARGS="$ARGS -ethUrl=${ETH_RPC_URL}" -ARGS="$ARGS -ethPassword=/data/.eth-password" -ARGS="$ARGS -datadir=/data" -ARGS="$ARGS -v=99" +set -- \ + -remoteSigner \ + "-network=${SIGNER_NETWORK}" \ + "-httpAddr=0.0.0.0:${SIGNER_PORT}" \ + "-cliAddr=127.0.0.1:4935" \ + "-ethUrl=${ETH_RPC_URL}" \ + "-ethPassword=/data/.eth-password" \ + "-datadir=/data" \ + -v=99 \ + "-remoteSignerWebhookUrl=${REMOTE_SIGNER_WEBHOOK_URL}" \ + "-remoteSignerWebhookHeaders=Authorization:Bearer ${WEBHOOK_SECRET}" \ + -monitor \ + "-kafkaBootstrapServers=${KAFKA_BROKERS}" \ + "-kafkaGatewayTopic=${KAFKA_GATEWAY_TOPIC}" if [ -n "${SIGNER_ETH_ADDR:-}" ]; then - ARGS="$ARGS -ethAcctAddr=${SIGNER_ETH_ADDR}" + set -- "$@" "-ethAcctAddr=${SIGNER_ETH_ADDR}" fi -ARGS="$ARGS -remoteSignerWebhookUrl=${REMOTE_SIGNER_WEBHOOK_URL}" -# X-Api-Key has no spaces — Authorization:Bearer ${WEBHOOK_SECRET} breaks when -# /usr/local/bin/livepeer $ARGS word-splits the secret into a stray argv token. -ARGS="$ARGS -remoteSignerWebhookHeaders=X-Api-Key:${WEBHOOK_SECRET}" - -ARGS="$ARGS -monitor" -ARGS="$ARGS -kafkaBootstrapServers=${KAFKA_BROKERS}" -ARGS="$ARGS -kafkaGatewayTopic=${KAFKA_GATEWAY_TOPIC}" - if [ "${SIGNER_REMOTE_DISCOVERY:-0}" = "1" ] || [ "${SIGNER_REMOTE_DISCOVERY:-0}" = "true" ]; then - ARGS="$ARGS -remoteDiscovery=true" + set -- "$@" -remoteDiscovery=true if [ -n "${ORCH_WEBHOOK_URL:-}" ]; then - ARGS="$ARGS -orchWebhookUrl=${ORCH_WEBHOOK_URL}" + set -- "$@" "-orchWebhookUrl=${ORCH_WEBHOOK_URL}" fi if [ -n "${LIVE_AI_CAP_REPORT_INTERVAL:-}" ]; then - ARGS="$ARGS -liveAICapReportInterval=${LIVE_AI_CAP_REPORT_INTERVAL}" + set -- "$@" "-liveAICapReportInterval=${LIVE_AI_CAP_REPORT_INTERVAL}" fi fi -echo "entrypoint: starting livepeer $ARGS" >&2 -exec /usr/local/bin/livepeer $ARGS +echo "entrypoint: starting livepeer remote-signer on 0.0.0.0:${SIGNER_PORT}" >&2 +exec /usr/local/bin/livepeer "$@" From 6132276a22422a741cf4efd2c28136ba89e065ad Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 14:44:37 -0400 Subject: [PATCH 05/17] docker(collector): pin docker image version for openmeterio/benthos-collector --- deploy/openmeter-collector/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/openmeter-collector/Dockerfile b/deploy/openmeter-collector/Dockerfile index 1a79f70..968cf01 100644 --- a/deploy/openmeter-collector/Dockerfile +++ b/deploy/openmeter-collector/Dockerfile @@ -1,5 +1,5 @@ # OpenMeter Benthos collector: Kafka create_signed_ticket events -> Konnect/OpenMeter ingest. -FROM ghcr.io/openmeterio/benthos-collector:latest +FROM ghcr.io/openmeterio/benthos-collector:main-6b60ab6-1782310960 COPY deploy/openmeter-collector/collector.yaml /config.yaml From 1005d32db097190a0599ed2ea52b05abbd450ae6 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 14:45:20 -0400 Subject: [PATCH 06/17] docker(signer): pin go-livepeer image to v0.8.11 --- deploy/remote-signer/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/remote-signer/Dockerfile b/deploy/remote-signer/Dockerfile index ec62004..24074f4 100644 --- a/deploy/remote-signer/Dockerfile +++ b/deploy/remote-signer/Dockerfile @@ -1,10 +1,10 @@ -# go-livepeer remote signer — extends livepeer/go-livepeer:latest (CUDA libs required by binary). +# go-livepeer remote signer — extends livepeer/go-livepeer:v0.8.11 (CUDA libs required by binary). # Do not copy livepeer into debian:slim; the binary links libnppig.so and other CUDA NPP libs. # # No Apache DMZ: go-livepeer runs directly in this container. Only the signing HTTP port # is EXPOSE'd and published. The CLI/admin port (-cliAddr) is bound to loopback inside # the container and is never exposed or published — not here, not in docker-compose. -FROM livepeer/go-livepeer:latest +FROM livepeer/go-livepeer:v0.8.11 RUN mkdir -p /data From 71eeb0bfcca3ed754d359ce6b7ffcbd22e59ee0c Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 14:50:19 -0400 Subject: [PATCH 07/17] chore(deploy): add .gitignore to exclude environment file --- deploy/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 deploy/.gitignore diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000..f10862a --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1 @@ +/.env From e93230b34a76a4ea2a1b1cde04a815c87c6bd270 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 14:55:50 -0400 Subject: [PATCH 08/17] docs(deploy): update README and docker-compose comments to reflect Redpanda integration Replaced references to Apache Kafka with Redpanda in both the README and docker-compose.yml. Clarified the architecture and identity validation process for the remote signer container. --- deploy/README.md | 13 ++++--------- deploy/docker-compose.yml | 8 ++++---- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 667656f..7df06b6 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -5,15 +5,13 @@ Docker Compose stack for the clearinghouse runtime: ## Design decisions -**No Apache DMZ.** The remote signer container runs `go-livepeer` directly — -there is no Apache reverse proxy or `mod_authnz_jwt` layer in front of it. -Identity validation is handled by the `-remoteSignerWebhookUrl` hook (your -`/authorize` endpoint) and the shared `WEBHOOK_SECRET`. +**Redpanda over Apache Kafka.** The stack uses Redpanda as the Kafka-compatible broker. Redpanda Kafka runs as a single-binary dev container with no ZooKeeper dependency and faster local startup. + +**Identity & auth.** The signer container runs `go-livepeer` directly. Every signing request is authorized by go-livepeer's `-remoteSignerWebhookUrl` hook, which calls your `/authorize` endpoint with `Authorization: Bearer ` — no reverse proxy or gateway in front of the signer. **CLI port not exposed.** go-livepeer's `-cliAddr` (admin/RPC) is bound to `127.0.0.1:4935` inside the container and is never published or mapped to the -host — not in `docker-compose.yml`. Only the signing HTTP port -(`8081`) is exposed. +host. Only the signing HTTP port (`8081`) is exposed. ## Local stack @@ -21,9 +19,6 @@ host — not in `docker-compose.yml`. Only the signing HTTP port cp deploy/.env.example deploy/.env $EDITOR deploy/.env -# If using clearinghouse-bootstrap (feat/go-bootstrap-cli), merge WEBHOOK_SECRET -# and Konnect vars from .env.livepeer into deploy/.env. - docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build docker compose -f deploy/docker-compose.yml --env-file deploy/.env logs -f docker compose -f deploy/docker-compose.yml --env-file deploy/.env down diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e718acf..393e0a7 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,8 +1,8 @@ -# Clearinghouse deploy stack: Kafka + go-livepeer remote signer + OpenMeter collector. +# Clearinghouse deploy stack: Redpanda + go-livepeer remote signer + OpenMeter collector. # -# No Apache DMZ — the signer container runs go-livepeer directly. Only its signing -# HTTP port (8081) is published to the host. The CLI/admin port (-cliAddr) is bound -# to loopback inside the container and is never exposed or published. +# Redpanda is the Kafka-compatible broker; The signer container runs go-livepeer directly — only its signing HTTP port +# (8081) is published to the host. The CLI/admin port (-cliAddr) is bound to loopback +# inside the container and is never exposed or published. # # Full stack: # docker compose -f deploy/docker-compose.yml --env-file .env up -d --build From 80648df8334c39f530bba1ef04b68b073dc6ce1b Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 15:31:25 -0400 Subject: [PATCH 09/17] docs(deploy): add notes for Redpanda development mode in entrypoint script Included comments in the entrypoint.sh to clarify the usage of Redpanda in development mode, highlighting the lack of security and persistence features, and linking to the relevant issue for tracking. --- deploy/kafka/entrypoint.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/kafka/entrypoint.sh b/deploy/kafka/entrypoint.sh index 2ec60bb..d366446 100644 --- a/deploy/kafka/entrypoint.sh +++ b/deploy/kafka/entrypoint.sh @@ -3,6 +3,13 @@ set -eu ADVERTISED="${KAFKA_ADVERTISED_ADDR:-kafka:9092}" +# NOTE: Deployment: See https://github.com/livepeer/clearinghouse/issues/43 for tracking. +# The --mode dev-container flag below launches Redpanda in "development container" mode: +# - No security/encryption (PLAINTEXT) +# - Not for production! This enables fast local startup, disables most persistence/durability, +# and relaxes networking for development and Docker Compose stacks. +# Update this script if production requirements change. + # Base image ENTRYPOINT is /entrypoint.sh -> exec /usr/bin/rpk "$@". # Shell-form CMD becomes /bin/sh -c ..., which rpk rejects (-c flag). # Exec-form CMD works; entrypoint allows runtime KAFKA_ADVERTISED_ADDR override. From e306c3ecb1ead05f3dff0ca98a8a6b7c0fdfed49 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 15:48:56 -0400 Subject: [PATCH 10/17] docs(deploy): clarify data directory setup in docker-compose and README Updated comments in docker-compose.yml to specify that the keystore and .eth-password should be copied into deploy/data before the first run. Revised README to streamline the description of the bootstrap process for meters and features. --- deploy/README.md | 4 ++-- deploy/docker-compose.yml | 2 +- deploy/kafka/Dockerfile | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 7df06b6..168de35 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -58,9 +58,9 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s ## OpenMeter/Konnect bootstrap Provision meters, features, and the default pay-per-use plan before starting the collector. -Use the Go `clearinghouse-bootstrap` CLI (`feat/go-bootstrap-cli`) or your existing Konnect setup. +Use the Go `clearinghouse-bootstrap` CLI or your existing Konnect setup. -Creates (additive — existing pymthouse objects are untouched): +Creates: | Object | Key | Purpose | |--------|-----|---------| diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 393e0a7..4a15a7f 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -69,4 +69,4 @@ services: ETH_USD_PRICE: ${ETH_USD_PRICE:-3500} # Signer datadir is bind-mounted from deploy/data (see SIGNER_DATA_DIR). -# Copy keystore + .eth-password from pymthouse/data before first run. +# Copy keystore + .eth-password into deploy/data before first run. diff --git a/deploy/kafka/Dockerfile b/deploy/kafka/Dockerfile index 1c353f9..d935f82 100644 --- a/deploy/kafka/Dockerfile +++ b/deploy/kafka/Dockerfile @@ -2,7 +2,6 @@ # KAFKA_ADVERTISED_ADDR defaults to kafka:9092 for local docker-compose. FROM docker.redpanda.com/redpandadata/redpanda:v24.2.4 -ENV CLUSTER_ID=6ac74f21-e6d3-453b-b5d6-de4fb811e8bb ENV KAFKA_ADVERTISED_ADDR=kafka:9092 USER root From 085d90472b193ff0898905e75c4c7e643f374f71 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 16:19:20 -0400 Subject: [PATCH 11/17] fix(deploy): require ETH_USD_PRICE environment variable in configuration Updated docker-compose.yml and collector.yaml to ensure ETH_USD_PRICE is a required environment variable, removing the default value. Revised README to reflect this change, indicating that the ETH/USD rate must be provided explicitly. --- deploy/README.md | 2 +- deploy/docker-compose.yml | 2 +- deploy/openmeter-collector/collector.yaml | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 168de35..b514dcb 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,7 +53,7 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s | `OPENMETER_URL` | yes | — | OpenMeter / Konnect base URL (from bootstrap) | | `OPENMETER_INGEST_URL` | yes | — | Ingest endpoint (`${OPENMETER_URL}/events` for Konnect) | | `OPENMETER_API_KEY` | yes | — | Konnect PAT (`kpat_…`) (from bootstrap) | -| `ETH_USD_PRICE` | no | `3500` | ETH/USD rate for Wei→USD micros conversion | +| `ETH_USD_PRICE` | yes | - | ETH/USD rate for Wei→USD micros conversion | ## OpenMeter/Konnect bootstrap diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 4a15a7f..ed997bc 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -66,7 +66,7 @@ services: OPENMETER_URL: ${OPENMETER_URL} OPENMETER_INGEST_URL: ${OPENMETER_INGEST_URL} OPENMETER_API_KEY: ${OPENMETER_API_KEY} - ETH_USD_PRICE: ${ETH_USD_PRICE:-3500} + ETH_USD_PRICE: ${ETH_USD_PRICE} # Signer datadir is bind-mounted from deploy/data (see SIGNER_DATA_DIR). # Copy keystore + .eth-password into deploy/data before first run. diff --git a/deploy/openmeter-collector/collector.yaml b/deploy/openmeter-collector/collector.yaml index d520b94..7d6a82c 100644 --- a/deploy/openmeter-collector/collector.yaml +++ b/deploy/openmeter-collector/collector.yaml @@ -31,7 +31,10 @@ pipeline: let client_id = if $is_compound { $auth_id.slice(0, $colon) } else { $auth_id } let external_user_id = if $is_compound { $auth_id.slice($colon + 1) } else { $auth_id } - let eth_usd = env("ETH_USD_PRICE").number().or(3500) + let eth_usd_str = env("ETH_USD_PRICE").or("").string().trim() + let _ = if $eth_usd_str == "" { throw("ETH_USD_PRICE is required") } + let eth_usd = $eth_usd_str.number() + let _ = if $eth_usd <= 0 { throw("ETH_USD_PRICE must be a positive number") } let fee_wei = $data.computed_fee.number().or(0) let fee_usd_micros = ($fee_wei * $eth_usd / 1000000000000).round() From 99d56b09750b773d3fc346efc0a7cde95bc758b1 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 16:32:45 -0400 Subject: [PATCH 12/17] feat(deploy): enhance Kafka configuration in environment and Docker setup Added KAFKA_ADVERTISED_ADDR and KAFKA_BROKERS to the .env.example file for better Kafka integration. Updated docker-compose.yml to utilize these environment variables, simplifying the configuration for the Kafka and signer services. Revised README to document the new environment variables and their purposes. --- deploy/.env.example | 2 ++ deploy/README.md | 2 ++ deploy/docker-compose.yml | 28 ++++++++++++++++------------ deploy/kafka/Dockerfile | 3 --- deploy/kafka/entrypoint.sh | 2 +- deploy/remote-signer/Dockerfile | 14 +++----------- deploy/remote-signer/entrypoint.sh | 10 +++++----- 7 files changed, 29 insertions(+), 32 deletions(-) diff --git a/deploy/.env.example b/deploy/.env.example index ed906e9..170d28b 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -17,6 +17,8 @@ SIGNER_REMOTE_DISCOVERY=0 # LIVE_AI_CAP_REPORT_INTERVAL= # --- Kafka --- +KAFKA_ADVERTISED_ADDR=kafka:9092 +KAFKA_BROKERS=kafka:9092 KAFKA_GATEWAY_TOPIC=livepeer-gateway-events # --- OpenMeter / Konnect (required for collector) --- diff --git a/deploy/README.md b/deploy/README.md index b514dcb..ff1e070 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -49,6 +49,8 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s | `ETH_RPC_URL` | no | public arb1 endpoint | Arbitrum RPC | | `SIGNER_ETH_ADDR` | no | — | Funded signer Ethereum address | | `SIGNER_HOST_PORT` | no | `8081` | Host port for the signing HTTP endpoint | +| `KAFKA_ADVERTISED_ADDR` | no | `kafka:9092` | Redpanda advertised Kafka address (broker container) | +| `KAFKA_BROKERS` | no | `kafka:9092` | Kafka bootstrap servers | | `KAFKA_GATEWAY_TOPIC` | no | `livepeer-gateway-events` | Kafka topic | | `OPENMETER_URL` | yes | — | OpenMeter / Konnect base URL (from bootstrap) | | `OPENMETER_INGEST_URL` | yes | — | Ingest endpoint (`${OPENMETER_URL}/events` for Konnect) | diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index ed997bc..e384732 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -10,6 +10,19 @@ # Kafka + signer only (no metering): # docker compose -f deploy/docker-compose.yml --env-file .env up -d --build kafka remote-signer +x-kafka-env: &kafka-env + KAFKA_BROKERS: ${KAFKA_BROKERS} + KAFKA_GATEWAY_TOPIC: ${KAFKA_GATEWAY_TOPIC} + +x-signer-env: &signer-env + SIGNER_NETWORK: ${SIGNER_NETWORK} + ETH_RPC_URL: ${ETH_RPC_URL} + SIGNER_PORT: "8081" + SIGNER_ETH_ADDR: ${SIGNER_ETH_ADDR:-} + SIGNER_REMOTE_DISCOVERY: ${SIGNER_REMOTE_DISCOVERY:-0} + ORCH_WEBHOOK_URL: ${ORCH_WEBHOOK_URL:-} + LIVE_AI_CAP_REPORT_INTERVAL: ${LIVE_AI_CAP_REPORT_INTERVAL:-} + services: kafka: build: @@ -17,7 +30,7 @@ services: dockerfile: deploy/kafka/Dockerfile restart: unless-stopped environment: - KAFKA_ADVERTISED_ADDR: kafka:9092 + KAFKA_ADVERTISED_ADDR: ${KAFKA_ADVERTISED_ADDR} healthcheck: test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy'"] interval: 10s @@ -40,15 +53,7 @@ services: volumes: - ${SIGNER_DATA_DIR:-./data}:/data environment: - SIGNER_NETWORK: ${SIGNER_NETWORK:-arbitrum-one-mainnet} - SIGNER_PORT: "8081" - ETH_RPC_URL: ${ETH_RPC_URL:-https://arb1.arbitrum.io/rpc} - SIGNER_ETH_ADDR: ${SIGNER_ETH_ADDR:-} - SIGNER_REMOTE_DISCOVERY: ${SIGNER_REMOTE_DISCOVERY:-0} - ORCH_WEBHOOK_URL: ${ORCH_WEBHOOK_URL:-} - LIVE_AI_CAP_REPORT_INTERVAL: ${LIVE_AI_CAP_REPORT_INTERVAL:-} - KAFKA_BROKERS: kafka:9092 - KAFKA_GATEWAY_TOPIC: ${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events} + <<: [*signer-env, *kafka-env] REMOTE_SIGNER_WEBHOOK_URL: ${REMOTE_SIGNER_WEBHOOK_URL} WEBHOOK_SECRET: ${WEBHOOK_SECRET} @@ -61,8 +66,7 @@ services: kafka: condition: service_healthy environment: - KAFKA_BROKERS: kafka:9092 - KAFKA_GATEWAY_TOPIC: ${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events} + <<: *kafka-env OPENMETER_URL: ${OPENMETER_URL} OPENMETER_INGEST_URL: ${OPENMETER_INGEST_URL} OPENMETER_API_KEY: ${OPENMETER_API_KEY} diff --git a/deploy/kafka/Dockerfile b/deploy/kafka/Dockerfile index d935f82..dd4b225 100644 --- a/deploy/kafka/Dockerfile +++ b/deploy/kafka/Dockerfile @@ -1,9 +1,6 @@ # Clearinghouse Kafka bus (Redpanda dev-container mode). -# KAFKA_ADVERTISED_ADDR defaults to kafka:9092 for local docker-compose. FROM docker.redpanda.com/redpandadata/redpanda:v24.2.4 -ENV KAFKA_ADVERTISED_ADDR=kafka:9092 - USER root COPY deploy/kafka/entrypoint.sh /clearinghouse-entrypoint.sh RUN chmod +x /clearinghouse-entrypoint.sh diff --git a/deploy/kafka/entrypoint.sh b/deploy/kafka/entrypoint.sh index d366446..259f11a 100644 --- a/deploy/kafka/entrypoint.sh +++ b/deploy/kafka/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu -ADVERTISED="${KAFKA_ADVERTISED_ADDR:-kafka:9092}" +ADVERTISED="${KAFKA_ADVERTISED_ADDR}" # NOTE: Deployment: See https://github.com/livepeer/clearinghouse/issues/43 for tracking. # The --mode dev-container flag below launches Redpanda in "development container" mode: diff --git a/deploy/remote-signer/Dockerfile b/deploy/remote-signer/Dockerfile index 24074f4..a1c395c 100644 --- a/deploy/remote-signer/Dockerfile +++ b/deploy/remote-signer/Dockerfile @@ -1,9 +1,9 @@ # go-livepeer remote signer — extends livepeer/go-livepeer:v0.8.11 (CUDA libs required by binary). # Do not copy livepeer into debian:slim; the binary links libnppig.so and other CUDA NPP libs. # -# No Apache DMZ: go-livepeer runs directly in this container. Only the signing HTTP port -# is EXPOSE'd and published. The CLI/admin port (-cliAddr) is bound to loopback inside -# the container and is never exposed or published — not here, not in docker-compose. +# go-livepeer runs directly in this container. Only the signing HTTP port is EXPOSE'd +# and published. The CLI/admin port (-cliAddr) is bound to loopback inside the +# container and is never exposed or published — not here, not in docker-compose. FROM livepeer/go-livepeer:v0.8.11 RUN mkdir -p /data @@ -11,14 +11,6 @@ RUN mkdir -p /data COPY deploy/remote-signer/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -ENV SIGNER_NETWORK=arbitrum-one-mainnet -ENV SIGNER_PORT=8081 -ENV ETH_RPC_URL=https://arb1.arbitrum.io/rpc -ENV SIGNER_ETH_ADDR="" -ENV SIGNER_REMOTE_DISCOVERY=0 -ENV KAFKA_BROKERS=kafka:9092 -ENV KAFKA_GATEWAY_TOPIC=livepeer-gateway-events - EXPOSE 8081 VOLUME ["/data"] diff --git a/deploy/remote-signer/entrypoint.sh b/deploy/remote-signer/entrypoint.sh index bdf5aba..e8b2462 100755 --- a/deploy/remote-signer/entrypoint.sh +++ b/deploy/remote-signer/entrypoint.sh @@ -1,11 +1,11 @@ #!/bin/sh set -eu -SIGNER_NETWORK="${SIGNER_NETWORK:-arbitrum-one-mainnet}" -SIGNER_PORT="${SIGNER_PORT:-8081}" -ETH_RPC_URL="${ETH_RPC_URL:-https://arb1.arbitrum.io/rpc}" -KAFKA_BROKERS="${KAFKA_BROKERS:-kafka:9092}" -KAFKA_GATEWAY_TOPIC="${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events}" +SIGNER_NETWORK="${SIGNER_NETWORK}" +SIGNER_PORT="${SIGNER_PORT}" +ETH_RPC_URL="${ETH_RPC_URL}" +KAFKA_BROKERS="${KAFKA_BROKERS}" +KAFKA_GATEWAY_TOPIC="${KAFKA_GATEWAY_TOPIC}" if [ -z "${REMOTE_SIGNER_WEBHOOK_URL:-}" ]; then echo "entrypoint: REMOTE_SIGNER_WEBHOOK_URL is required (identity webhook URL)" >&2 From 6d1858a7510d863fdf01b32a5c1d1beefd77dabb Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 16:37:55 -0400 Subject: [PATCH 13/17] feat(deploy): add SIGNER_ETH_KEYSTORE_PATH to configuration Introduced SIGNER_ETH_KEYSTORE_PATH in docker-compose.yml to allow specifying an optional keystore directory or keyfile path. Updated entrypoint.sh to pass this new environment variable to the signer service. Revised README to document the new variable and its purpose. --- deploy/.env.example | 1 + deploy/README.md | 1 + deploy/docker-compose.yml | 1 + deploy/remote-signer/entrypoint.sh | 4 ++++ 4 files changed, 7 insertions(+) diff --git a/deploy/.env.example b/deploy/.env.example index 170d28b..f1f6408 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -11,6 +11,7 @@ WEBHOOK_SECRET= SIGNER_NETWORK=arbitrum-one-mainnet ETH_RPC_URL=https://arb1.arbitrum.io/rpc SIGNER_ETH_ADDR= +SIGNER_ETH_KEYSTORE_PATH= SIGNER_HOST_PORT=8081 SIGNER_REMOTE_DISCOVERY=0 # ORCH_WEBHOOK_URL= diff --git a/deploy/README.md b/deploy/README.md index ff1e070..149e3ef 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -48,6 +48,7 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s | `SIGNER_NETWORK` | no | `arbitrum-one-mainnet` | go-livepeer `-network` | | `ETH_RPC_URL` | no | public arb1 endpoint | Arbitrum RPC | | `SIGNER_ETH_ADDR` | no | — | Funded signer Ethereum address | +| `SIGNER_ETH_KEYSTORE_PATH` | no | — | Optional keystore directory or keyfile path passed to `-ethKeystorePath` | | `SIGNER_HOST_PORT` | no | `8081` | Host port for the signing HTTP endpoint | | `KAFKA_ADVERTISED_ADDR` | no | `kafka:9092` | Redpanda advertised Kafka address (broker container) | | `KAFKA_BROKERS` | no | `kafka:9092` | Kafka bootstrap servers | diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e384732..785bb6d 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -19,6 +19,7 @@ x-signer-env: &signer-env ETH_RPC_URL: ${ETH_RPC_URL} SIGNER_PORT: "8081" SIGNER_ETH_ADDR: ${SIGNER_ETH_ADDR:-} + SIGNER_ETH_KEYSTORE_PATH: ${SIGNER_ETH_KEYSTORE_PATH:-} SIGNER_REMOTE_DISCOVERY: ${SIGNER_REMOTE_DISCOVERY:-0} ORCH_WEBHOOK_URL: ${ORCH_WEBHOOK_URL:-} LIVE_AI_CAP_REPORT_INTERVAL: ${LIVE_AI_CAP_REPORT_INTERVAL:-} diff --git a/deploy/remote-signer/entrypoint.sh b/deploy/remote-signer/entrypoint.sh index e8b2462..8cf52b3 100755 --- a/deploy/remote-signer/entrypoint.sh +++ b/deploy/remote-signer/entrypoint.sh @@ -39,6 +39,10 @@ if [ -n "${SIGNER_ETH_ADDR:-}" ]; then set -- "$@" "-ethAcctAddr=${SIGNER_ETH_ADDR}" fi +if [ -n "${SIGNER_ETH_KEYSTORE_PATH:-}" ]; then + set -- "$@" "-ethKeystorePath=${SIGNER_ETH_KEYSTORE_PATH}" +fi + if [ "${SIGNER_REMOTE_DISCOVERY:-0}" = "1" ] || [ "${SIGNER_REMOTE_DISCOVERY:-0}" = "true" ]; then set -- "$@" -remoteDiscovery=true if [ -n "${ORCH_WEBHOOK_URL:-}" ]; then From 718e9d533dd0ae6d9dee0f8674ef4da95a94f3ce Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 16:46:21 -0400 Subject: [PATCH 14/17] feat(deploy): add SIGNER_DATA_DIR and update README for remote discovery Introduced SIGNER_DATA_DIR in .env.example to specify the host directory for signer state and keystore. Updated README to document this new variable and added details for SIGNER_REMOTE_DISCOVERY and ORCH_WEBHOOK_URL, enhancing clarity on their usage. --- deploy/.env.example | 1 + deploy/README.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/deploy/.env.example b/deploy/.env.example index f1f6408..bdee875 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -12,6 +12,7 @@ SIGNER_NETWORK=arbitrum-one-mainnet ETH_RPC_URL=https://arb1.arbitrum.io/rpc SIGNER_ETH_ADDR= SIGNER_ETH_KEYSTORE_PATH= +SIGNER_DATA_DIR=./data SIGNER_HOST_PORT=8081 SIGNER_REMOTE_DISCOVERY=0 # ORCH_WEBHOOK_URL= diff --git a/deploy/README.md b/deploy/README.md index 149e3ef..58861b3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -49,7 +49,11 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s | `ETH_RPC_URL` | no | public arb1 endpoint | Arbitrum RPC | | `SIGNER_ETH_ADDR` | no | — | Funded signer Ethereum address | | `SIGNER_ETH_KEYSTORE_PATH` | no | — | Optional keystore directory or keyfile path passed to `-ethKeystorePath` | +| `SIGNER_DATA_DIR` | no | `./data` | Host directory bind-mounted to `/data` for signer state, keystore, and password file | | `SIGNER_HOST_PORT` | no | `8081` | Host port for the signing HTTP endpoint | +| `SIGNER_REMOTE_DISCOVERY` | no | `0` | Enables remote signer orchestrator discovery when set to `1` or `true` | +| `ORCH_WEBHOOK_URL` | no | — | Optional orchestrator discovery webhook passed to `-orchWebhookUrl` when remote discovery is enabled | +| `LIVE_AI_CAP_REPORT_INTERVAL` | no | — | Optional capacity report interval passed to `-liveAICapReportInterval` when remote discovery is enabled | | `KAFKA_ADVERTISED_ADDR` | no | `kafka:9092` | Redpanda advertised Kafka address (broker container) | | `KAFKA_BROKERS` | no | `kafka:9092` | Kafka bootstrap servers | | `KAFKA_GATEWAY_TOPIC` | no | `livepeer-gateway-events` | Kafka topic | From 622de5adc3a214d686d085c756ccd275fed7b29b Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 17:06:14 -0400 Subject: [PATCH 15/17] feat(deploy): update OpenMeter configuration Revised .env.example to clarify OpenMeter URLs for Konnect and self-hosted setups. Updated docker-compose.yml to remove the deprecated OPENMETER_URL. Updated README to provide clearer instructions for setting up the environment. --- deploy/.env.example | 3 +- deploy/README.md | 61 ++++++++++++++++------- deploy/docker-compose.yml | 1 - deploy/openmeter-collector/collector.yaml | 4 +- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/deploy/.env.example b/deploy/.env.example index bdee875..12e98ba 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -24,7 +24,8 @@ KAFKA_BROKERS=kafka:9092 KAFKA_GATEWAY_TOPIC=livepeer-gateway-events # --- OpenMeter / Konnect (required for collector) --- -OPENMETER_URL=https://us.api.konghq.com/v3/openmeter +# Konnect: https://.api.konghq.com/v3/openmeter/events +# Self-hosted: https:///api/v1/events OPENMETER_INGEST_URL=https://us.api.konghq.com/v3/openmeter/events OPENMETER_API_KEY= ETH_USD_PRICE=3500 diff --git a/deploy/README.md b/deploy/README.md index 58861b3..68295ad 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,7 +1,26 @@ # clearinghouse deploy/ Docker Compose stack for the clearinghouse runtime: -**Kafka/Redpanda → go-livepeer remote signer → OpenMeter/Benthos collector → Konnect metering**. +**Redpanda → go-livepeer remote signer → OpenMeter/Benthos collector → Konnect metering**. + +## Components + +| Service | Role | Docs | +| --- | --- | --- | +| **Redpanda** (`kafka`) | Kafka-compatible event bus. The signer publishes gateway events; the collector consumes them. | [Redpanda docs](https://docs.redpanda.com/) | +| **go-livepeer remote signer** (`remote-signer`) | Signs Livepeer payment tickets and emits `create_signed_ticket` events to Kafka. | [go-livepeer](https://github.com/livepeer/go-livepeer) | +| **OpenMeter collector** (`openmeter-collector`) | Benthos pipeline: filters Kafka events, converts fees to USD micros, POSTs CloudEvents to OpenMeter ingest. | [OpenMeter collector](https://openmeter.io/docs/collectors) | +| **Konnect / OpenMeter** (external) | Hosted metering and billing API. Set `OPENMETER_INGEST_URL` to your ingest endpoint. | [Konnect OpenMeter](https://docs.konghq.com/konnect/openmeter/), [self-hosted OpenMeter](https://openmeter.io/docs/deploy/kubernetes) | + +Data flow: + +```text +Signer HTTP request + → identity webhook (/authorize) + → signed ticket + Kafka create_signed_ticket event + → collector transforms event + → OpenMeter ingest API +``` ## Design decisions @@ -9,25 +28,20 @@ Docker Compose stack for the clearinghouse runtime: **Identity & auth.** The signer container runs `go-livepeer` directly. Every signing request is authorized by go-livepeer's `-remoteSignerWebhookUrl` hook, which calls your `/authorize` endpoint with `Authorization: Bearer ` — no reverse proxy or gateway in front of the signer. -**CLI port not exposed.** go-livepeer's `-cliAddr` (admin/RPC) is bound to -`127.0.0.1:4935` inside the container and is never published or mapped to the -host. Only the signing HTTP port (`8081`) is exposed. +**CLI port not exposed.** go-livepeer's `-cliAddr` (admin/RPC) is bound to `127.0.0.1:4935` inside the container and is never published or mapped to the host. Only the signing HTTP port (`8081`) is exposed. ## Local stack +### 1. Quick check — Kafka + signer + +Start with the minimum services to confirm the signer and broker are alive. You still need a real identity webhook URL. + ```bash cp deploy/.env.example deploy/.env $EDITOR deploy/.env -docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build -docker compose -f deploy/docker-compose.yml --env-file deploy/.env logs -f -docker compose -f deploy/docker-compose.yml --env-file deploy/.env down -``` - -Kafka + signer only (no metering): - -```bash docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build kafka remote-signer +docker compose -f deploy/docker-compose.yml --env-file deploy/.env logs -f remote-signer ``` Verify CLI port is not published: @@ -39,12 +53,22 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s # expected: 0.0.0.0:8081 ``` +### 2. Full stack — add metering + +Provision OpenMeter meters/features (see [OpenMeter/Konnect bootstrap](#openmeterkonnect-bootstrap)), then set `OPENMETER_INGEST_URL`, `OPENMETER_API_KEY`, and `ETH_USD_PRICE` in `deploy/.env`. + +```bash +docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build +docker compose -f deploy/docker-compose.yml --env-file deploy/.env logs -f +docker compose -f deploy/docker-compose.yml --env-file deploy/.env down +``` + ## Environment variables | Variable | Required | Default | Description | -|---|---|---|---| +| --- | --- | --- | --- | | `REMOTE_SIGNER_WEBHOOK_URL` | yes | — | Identity webhook URL (`/authorize` endpoint) | -| `WEBHOOK_SECRET` | yes | — | Shared secret passed as `Authorization:Bearer ` to the webhook (from bootstrap) | +| `WEBHOOK_SECRET` | yes | — | Shared secret passed as `Authorization: Bearer ` to the webhook | | `SIGNER_NETWORK` | no | `arbitrum-one-mainnet` | go-livepeer `-network` | | `ETH_RPC_URL` | no | public arb1 endpoint | Arbitrum RPC | | `SIGNER_ETH_ADDR` | no | — | Funded signer Ethereum address | @@ -57,10 +81,9 @@ docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-s | `KAFKA_ADVERTISED_ADDR` | no | `kafka:9092` | Redpanda advertised Kafka address (broker container) | | `KAFKA_BROKERS` | no | `kafka:9092` | Kafka bootstrap servers | | `KAFKA_GATEWAY_TOPIC` | no | `livepeer-gateway-events` | Kafka topic | -| `OPENMETER_URL` | yes | — | OpenMeter / Konnect base URL (from bootstrap) | -| `OPENMETER_INGEST_URL` | yes | — | Ingest endpoint (`${OPENMETER_URL}/events` for Konnect) | -| `OPENMETER_API_KEY` | yes | — | Konnect PAT (`kpat_…`) (from bootstrap) | -| `ETH_USD_PRICE` | yes | - | ETH/USD rate for Wei→USD micros conversion | +| `OPENMETER_INGEST_URL` | collector only | — | Ingest endpoint. Konnect: `https://.api.konghq.com/v3/openmeter/events`. Self-hosted: `https:///api/v1/events` | +| `OPENMETER_API_KEY` | collector only | — | Konnect PAT (`kpat_…`) or self-hosted API key | +| `ETH_USD_PRICE` | collector only | — | ETH/USD rate for Wei→USD micros conversion | ## OpenMeter/Konnect bootstrap @@ -70,7 +93,7 @@ Use the Go `clearinghouse-bootstrap` CLI or your existing Konnect setup. Creates: | Object | Key | Purpose | -|--------|-----|---------| +| --- | --- | --- | | Meter | `network_fee_usd_micros` | Raw network cost from signer | | Meter | `billable_usd_micros` | Post-markup billable amount (collector phase 2) | | Meter | `signed_ticket_count` | Request counts | diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 785bb6d..d5d8c01 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -68,7 +68,6 @@ services: condition: service_healthy environment: <<: *kafka-env - OPENMETER_URL: ${OPENMETER_URL} OPENMETER_INGEST_URL: ${OPENMETER_INGEST_URL} OPENMETER_API_KEY: ${OPENMETER_API_KEY} ETH_USD_PRICE: ${ETH_USD_PRICE} diff --git a/deploy/openmeter-collector/collector.yaml b/deploy/openmeter-collector/collector.yaml index 7d6a82c..0f2ed2f 100644 --- a/deploy/openmeter-collector/collector.yaml +++ b/deploy/openmeter-collector/collector.yaml @@ -4,8 +4,8 @@ # - Topic carries multiple event types; filter to create_signed_ticket only. # - auth_id is expected as "client_id:external_user_id" from the signer webhook response. # - Uses first-colon split semantics to preserve compatibility with builder-sdk. -# - Konnect ingest: POST {OPENMETER_INGEST_URL} (set to ${OPENMETER_URL}/events). -# For self-hosted OpenMeter set OPENMETER_INGEST_URL to ${OPENMETER_URL}/api/v1/events. +# - Konnect ingest: POST OPENMETER_INGEST_URL (e.g. https://.api.konghq.com/v3/openmeter/events). +# For self-hosted OpenMeter use https:///api/v1/events. input: kafka: From 09333e8db54ab83a73058366c9287778ba28904c Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 17:31:01 -0400 Subject: [PATCH 16/17] refactor(deploy): remove deploy folder, shift contents to root for docker Compose setup for clearinghouse stack Added a new docker-compose.yml file to define the services for the clearinghouse stack, including Redpanda, go-livepeer remote signer, and OpenMeter collector. Introduced a .gitignore file to exclude sensitive environment files and created example .env files for each service to facilitate configuration. Updated README to outline the components and setup instructions for the stack. --- .gitignore | 4 + deploy/README.md => README.md | 73 ++++++++++-------- deploy/.env.example | 31 -------- deploy/.gitignore | 1 - deploy/docker-compose.yml | 76 ------------------- deploy/openmeter-collector/Dockerfile | 6 -- docker-compose.yml | 58 ++++++++++++++ kafka/.env.example | 5 ++ {deploy/kafka => kafka}/Dockerfile | 2 +- {deploy/kafka => kafka}/entrypoint.sh | 11 ++- openmeter-collector/.env.example | 15 ++++ openmeter-collector/Dockerfile | 8 ++ .../collector.yaml | 0 openmeter-collector/entrypoint.sh | 11 +++ remote-signer/.env.example | 27 +++++++ .../Dockerfile | 2 +- .../entrypoint.sh | 21 +++-- 17 files changed, 196 insertions(+), 155 deletions(-) create mode 100644 .gitignore rename deploy/README.md => README.md (56%) delete mode 100644 deploy/.env.example delete mode 100644 deploy/.gitignore delete mode 100644 deploy/docker-compose.yml delete mode 100644 deploy/openmeter-collector/Dockerfile create mode 100644 docker-compose.yml create mode 100644 kafka/.env.example rename {deploy/kafka => kafka}/Dockerfile (78%) rename {deploy/kafka => kafka}/entrypoint.sh (82%) create mode 100644 openmeter-collector/.env.example create mode 100644 openmeter-collector/Dockerfile rename {deploy/openmeter-collector => openmeter-collector}/collector.yaml (100%) create mode 100644 openmeter-collector/entrypoint.sh create mode 100644 remote-signer/.env.example rename {deploy/remote-signer => remote-signer}/Dockerfile (91%) rename {deploy/remote-signer => remote-signer}/entrypoint.sh (75%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7501e7f --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +kafka/.env +remote-signer/.env +openmeter-collector/.env +remote-signer/data/ diff --git a/deploy/README.md b/README.md similarity index 56% rename from deploy/README.md rename to README.md index 68295ad..734763c 100644 --- a/deploy/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# clearinghouse deploy/ +# clearinghouse Docker Compose stack for the clearinghouse runtime: **Redpanda → go-livepeer remote signer → OpenMeter/Benthos collector → Konnect metering**. @@ -24,12 +24,14 @@ Signer HTTP request ## Design decisions -**Redpanda over Apache Kafka.** The stack uses Redpanda as the Kafka-compatible broker. Redpanda Kafka runs as a single-binary dev container with no ZooKeeper dependency and faster local startup. +**Redpanda over Apache Kafka.** The stack uses Redpanda as the Kafka-compatible broker. Redpanda runs as a single-binary dev container with no ZooKeeper dependency and faster local startup. **Identity & auth.** The signer container runs `go-livepeer` directly. Every signing request is authorized by go-livepeer's `-remoteSignerWebhookUrl` hook, which calls your `/authorize` endpoint with `Authorization: Bearer ` — no reverse proxy or gateway in front of the signer. **CLI port not exposed.** go-livepeer's `-cliAddr` (admin/RPC) is bound to `127.0.0.1:4935` inside the container and is never published or mapped to the host. Only the signing HTTP port (`8081`) is exposed. +**Per-service configuration.** Each service reads a local `.env` file mounted at `/service/.env` and sourced by its entrypoint. Copy the `.env.example` in each service directory before starting the stack. + ## Local stack ### 1. Quick check — Kafka + signer @@ -37,53 +39,60 @@ Signer HTTP request Start with the minimum services to confirm the signer and broker are alive. You still need a real identity webhook URL. ```bash -cp deploy/.env.example deploy/.env -$EDITOR deploy/.env +cp kafka/.env.example kafka/.env +cp remote-signer/.env.example remote-signer/.env +$EDITOR remote-signer/.env -docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build kafka remote-signer -docker compose -f deploy/docker-compose.yml --env-file deploy/.env logs -f remote-signer +docker compose up -d --build kafka remote-signer +docker compose logs -f remote-signer ``` Verify CLI port is not published: ```bash -docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-signer 4935 +docker compose port remote-signer 4935 # expected: no output / error (port is not mapped) -docker compose -f deploy/docker-compose.yml --env-file deploy/.env port remote-signer 8081 +docker compose port remote-signer 8081 # expected: 0.0.0.0:8081 ``` ### 2. Full stack — add metering -Provision OpenMeter meters/features (see [OpenMeter/Konnect bootstrap](#openmeterkonnect-bootstrap)), then set `OPENMETER_INGEST_URL`, `OPENMETER_API_KEY`, and `ETH_USD_PRICE` in `deploy/.env`. +Provision OpenMeter meters/features (see [OpenMeter/Konnect bootstrap](#openmeterkonnect-bootstrap)), then configure the collector: ```bash -docker compose -f deploy/docker-compose.yml --env-file deploy/.env up -d --build -docker compose -f deploy/docker-compose.yml --env-file deploy/.env logs -f -docker compose -f deploy/docker-compose.yml --env-file deploy/.env down +cp openmeter-collector/.env.example openmeter-collector/.env +$EDITOR openmeter-collector/.env + +docker compose up -d --build +docker compose logs -f +docker compose down ``` ## Environment variables -| Variable | Required | Default | Description | -| --- | --- | --- | --- | -| `REMOTE_SIGNER_WEBHOOK_URL` | yes | — | Identity webhook URL (`/authorize` endpoint) | -| `WEBHOOK_SECRET` | yes | — | Shared secret passed as `Authorization: Bearer ` to the webhook | -| `SIGNER_NETWORK` | no | `arbitrum-one-mainnet` | go-livepeer `-network` | -| `ETH_RPC_URL` | no | public arb1 endpoint | Arbitrum RPC | -| `SIGNER_ETH_ADDR` | no | — | Funded signer Ethereum address | -| `SIGNER_ETH_KEYSTORE_PATH` | no | — | Optional keystore directory or keyfile path passed to `-ethKeystorePath` | -| `SIGNER_DATA_DIR` | no | `./data` | Host directory bind-mounted to `/data` for signer state, keystore, and password file | -| `SIGNER_HOST_PORT` | no | `8081` | Host port for the signing HTTP endpoint | -| `SIGNER_REMOTE_DISCOVERY` | no | `0` | Enables remote signer orchestrator discovery when set to `1` or `true` | -| `ORCH_WEBHOOK_URL` | no | — | Optional orchestrator discovery webhook passed to `-orchWebhookUrl` when remote discovery is enabled | -| `LIVE_AI_CAP_REPORT_INTERVAL` | no | — | Optional capacity report interval passed to `-liveAICapReportInterval` when remote discovery is enabled | -| `KAFKA_ADVERTISED_ADDR` | no | `kafka:9092` | Redpanda advertised Kafka address (broker container) | -| `KAFKA_BROKERS` | no | `kafka:9092` | Kafka bootstrap servers | -| `KAFKA_GATEWAY_TOPIC` | no | `livepeer-gateway-events` | Kafka topic | -| `OPENMETER_INGEST_URL` | collector only | — | Ingest endpoint. Konnect: `https://.api.konghq.com/v3/openmeter/events`. Self-hosted: `https:///api/v1/events` | -| `OPENMETER_API_KEY` | collector only | — | Konnect PAT (`kpat_…`) or self-hosted API key | -| `ETH_USD_PRICE` | collector only | — | ETH/USD rate for Wei→USD micros conversion | +Each service documents its variables in its own `.env.example`: + +| Service | Config file | Key variables | +| --- | --- | --- | +| `kafka` | [`kafka/.env.example`](kafka/.env.example) | `KAFKA_ADVERTISED_ADDR` | +| `remote-signer` | [`remote-signer/.env.example`](remote-signer/.env.example) | `REMOTE_SIGNER_WEBHOOK_URL`, `WEBHOOK_SECRET`, `SIGNER_*`, `KAFKA_BROKERS`, `KAFKA_GATEWAY_TOPIC` | +| `openmeter-collector` | [`openmeter-collector/.env.example`](openmeter-collector/.env.example) | `KAFKA_BROKERS`, `KAFKA_GATEWAY_TOPIC`, `OPENMETER_INGEST_URL`, `OPENMETER_API_KEY`, `ETH_USD_PRICE` | + +Signer state (keystore, `.eth-password`, chain DB) is stored under [`remote-signer/data/`](remote-signer/data/), bind-mounted to `/data` in the container. + +```bash +mkdir -p remote-signer/data/keystore +cp /path/to/your/keystore/* remote-signer/data/keystore/ +cp /path/to/your/.eth-password remote-signer/data/.eth-password + +cp remote-signer/.env.example remote-signer/.env +$EDITOR remote-signer/.env +``` + +Set `SIGNER_ETH_KEYSTORE_PATH=/data/keystore` (container path) and `SIGNER_ETH_ADDR` to your funded signer address. If `SIGNER_ETH_KEYSTORE_PATH` is unset, the entrypoint uses `/data/keystore` when that directory exists. + +To change the host signing port from `8081`, use a Compose override file. ## OpenMeter/Konnect bootstrap @@ -114,7 +123,7 @@ Signer computed_fee (wei) ``` Markup rules are defined in the bootstrap CLI catalog. Collector -pipeline config: [`deploy/openmeter-collector/collector.yaml`](openmeter-collector/collector.yaml). +pipeline config: [`openmeter-collector/collector.yaml`](openmeter-collector/collector.yaml). The collector does not yet emit `billable_usd_micros` (phase 2); until then the billable meter stays empty while the catalog is ready. diff --git a/deploy/.env.example b/deploy/.env.example deleted file mode 100644 index 12e98ba..0000000 --- a/deploy/.env.example +++ /dev/null @@ -1,31 +0,0 @@ -# Runtime stack env for deploy/docker-compose.yml -# -# cp deploy/.env.example deploy/.env -# $EDITOR deploy/.env - -# --- Identity webhook (required) --- -REMOTE_SIGNER_WEBHOOK_URL=https://your-platform.vercel.app/api/signer/authorize -WEBHOOK_SECRET= - -# --- Signer --- -SIGNER_NETWORK=arbitrum-one-mainnet -ETH_RPC_URL=https://arb1.arbitrum.io/rpc -SIGNER_ETH_ADDR= -SIGNER_ETH_KEYSTORE_PATH= -SIGNER_DATA_DIR=./data -SIGNER_HOST_PORT=8081 -SIGNER_REMOTE_DISCOVERY=0 -# ORCH_WEBHOOK_URL= -# LIVE_AI_CAP_REPORT_INTERVAL= - -# --- Kafka --- -KAFKA_ADVERTISED_ADDR=kafka:9092 -KAFKA_BROKERS=kafka:9092 -KAFKA_GATEWAY_TOPIC=livepeer-gateway-events - -# --- OpenMeter / Konnect (required for collector) --- -# Konnect: https://.api.konghq.com/v3/openmeter/events -# Self-hosted: https:///api/v1/events -OPENMETER_INGEST_URL=https://us.api.konghq.com/v3/openmeter/events -OPENMETER_API_KEY= -ETH_USD_PRICE=3500 diff --git a/deploy/.gitignore b/deploy/.gitignore deleted file mode 100644 index f10862a..0000000 --- a/deploy/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/.env diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml deleted file mode 100644 index d5d8c01..0000000 --- a/deploy/docker-compose.yml +++ /dev/null @@ -1,76 +0,0 @@ -# Clearinghouse deploy stack: Redpanda + go-livepeer remote signer + OpenMeter collector. -# -# Redpanda is the Kafka-compatible broker; The signer container runs go-livepeer directly — only its signing HTTP port -# (8081) is published to the host. The CLI/admin port (-cliAddr) is bound to loopback -# inside the container and is never exposed or published. -# -# Full stack: -# docker compose -f deploy/docker-compose.yml --env-file .env up -d --build -# -# Kafka + signer only (no metering): -# docker compose -f deploy/docker-compose.yml --env-file .env up -d --build kafka remote-signer - -x-kafka-env: &kafka-env - KAFKA_BROKERS: ${KAFKA_BROKERS} - KAFKA_GATEWAY_TOPIC: ${KAFKA_GATEWAY_TOPIC} - -x-signer-env: &signer-env - SIGNER_NETWORK: ${SIGNER_NETWORK} - ETH_RPC_URL: ${ETH_RPC_URL} - SIGNER_PORT: "8081" - SIGNER_ETH_ADDR: ${SIGNER_ETH_ADDR:-} - SIGNER_ETH_KEYSTORE_PATH: ${SIGNER_ETH_KEYSTORE_PATH:-} - SIGNER_REMOTE_DISCOVERY: ${SIGNER_REMOTE_DISCOVERY:-0} - ORCH_WEBHOOK_URL: ${ORCH_WEBHOOK_URL:-} - LIVE_AI_CAP_REPORT_INTERVAL: ${LIVE_AI_CAP_REPORT_INTERVAL:-} - -services: - kafka: - build: - context: .. - dockerfile: deploy/kafka/Dockerfile - restart: unless-stopped - environment: - KAFKA_ADVERTISED_ADDR: ${KAFKA_ADVERTISED_ADDR} - healthcheck: - test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy'"] - interval: 10s - timeout: 5s - retries: 12 - start_period: 15s - - remote-signer: - build: - context: .. - dockerfile: deploy/remote-signer/Dockerfile - restart: unless-stopped - depends_on: - kafka: - condition: service_healthy - extra_hosts: - - "host.docker.internal:host-gateway" - ports: - - "${SIGNER_HOST_PORT:-8081}:8081" - volumes: - - ${SIGNER_DATA_DIR:-./data}:/data - environment: - <<: [*signer-env, *kafka-env] - REMOTE_SIGNER_WEBHOOK_URL: ${REMOTE_SIGNER_WEBHOOK_URL} - WEBHOOK_SECRET: ${WEBHOOK_SECRET} - - openmeter-collector: - build: - context: .. - dockerfile: deploy/openmeter-collector/Dockerfile - restart: unless-stopped - depends_on: - kafka: - condition: service_healthy - environment: - <<: *kafka-env - OPENMETER_INGEST_URL: ${OPENMETER_INGEST_URL} - OPENMETER_API_KEY: ${OPENMETER_API_KEY} - ETH_USD_PRICE: ${ETH_USD_PRICE} - -# Signer datadir is bind-mounted from deploy/data (see SIGNER_DATA_DIR). -# Copy keystore + .eth-password into deploy/data before first run. diff --git a/deploy/openmeter-collector/Dockerfile b/deploy/openmeter-collector/Dockerfile deleted file mode 100644 index 968cf01..0000000 --- a/deploy/openmeter-collector/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# OpenMeter Benthos collector: Kafka create_signed_ticket events -> Konnect/OpenMeter ingest. -FROM ghcr.io/openmeterio/benthos-collector:main-6b60ab6-1782310960 - -COPY deploy/openmeter-collector/collector.yaml /config.yaml - -CMD ["-c", "/config.yaml"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e0969fa --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,58 @@ +# Clearinghouse stack: Redpanda + go-livepeer remote signer + OpenMeter collector. +# +# Redpanda is the Kafka-compatible broker. The signer container runs go-livepeer directly — +# only its signing HTTP port (8081) is published to the host. The CLI/admin port (-cliAddr) +# is bound to loopback inside the container and is never exposed or published. +# +# Per-service config: copy each service's .env.example to .env before starting. +# +# Full stack: +# docker compose up -d --build +# +# Kafka + signer only (no metering): +# docker compose up -d --build kafka remote-signer + +services: + kafka: + build: + context: . + dockerfile: kafka/Dockerfile + restart: unless-stopped + volumes: + - ./kafka/.env:/service/.env:ro + healthcheck: + test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy'"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 15s + + remote-signer: + build: + context: . + dockerfile: remote-signer/Dockerfile + restart: unless-stopped + depends_on: + kafka: + condition: service_healthy + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "8081:8081" + volumes: + - ./remote-signer/.env:/service/.env:ro + - ./remote-signer/data:/data + + openmeter-collector: + build: + context: . + dockerfile: openmeter-collector/Dockerfile + restart: unless-stopped + depends_on: + kafka: + condition: service_healthy + volumes: + - ./openmeter-collector/.env:/service/.env:ro + +# Signer datadir is bind-mounted from remote-signer/data. +# Copy keystore + .eth-password into remote-signer/data before first run. diff --git a/kafka/.env.example b/kafka/.env.example new file mode 100644 index 0000000..ba6e6b3 --- /dev/null +++ b/kafka/.env.example @@ -0,0 +1,5 @@ +# Kafka (Redpanda) service env — copied to kafka/.env at runtime. +# +# cp kafka/.env.example kafka/.env + +KAFKA_ADVERTISED_ADDR=kafka:9092 diff --git a/deploy/kafka/Dockerfile b/kafka/Dockerfile similarity index 78% rename from deploy/kafka/Dockerfile rename to kafka/Dockerfile index dd4b225..0db81a5 100644 --- a/deploy/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -2,7 +2,7 @@ FROM docker.redpanda.com/redpandadata/redpanda:v24.2.4 USER root -COPY deploy/kafka/entrypoint.sh /clearinghouse-entrypoint.sh +COPY kafka/entrypoint.sh /clearinghouse-entrypoint.sh RUN chmod +x /clearinghouse-entrypoint.sh USER redpanda diff --git a/deploy/kafka/entrypoint.sh b/kafka/entrypoint.sh similarity index 82% rename from deploy/kafka/entrypoint.sh rename to kafka/entrypoint.sh index 259f11a..9e61f20 100644 --- a/deploy/kafka/entrypoint.sh +++ b/kafka/entrypoint.sh @@ -1,12 +1,19 @@ #!/bin/sh set -eu -ADVERTISED="${KAFKA_ADVERTISED_ADDR}" +if [ -f /service/.env ]; then + set -a + # shellcheck disable=SC1091 + . /service/.env + set +a +fi + +ADVERTISED="${KAFKA_ADVERTISED_ADDR:-kafka:9092}" # NOTE: Deployment: See https://github.com/livepeer/clearinghouse/issues/43 for tracking. # The --mode dev-container flag below launches Redpanda in "development container" mode: # - No security/encryption (PLAINTEXT) -# - Not for production! This enables fast local startup, disables most persistence/durability, +# - Not for production! This enables fast local startup, disables most persistence/durability, # and relaxes networking for development and Docker Compose stacks. # Update this script if production requirements change. diff --git a/openmeter-collector/.env.example b/openmeter-collector/.env.example new file mode 100644 index 0000000..d57e5ec --- /dev/null +++ b/openmeter-collector/.env.example @@ -0,0 +1,15 @@ +# OpenMeter collector service env — copied to openmeter-collector/.env at runtime. +# +# cp openmeter-collector/.env.example openmeter-collector/.env +# $EDITOR openmeter-collector/.env + +# --- Kafka (event consume) --- +KAFKA_BROKERS=kafka:9092 +KAFKA_GATEWAY_TOPIC=livepeer-gateway-events + +# --- OpenMeter / Konnect (required) --- +# Konnect: https://.api.konghq.com/v3/openmeter/events +# Self-hosted: https:///api/v1/events +OPENMETER_INGEST_URL=https://us.api.konghq.com/v3/openmeter/events +OPENMETER_API_KEY= +ETH_USD_PRICE=3500 diff --git a/openmeter-collector/Dockerfile b/openmeter-collector/Dockerfile new file mode 100644 index 0000000..878989c --- /dev/null +++ b/openmeter-collector/Dockerfile @@ -0,0 +1,8 @@ +# OpenMeter Benthos collector: Kafka create_signed_ticket events -> Konnect/OpenMeter ingest. +FROM ghcr.io/openmeterio/benthos-collector:main-6b60ab6-1782310960 + +COPY openmeter-collector/collector.yaml /config.yaml +COPY openmeter-collector/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/deploy/openmeter-collector/collector.yaml b/openmeter-collector/collector.yaml similarity index 100% rename from deploy/openmeter-collector/collector.yaml rename to openmeter-collector/collector.yaml diff --git a/openmeter-collector/entrypoint.sh b/openmeter-collector/entrypoint.sh new file mode 100644 index 0000000..15517be --- /dev/null +++ b/openmeter-collector/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +if [ -f /service/.env ]; then + set -a + # shellcheck disable=SC1091 + . /service/.env + set +a +fi + +exec /usr/local/bin/benthos -c /config.yaml diff --git a/remote-signer/.env.example b/remote-signer/.env.example new file mode 100644 index 0000000..6f74811 --- /dev/null +++ b/remote-signer/.env.example @@ -0,0 +1,27 @@ +# Remote signer service env — copied to remote-signer/.env at runtime. +# +# cp remote-signer/.env.example remote-signer/.env +# $EDITOR remote-signer/.env + +# --- Identity webhook (required) --- +REMOTE_SIGNER_WEBHOOK_URL=https://your-platform.vercel.app/api/signer/authorize +WEBHOOK_SECRET= + +# --- Signer --- +# Paths below are inside the container. Host files live under remote-signer/data/ +# (bind-mounted to /data). Copy your keystore directory and .eth-password there +# before first run. +SIGNER_NETWORK=arbitrum-one-mainnet +SIGNER_PORT=8081 +ETH_RPC_URL=https://arb1.arbitrum.io/rpc +SIGNER_ETH_ADDR= +# Directory or keyfile under /data — e.g. /data/keystore or /data/keystore/UTC--...json +# Defaults to /data/keystore when that directory exists and this is unset. +SIGNER_ETH_KEYSTORE_PATH=/data/keystore +SIGNER_REMOTE_DISCOVERY=0 +# ORCH_WEBHOOK_URL= +# LIVE_AI_CAP_REPORT_INTERVAL= + +# --- Kafka (event publish) --- +KAFKA_BROKERS=kafka:9092 +KAFKA_GATEWAY_TOPIC=livepeer-gateway-events diff --git a/deploy/remote-signer/Dockerfile b/remote-signer/Dockerfile similarity index 91% rename from deploy/remote-signer/Dockerfile rename to remote-signer/Dockerfile index a1c395c..0f55fbd 100644 --- a/deploy/remote-signer/Dockerfile +++ b/remote-signer/Dockerfile @@ -8,7 +8,7 @@ FROM livepeer/go-livepeer:v0.8.11 RUN mkdir -p /data -COPY deploy/remote-signer/entrypoint.sh /entrypoint.sh +COPY remote-signer/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh EXPOSE 8081 diff --git a/deploy/remote-signer/entrypoint.sh b/remote-signer/entrypoint.sh similarity index 75% rename from deploy/remote-signer/entrypoint.sh rename to remote-signer/entrypoint.sh index 8cf52b3..e35e975 100755 --- a/deploy/remote-signer/entrypoint.sh +++ b/remote-signer/entrypoint.sh @@ -1,11 +1,18 @@ #!/bin/sh set -eu -SIGNER_NETWORK="${SIGNER_NETWORK}" -SIGNER_PORT="${SIGNER_PORT}" -ETH_RPC_URL="${ETH_RPC_URL}" -KAFKA_BROKERS="${KAFKA_BROKERS}" -KAFKA_GATEWAY_TOPIC="${KAFKA_GATEWAY_TOPIC}" +if [ -f /service/.env ]; then + set -a + # shellcheck disable=SC1091 + . /service/.env + set +a +fi + +SIGNER_NETWORK="${SIGNER_NETWORK:-arbitrum-one-mainnet}" +SIGNER_PORT="${SIGNER_PORT:-8081}" +ETH_RPC_URL="${ETH_RPC_URL:-https://arb1.arbitrum.io/rpc}" +KAFKA_BROKERS="${KAFKA_BROKERS:-kafka:9092}" +KAFKA_GATEWAY_TOPIC="${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events}" if [ -z "${REMOTE_SIGNER_WEBHOOK_URL:-}" ]; then echo "entrypoint: REMOTE_SIGNER_WEBHOOK_URL is required (identity webhook URL)" >&2 @@ -20,6 +27,10 @@ if [ ! -f /data/.eth-password ]; then echo "" >/data/.eth-password fi +if [ -z "${SIGNER_ETH_KEYSTORE_PATH:-}" ] && [ -d /data/keystore ]; then + SIGNER_ETH_KEYSTORE_PATH=/data/keystore +fi + set -- \ -remoteSigner \ "-network=${SIGNER_NETWORK}" \ From eec8c7a8c0c2c0a91c027bc9927f491f900c055c Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 24 Jun 2026 17:38:56 -0400 Subject: [PATCH 17/17] docs: update README and .env.example for identity webhook configuration Enhanced the README to clarify the identity and authorization process for the remote signer, including instructions for local alive checks without a webhook. Updated .env.example to indicate that leaving REMOTE_SIGNER_WEBHOOK_URL empty allows starting the signer without webhook authorization, while emphasizing the necessity of this variable in production. --- README.md | 11 ++++++++--- remote-signer/.env.example | 4 +++- remote-signer/entrypoint.sh | 23 ++++++++++++----------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 734763c..5b6e2cf 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Signer HTTP request **Redpanda over Apache Kafka.** The stack uses Redpanda as the Kafka-compatible broker. Redpanda runs as a single-binary dev container with no ZooKeeper dependency and faster local startup. -**Identity & auth.** The signer container runs `go-livepeer` directly. Every signing request is authorized by go-livepeer's `-remoteSignerWebhookUrl` hook, which calls your `/authorize` endpoint with `Authorization: Bearer ` — no reverse proxy or gateway in front of the signer. +**Identity & auth.** The signer container runs `go-livepeer` directly. In the normal path, every signing request is authorized by go-livepeer's `-remoteSignerWebhookUrl` hook, which calls your `/authorize` endpoint with `Authorization: Bearer ` — no reverse proxy or gateway in front of the signer. For local alive checks only, leave `REMOTE_SIGNER_WEBHOOK_URL` empty to omit the webhook hook. **CLI port not exposed.** go-livepeer's `-cliAddr` (admin/RPC) is bound to `127.0.0.1:4935` inside the container and is never published or mapped to the host. Only the signing HTTP port (`8081`) is exposed. @@ -36,17 +36,22 @@ Signer HTTP request ### 1. Quick check — Kafka + signer -Start with the minimum services to confirm the signer and broker are alive. You still need a real identity webhook URL. +Start here before wiring identity or metering. This runs only the Kafka broker and remote signer so you can confirm the core path is alive. ```bash cp kafka/.env.example kafka/.env cp remote-signer/.env.example remote-signer/.env $EDITOR remote-signer/.env +# For a local alive check without an identity webhook: +# REMOTE_SIGNER_WEBHOOK_URL= +# WEBHOOK_SECRET= docker compose up -d --build kafka remote-signer docker compose logs -f remote-signer ``` +Expected result: `remote-signer` starts cleanly, connects to Kafka, and serves the signing HTTP port. + Verify CLI port is not published: ```bash @@ -58,7 +63,7 @@ docker compose port remote-signer 8081 ### 2. Full stack — add metering -Provision OpenMeter meters/features (see [OpenMeter/Konnect bootstrap](#openmeterkonnect-bootstrap)), then configure the collector: +After the quick check passes, add the OpenMeter collector and hosted metering configuration. Provision OpenMeter meters/features (see [OpenMeter/Konnect bootstrap](#openmeterkonnect-bootstrap)), then configure the collector: ```bash cp openmeter-collector/.env.example openmeter-collector/.env diff --git a/remote-signer/.env.example b/remote-signer/.env.example index 6f74811..1c1bcc7 100644 --- a/remote-signer/.env.example +++ b/remote-signer/.env.example @@ -3,9 +3,11 @@ # cp remote-signer/.env.example remote-signer/.env # $EDITOR remote-signer/.env -# --- Identity webhook (required) --- +# --- Identity webhook --- REMOTE_SIGNER_WEBHOOK_URL=https://your-platform.vercel.app/api/signer/authorize WEBHOOK_SECRET= +# Leave REMOTE_SIGNER_WEBHOOK_URL empty to start signer without webhook authorization. +# REMOTE_SIGNER_WEBHOOK_URL should always be set in production. # --- Signer --- # Paths below are inside the container. Host files live under remote-signer/data/ diff --git a/remote-signer/entrypoint.sh b/remote-signer/entrypoint.sh index e35e975..610e4cc 100755 --- a/remote-signer/entrypoint.sh +++ b/remote-signer/entrypoint.sh @@ -14,15 +14,6 @@ ETH_RPC_URL="${ETH_RPC_URL:-https://arb1.arbitrum.io/rpc}" KAFKA_BROKERS="${KAFKA_BROKERS:-kafka:9092}" KAFKA_GATEWAY_TOPIC="${KAFKA_GATEWAY_TOPIC:-livepeer-gateway-events}" -if [ -z "${REMOTE_SIGNER_WEBHOOK_URL:-}" ]; then - echo "entrypoint: REMOTE_SIGNER_WEBHOOK_URL is required (identity webhook URL)" >&2 - exit 1 -fi -if [ -z "${WEBHOOK_SECRET:-}" ]; then - echo "entrypoint: WEBHOOK_SECRET is required" >&2 - exit 1 -fi - if [ ! -f /data/.eth-password ]; then echo "" >/data/.eth-password fi @@ -40,12 +31,22 @@ set -- \ "-ethPassword=/data/.eth-password" \ "-datadir=/data" \ -v=99 \ - "-remoteSignerWebhookUrl=${REMOTE_SIGNER_WEBHOOK_URL}" \ - "-remoteSignerWebhookHeaders=Authorization:Bearer ${WEBHOOK_SECRET}" \ -monitor \ "-kafkaBootstrapServers=${KAFKA_BROKERS}" \ "-kafkaGatewayTopic=${KAFKA_GATEWAY_TOPIC}" +if [ -n "${REMOTE_SIGNER_WEBHOOK_URL:-}" ]; then + if [ -z "${WEBHOOK_SECRET:-}" ]; then + echo "entrypoint: WEBHOOK_SECRET is required when REMOTE_SIGNER_WEBHOOK_URL is set" >&2 + exit 1 + fi + set -- "$@" \ + "-remoteSignerWebhookUrl=${REMOTE_SIGNER_WEBHOOK_URL}" \ + "-remoteSignerWebhookHeaders=Authorization:Bearer ${WEBHOOK_SECRET}" +else + echo "entrypoint: WARNING: starting remote signer without identity webhook authorization" >&2 +fi + if [ -n "${SIGNER_ETH_ADDR:-}" ]; then set -- "$@" "-ethAcctAddr=${SIGNER_ETH_ADDR}" fi