fix: fail fast on signer HTTP 483 insufficient balance

examples/write_frames.py

@@ -4,6 +4,7 @@
 
 import av
 
+from livepeer_gateway import configure_logging
 from livepeer_gateway.errors import LivepeerGatewayError
 from livepeer_gateway.lv2v import StartJobRequest, start_lv2v
 from livepeer_gateway.media_publish import MediaPublishConfig, VideoOutputConfig
@@ -49,6 +50,7 @@ def _solid_rgb_frame(width: int, height: int, rgb: tuple[int, int, int]) -> av.V
 
 
 async def main() -> None:
+    configure_logging()
     args = _parse_args()
     frame_interval = 1.0 / max(1e-6, args.fps)
 
@@ -88,4 +90,3 @@ async def main() -> None:
 
 if __name__ == "__main__":
     asyncio.run(main())
-

src/livepeer_gateway/__init__.py

@@ -15,7 +15,14 @@
     wait_for_training,
     list_capabilities,
 )
-from .errors import LivepeerGatewayError, NoOrchestratorAvailableError, PaymentError
+from .errors import (
+    InsufficientBalance,
+    LivepeerGatewayError,
+    NoOrchestratorAvailableError,
+    OrchestratorRejection,
+    PaymentError,
+)
+from .logging_config import apply_package_log_level, configure_logging
 from .events import Events
 from .media_publish import (
     AudioOutputConfig,
@@ -33,7 +40,6 @@
     VideoDecodedMediaFrame,
 )
 from .media_output import MediaOutput, MediaOutputStats
-from .errors import OrchestratorRejection
 from .lv2v import LiveVideoToVideo, StartJobRequest, start_lv2v
 from .orch_info import get_orch_info
 from .orchestrator import discover_orchestrators
@@ -62,9 +68,11 @@
     "get_orch_info",
     "LiveVideoToVideo",
     "LivepeerGatewayError",
+    "InsufficientBalance",
     "NoOrchestratorAvailableError",
     "OrchestratorRejection",
     "PaymentError",
+    "configure_logging",
     "MediaPublish",
     "MediaPublishConfig",
     "MediaPublishTrack",
@@ -99,3 +107,6 @@
     "TrickleSubscriberStats",
     "VideoDecodedMediaFrame",
 ]
+
+# Apply LOG_LEVEL to the package logger on import (no basicConfig).
+apply_package_log_level()

src/livepeer_gateway/byoc.py

@@ -42,7 +42,12 @@
 from urllib.request import Request, urlopen
 
 from .orchestrator import _http_origin, _parse_http_url, discover_orchestrators
-from .errors import LivepeerGatewayError, NoOrchestratorAvailableError, OrchestratorRejection
+from .errors import (
+    InsufficientBalance,
+    LivepeerGatewayError,
+    NoOrchestratorAvailableError,
+    OrchestratorRejection,
+)
 
 _LOG = logging.getLogger(__name__)
 
@@ -394,6 +399,8 @@ def submit_byoc_job(
                 )
                 headers.update(payment_headers)
                 _LOG.info("BYOC job %s: payment tickets created for %s", job_id, orch_origin)
+            except InsufficientBalance:
+                raise
             except Exception as e:
                 _LOG.warning("BYOC job %s: payment creation failed for %s: %s", job_id, orch_origin, e)
                 rejections.append(OrchestratorRejection(url=orch_origin, reason=f"payment failed: {e}"))
@@ -678,6 +685,8 @@ def submit_training_job(
                 headers.update(payment_headers)
                 _LOG.info("Training job %s: payment tickets created for %s",
                           job_id, orch_origin)
+            except InsufficientBalance:
+                raise
             except Exception as e:
                 _LOG.warning("Training job %s: payment creation failed for %s: %s",
                              job_id, orch_origin, e)

src/livepeer_gateway/errors.py

@@ -31,5 +31,9 @@ class SkipPaymentCycle(LivepeerGatewayError):
     """Raised when the signer returns HTTP 482 to skip a payment cycle."""
 
 
+class InsufficientBalance(LivepeerGatewayError):
+    """Raised when the signer returns HTTP 483 (insufficient balance / allowance)."""
+
+
 class PaymentError(LivepeerGatewayError):
     """Raised when a PaymentSession operation fails."""

src/livepeer_gateway/logging_config.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import logging
+import os
+
+
+_PACKAGE_LOGGER = "livepeer_gateway"
+_DEFAULT_LEVEL = logging.INFO
+_LOG_FORMAT = "%(levelname)s %(name)s: %(message)s"
+
+
+def _resolve_level(level: str | int | None = None) -> int:
+    if level is None:
+        level = os.environ.get("LOG_LEVEL", "INFO")
+    if isinstance(level, int):
+        return level
+    return getattr(logging, str(level).upper(), _DEFAULT_LEVEL)
+
+
+def apply_package_log_level(level: str | int | None = None) -> None:
+    """Set the livepeer_gateway logger level from arg or LOG_LEVEL env (default INFO)."""
+    logging.getLogger(_PACKAGE_LOGGER).setLevel(_resolve_level(level))
+
+
+def configure_logging(level: str | int | None = None) -> None:
+    """Configure livepeer_gateway log level from arg or LOG_LEVEL env (default INFO).
+
+    Ensures a basic root handler exists so package logs are visible.
+    """
+    resolved = _resolve_level(level)
+    apply_package_log_level(resolved)
+    if not logging.getLogger().handlers:
+        logging.basicConfig(level=resolved, format=_LOG_FORMAT)

src/livepeer_gateway/lv2v.py

@@ -11,6 +11,7 @@
 from .channel_writer import ChannelWriter
 from .control import Control, ControlConfig, ControlMode
 from .errors import (
+    InsufficientBalance,
     LivepeerGatewayError,
     NoOrchestratorAvailableError,
     OrchestratorRejection,
@@ -380,6 +381,9 @@ def start_lv2v(
             if mode == ControlMode.MESSAGE and job.control is not None:
                 job.control.start_keepalive()
             return job
+        except InsufficientBalance:
+            # Account-level signer denial — other orch candidates cannot succeed.
+            raise
         except LivepeerGatewayError as e:
             _LOG.debug(
                 "start_lv2v candidate failed, trying fallback if available: %s (%s)",

src/livepeer_gateway/orchestrator.py

@@ -13,6 +13,7 @@
 from .capabilities import capabilities_to_query
 
 from .errors import (
+    InsufficientBalance,
     LivepeerGatewayError,
     SignerRefreshRequired,
     SkipPaymentCycle,
@@ -116,6 +117,10 @@ def request_json(
             raise SkipPaymentCycle(
                 f"Signer returned HTTP 482 (skip payment cycle) (url={url}){body_part}"
             ) from e
+        if e.code == 483:
+            raise InsufficientBalance(
+                f"Signer returned HTTP 483 (insufficient balance) (url={url}){body_part}"
+            ) from e
         raise LivepeerGatewayError(
             f"HTTP JSON error: HTTP {e.code} from endpoint (url={url}){body_part}"
         ) from e

tests/test_insufficient_balance.py

@@ -0,0 +1,66 @@
+"""HTTP 483 (insufficient balance) must fail fast — no orch fallback."""
+from __future__ import annotations
+
+from io import BytesIO
+from unittest.mock import MagicMock, patch
+from urllib.error import HTTPError
+
+import pytest
+
+from livepeer_gateway.errors import InsufficientBalance, NoOrchestratorAvailableError
+from livepeer_gateway.lv2v import StartJobRequest, start_lv2v
+from livepeer_gateway.orchestrator import request_json
+
+
+def test_request_json_maps_483_to_insufficient_balance() -> None:
+    body = b'{"error":"Starter allowance exhausted"}'
+    err = HTTPError(
+        "https://signer.example/generate-live-payment",
+        483,
+        "Insufficient Balance",
+        hdrs=None,  # type: ignore[arg-type]
+        fp=BytesIO(body),
+    )
+    with patch("livepeer_gateway.orchestrator.urlopen", side_effect=err):
+        with pytest.raises(InsufficientBalance, match="Starter allowance exhausted"):
+            request_json(
+                "https://signer.example/generate-live-payment",
+                payload={},
+            )
+
+
+def test_start_lv2v_fails_fast_on_483_without_orch_fallback() -> None:
+    info_a = MagicMock()
+    info_a.transcoder = "https://orch-a.example:8935"
+    info_b = MagicMock()
+    info_b.transcoder = "https://orch-b.example:8935"
+
+    cursor = MagicMock()
+    cursor.next.side_effect = [
+        ("https://orch-a.example:8935", info_a),
+        ("https://orch-b.example:8935", info_b),
+        NoOrchestratorAvailableError("exhausted", rejections=[]),
+    ]
+
+    payment_session = MagicMock()
+    payment_session.get_payment.side_effect = InsufficientBalance(
+        "Signer returned HTTP 483 (insufficient balance) "
+        "(url=https://signer.example/generate-live-payment); "
+        "body='Starter allowance exhausted'"
+    )
+
+    with (
+        patch("livepeer_gateway.lv2v.orchestrator_selector", return_value=cursor),
+        patch("livepeer_gateway.lv2v.PaymentSession", return_value=payment_session),
+        patch("livepeer_gateway.lv2v.build_capabilities"),
+    ):
+        with pytest.raises(InsufficientBalance, match="Starter allowance exhausted"):
+            start_lv2v(
+                None,
+                StartJobRequest(model_id="noop"),
+                signer_url="https://signer.example",
+                discovery_url="https://discovery.example/raw",
+            )
+
+    assert cursor.next.call_count == 1
+    assert payment_session.get_payment.call_count == 1