feat(gateway): adopt the unified route surface, and record what the executors actually did

.gitignore

@@ -171,7 +171,18 @@ gateway-files/
 # Hummingbot credentials and local data
 bots/credentials/
 bots/instances/
-bots/conf/
+# Archived bot instances: hummingbot-api moves a stopped bot's whole working
+# directory here, credentials included — conf/connectors/*.yml holds encrypted
+# API keys and conf/.password_verification the password check. Nothing under it
+# belongs in git, and `git add -A` will sweep the lot without this line.
+bots/archived/
+# Any bot's conf directory, at any depth and whatever its state — active,
+# archived, or a layout that does not exist yet. A running bot's conf/ holds the
+# same credentials an archived one does: conf/connectors/*.yml carries encrypted
+# API keys and conf/.password_verification the password check. Naming the parent
+# directories one at a time is how bots/archived/ was missed while
+# bots/instances/ was covered.
+conf/
 
 # Local MCP configuration (project-specific overrides)
 .mcp.json
@@ -181,3 +192,5 @@ bots/conf/
 .idea/
 improvements
 bots/gateway-files/
+.DS_Store
+Dockerfile.patched-hummingbot

Makefile

@@ -1,4 +1,4 @@
-.PHONY: setup run deploy stop install uninstall build install-pre-commit tailscale-status reset
+.PHONY: setup run deploy stop install uninstall build install-pre-commit tailscale-status reset gateway-models
 
 SETUP_SENTINEL := .setup-complete
 
@@ -86,6 +86,36 @@ install-pre-commit:
 	conda run -n hummingbot-api pip install pre-commit
 	conda run -n hummingbot-api pre-commit install
 
+# Header stamped onto the generated models. `#` starts a comment in a Makefile, so it
+# has to reach the recipe through a variable.
+HASH := \#
+define GATEWAY_MODELS_HEADER
+$(HASH) Generated from gateway-openapi.json by 'make gateway-models'. Do not edit.
+$(HASH) flake8: noqa: E501
+endef
+export GATEWAY_MODELS_HEADER
+
+# --ignore-enum-constraints keeps `connector` and `network` as plain strings. Gateway
+# constrains them by enum so its docs can offer dropdowns, but generating those as Python
+# enums would bake a connector and network roster into this service: a venue Gateway added
+# after the last spec refresh would be rejected before the request left the process. It
+# also removes the only classes the generator had to number (Connector9, Connector15, ...),
+# which were renamed by any unrelated route insertion.
+#
+# Regenerate models/gateway_generated.py from the vendored Gateway spec.
+# Adopting a Gateway change is two steps — refresh the spec, then rerun this:
+#   cd ../gateway && pnpm generate:openapi && cp openapi.json ../hummingbot-api/gateway-openapi.json
+#   make gateway-models
+# test/test_gateway_models_match_spec.py fails if the committed models drift from the spec.
+gateway-models:
+	conda run --no-capture-output -n hummingbot-api python -m datamodel_code_generator \
+		--input gateway-openapi.json --input-file-type openapi --openapi-scopes schemas \
+		--output models/gateway_generated.py --output-model-type pydantic_v2.BaseModel \
+		--snake-case-field --target-python-version 3.12 --disable-timestamp \
+		--ignore-enum-constraints \
+		--formatters black --formatters isort \
+		--custom-file-header "$$GATEWAY_MODELS_HEADER"
+
 # Build Docker image
 build:
 	docker build -t hummingbot/hummingbot-api:latest .

bots/controllers/generic/lp_rebalancer/__init__.py

@@ -1,3 +1,9 @@
-from controllers.generic.lp_rebalancer.lp_rebalancer import LPRebalancer, LPRebalancerConfig
+# Relative, not `from controllers.generic....`. The same tree is imported under two
+# different package roots: `controllers.*` inside a bot container, and
+# `bots.controllers.*` by hummingbot-api, which mounts it one level deeper. An absolute
+# path pins the module to one of them and breaks under the other — and because this
+# package's __init__ runs before either candidate module path can be reached, the failure
+# took the whole controller down rather than one import of it.
+from .lp_rebalancer import LPRebalancer, LPRebalancerConfig
 
 __all__ = ["LPRebalancer", "LPRebalancerConfig"]

bots/controllers/generic/lp_rebalancer/lp_rebalancer.py

@@ -58,7 +58,8 @@ class LPRebalancerConfig(ControllerConfigBase):
     position_offset_pct: Decimal = Field(
         default=Decimal("0.01"),
         json_schema_extra={"is_updatable": True},
-        description="Offset from current price. Positive = out-of-range (single-sided). Negative = in-range (needs both tokens, autoswap will convert |offset|%)"
+        description="Offset from current price. Positive = out-of-range (single-sided). "
+                    "Negative = in-range (needs both tokens, autoswap will convert |offset|%)"
     )
 
     # Rebalance threshold - used to set LP executor's limit prices
@@ -173,9 +174,7 @@ def __init__(self, config: LPRebalancerConfig, *args, **kwargs):
         self.config: LPRebalancerConfig = config
 
         # Parse lp_provider into dex_name and trading_type for gateway calls
-        self.lp_dex_name, self.lp_trading_type = parse_provider(
-            config.lp_provider, default_trading_type="clmm"
-        )
+        self.lp_dex_name, self.lp_trading_type = parse_provider(config.lp_provider)
 
         # Parse token symbols from trading pair
         parts = config.trading_pair.split("-")
@@ -184,6 +183,9 @@ def __init__(self, config: LPRebalancerConfig, *args, **kwargs):
 
         # Track the executor we created
         self._current_executor_id: Optional[str] = None
+        # Set when a FAILED LP executor still reports a live on-chain position; the
+        # controller halts new position creation until it is recovered manually
+        self._orphaned_position_address: Optional[str] = None
 
         # Track amounts from last closed position (for autoswap sizing)
         self._last_closed_base_amount: Optional[Decimal] = None
@@ -416,6 +418,16 @@ def determine_executor_actions(self) -> List[ExecutorAction]:
 
         actions = []
 
+        # An orphaned on-chain position (FAILED close) must be recovered before any new
+        # position is opened - creating a fresh executor here would stack live exposure
+        # on top of the stranded one.
+        if self._orphaned_position_address:
+            self.logger().debug(
+                f"Halted: position {self._orphaned_position_address} from a FAILED executor is "
+                "still open on-chain and requires manual recovery"
+            )
+            return actions
+
         # Handle order executor tracking and completion (for autoswap)
         if self._pending_swap_side is not None:
             if not self._swap_executor_id:
@@ -509,10 +521,15 @@ def determine_executor_actions(self) -> List[ExecutorAction]:
             # Previous executor terminated - capture final amounts and update position_hold
             terminated_executor = self.get_tracked_executor()
             if terminated_executor:
-                # Skip position_hold update if executor failed (no tokens were actually deposited/returned)
-                if terminated_executor.close_type == CloseType.FAILED:
+                # Skip position_hold update if the executor failed (nothing deposited or
+                # returned) or ended as an involuntary hold (close exhausted): in the
+                # latter case base/quote amounts are pool balances of a still-open
+                # position, and booking them as returned tokens would corrupt the hold.
+                if (terminated_executor.close_type == CloseType.FAILED
+                        or terminated_executor.custom_info.get("hold_reason")):
                     self.logger().warning(
-                        f"Executor {terminated_executor.id} FAILED - skipping position_hold update"
+                        f"Executor {terminated_executor.id} ended {terminated_executor.close_type} "
+                        "without returning tokens - skipping position_hold update"
                     )
                 else:
                     self._last_closed_base_amount = Decimal(str(terminated_executor.custom_info.get("base_amount", 0)))
@@ -539,27 +556,44 @@ def determine_executor_actions(self) -> List[ExecutorAction]:
                         f"Position hold total: base={self._position_hold_base}, quote={self._position_hold_quote}"
                     )
 
-            # Check if executor FAILED - retry with same side from executor's config
+            # Check if the executor went terminal abnormally - FAILED (nothing on-chain)
+            # or an involuntary POSITION_HOLD (close retries exhausted, hold_reason set)
             executor_failed = terminated_executor and terminated_executor.close_type == CloseType.FAILED
+            involuntary_hold = bool(terminated_executor and terminated_executor.custom_info.get("hold_reason"))
             failed_executor_side = None
-            if executor_failed:
+            if executor_failed or involuntary_hold:
                 failed_executor_side = terminated_executor.custom_info.get("side")
+                # A terminal executor that still reports a position address went down on
+                # the CLOSE side: its deposit is still on-chain (involuntary hold, or a
+                # legacy FAILED-with-position from a force-stop). Re-opening would stack
+                # a second position on top of the stranded one.
+                orphaned_position = terminated_executor.custom_info.get("position_address")
+                if orphaned_position:
+                    self._orphaned_position_address = orphaned_position
+                    self._current_executor_id = None
+                    self.logger().error(
+                        f"Executor {terminated_executor.id} ended {terminated_executor.close_type} "
+                        f"with position {orphaned_position} still open on-chain. Halting new "
+                        "position creation until the position is closed or recovered manually."
+                    )
+                    return actions
 
             # Capture closed position bounds for side determination (only for successful closes)
             closed_lower_price = None
             closed_upper_price = None
-            if terminated_executor and not executor_failed:
+            if terminated_executor and not executor_failed and not involuntary_hold:
                 closed_lower_price = Decimal(str(terminated_executor.custom_info.get("lower_price", 0)))
                 closed_upper_price = Decimal(str(terminated_executor.custom_info.get("upper_price", 0)))
 
             # Clear tracking
             self._current_executor_id = None
 
             # Determine side for new position
-            if executor_failed and failed_executor_side is not None:
-                # Retry with same side on failure
+            if (executor_failed or involuntary_hold) and failed_executor_side is not None:
+                # Retry with same side after any abnormal terminal (FAILED, or an
+                # involuntary hold that left no on-chain position to recover)
                 side = failed_executor_side
-                self.logger().info(f"Retrying with same side={side} after executor failure")
+                self.logger().info(f"Retrying with same side={side} after abnormal executor end")
             elif not self._initial_position_created:
                 # Initial position: use configured side
                 side = self.config.side
@@ -576,7 +610,10 @@ def determine_executor_actions(self) -> List[ExecutorAction]:
                 else:
                     # Price is within old bounds (shouldn't happen with limit-price auto-close)
                     side = self._determine_side_from_price(self._pool_price)
-                    self.logger().info(f"Price {self._pool_price} in range [{closed_lower_price}, {closed_upper_price}] → side={side} from limits")
+                    self.logger().info(
+                        f"Price {self._pool_price} in range [{closed_lower_price}, {closed_upper_price}] "
+                        f"→ side={side} from limits"
+                    )
             else:
                 # Fallback to price limits
                 if not self._pool_price:
@@ -784,24 +821,25 @@ def _is_price_within_limits(self, price: Decimal, side: TradeType) -> bool:
         """
         Check if price is within configured limits for the position type.
         """
+        # `is not None`: a limit set to exactly 0 is a real bound, not "unset"
         if side == TradeType.SELL:
-            if self.config.sell_price_min and price < self.config.sell_price_min:
+            if self.config.sell_price_min is not None and price < self.config.sell_price_min:
                 return False
-            if self.config.sell_price_max and price > self.config.sell_price_max:
+            if self.config.sell_price_max is not None and price > self.config.sell_price_max:
                 return False
         elif side == TradeType.BUY:
-            if self.config.buy_price_min and price < self.config.buy_price_min:
+            if self.config.buy_price_min is not None and price < self.config.buy_price_min:
                 return False
-            if self.config.buy_price_max and price > self.config.buy_price_max:
+            if self.config.buy_price_max is not None and price > self.config.buy_price_max:
                 return False
         else:  # RANGE
-            if self.config.buy_price_min and price < self.config.buy_price_min:
+            if self.config.buy_price_min is not None and price < self.config.buy_price_min:
                 return False
-            if self.config.buy_price_max and price > self.config.buy_price_max:
+            if self.config.buy_price_max is not None and price > self.config.buy_price_max:
                 return False
-            if self.config.sell_price_min and price < self.config.sell_price_min:
+            if self.config.sell_price_min is not None and price < self.config.sell_price_min:
                 return False
-            if self.config.sell_price_max and price > self.config.sell_price_max:
+            if self.config.sell_price_max is not None and price > self.config.sell_price_max:
                 return False
         return True
 
@@ -941,7 +979,8 @@ def to_format_status(self) -> List[str]:
         width = self.config.position_width_pct
         offset = self.config.position_offset_pct
         threshold = self.config.rebalance_threshold_pct
-        line = f"| Config: side={side_str}, amount={amt} {self._quote_token}, width={width}%, offset={offset}%, threshold={threshold}%"
+        line = (f"| Config: side={side_str}, amount={amt} {self._quote_token}, "
+                f"width={width}%, offset={offset}%, threshold={threshold}%")
         status.append(line + " " * (box_width - len(line) + 1) + "|")
 
         status.append("|" + " " * box_width + "|")
@@ -985,7 +1024,8 @@ def to_format_status(self) -> List[str]:
                 lower_limit = Decimal(str(lower_price)) * (Decimal("1") - threshold_pct)
                 upper_limit = Decimal(str(upper_price)) * (Decimal("1") + threshold_pct)
 
-                line = f"| Price: {float(self._pool_price):.{price_decimals}f}  |  Auto-close if: <{float(lower_limit):.{price_decimals}f} or >{float(upper_limit):.{price_decimals}f}"
+                line = (f"| Price: {float(self._pool_price):.{price_decimals}f}  |  Auto-close if: "
+                        f"<{float(lower_limit):.{price_decimals}f} or >{float(upper_limit):.{price_decimals}f}")
                 status.append(line + " " * (box_width - len(line) + 1) + "|")
 
                 state = custom.get("state", "UNKNOWN")
@@ -1070,7 +1110,8 @@ def to_format_status(self) -> List[str]:
             line = f"| Swaps Executed: {len(closed_swaps)}"
             status.append(line + " " * (box_width - len(line) + 1) + "|")
 
-        line = f"| Fees Collected: {float(total_fees_base):.6f} {self._base_token} + {float(total_fees_quote):.6f} {self._quote_token} = {float(total_fees_value):.6f} {self._quote_token}"
+        line = (f"| Fees Collected: {float(total_fees_base):.6f} {self._base_token} + "
+                f"{float(total_fees_quote):.6f} {self._quote_token} = {float(total_fees_value):.6f} {self._quote_token}")
         status.append(line + " " * (box_width - len(line) + 1) + "|")
 
         status.append("+" + "-" * box_width + "+")

database/connection.py

@@ -74,6 +74,41 @@ async def _run_migrations(self, conn):
                 "position_holds", "cum_fees_quote",
                 "ALTER TABLE position_holds ADD COLUMN cum_fees_quote NUMERIC(30,18) NOT NULL DEFAULT 0"
             ),
+            # Position-account rent, locked on open and refunded on close. create_all only
+            # creates missing tables, so a model gaining a column reaches an existing
+            # database only through this list. Both position tables carry the pair: DAMM v2
+            # positions are NFTs with their own account, exactly like CLMM ones.
+            (
+                "gateway_clmm_positions", "position_rent",
+                "ALTER TABLE gateway_clmm_positions ADD COLUMN position_rent NUMERIC(30,18)"
+            ),
+            (
+                "gateway_clmm_positions", "position_rent_refunded",
+                "ALTER TABLE gateway_clmm_positions ADD COLUMN position_rent_refunded NUMERIC(30,18)"
+            ),
+            (
+                "gateway_amm_positions", "position_rent",
+                "ALTER TABLE gateway_amm_positions ADD COLUMN position_rent NUMERIC(30,18)"
+            ),
+            (
+                "gateway_amm_positions", "position_rent_refunded",
+                "ALTER TABLE gateway_amm_positions ADD COLUMN position_rent_refunded NUMERIC(30,18)"
+            ),
+            # Volume generated, split from capital deployed. Existing rows are backfilled
+            # from filled_amount_quote for every executor type EXCEPT lp_executor, because
+            # for an executor that places orders the amount it filled IS the volume it
+            # traded, while an LP position's filled amount is the capital it put up. The
+            # real volume of a historical LP position is not recoverable — its fees were
+            # never stored — so those rows keep 0 rather than a number that was wrong.
+            (
+                "executors", "volume_traded_quote",
+                (
+                    "ALTER TABLE executors ADD COLUMN volume_traded_quote "
+                    "NUMERIC(30,18) NOT NULL DEFAULT 0",
+                    "UPDATE executors SET volume_traded_quote = filled_amount_quote "
+                    "WHERE executor_type <> 'lp_executor'",
+                ),
+            ),
         ]
         for table, column, sql in migrations:
             try:
@@ -86,7 +121,10 @@ async def _run_migrations(self, conn):
                     {"table": table, "column": column}
                 )
                 if result.fetchone() is None:
-                    await conn.execute(text(sql))
+                    # A migration may need more than the ALTER — a backfill, say — so an
+                    # entry can carry several statements, run in order.
+                    for statement in ((sql,) if isinstance(sql, str) else sql):
+                        await conn.execute(text(statement))
                     logger.info(f"Migration: added {column} to {table}")
             except Exception as e:
                 # Column-already-exists is expected on repeat startups