feat: add centralized inference gateway

.github/workflows/ci.yml

@@ -37,6 +37,9 @@ jobs:
             exit 1
           fi
 
+      - name: Validate gateway Compose example
+        run: go test ./ops -run '^TestGatewayComposeExampleParsesAndHasHealthGating$'
+
       - name: Build
         run: go build ./...
 

cmd/flags.go

@@ -260,7 +260,7 @@ func AddMaxOutputTokensFlag(cmd *cobra.Command, dest *int) {
 
 // AddToolFlags adds tool-related flags (--tools, --read-dir, --write-dir, --shell-allow)
 func AddToolFlags(cmd *cobra.Command, tools *string, readDirs, writeDirs, shellAllow *[]string) {
-	cmd.Flags().StringVar(tools, "tools", "", "Enable local tools (comma-separated, or 'all'): read_file,write_file,edit_file,shell,grep,glob,view_image,show_image,image_generate,ask_user,spawn_agent,queue_agent,wait_for_jobs")
+	cmd.Flags().StringVar(tools, "tools", "", "Enable local tools (comma-separated, 'all', or 'none'): read_file,write_file,edit_file,shell,grep,glob,view_image,show_image,image_generate,ask_user,spawn_agent,queue_agent,wait_for_jobs")
 	cmd.Flags().StringArrayVar(readDirs, "read-dir", nil, "Directories for read_file/grep/glob/view_image tools (repeatable)")
 	cmd.Flags().StringArrayVar(writeDirs, "write-dir", nil, "Directories for write_file/edit_file tools (repeatable)")
 	cmd.Flags().StringArrayVar(shellAllow, "shell-allow", nil, "Shell command patterns to allow (repeatable, glob syntax)")

cmd/gateway_enroll_test.go

@@ -0,0 +1,119 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/samsaffron/term-llm/internal/config"
+	"github.com/samsaffron/term-llm/internal/gateway"
+	"github.com/spf13/viper"
+)
+
+func TestGatewayEnrollWritesParseableConfigAndSecureTokenByDefault(t *testing.T) {
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+	configHome := t.TempDir()
+	t.Setenv("XDG_CONFIG_HOME", configHome)
+
+	stateDir := t.TempDir()
+	store, err := gateway.OpenClientStore(filepath.Join(stateDir, "clients.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, bootstrap, err := store.CreateEnrollment("satellite-test", gateway.Policy{AllowProviders: []string{"debug"}}, time.Minute)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sealer, err := gateway.OpenStateSealer(filepath.Join(stateDir, "state.key"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	server, err := gateway.NewServer(gateway.ServerConfig{Config: &config.Config{Providers: map[string]config.ProviderConfig{}}, Clients: store, Sealer: sealer})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ts := httptest.NewServer(server.Handler())
+	defer ts.Close()
+
+	oldName, oldWrite, oldTokenFile, oldPrint := gatewayEnrollName, gatewayEnrollWrite, gatewayEnrollTokenFile, gatewayEnrollPrintOnly
+	t.Cleanup(func() {
+		gatewayEnrollName, gatewayEnrollWrite, gatewayEnrollTokenFile, gatewayEnrollPrintOnly = oldName, oldWrite, oldTokenFile, oldPrint
+	})
+	gatewayEnrollName = "satellite-test"
+	gatewayEnrollWrite = true
+	gatewayEnrollTokenFile = ""
+	gatewayEnrollPrintOnly = false
+	var output bytes.Buffer
+	gatewayEnrollCmd.SetOut(&output)
+	gatewayEnrollCmd.SetContext(t.Context())
+	t.Cleanup(func() {
+		gatewayEnrollCmd.SetOut(nil)
+		gatewayEnrollCmd.SetContext(context.Background())
+	})
+	if err := runGatewayEnroll(gatewayEnrollCmd, []string{ts.URL, bootstrap}); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(output.String(), "tlg1_") {
+		t.Fatalf("default enrollment printed client token: %q", output.String())
+	}
+
+	tokenPath := filepath.Join(configHome, "term-llm", "gateway-token")
+	info, err := os.Stat(tokenPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Mode().Perm() != 0o600 {
+		t.Fatalf("token mode = %o, want 600", info.Mode().Perm())
+	}
+	cfg, err := config.Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.Gateway.URL != ts.URL || cfg.Gateway.TokenFile != tokenPath || cfg.Gateway.Token != "" {
+		t.Fatalf("enrolled gateway config = %+v", cfg.Gateway)
+	}
+	token, err := cfg.Gateway.ResolveToken()
+	if err != nil || !strings.HasPrefix(token, "tlg1_") {
+		t.Fatalf("resolved enrolled token = %q, %v", token, err)
+	}
+}
+
+func TestGatewayEnrollPrintOnlyIsExplicitAndDoesNotWrite(t *testing.T) {
+	configHome := t.TempDir()
+	t.Setenv("XDG_CONFIG_HOME", configHome)
+	stateDir := t.TempDir()
+	store, _ := gateway.OpenClientStore(filepath.Join(stateDir, "clients.json"))
+	_, bootstrap, _ := store.CreateEnrollment("print-test", gateway.Policy{AllowProviders: []string{"debug"}}, time.Minute)
+	sealer, _ := gateway.OpenStateSealer(filepath.Join(stateDir, "state.key"))
+	server, _ := gateway.NewServer(gateway.ServerConfig{Config: &config.Config{Providers: map[string]config.ProviderConfig{}}, Clients: store, Sealer: sealer})
+	ts := httptest.NewServer(server.Handler())
+	defer ts.Close()
+
+	oldName, oldWrite, oldTokenFile, oldPrint := gatewayEnrollName, gatewayEnrollWrite, gatewayEnrollTokenFile, gatewayEnrollPrintOnly
+	t.Cleanup(func() {
+		gatewayEnrollName, gatewayEnrollWrite, gatewayEnrollTokenFile, gatewayEnrollPrintOnly = oldName, oldWrite, oldTokenFile, oldPrint
+	})
+	gatewayEnrollName, gatewayEnrollWrite, gatewayEnrollTokenFile, gatewayEnrollPrintOnly = "print-test", true, "", true
+	var output bytes.Buffer
+	gatewayEnrollCmd.SetOut(&output)
+	gatewayEnrollCmd.SetContext(t.Context())
+	t.Cleanup(func() {
+		gatewayEnrollCmd.SetOut(nil)
+		gatewayEnrollCmd.SetContext(context.Background())
+	})
+	if err := runGatewayEnroll(gatewayEnrollCmd, []string{ts.URL, bootstrap}); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(output.String(), "token: tlg1_") {
+		t.Fatalf("print-only output omitted token: %q", output.String())
+	}
+	if _, err := os.Stat(filepath.Join(configHome, "term-llm", "config.yaml")); !os.IsNotExist(err) {
+		t.Fatalf("print-only wrote config: %v", err)
+	}
+}

cmd/gateway_session_test.go

@@ -0,0 +1,19 @@
+package cmd
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestGatewayProviderSessionIdleTimeoutFlag(t *testing.T) {
+	flag := gatewayServeCmd.Flags().Lookup("provider-session-idle-timeout")
+	if flag == nil {
+		t.Fatal("gateway serve is missing --provider-session-idle-timeout")
+	}
+	if flag.DefValue != "30s" {
+		t.Fatalf("provider session idle timeout default = %q, want 30s", flag.DefValue)
+	}
+	if !strings.Contains(flag.Usage, "0 disables") || !strings.Contains(flag.Usage, "WebSocket") {
+		t.Fatalf("provider session idle timeout help is incomplete: %q", flag.Usage)
+	}
+}

cmd/gateway_tools_test.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/samsaffron/term-llm/internal/config"
+)
+
+func TestRequiredGatewayFetchKeepsLegibleReadURLStub(t *testing.T) {
+	cfg := &config.Config{Gateway: config.GatewayConfig{URL: "https://gateway.invalid", Required: true}}
+	if tool := newReadURLToolForConfig(cfg); tool == nil {
+		t.Fatal("gateway outage silently removed read_url")
+	}
+	_, err := (unavailableGatewayFetcher{err: context.DeadlineExceeded}).FetchURL(context.Background(), "https://example.com")
+	if err == nil || !strings.Contains(err.Error(), "gateway read_url unavailable") || !strings.Contains(err.Error(), "gateway.fetch: false") {
+		t.Fatalf("gateway read_url stub error = %v", err)
+	}
+}
+
+func TestGatewaySearchFailureDoesNotSilentlyFallBack(t *testing.T) {
+	searcher := unavailableGatewaySearcher{err: context.DeadlineExceeded}
+	_, err := searcher.Search(context.Background(), "query", 10)
+	if err == nil || !strings.Contains(err.Error(), "gateway search unavailable") || !strings.Contains(err.Error(), "gateway.search: false") {
+		t.Fatalf("gateway search stub error = %v", err)
+	}
+}

cmd/gateway_usage_test.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"bytes"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/samsaffron/term-llm/internal/gateway"
+)
+
+func TestGatewayUsageCommandReadsAttributedRecords(t *testing.T) {
+	oldStateDir, oldClient, oldJSON := gatewayStateDir, gatewayUsageClient, gatewayUsageJSON
+	t.Cleanup(func() {
+		gatewayStateDir, gatewayUsageClient, gatewayUsageJSON = oldStateDir, oldClient, oldJSON
+	})
+	gatewayStateDir = t.TempDir()
+	gatewayUsageClient = "satellite-a"
+	gatewayUsageJSON = false
+	recorder := &gateway.JSONLUsageRecorder{Path: filepath.Join(gatewayStateDir, "usage.jsonl")}
+	if err := recorder.Record(gateway.UsageRecord{
+		StartedAt: time.Now().Add(-time.Second), CompletedAt: time.Now(), ClientID: "client-a", ClientName: "satellite-a",
+		ProviderKey: "openai", Model: "gpt", RequestID: "req-1", InputTokens: 10, OutputTokens: 2,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	var output bytes.Buffer
+	gatewayUsageCmd.SetOut(&output)
+	t.Cleanup(func() { gatewayUsageCmd.SetOut(nil) })
+	if err := runGatewayUsage(gatewayUsageCmd, nil); err != nil {
+		t.Fatal(err)
+	}
+	text := output.String()
+	if !strings.Contains(text, "satellite-a") || !strings.Contains(text, "openai:gpt") || !strings.Contains(text, "requests=1") {
+		t.Fatalf("gateway usage output = %q", text)
+	}
+}

cmd/models.go

@@ -76,6 +76,20 @@ func runModels(cmd *cobra.Command, args []string) error {
 	if providerName == "" {
 		providerName = cfg.DefaultProvider
 	}
+	if cfg.Gateway.Enabled() && !cfg.IsLocalProvider(providerName) {
+		provider, routeErr := llm.NewProviderByName(cfg, providerName, "")
+		if routeErr != nil {
+			return routeErr
+		} else if remote, ok := provider.(*llm.GatewayProvider); ok {
+			ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
+			defer cancel()
+			models, listErr := remote.ListModels(ctx)
+			if listErr != nil {
+				return fmt.Errorf("failed to list gateway models: %w", listErr)
+			}
+			return outputListedModels(providerName, models, true)
+		}
+	}
 
 	// Get provider config - handle built-in providers that may not be explicitly configured
 	providerCfg, ok := cfg.Providers[providerName]
@@ -225,6 +239,12 @@ func runModels(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to list models: %w", err)
 	}
 
+	// Only these providers return or have known pricing info.
+	providerHasPricing := providerType == config.ProviderTypeOpenRouter || providerType == config.ProviderTypeZen || providerType == config.ProviderTypeNearAI || providerType == config.ProviderTypeSambaNova
+	return outputListedModels(providerName, models, providerHasPricing)
+}
+
+func outputListedModels(providerName string, models []llm.ModelInfo, providerHasPricing bool) error {
 	if len(models) == 0 {
 		fmt.Println("No models found.")
 		return nil
@@ -236,12 +256,7 @@ func runModels(cmd *cobra.Command, args []string) error {
 		return enc.Encode(models)
 	}
 
-	// Pretty print
 	fmt.Printf("Available models from %s:\n\n", providerName)
-
-	// Only these providers return or have known pricing info
-	providerHasPricing := providerType == config.ProviderTypeOpenRouter || providerType == config.ProviderTypeZen || providerType == config.ProviderTypeNearAI || providerType == config.ProviderTypeSambaNova
-
 	for _, m := range models {
 		if m.DisplayName != "" {
 			fmt.Printf("  %s (%s)", m.ID, m.DisplayName)
@@ -255,10 +270,8 @@ func runModels(cmd *cobra.Command, args []string) error {
 		}
 
 		// Show pricing info only if provider returns it
-		if providerHasPricing {
-			if m.InputPrice < 0 || m.OutputPrice < 0 {
-				fmt.Printf(" [pricing unknown]")
-			} else if m.InputPrice == 0 && m.OutputPrice == 0 {
+		if providerHasPricing && m.InputPrice >= 0 && m.OutputPrice >= 0 {
+			if m.InputPrice == 0 && m.OutputPrice == 0 {
 				fmt.Printf(" [FREE]")
 			} else {
 				fmt.Printf(" [$%.2f/$%.2f per 1M tokens]", m.InputPrice, m.OutputPrice)

cmd/runner.go

@@ -188,8 +188,18 @@ func (r *cmdRunner) prepare(ctx context.Context, req runpkg.Request, sink runpkg
 		return nil, err
 	}
 	if model := strings.TrimSpace(req.Model); model != "" {
-		if err := applyAgentModelOverride(cfg, model); err != nil {
-			return nil, fmt.Errorf("apply model override %q: %w", model, err)
+		// A provider:model CLI selection is already concrete. In particular,
+		// debug:fast means the literal gateway catalog model "fast", not the
+		// special agent-level fast-model alias.
+		parts := strings.SplitN(providerFlag, ":", 2)
+		explicitModel := ""
+		if len(parts) == 2 {
+			explicitModel = strings.TrimSpace(parts[1])
+		}
+		if strings.TrimSpace(providerFlag) == "" || explicitModel == "" || explicitModel != model {
+			if err := applyAgentModelOverride(cfg, model); err != nil {
+				return nil, fmt.Errorf("apply model override %q: %w", model, err)
+			}
 		}
 	}
 

cmd/tools.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"context"
+	"fmt"
 	"log"
 
 	"github.com/samsaffron/term-llm/internal/config"
@@ -9,12 +11,29 @@ import (
 	"github.com/samsaffron/term-llm/internal/tools"
 )
 
+type unavailableGatewaySearcher struct{ err error }
+
+func (s unavailableGatewaySearcher) Search(context.Context, string, int) ([]search.Result, error) {
+	return nil, fmt.Errorf("gateway search unavailable: %w; check gateway URL/network/token or set gateway.search: false to use local search", s.err)
+}
+
+type unavailableGatewayFetcher struct{ err error }
+
+func (f unavailableGatewayFetcher) FetchURL(context.Context, string) (string, error) {
+	return "", fmt.Errorf("gateway read_url unavailable: %w; check gateway URL/network/token or set gateway.fetch: false to use local fetch", f.err)
+}
+
 func defaultToolRegistry(cfg *config.Config) *llm.ToolRegistry {
 	registry := llm.NewToolRegistry()
 	searcher, err := search.NewSearcher(cfg)
 	if err != nil {
-		log.Printf("Warning: search provider error: %v, falling back to DuckDuckGo", err)
-		searcher = search.NewDuckDuckGoLite(nil)
+		if cfg != nil && cfg.Gateway.Enabled() && cfg.Gateway.RouteSearch() {
+			log.Printf("Warning: gateway search unavailable: %v", err)
+			searcher = unavailableGatewaySearcher{err: err}
+		} else {
+			log.Printf("Warning: search provider error: %v, falling back to DuckDuckGo", err)
+			searcher = search.NewDuckDuckGoLite(nil)
+		}
 	}
 	registry.Register(llm.NewWebSearchTool(searcher))
 	if readURLTool := newReadURLToolForConfig(cfg); readURLTool != nil {
@@ -24,6 +43,14 @@ func defaultToolRegistry(cfg *config.Config) *llm.ToolRegistry {
 }
 
 func newReadURLToolForConfig(cfg *config.Config) *llm.ReadURLTool {
+	if cfg.Gateway.Enabled() && cfg.Gateway.RouteFetch() {
+		client, err := search.NewGatewayClient(cfg.Gateway)
+		if err != nil {
+			log.Printf("Warning: gateway fetch unavailable: %v", err)
+			return llm.NewReadURLToolWithFetcher(unavailableGatewayFetcher{err: err})
+		}
+		return llm.NewReadURLToolWithFetcher(client)
+	}
 	switch cfg.Search.FetchProvider {
 	case "", "jina":
 		return llm.NewReadURLTool()

cmd/usage.go

@@ -67,7 +67,7 @@ func init() {
 	usageCmd.Flags().StringVar(&usageUntil, "until", "", "End date (YYYYMMDD)")
 	usageCmd.Flags().BoolVar(&usageJSON, "json", false, "Output as JSON")
 	usageCmd.Flags().BoolVar(&usageBreakdown, "breakdown", false, "Show per-model breakdown")
-	usageCmd.Flags().BoolVar(&usageIncludeExternal, "include-external", false, "Include externally-tracked term-llm usage (claude-bin, codex, gemini-cli calls)")
+	usageCmd.Flags().BoolVar(&usageIncludeExternal, "include-external", false, "Include externally-tracked term-llm usage (CLI-provider and gateway calls) in any provider view")
 	usageCmd.Flags().StringVar(&usageCopilotScope, "copilot-scope", "user", "Copilot billing scope (user, org, enterprise)")
 	usageCmd.Flags().StringVar(&usageCopilotEntity, "copilot-entity", "", "Copilot billing entity (username, organization, or enterprise slug; defaults to authenticated user for user scope)")
 	usageCmd.Flags().IntVar(&usageCopilotYear, "year", 0, "Copilot usage year (YYYY; defaults to GitHub API current year)")