Add optional OpenTelemetry trace export for job lifecycle

.gitignore

@@ -9,6 +9,9 @@ release
 *.dylib
 bin
 
+# Locally built command binaries (e.g. `go build ./cmd/ghalistener`)
+/ghalistener
+
 # Test binary, build with `go test -c`
 *.test
 

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -81,6 +81,11 @@ spec:
         - "--listener-metrics-endpoint="
         - "--metrics-addr=0"
         {{- end }}
+        {{- with .Values.otel }}
+        {{- with .listenerEndpoint }}
+        - "--listener-otel-endpoint={{ . }}"
+        {{- end }}
+        {{- end }}
         {{- if .Values.pprof.addr }}
         - "--pprof-addr={{ .Values.pprof.addr }}"
         {{- end }}

charts/gha-runner-scale-set-controller/tests/template_test.go

@@ -1218,3 +1218,59 @@ func TestNamespaceOverride(t *testing.T) {
 		})
 	}
 }
+
+func TestTemplate_ControllerDeployment_ListenerOTelEndpoint(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller")
+	require.NoError(t, err)
+
+	releaseName := "test-arc"
+	namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+	options := &helm.Options{
+		Logger: logger.Discard,
+		SetValues: map[string]string{
+			"image.tag":             "dev",
+			"otel.listenerEndpoint": "http://otel-collector:4318",
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/deployment.yaml"})
+
+	var deployment appsv1.Deployment
+	helm.UnmarshalK8SYaml(t, output, &deployment)
+
+	assert.Contains(t, deployment.Spec.Template.Spec.Containers[0].Args,
+		"--listener-otel-endpoint=http://otel-collector:4318")
+}
+
+func TestTemplate_ControllerDeployment_NoOTelByDefault(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller")
+	require.NoError(t, err)
+
+	releaseName := "test-arc"
+	namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+	options := &helm.Options{
+		Logger: logger.Discard,
+		SetValues: map[string]string{
+			"image.tag": "dev",
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/deployment.yaml"})
+
+	var deployment appsv1.Deployment
+	helm.UnmarshalK8SYaml(t, output, &deployment)
+
+	for _, arg := range deployment.Spec.Template.Spec.Containers[0].Args {
+		assert.NotContains(t, arg, "--listener-otel-endpoint")
+	}
+}

charts/gha-runner-scale-set-controller/values.yaml

@@ -98,6 +98,13 @@ priorityClassName: ""
 pprof: {}
 #   addr: ":6060"
 
+## To export OpenTelemetry traces of job lifecycle spans (queue,
+## startup, execution) from the listener pods, uncomment the following
+## lines and set the OTLP/HTTP collector base URL. TLS follows the URL
+## scheme.
+# otel:
+#   listenerEndpoint: "http://otel-collector:4318"
+
 flags:
   ## Log level can be set here with one of the following values: "debug", "info", "warn", "error".
   ## Defaults to "debug".

cmd/ghalistener/config/config.go

@@ -44,6 +44,11 @@ type Config struct {
 	MetricsAddr                 string                  `json:"metrics_addr"`
 	MetricsEndpoint             string                  `json:"metrics_endpoint"`
 	Metrics                     *v1alpha1.MetricsConfig `json:"metrics"`
+	// OTelEndpoint enables OTLP trace export of job lifecycle spans.
+	// Full base URL (like OTEL_EXPORTER_OTLP_ENDPOINT), e.g.
+	// http://collector:4318; TLS follows the URL scheme. When empty,
+	// the standard OTEL_EXPORTER_OTLP_* env vars still apply.
+	OTelEndpoint string `json:"otel_endpoint"`
 }
 
 func Read(ctx context.Context, configPath string) (*Config, error) {

cmd/ghalistener/main.go

@@ -4,16 +4,20 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/actions/actions-runner-controller/cmd/ghalistener/config"
 	"github.com/actions/actions-runner-controller/cmd/ghalistener/metrics"
 	"github.com/actions/actions-runner-controller/cmd/ghalistener/scaler"
 	"github.com/actions/actions-runner-controller/github/actions"
 	"github.com/actions/scaleset/listener"
 	"github.com/google/uuid"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -39,6 +43,29 @@ func main() {
 	}
 }
 
+// newOTLPTraceExporter builds the OTLP/HTTP span exporter. A non-empty
+// endpoint is a full base URL (the OTLP spec form of
+// OTEL_EXPORTER_OTLP_ENDPOINT, e.g. http://collector:4318); the
+// /v1/traces signal path is appended when the URL has no path. With an
+// empty endpoint the SDK's spec-compliant OTEL_EXPORTER_OTLP_* env
+// handling applies (endpoint, headers, TLS, ...).
+func newOTLPTraceExporter(ctx context.Context, endpoint string) (sdktrace.SpanExporter, error) {
+	if endpoint == "" {
+		return otlptracehttp.New(ctx)
+	}
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("invalid OTel endpoint %q: %w", endpoint, err)
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, fmt.Errorf("invalid OTel endpoint %q: scheme must be http or https", endpoint)
+	}
+	if u.Path == "" || u.Path == "/" {
+		u.Path = "/v1/traces"
+	}
+	return otlptracehttp.New(ctx, otlptracehttp.WithEndpointURL(u.String()))
+}
+
 func run(ctx context.Context, config *config.Config) error {
 	ghConfig, err := actions.ParseGitHubConfigFromURL(config.ConfigureURL)
 	if err != nil {
@@ -65,6 +92,42 @@ func run(ctx context.Context, config *config.Config) error {
 		})
 	}
 
+	// OTel trace recorder (optional). Enabled by the listener config
+	// (otel_endpoint, wired from the controller) or by the standard
+	// OTEL_EXPORTER_OTLP_* environment variables.
+	var otelRecorder *metrics.OTelRecorder
+	otelEnabled := config.OTelEndpoint != "" ||
+		os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") != "" ||
+		os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != ""
+	if otelEnabled {
+		otlpExporter, err := newOTLPTraceExporter(ctx, config.OTelEndpoint)
+		if err != nil {
+			return fmt.Errorf("failed to create OTel exporter: %w", err)
+		}
+		otelRecorder = metrics.NewOTelRecorder(metrics.OTelRecorderConfig{
+			Exporter:          otlpExporter,
+			Logger:            logger.With("component", "otel recorder"),
+			ServerURL:         ghConfig.ConfigURL.Scheme + "://" + ghConfig.ConfigURL.Host,
+			ScaleSetName:      config.EphemeralRunnerSetName,
+			ScaleSetNamespace: config.EphemeralRunnerSetNamespace,
+		})
+		// Shut down with a fresh bounded context: the signal ctx is
+		// already canceled on SIGINT/SIGTERM, which would abort the
+		// final flush of queued spans.
+		defer func() {
+			sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			if err := otelRecorder.Shutdown(sctx); err != nil {
+				logger.Error("Failed to shut down OTel recorder", "error", err)
+			}
+		}()
+		endpointSource := config.OTelEndpoint
+		if endpointSource == "" {
+			endpointSource = "OTEL_EXPORTER_OTLP_* environment"
+		}
+		logger.Info("OTel trace recorder enabled", "endpoint", endpointSource)
+	}
+
 	hostname, err := os.Hostname()
 	if err != nil {
 		hostname = uuid.NewString()
@@ -91,13 +154,25 @@ func run(ctx context.Context, config *config.Config) error {
 	}()
 
 	var listenerOptions []listener.Option
-	if metricsExporter != nil {
+
+	// Build the metrics recorder: Prometheus, OTel, or both
+	var recorder listener.MetricsRecorder
+	switch {
+	case metricsExporter != nil && otelRecorder != nil:
+		recorder = metrics.NewComposite(metricsExporter, otelRecorder)
+	case metricsExporter != nil:
+		recorder = metricsExporter
+	case otelRecorder != nil:
+		recorder = otelRecorder
+	}
+
+	if recorder != nil {
 		listenerOptions = append(
 			listenerOptions,
-			listener.WithMetricsRecorder(
-				metricsExporter,
-			),
+			listener.WithMetricsRecorder(recorder),
 		)
+	}
+	if metricsExporter != nil {
 		metricsExporter.RecordStatic(config.MinRunners, config.MaxRunners)
 	}
 

cmd/ghalistener/main_test.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+type otlpCapture struct {
+	mu    sync.Mutex
+	paths []string
+}
+
+func (c *otlpCapture) handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		c.mu.Lock()
+		c.paths = append(c.paths, r.URL.Path)
+		c.mu.Unlock()
+		w.Header().Set("Content-Type", "application/x-protobuf")
+		w.WriteHeader(http.StatusOK)
+	})
+}
+
+func (c *otlpCapture) Paths() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	out := make([]string, len(c.paths))
+	copy(out, c.paths)
+	return out
+}
+
+func exportOneSpan(t *testing.T, ctx context.Context, endpoint string) {
+	t.Helper()
+	exp, err := newOTLPTraceExporter(ctx, endpoint)
+	require.NoError(t, err)
+	stubs := tracetest.SpanStubs{{Name: "test"}}
+	require.NoError(t, exp.ExportSpans(ctx, stubs.Snapshots()))
+	require.NoError(t, exp.Shutdown(ctx))
+}
+
+// The listener config carries a full base URL (the OTLP spec form of
+// OTEL_EXPORTER_OTLP_ENDPOINT, e.g. http://collector:4318). The
+// exporter must parse it and append the /v1/traces signal path — not
+// treat it as a bare host:port.
+func TestNewOTLPTraceExporter_ConfigEndpointFullURL(t *testing.T) {
+	capture := &otlpCapture{}
+	srv := httptest.NewServer(capture.handler())
+	defer srv.Close()
+
+	exportOneSpan(t, context.Background(), srv.URL)
+
+	require.NotEmpty(t, capture.Paths(), "collector received no export")
+	assert.Equal(t, "/v1/traces", capture.Paths()[0])
+}
+
+func TestNewOTLPTraceExporter_ConfigEndpointExplicitPath(t *testing.T) {
+	capture := &otlpCapture{}
+	srv := httptest.NewServer(capture.handler())
+	defer srv.Close()
+
+	exportOneSpan(t, context.Background(), srv.URL+"/custom/v1/traces")
+
+	require.NotEmpty(t, capture.Paths(), "collector received no export")
+	assert.Equal(t, "/custom/v1/traces", capture.Paths()[0])
+}
+
+// With no config endpoint, the SDK's spec-compliant OTEL_EXPORTER_OTLP_*
+// env handling applies (endpoint, headers, TLS, ...).
+func TestNewOTLPTraceExporter_StandardEnvVars(t *testing.T) {
+	capture := &otlpCapture{}
+	srv := httptest.NewServer(capture.handler())
+	defer srv.Close()
+
+	t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", srv.URL)
+
+	exportOneSpan(t, context.Background(), "")
+
+	require.NotEmpty(t, capture.Paths(), "collector received no export")
+	assert.Equal(t, "/v1/traces", capture.Paths()[0])
+}
+
+func TestNewOTLPTraceExporter_RejectsInvalidEndpoint(t *testing.T) {
+	_, err := newOTLPTraceExporter(context.Background(), "not a url")
+	assert.Error(t, err)
+}

cmd/ghalistener/metrics/composite.go

@@ -0,0 +1,44 @@
+package metrics
+
+import (
+	"github.com/actions/scaleset"
+	"github.com/actions/scaleset/listener"
+)
+
+var _ listener.MetricsRecorder = &CompositeRecorder{}
+
+// CompositeRecorder delegates every MetricsRecorder call to all
+// wrapped recorders. This allows the Prometheus exporter and OTel
+// recorder to operate side by side.
+type CompositeRecorder struct {
+	recorders []listener.MetricsRecorder
+}
+
+// NewComposite creates a MetricsRecorder that fans out to all given recorders.
+func NewComposite(recorders ...listener.MetricsRecorder) *CompositeRecorder {
+	return &CompositeRecorder{recorders: recorders}
+}
+
+func (c *CompositeRecorder) RecordStatistics(stats *scaleset.RunnerScaleSetStatistic) {
+	for _, r := range c.recorders {
+		r.RecordStatistics(stats)
+	}
+}
+
+func (c *CompositeRecorder) RecordJobStarted(msg *scaleset.JobStarted) {
+	for _, r := range c.recorders {
+		r.RecordJobStarted(msg)
+	}
+}
+
+func (c *CompositeRecorder) RecordJobCompleted(msg *scaleset.JobCompleted) {
+	for _, r := range c.recorders {
+		r.RecordJobCompleted(msg)
+	}
+}
+
+func (c *CompositeRecorder) RecordDesiredRunners(count int) {
+	for _, r := range c.recorders {
+		r.RecordDesiredRunners(count)
+	}
+}