diff --git a/controller/cmd/exporter-set-controller/main.go b/controller/cmd/exporter-set-controller/main.go index 6bcf56d69..4aab5134d 100644 --- a/controller/cmd/exporter-set-controller/main.go +++ b/controller/cmd/exporter-set-controller/main.go @@ -29,6 +29,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -38,6 +39,7 @@ import ( "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset" "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/cuttlefish" "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/qemu" + qemussh "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/qemu-ssh" ) var ( @@ -102,13 +104,6 @@ func main() { os.Exit(1) } - // Select the provisioner implementation based on the flag - prov, err := selectProvisioner(provisioner) - if err != nil { - setupLog.Error(err, "unsupported provisioner", "provisioner", provisioner) - os.Exit(1) - } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Cache: cache.Options{ @@ -129,6 +124,16 @@ func main() { os.Exit(1) } + // Select the provisioner implementation based on the flag. + // Some provisioners (e.g. qemu-ssh) need a Kubernetes client for + // reading Secrets and updating annotations, so we create them after + // the manager is initialized. + prov, err := selectProvisioner(provisioner, mgr.GetClient()) + if err != nil { + setupLog.Error(err, "unsupported provisioner", "provisioner", provisioner) + os.Exit(1) + } + if err = (&exporterset.ExporterSetReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -158,16 +163,18 @@ func main() { // selectProvisioner returns the Provisioner implementation for the given name. // Add new provisioners here as they are implemented. -func selectProvisioner(name string) (exporterset.Provisioner, error) { +func selectProvisioner(name string, c client.Client) (exporterset.Provisioner, error) { switch name { case cuttlefish.ProvisionerName: return cuttlefish.New(version), nil case qemu.ProvisionerName: return qemu.New(version), nil + case qemussh.ProvisionerName: + return qemussh.New(version, c), nil default: return nil, fmt.Errorf( - "unknown provisioner %q; supported: %s, %s", - name, qemu.ProvisionerName, cuttlefish.ProvisionerName, + "unknown provisioner %q; supported: %s, %s, %s", + name, qemu.ProvisionerName, cuttlefish.ProvisionerName, qemussh.ProvisionerName, ) } } diff --git a/controller/config/samples/operator_jumpstarter_with_qemu_ssh.yaml b/controller/config/samples/operator_jumpstarter_with_qemu_ssh.yaml new file mode 100644 index 000000000..33da52b80 --- /dev/null +++ b/controller/config/samples/operator_jumpstarter_with_qemu_ssh.yaml @@ -0,0 +1,23 @@ +# Jumpstarter operator CR — enabling both in-cluster QEMU and off-cluster +# QEMU-SSH provisioners. +# +# Apply with: kubectl apply -f operator_jumpstarter_with_qemu_ssh.yaml +apiVersion: operator.jumpstarter.dev/v1alpha1 +kind: Jumpstarter +metadata: + name: jumpstarter + namespace: jumpstarter +spec: + exporterSets: + # Container image for the exporter-set controller binary. + # All provisioners share the same binary; --provisioner selects + # the backend at startup. + image: quay.io/jumpstarter-dev/exporter-set-controller:latest + provisioners: + # In-cluster QEMU provisioner (creates Pods with QEMU sidecar) + - name: qemu.jumpstarter.dev + enabled: true + # Off-cluster QEMU provisioner (deploys containers on remote + # hosts via SSH using Podman quadlets) + - name: qemu-ssh.jumpstarter.dev + enabled: true diff --git a/controller/config/samples/secret_ssh_credentials.yaml b/controller/config/samples/secret_ssh_credentials.yaml new file mode 100644 index 000000000..55b6866b0 --- /dev/null +++ b/controller/config/samples/secret_ssh_credentials.yaml @@ -0,0 +1,26 @@ +# SSH credentials Secret — required by the qemu-ssh.jumpstarter.dev +# provisioner for authenticating to remote lab hosts. +# +# The credentialsSecretRef in the VirtualTargetClass points to this +# Secret. It uses the standard kubernetes.io/ssh-auth type. +# +# Note: kubernetes.io/ssh-auth only carries the private key. The SSH +# username is configured in the VirtualTargetClass parameters +# (parameters.ssh.user or parameters.host.user), not in this Secret. +# +# Create from a file (preferred): +# kubectl create secret generic lab-ssh-key \ +# --from-file=ssh-privatekey=$HOME/.ssh/id_ed25519 \ +# -n jumpstarter +# +# Or apply this manifest after base64-encoding your key: +# cat ~/.ssh/id_ed25519 | base64 -w0 +apiVersion: v1 +kind: Secret +metadata: + name: lab-ssh-key + namespace: jumpstarter +type: kubernetes.io/ssh-auth +data: + # Replace with your base64-encoded SSH private key + ssh-privatekey: diff --git a/controller/config/samples/v1alpha1_exporterset_qemu_ssh.yaml b/controller/config/samples/v1alpha1_exporterset_qemu_ssh.yaml new file mode 100644 index 000000000..862c78f80 --- /dev/null +++ b/controller/config/samples/v1alpha1_exporterset_qemu_ssh.yaml @@ -0,0 +1,44 @@ +# ExporterSet — off-cluster QEMU aarch64 pool via SSH +# +# This example creates a scalable pool of aarch64 virtual targets +# running as Podman containers on remote lab hosts. The +# exporter-set controller manages the lifecycle via SSH. +# +# Uses the qemu-ssh-aarch64 VirtualTargetClass (see +# v1alpha1_virtualtargetclass_qemu_ssh.yaml). +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: ExporterSet +metadata: + name: aarch64-ssh-pool + namespace: jumpstarter +spec: + minReplicas: 0 + maxReplicas: 4 + minAvailableReplicas: 1 + scaleDownCooldown: 5m + recycleStrategy: ExitAndReplace + virtualTargetClassName: qemu-ssh-aarch64 + # Override class-level resource defaults (deep-merged) + parameters: + resources: + memory: 8Gi + selector: + matchLabels: + board: aarch64-qemu + virtual: "true" + template: + metadata: + labels: + board: aarch64-qemu + arch: aarch64 + virtual: "true" + spec: + drivers: + - name: qemu + type: jumpstarter_driver_qemu.driver.Qemu + - name: power + type: jumpstarter_driver_power.driver.QemuPower + - name: serial + type: jumpstarter_driver_serial.driver.QemuSerial + # tcp driver is auto-injected by the provisioner (ssh + # hostfwd on port 2222) — no need to specify it explicitly diff --git a/controller/config/samples/v1alpha1_virtualtargetclass_qemu_ssh.yaml b/controller/config/samples/v1alpha1_virtualtargetclass_qemu_ssh.yaml new file mode 100644 index 000000000..4f9b87c0e --- /dev/null +++ b/controller/config/samples/v1alpha1_virtualtargetclass_qemu_ssh.yaml @@ -0,0 +1,46 @@ +# VirtualTargetClass — off-cluster QEMU via SSH +# +# This example defines a virtual target profile for aarch64 targets +# running on a remote lab host (bare metal or VM with KVM). The +# exporter-set controller connects via SSH and deploys Podman +# containers (quadlets) on the host. +# +# Each ExporterSet manages one host. To use multiple hosts, create +# additional ExporterSets under this same VirtualTargetClass. +# +# Prerequisites: +# 1. Create the SSH credentials Secret: +# kubectl create secret generic lab-ssh-key \ +# --from-file=ssh-privatekey=$HOME/.ssh/id_ed25519 \ +# -n jumpstarter +# 2. Ensure the remote host has Podman and systemd installed. +# 3. Enable qemu-ssh.jumpstarter.dev provisioner in the Jumpstarter CR. +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: VirtualTargetClass +metadata: + name: qemu-ssh-aarch64 + namespace: jumpstarter +spec: + provisioner: qemu-ssh.jumpstarter.dev + credentialsSecretRef: + name: lab-ssh-key + bindingMode: Immediate + reclaimPolicy: Delete + parameters: + # SSH connection defaults (per-host overrides available in host) + ssh: + user: root + port: 22 + # Remote lab host for this ExporterSet + host: + name: lab-host-01.example.com + # Runtime container configuration + runtime: + kvm: true # pass /dev/kvm into the QEMU runtime container + # Default resource allocation per virtual target + arch: aarch64 + resources: + cpu: 4 + memory: 4Gi + storage: + size: 16Gi diff --git a/controller/deploy/operator/go.mod b/controller/deploy/operator/go.mod index 3db24dc34..7f370be00 100644 --- a/controller/deploy/operator/go.mod +++ b/controller/deploy/operator/go.mod @@ -4,7 +4,6 @@ go 1.26.3 require ( github.com/cert-manager/cert-manager v1.18.6 - github.com/go-logr/logr v1.4.3 github.com/google/uuid v1.6.0 github.com/jumpstarter-dev/jumpstarter/controller v0.7.1 github.com/onsi/ginkgo/v2 v2.22.2 @@ -15,7 +14,6 @@ require ( k8s.io/apimachinery v0.34.1 k8s.io/apiserver v0.34.1 k8s.io/client-go v0.34.1 - k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d sigs.k8s.io/controller-runtime v0.22.3 sigs.k8s.io/yaml v1.6.0 ) @@ -48,6 +46,7 @@ require ( github.com/gin-gonic/gin v1.10.0 // indirect github.com/go-chi/chi/v5 v5.2.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.1 // indirect @@ -111,16 +110,16 @@ require ( go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.47.0 // indirect + golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/term v0.39.0 // indirect - golang.org/x/text v0.33.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.13.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.49.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect @@ -134,6 +133,7 @@ require ( k8s.io/component-base v0.34.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 // indirect sigs.k8s.io/gateway-api v1.4.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect diff --git a/controller/deploy/operator/go.sum b/controller/deploy/operator/go.sum index 90eb2f8e6..54bba39d3 100644 --- a/controller/deploy/operator/go.sum +++ b/controller/deploy/operator/go.sum @@ -251,8 +251,8 @@ golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -261,37 +261,37 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/controller/go.mod b/controller/go.mod index 03fd061e2..421af180c 100644 --- a/controller/go.mod +++ b/controller/go.mod @@ -13,12 +13,14 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 + github.com/pkg/sftp v1.13.11 github.com/prometheus/client_golang v1.22.0 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.62.0 github.com/zitadel/oidc/v3 v3.34.1 + golang.org/x/crypto v0.57.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/sync v0.19.0 + golang.org/x/sync v0.23.0 google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 @@ -66,13 +68,14 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.23.2 // indirect github.com/google/gnostic-models v0.6.9 // indirect - github.com/google/go-cmp v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -102,14 +105,13 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.47.0 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/term v0.39.0 // indirect - golang.org/x/text v0.33.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.49.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/controller/go.sum b/controller/go.sum index bc4fe72a5..e0985d380 100644 --- a/controller/go.sum +++ b/controller/go.sum @@ -123,6 +123,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -157,6 +159,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.11 h1:0N92SLTB8JqASJB14ZLHHzFnBV8mG9zw4K7jghEFWuE= +github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hsN0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= @@ -235,8 +239,8 @@ golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -245,37 +249,37 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/controller/internal/exporterset/deployer.go b/controller/internal/exporterset/deployer.go new file mode 100644 index 000000000..110e03ad6 --- /dev/null +++ b/controller/internal/exporterset/deployer.go @@ -0,0 +1,54 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exporterset + +import ( + "context" + + jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" +) + +// Deployer is an optional interface that off-cluster provisioners +// implement to manage exporter instances outside the Kubernetes +// cluster. When a Provisioner also implements Deployer, the +// reconciler calls Deploy/IsDeployed instead of creating Pods. +// +// In-cluster provisioners (e.g. qemu.jumpstarter.dev) do not +// implement this interface — they use RenderPod and standard Pod +// lifecycle. +type Deployer interface { + // Deploy sets up the exporter on the remote host. Called after + // the Exporter CR has credentials (endpoint + token) and the + // config Secret has been synced. + Deploy( + ctx context.Context, + es *virtualtargetv1alpha1.ExporterSet, + vtc *virtualtargetv1alpha1.VirtualTargetClass, + mergedParameters map[string]any, + images *virtualtargetv1alpha1.ImageOverrides, + exporter *jumpstarterdevv1alpha1.Exporter, + caBundle string, + ) error + + // IsDeployed reports whether the exporter instance is already + // running on its assigned remote host. + IsDeployed( + ctx context.Context, + exporter *jumpstarterdevv1alpha1.Exporter, + ) (bool, error) +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/enrich.go b/controller/internal/exporterset/provisioners/qemu-ssh/enrich.go new file mode 100644 index 000000000..f9b3fb0fc --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/enrich.go @@ -0,0 +1,182 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "encoding/json" + "fmt" + + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +const ( + qemuDriverType = "jumpstarter_driver_qemu.driver.Qemu" + tcpDriverType = "jumpstarter_driver_network.driver.TcpNetwork" +) + +// enrichExporterExport adjusts driver configuration for off-cluster +// deployment. The logic mirrors the in-cluster QEMU provisioner but +// paths match the quadlet container layout. +func enrichExporterExport( + drivers []virtualtargetv1alpha1.DriverConfig, + mergedParameters map[string]any, +) ([]virtualtargetv1alpha1.DriverConfig, error) { + result := make([]virtualtargetv1alpha1.DriverConfig, 0, len(drivers)+1) + hasTCP := false + + for _, d := range drivers { + if d.Type == tcpDriverType { + hasTCP = true + } + + if d.Type == qemuDriverType { + var err error + d, err = enrichQemuDriver(d, mergedParameters) + if err != nil { + return nil, err + } + } + result = append(result, d) + } + + if !hasTCP { + result = append(result, virtualtargetv1alpha1.DriverConfig{ + Name: "tcp", + Type: tcpDriverType, + Config: mustJSON(map[string]any{ + "host": "127.0.0.1", + "port": 2222, + }), + }) + } + + return result, nil +} + +func enrichQemuDriver( + d virtualtargetv1alpha1.DriverConfig, + params map[string]any, +) (virtualtargetv1alpha1.DriverConfig, error) { + config := make(map[string]any) + if d.Config != nil && d.Config.Raw != nil { + if err := json.Unmarshal(d.Config.Raw, &config); err != nil { + return d, fmt.Errorf("unmarshal QEMU driver config: %w", err) + } + } + + config["launcher_socket"] = launcherSocketPath + + setDefault(config, "arch", params, "arch") + setDefault(config, "smp", params, "resources.cpu") + setDefault(config, "mem", params, "resources.memory") + setDefault(config, "disk_size", params, "storage.size") + + if _, hasPartitions := config["default_partitions"]; !hasPartitions { + arch, _ := config["arch"].(string) + config["default_partitions"] = defaultPartitionsForArch(arch) + } + + hostfwd, _ := config["hostfwd"].(map[string]any) + if hostfwd == nil { + hostfwd = make(map[string]any) + } + if _, hasSSH := hostfwd["ssh"]; !hasSSH { + hostfwd["ssh"] = map[string]any{ + "hostaddr": "127.0.0.1", + "hostport": 2222, + "guestport": 22, + } + config["hostfwd"] = hostfwd + } + + raw, _ := json.Marshal(config) + d.Config = &apiextensionsv1.JSON{Raw: raw} + return d, nil +} + +func defaultPartitionsForArch(arch string) map[string]string { + switch arch { + case "aarch64": + return map[string]string{ + "OVMF_CODE.fd": "/usr/share/AAVMF/AAVMF_CODE.fd", + "OVMF_VARS.fd": "/usr/share/AAVMF/AAVMF_VARS.fd", + } + default: + return map[string]string{ + "OVMF_CODE.fd": "/usr/share/edk2/ovmf/OVMF_CODE.fd", + "OVMF_VARS.fd": "/usr/share/edk2/ovmf/OVMF_VARS.fd", + } + } +} + +func setDefault(config map[string]any, key string, params map[string]any, paramPath string) { + if _, exists := config[key]; exists { + return + } + + parts := splitDot(paramPath) + var val any = params + for _, p := range parts { + m, ok := val.(map[string]any) + if !ok { + return + } + val = m[p] + } + + if val != nil { + if key == "disk_size" || key == "mem" { + val = normalizeQemuSize(val) + } + config[key] = val + } +} + +func normalizeQemuSize(v any) any { + s, ok := v.(string) + if !ok || len(s) < 2 { + return v + } + if s[len(s)-1] != 'i' { + return v + } + switch s[len(s)-2] { + case 'K', 'M', 'G', 'T', 'k', 'm', 'g', 't': + return s[:len(s)-1] + default: + return v + } +} + +func splitDot(s string) []string { + result := make([]string, 0, 2) + start := 0 + for i := range s { + if s[i] == '.' { + result = append(result, s[start:i]) + start = i + 1 + } + } + result = append(result, s[start:]) + return result +} + +func mustJSON(v any) *apiextensionsv1.JSON { + raw, _ := json.Marshal(v) + return &apiextensionsv1.JSON{Raw: raw} +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/enrich_test.go b/controller/internal/exporterset/provisioners/qemu-ssh/enrich_test.go new file mode 100644 index 000000000..1ddb743c2 --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/enrich_test.go @@ -0,0 +1,213 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "encoding/json" + "testing" + + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func TestEnrich_injectsLauncherSocket(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + { + Name: "qemu", + Type: qemuDriverType, + Config: mustJSON(map[string]any{"arch": "x86_64"}), + }, + } + + result, err := enrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + + config := unmarshalConfig(t, findDriver(result, "qemu").Config) + if got := config["launcher_socket"]; got != launcherSocketPath { + t.Errorf("launcher_socket = %v, want %v", got, launcherSocketPath) + } +} + +func TestEnrich_autoInjectsTCP(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + { + Name: "qemu", + Type: qemuDriverType, + Config: mustJSON(map[string]any{"arch": "x86_64"}), + }, + } + + result, err := enrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + + tcp := findDriver(result, "tcp") + if tcp == nil { + t.Fatal("tcp driver not auto-injected") + } + if tcp.Type != tcpDriverType { + t.Errorf("tcp driver type = %q", tcp.Type) + } +} + +func TestEnrich_doesNotDuplicateTCP(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + { + Name: "qemu", + Type: qemuDriverType, + Config: mustJSON(map[string]any{"arch": "x86_64"}), + }, + { + Name: "tcp", + Type: tcpDriverType, + Config: mustJSON(map[string]any{"host": "10.0.0.1", "port": 3333}), + }, + } + + result, err := enrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + + count := 0 + for _, d := range result { + if d.Type == tcpDriverType { + count++ + } + } + if count != 1 { + t.Errorf("tcp driver count = %d, want 1", count) + } +} + +func TestEnrich_defaultsFromParameters(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + { + Name: "qemu", + Type: qemuDriverType, + Config: mustJSON(map[string]any{}), + }, + } + + params := map[string]any{ + "arch": "aarch64", + "resources": map[string]any{ + "cpu": 4, + "memory": "4Gi", + }, + "storage": map[string]any{ + "size": "16Gi", + }, + } + + result, err := enrichExporterExport(drivers, params) + if err != nil { + t.Fatal(err) + } + + config := unmarshalConfig(t, findDriver(result, "qemu").Config) + if got := config["arch"]; got != "aarch64" { + t.Errorf("arch = %v, want aarch64", got) + } + if got := config["smp"]; got != float64(4) { + t.Errorf("smp = %v, want 4", got) + } + if got := config["mem"]; got != "4G" { + t.Errorf("mem = %v, want 4G (normalized from 4Gi)", got) + } + if got := config["disk_size"]; got != "16G" { + t.Errorf("disk_size = %v, want 16G", got) + } +} + +func TestEnrich_defaultPartitionsAarch64(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + { + Name: "qemu", + Type: qemuDriverType, + Config: mustJSON(map[string]any{"arch": "aarch64"}), + }, + } + + result, err := enrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + + config := unmarshalConfig(t, findDriver(result, "qemu").Config) + partitions, ok := config["default_partitions"].(map[string]any) + if !ok { + t.Fatalf("default_partitions not a map: %T", config["default_partitions"]) + } + if got := partitions["OVMF_CODE.fd"]; got != "/usr/share/AAVMF/AAVMF_CODE.fd" { + t.Errorf("OVMF_CODE.fd = %v", got) + } +} + +func TestEnrich_hostfwdSSH(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + { + Name: "qemu", + Type: qemuDriverType, + Config: mustJSON(map[string]any{"arch": "x86_64"}), + }, + } + + result, err := enrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + + config := unmarshalConfig(t, findDriver(result, "qemu").Config) + hostfwd, ok := config["hostfwd"].(map[string]any) + if !ok { + t.Fatalf("hostfwd not a map: %T", config["hostfwd"]) + } + sshFwd, ok := hostfwd["ssh"].(map[string]any) + if !ok { + t.Fatalf("hostfwd.ssh not a map: %T", hostfwd["ssh"]) + } + if got := sshFwd["hostport"].(float64); got != 2222 { + t.Errorf("hostfwd.ssh.hostport = %v", got) + } +} + +// --- helpers --- + +func findDriver(drivers []virtualtargetv1alpha1.DriverConfig, name string) *virtualtargetv1alpha1.DriverConfig { + for i := range drivers { + if drivers[i].Name == name { + return &drivers[i] + } + } + return nil +} + +func unmarshalConfig(t *testing.T, raw *apiextensionsv1.JSON) map[string]any { + t.Helper() + if raw == nil || raw.Raw == nil { + t.Fatal("config is nil") + } + var config map[string]any + if err := json.Unmarshal(raw.Raw, &config); err != nil { + t.Fatalf("unmarshal config: %v", err) + } + return config +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/host.go b/controller/internal/exporterset/provisioners/qemu-ssh/host.go new file mode 100644 index 000000000..61d5bb58c --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/host.go @@ -0,0 +1,149 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "encoding/json" + "fmt" +) + +const ( + // AnnotationHost is the annotation key on Exporter CRs that + // records which remote host an instance was assigned to. + AnnotationHost = "qemu-ssh.jumpstarter.dev/host" +) + +// HostConfig describes the single remote lab host for an ExporterSet +// using the qemu-ssh provisioner. Each ExporterSet manages one host; +// additional hosts are modeled as additional ExporterSets under the +// same VirtualTargetClass. Capacity is controlled via ExporterSet +// replica bounds, not per-host slot accounting. +type HostConfig struct { + // Name is the FQDN or IP of the remote host. + Name string `json:"name"` + + // Port is the SSH port. Defaults to the parameters-level SSH port + // or 22 if unset. + Port int `json:"port,omitempty"` + + // User is a per-host SSH user override. Falls back to the + // parameters-level SSH user. + User string `json:"user,omitempty"` +} + +// SSHConfig holds parameters-level SSH defaults parsed from merged +// parameters. +type SSHConfig struct { + User string `json:"user,omitempty"` + Port int `json:"port,omitempty"` +} + +// ParseHost extracts the single host from merged parameters. +// Expected structure: parameters.host: {name, port?, user?} +func ParseHost(mergedParameters map[string]any) (HostConfig, error) { + hostRaw, ok := mergedParameters["host"] + if !ok { + return HostConfig{}, fmt.Errorf("parameters.host is required for qemu-ssh provisioner") + } + + data, err := json.Marshal(hostRaw) + if err != nil { + return HostConfig{}, fmt.Errorf("marshal host: %w", err) + } + + var host HostConfig + if err := json.Unmarshal(data, &host); err != nil { + return HostConfig{}, fmt.Errorf("unmarshal host: %w", err) + } + + if host.Name == "" { + return HostConfig{}, fmt.Errorf("parameters.host.name is required") + } + + return host, nil +} + +// ParseSSHConfig extracts parameters-level SSH defaults from merged +// parameters. +func ParseSSHConfig(mergedParameters map[string]any) (SSHConfig, error) { + var cfg SSHConfig + sshRaw, ok := mergedParameters["ssh"] + if !ok { + return cfg, nil + } + + data, err := json.Marshal(sshRaw) + if err != nil { + return cfg, fmt.Errorf("marshal ssh config: %w", err) + } + if err := json.Unmarshal(data, &cfg); err != nil { + return cfg, fmt.Errorf("unmarshal ssh config: %w", err) + } + return cfg, nil +} + +// ResolveSSHUser returns the effective SSH user for a host, falling +// back to parameters-level defaults. +func ResolveSSHUser(host HostConfig, ssh SSHConfig) string { + if host.User != "" { + return host.User + } + if ssh.User != "" { + return ssh.User + } + return "root" +} + +// ResolveSSHPort returns the effective SSH port for a host, falling +// back to parameters-level defaults, then 22. +func ResolveSSHPort(host HostConfig, ssh SSHConfig) int { + if host.Port > 0 { + return host.Port + } + if ssh.Port > 0 { + return ssh.Port + } + return 22 +} + +// ParseRuntimeConfig extracts runtime-specific settings from merged +// parameters (e.g. KVM enablement, extra devices). +func ParseRuntimeConfig(mergedParameters map[string]any) (kvm bool, extraDevices []string) { + runtimeRaw, ok := mergedParameters["runtime"] + if !ok { + return false, nil + } + + runtimeMap, ok := runtimeRaw.(map[string]any) + if !ok { + return false, nil + } + + if v, ok := runtimeMap["kvm"].(bool); ok { + kvm = v + } + + if devs, ok := runtimeMap["devices"].([]any); ok { + for _, d := range devs { + if s, ok := d.(string); ok { + extraDevices = append(extraDevices, s) + } + } + } + + return kvm, extraDevices +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/host_test.go b/controller/internal/exporterset/provisioners/qemu-ssh/host_test.go new file mode 100644 index 000000000..a578e23cb --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/host_test.go @@ -0,0 +1,214 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "testing" +) + +func TestParseHost_valid(t *testing.T) { + params := map[string]any{ + "host": map[string]any{ + "name": "bench-01.lab.example.com", + "port": float64(2222), + "user": "admin", + }, + } + + host, err := ParseHost(params) + if err != nil { + t.Fatalf("ParseHost() error = %v", err) + } + + if host.Name != "bench-01.lab.example.com" { + t.Errorf("host.Name = %q", host.Name) + } + if host.Port != 2222 { + t.Errorf("host.Port = %d", host.Port) + } + if host.User != "admin" { + t.Errorf("host.User = %q", host.User) + } +} + +func TestParseHost_nameOnly(t *testing.T) { + params := map[string]any{ + "host": map[string]any{ + "name": "bench-01.lab.example.com", + }, + } + + host, err := ParseHost(params) + if err != nil { + t.Fatalf("ParseHost() error = %v", err) + } + if host.Name != "bench-01.lab.example.com" { + t.Errorf("host.Name = %q", host.Name) + } + if host.Port != 0 || host.User != "" { + t.Errorf("expected empty port/user overrides, got %+v", host) + } +} + +func TestParseHost_missingHostKey(t *testing.T) { + _, err := ParseHost(map[string]any{}) + if err == nil { + t.Fatal("ParseHost() expected error for missing host key") + } +} + +func TestParseHost_missingName(t *testing.T) { + params := map[string]any{ + "host": map[string]any{ + "port": float64(2222), + }, + } + + _, err := ParseHost(params) + if err == nil { + t.Fatal("ParseHost() expected error for missing name") + } +} + +func TestParseSSHConfig(t *testing.T) { + params := map[string]any{ + "ssh": map[string]any{ + "user": "jumpstarter", + "port": float64(2222), + }, + } + + cfg, err := ParseSSHConfig(params) + if err != nil { + t.Fatalf("ParseSSHConfig() error = %v", err) + } + if cfg.User != "jumpstarter" { + t.Errorf("User = %q, want jumpstarter", cfg.User) + } + if cfg.Port != 2222 { + t.Errorf("Port = %d, want 2222", cfg.Port) + } +} + +func TestParseSSHConfig_missing(t *testing.T) { + cfg, err := ParseSSHConfig(map[string]any{}) + if err != nil { + t.Fatalf("ParseSSHConfig() error = %v", err) + } + if cfg.User != "" || cfg.Port != 0 { + t.Errorf("expected zero SSHConfig, got %+v", cfg) + } +} + +func TestParseSSHConfig_invalidType(t *testing.T) { + params := map[string]any{ + "ssh": map[string]any{ + "user": 12345, + }, + } + + _, err := ParseSSHConfig(params) + if err == nil { + t.Fatal("ParseSSHConfig() expected error for invalid user type") + } +} + +func TestResolveSSHUser(t *testing.T) { + cases := []struct { + name string + host HostConfig + ssh SSHConfig + wantUser string + }{ + {"host override", HostConfig{User: "admin"}, SSHConfig{User: "default"}, "admin"}, + {"ssh default", HostConfig{}, SSHConfig{User: "default"}, "default"}, + {"fallback root", HostConfig{}, SSHConfig{}, "root"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ResolveSSHUser(tc.host, tc.ssh); got != tc.wantUser { + t.Errorf("ResolveSSHUser() = %q, want %q", got, tc.wantUser) + } + }) + } +} + +func TestResolveSSHPort(t *testing.T) { + cases := []struct { + name string + host HostConfig + ssh SSHConfig + wantPort int + }{ + {"host override", HostConfig{Port: 2222}, SSHConfig{Port: 3333}, 2222}, + {"ssh default", HostConfig{}, SSHConfig{Port: 3333}, 3333}, + {"fallback 22", HostConfig{}, SSHConfig{}, 22}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ResolveSSHPort(tc.host, tc.ssh); got != tc.wantPort { + t.Errorf("ResolveSSHPort() = %d, want %d", got, tc.wantPort) + } + }) + } +} + +func TestParseRuntimeConfig_kvmEnabled(t *testing.T) { + params := map[string]any{ + "runtime": map[string]any{ + "kvm": true, + }, + } + + kvm, devices := ParseRuntimeConfig(params) + if !kvm { + t.Error("kvm = false, want true") + } + if len(devices) != 0 { + t.Errorf("devices = %v, want empty", devices) + } +} + +func TestParseRuntimeConfig_withDevices(t *testing.T) { + params := map[string]any{ + "runtime": map[string]any{ + "kvm": true, + "devices": []any{"/dev/vhost-net", "/dev/net/tun"}, + }, + } + + kvm, devices := ParseRuntimeConfig(params) + if !kvm { + t.Error("kvm = false, want true") + } + if len(devices) != 2 { + t.Fatalf("devices = %v, want 2 entries", devices) + } + if devices[0] != "/dev/vhost-net" || devices[1] != "/dev/net/tun" { + t.Errorf("devices = %v", devices) + } +} + +func TestParseRuntimeConfig_missing(t *testing.T) { + kvm, devices := ParseRuntimeConfig(map[string]any{}) + if kvm { + t.Error("kvm = true, want false") + } + if devices != nil { + t.Errorf("devices = %v, want nil", devices) + } +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/qemu_ssh.go b/controller/internal/exporterset/provisioners/qemu-ssh/qemu_ssh.go new file mode 100644 index 000000000..017fe23b9 --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/qemu_ssh.go @@ -0,0 +1,585 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "path/filepath" + "strings" + + jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + corev1 "k8s.io/api/core/v1" + sigsyaml "sigs.k8s.io/yaml" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // ProvisionerName is the provisioner identifier for + // off-cluster QEMU targets deployed via SSH. + ProvisionerName = "qemu-ssh.jumpstarter.dev" + + // DefaultExporterImage is the exporter container image. + DefaultExporterImage = "quay.io/jumpstarter-dev/jumpstarter:latest" + + // DefaultQEMURuntimeImage is the QEMU runtime container image. + DefaultQEMURuntimeImage = "quay.io/jumpstarter-dev/virtual/qemu-runtime:latest" + + // sshPrivateKeyField is the Secret data key for SSH private + // keys (standard kubernetes.io/ssh-auth type). + sshPrivateKeyField = "ssh-privatekey" +) + +// Provisioner implements the qemu-ssh.jumpstarter.dev provisioner +// for off-cluster QEMU targets deployed via SSH to remote lab hosts. +// +// It implements both the exporterset.Provisioner interface (Name, +// RenderPod, EnrichExporterExport, Cleanup) and the +// exporterset.Deployer interface (Deploy, IsDeployed). +type Provisioner struct { + Version string + Client client.Client +} + +// New creates a new qemu-ssh provisioner. +func New(version string, c client.Client) *Provisioner { + return &Provisioner{Version: version, Client: c} +} + +// Name returns the provisioner identifier. +func (p *Provisioner) Name() string { + return ProvisionerName +} + +// RenderPod returns nil — off-cluster provisioners don't create Pods. +// The reconciler detects the Deployer interface and calls Deploy +// instead. +func (p *Provisioner) RenderPod( + _ context.Context, + _ *virtualtargetv1alpha1.ExporterSet, + _ *virtualtargetv1alpha1.VirtualTargetClass, + _ map[string]any, + _ *virtualtargetv1alpha1.ImageOverrides, + _ *jumpstarterdevv1alpha1.Exporter, +) (*corev1.Pod, error) { + return nil, nil +} + +// EnrichExporterExport adjusts driver config for off-cluster +// deployment (launcher_socket, defaults, firmware paths, hostfwd). +func (p *Provisioner) EnrichExporterExport( + drivers []virtualtargetv1alpha1.DriverConfig, + mergedParameters map[string]any, +) ([]virtualtargetv1alpha1.DriverConfig, error) { + return enrichExporterExport(drivers, mergedParameters) +} + +// Deploy sets up the exporter on a remote host via SSH: +// 1. Read SSH key from credentialsSecretRef +// 2. Select a host with free capacity +// 3. Connect via SSH +// 4. Write exporter config, quadlet files +// 5. Create shared volume, reload systemd, start containers +// 6. Annotate the Exporter CR with the host assignment +func (p *Provisioner) Deploy( + ctx context.Context, + es *virtualtargetv1alpha1.ExporterSet, + vtc *virtualtargetv1alpha1.VirtualTargetClass, + mergedParameters map[string]any, + images *virtualtargetv1alpha1.ImageOverrides, + exporter *jumpstarterdevv1alpha1.Exporter, + caBundle string, +) error { + logger := log.FromContext(ctx) + + privateKey, err := p.readSSHKey(ctx, vtc) + if err != nil { + return err + } + + host, err := ParseHost(mergedParameters) + if err != nil { + return fmt.Errorf("parse host: %w", err) + } + + sshCfg, err := ParseSSHConfig(mergedParameters) + if err != nil { + return fmt.Errorf("parse ssh config: %w", err) + } + + logger.Info("deploying exporter to host", + "exporter", exporter.Name, + "host", host.Name, + ) + + conn, err := Connect(SSHConnectConfig{ + Host: host.Name, + Port: ResolveSSHPort(host, sshCfg), + User: ResolveSSHUser(host, sshCfg), + PrivateKey: privateKey, + }) + if err != nil { + return fmt.Errorf("SSH connect to %s: %w", host.Name, err) + } + defer conn.Close() //nolint:errcheck + + if err := p.deployInstance(ctx, conn, es, mergedParameters, images, exporter, caBundle); err != nil { + return err + } + + if err := p.annotateHost(ctx, exporter, host.Name); err != nil { + return fmt.Errorf("annotate exporter %s with host: %w", exporter.Name, err) + } + + return nil +} + +// IsDeployed checks whether the exporter has a host assignment +// annotation (meaning Deploy was previously called successfully). +func (p *Provisioner) IsDeployed( + _ context.Context, + exporter *jumpstarterdevv1alpha1.Exporter, +) (bool, error) { + if exporter.Annotations == nil { + return false, nil + } + _, ok := exporter.Annotations[AnnotationHost] + return ok, nil +} + +// Cleanup tears down the remote containers via SSH and removes the +// host annotation. +func (p *Provisioner) Cleanup( + ctx context.Context, + es *virtualtargetv1alpha1.ExporterSet, + exporter *jumpstarterdevv1alpha1.Exporter, +) error { + logger := log.FromContext(ctx) + + hostName := "" + if exporter.Annotations != nil { + hostName = exporter.Annotations[AnnotationHost] + } + if hostName == "" { + logger.V(1).Info("no host annotation on exporter, skipping cleanup", + "exporter", exporter.Name) + return nil + } + + vtcKey := client.ObjectKey{ + Namespace: es.Namespace, + Name: es.Spec.VirtualTargetClassName, + } + var vtc virtualtargetv1alpha1.VirtualTargetClass + if err := p.Client.Get(ctx, vtcKey, &vtc); err != nil { + return fmt.Errorf("get VirtualTargetClass for cleanup: %w", err) + } + + privateKey, err := p.readSSHKey(ctx, &vtc) + if err != nil { + return fmt.Errorf("read SSH key for cleanup: %w", err) + } + + mergedParams := map[string]any{} + if vtc.Spec.Parameters != nil && vtc.Spec.Parameters.Raw != nil { + _ = sigsyaml.Unmarshal(vtc.Spec.Parameters.Raw, &mergedParams) + } + sshCfg, _ := ParseSSHConfig(mergedParams) + host, _ := ParseHost(mergedParams) + + port := ResolveSSHPort(host, sshCfg) + user := ResolveSSHUser(host, sshCfg) + + conn, err := Connect(SSHConnectConfig{ + Host: hostName, + Port: port, + User: user, + PrivateKey: privateKey, + }) + if err != nil { + logger.Error(err, "SSH connect for cleanup failed", + "exporter", exporter.Name, "host", hostName) + return nil + } + defer conn.Close() //nolint:errcheck + + p.teardownInstance(ctx, conn, exporter.Name) + + logger.Info("cleaned up remote exporter", + "exporter", exporter.Name, "host", hostName) + + return nil +} + +// deployInstance performs the actual SSH operations to set up the +// exporter on the remote host. +func (p *Provisioner) deployInstance( + ctx context.Context, + conn RemoteHost, + es *virtualtargetv1alpha1.ExporterSet, + mergedParameters map[string]any, + images *virtualtargetv1alpha1.ImageOverrides, + exporter *jumpstarterdevv1alpha1.Exporter, + caBundle string, +) error { + logger := log.FromContext(ctx) + name := exporter.Name + + if err := conn.MkdirAll(ctx, ExporterConfigDir); err != nil { + return fmt.Errorf("mkdir %s: %w", ExporterConfigDir, err) + } + if err := conn.MkdirAll(ctx, QuadletDir); err != nil { + return fmt.Errorf("mkdir %s: %w", QuadletDir, err) + } + + exporterConfigContent, err := p.buildExporterConfig(ctx, es, exporter, caBundle, mergedParameters) + if err != nil { + return fmt.Errorf("build exporter config: %w", err) + } + + configPath := filepath.Join(ExporterConfigDir, name+".yaml") + changed, diff, err := conn.ReconcileFile(ctx, configPath, exporterConfigContent) + if err != nil { + return fmt.Errorf("reconcile exporter config: %w", err) + } + if changed { + logger.Info("exporter config written", "path", configPath, "diff", diff) + } + + kvm, extraDevices := ParseRuntimeConfig(mergedParameters) + exporterImage, runtimeImage := p.resolveImages(images) + + quadletCfg := QuadletConfig{ + Name: name, + Namespace: es.Namespace, + ExporterImage: exporterImage, + RuntimeImage: runtimeImage, + KVM: kvm, + ExtraDevices: extraDevices, + } + + runtimeQuadlet, err := RuntimeContainerFile(quadletCfg) + if err != nil { + return fmt.Errorf("generate runtime quadlet: %w", err) + } + runtimePath := filepath.Join(QuadletDir, RuntimeContainerFileName(name)) + changed, diff, err = conn.ReconcileFile(ctx, runtimePath, runtimeQuadlet) + if err != nil { + return fmt.Errorf("reconcile runtime quadlet: %w", err) + } + if changed { + logger.Info("runtime quadlet written", "path", runtimePath, "diff", diff) + } + + exporterQuadlet, err := ExporterContainerFile(quadletCfg) + if err != nil { + return fmt.Errorf("generate exporter quadlet: %w", err) + } + exporterPath := filepath.Join(QuadletDir, ExporterContainerFileName(name)) + changed, diff, err = conn.ReconcileFile(ctx, exporterPath, exporterQuadlet) + if err != nil { + return fmt.Errorf("reconcile exporter quadlet: %w", err) + } + if changed { + logger.Info("exporter quadlet written", "path", exporterPath, "diff", diff) + } + + volumeName := PodmanVolumeName(name) + if res, err := conn.RunCommand(ctx, + fmt.Sprintf("podman volume inspect %s >/dev/null 2>&1 || podman volume create %s", + volumeName, volumeName)); err != nil { + return fmt.Errorf("create shared volume: %w", err) + } else if res.ExitCode != 0 { + return fmt.Errorf("create shared volume: exit %d: %s", res.ExitCode, res.Stderr) + } + + if res, err := conn.RunCommand(ctx, "systemctl daemon-reload"); err != nil { + return fmt.Errorf("systemctl daemon-reload: %w", err) + } else if res.ExitCode != 0 { + return fmt.Errorf("systemctl daemon-reload: exit %d: %s", res.ExitCode, res.Stderr) + } + + runtimeSvc := RuntimeServiceName(name) + exporterSvc := ExporterServiceName(name) + + if res, err := conn.RunCommand(ctx, + fmt.Sprintf("systemctl enable --now %s %s", runtimeSvc, exporterSvc)); err != nil { + return fmt.Errorf("start services: %w", err) + } else if res.ExitCode != 0 { + return fmt.Errorf("start services %s %s: exit %d: %s", + runtimeSvc, exporterSvc, res.ExitCode, res.Stderr) + } + + return nil +} + +// teardownInstance stops and removes all remote resources for an +// exporter instance. +func (p *Provisioner) teardownInstance( + ctx context.Context, + conn RemoteHost, + name string, +) { + logger := log.FromContext(ctx) + + runtimeSvc := RuntimeServiceName(name) + exporterSvc := ExporterServiceName(name) + + if _, err := conn.RunCommand(ctx, + fmt.Sprintf("systemctl disable --now %s %s 2>/dev/null || true", + exporterSvc, runtimeSvc)); err != nil { + logger.Error(err, "failed to stop services", "exporter", name) + } + + for _, path := range []string{ + filepath.Join(QuadletDir, ExporterContainerFileName(name)), + filepath.Join(QuadletDir, RuntimeContainerFileName(name)), + filepath.Join(ExporterConfigDir, name+".yaml"), + } { + if err := conn.RemoveFile(ctx, path); err != nil { + logger.Error(err, "failed to remove file", "path", path) + } + } + + volumeName := PodmanVolumeName(name) + if _, err := conn.RunCommand(ctx, + fmt.Sprintf("podman volume rm %s 2>/dev/null || true", volumeName)); err != nil { + logger.Error(err, "failed to remove volume", "volume", volumeName) + } + + if _, err := conn.RunCommand(ctx, "systemctl daemon-reload"); err != nil { + logger.Error(err, "failed to reload systemd after cleanup") + } +} + +// readSSHKey reads the SSH private key from the VTC's +// credentialsSecretRef. +func (p *Provisioner) readSSHKey( + ctx context.Context, + vtc *virtualtargetv1alpha1.VirtualTargetClass, +) ([]byte, error) { + if vtc.Spec.CredentialsSecretRef == nil { + return nil, fmt.Errorf("VirtualTargetClass %s/%s has no credentialsSecretRef (required for SSH)", + vtc.Namespace, vtc.Name) + } + + var secret corev1.Secret + if err := p.Client.Get(ctx, client.ObjectKey{ + Name: vtc.Spec.CredentialsSecretRef.Name, + Namespace: vtc.Namespace, + }, &secret); err != nil { + return nil, fmt.Errorf("get SSH credentials Secret %q: %w", + vtc.Spec.CredentialsSecretRef.Name, err) + } + + key, ok := secret.Data[sshPrivateKeyField] + if !ok { + return nil, fmt.Errorf("credentials Secret %q missing %q key", + vtc.Spec.CredentialsSecretRef.Name, sshPrivateKeyField) + } + + return key, nil +} + +// annotateHost sets the host assignment annotation on the Exporter CR. +func (p *Provisioner) annotateHost( + ctx context.Context, + exporter *jumpstarterdevv1alpha1.Exporter, + hostName string, +) error { + if exporter.Annotations == nil { + exporter.Annotations = make(map[string]string) + } + exporter.Annotations[AnnotationHost] = hostName + return p.Client.Update(ctx, exporter) +} + +// resolveImages returns the exporter and runtime images, applying +// overrides and version resolution. +func (p *Provisioner) resolveImages( + images *virtualtargetv1alpha1.ImageOverrides, +) (exporterImage, runtimeImage string) { + exporterImage = resolveImage(p.Version, DefaultExporterImage) + runtimeImage = resolveImage(p.Version, DefaultQEMURuntimeImage) + + if images != nil { + if images.Exporter != nil && images.Exporter.Image != "" { + exporterImage = images.Exporter.Image + } + if images.Runtime != nil && images.Runtime.Image != "" { + runtimeImage = images.Runtime.Image + } + } + + return exporterImage, runtimeImage +} + +// resolveImage replaces :latest with the controller version tag. +func resolveImage(version, image string) string { + if version == "" || version == "dev" || strings.Contains(version, "-g") { + return image + } + v := strings.TrimPrefix(version, "v") + if base, ok := strings.CutSuffix(image, ":latest"); ok { + return base + ":" + v + } + return image +} + +// buildExporterConfig generates the ExporterConfig YAML that will be +// written to the remote host. +func (p *Provisioner) buildExporterConfig( + ctx context.Context, + es *virtualtargetv1alpha1.ExporterSet, + exporter *jumpstarterdevv1alpha1.Exporter, + caBundle string, + mergedParameters map[string]any, +) (string, error) { + token, err := p.readCredentialToken(ctx, exporter) + if err != nil { + return "", err + } + + caBase64 := base64.StdEncoding.EncodeToString([]byte(caBundle)) + + drivers := es.Spec.Template.Spec.Drivers + drivers, err = enrichExporterExport(drivers, mergedParameters) + if err != nil { + return "", fmt.Errorf("enrich drivers: %w", err) + } + + exportMap, err := buildExportMap(drivers) + if err != nil { + return "", fmt.Errorf("build export map: %w", err) + } + + exitOnLeaseEnd := es.Spec.RecycleStrategy != virtualtargetv1alpha1.RecycleStrategyInPlaceReuse + + cfg := exporterConfig{ + APIVersion: "jumpstarter.dev/v1alpha1", + Kind: "ExporterConfig", + Metadata: exporterConfigMetadata{ + Name: exporter.Name, + Namespace: exporter.Namespace, + }, + Endpoint: exporter.Status.Endpoint, + TLS: &exporterConfigTLS{ + CA: caBase64, + }, + Token: token, + Export: exportMap, + ExitOnLeaseEnd: exitOnLeaseEnd, + } + + cfgYAML, err := sigsyaml.Marshal(cfg) + if err != nil { + return "", fmt.Errorf("marshal ExporterConfig: %w", err) + } + + return string(cfgYAML), nil +} + +// readCredentialToken reads the JWT from the Exporter's credential +// Secret. +func (p *Provisioner) readCredentialToken( + ctx context.Context, + exporter *jumpstarterdevv1alpha1.Exporter, +) (string, error) { + var secret corev1.Secret + if err := p.Client.Get(ctx, client.ObjectKey{ + Name: exporter.Status.Credential.Name, + Namespace: exporter.Namespace, + }, &secret); err != nil { + return "", fmt.Errorf("get credential Secret %q: %w", + exporter.Status.Credential.Name, err) + } + + token, ok := secret.Data["token"] + if !ok { + return "", fmt.Errorf("credential Secret %q missing 'token' key", + exporter.Status.Credential.Name) + } + + return string(token), nil +} + +// --- ExporterConfig types (mirrors exporterconfig.go in parent package) --- + +type exporterConfig struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata exporterConfigMetadata `json:"metadata"` + Endpoint string `json:"endpoint"` + TLS *exporterConfigTLS `json:"tls,omitempty"` + Token string `json:"token"` + Export map[string]exporterConfigDriver `json:"export,omitempty"` + ExitOnLeaseEnd bool `json:"exitOnLeaseEnd"` +} + +type exporterConfigMetadata struct { + Name string `json:"name"` + Namespace string `json:"namespace"` +} + +type exporterConfigTLS struct { + CA string `json:"ca"` +} + +type exporterConfigDriver struct { + Type string `json:"type,omitempty"` + Ref string `json:"ref,omitempty"` + Config any `json:"config,omitempty"` + Children map[string]exporterConfigDriver `json:"children,omitempty"` +} + +// buildExportMap converts DriverConfigs to the export map (mirrors +// the parent package's buildExportMap). +func buildExportMap( + drivers []virtualtargetv1alpha1.DriverConfig, +) (map[string]exporterConfigDriver, error) { + exportMap := make(map[string]exporterConfigDriver, len(drivers)) + + for _, d := range drivers { + if _, exists := exportMap[d.Name]; exists { + return nil, fmt.Errorf("duplicate driver key %q", d.Name) + } + + if d.Ref != "" { + exportMap[d.Name] = exporterConfigDriver{Ref: d.Ref} + continue + } + + var config any + if d.Config != nil && d.Config.Raw != nil { + if err := json.Unmarshal(d.Config.Raw, &config); err != nil { + return nil, fmt.Errorf("unmarshal config for driver %q: %w", d.Name, err) + } + } + + exportMap[d.Name] = exporterConfigDriver{ + Type: d.Type, + Config: config, + } + } + + return exportMap, nil +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/qemu_ssh_test.go b/controller/internal/exporterset/provisioners/qemu-ssh/qemu_ssh_test.go new file mode 100644 index 000000000..aff60eb34 --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/qemu_ssh_test.go @@ -0,0 +1,220 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "context" + "testing" + + jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestProvisionerName(t *testing.T) { + p := New("v1.0.0", nil) + if got := p.Name(); got != ProvisionerName { + t.Errorf("Name() = %q, want %q", got, ProvisionerName) + } +} + +func TestRenderPod_returnsNil(t *testing.T) { + p := New("v1.0.0", nil) + pod, err := p.RenderPod(context.Background(), nil, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if pod != nil { + t.Errorf("RenderPod() should return nil for off-cluster provisioner") + } +} + +func TestIsDeployed_noAnnotation(t *testing.T) { + p := New("v1.0.0", nil) + exporter := &jumpstarterdevv1alpha1.Exporter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-exp"}, + } + deployed, err := p.IsDeployed(context.Background(), exporter) + if err != nil { + t.Fatal(err) + } + if deployed { + t.Error("IsDeployed should be false without annotation") + } +} + +func TestIsDeployed_withAnnotation(t *testing.T) { + p := New("v1.0.0", nil) + exporter := &jumpstarterdevv1alpha1.Exporter{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-exp", + Annotations: map[string]string{ + AnnotationHost: "lab-host-1.example.com", + }, + }, + } + deployed, err := p.IsDeployed(context.Background(), exporter) + if err != nil { + t.Fatal(err) + } + if !deployed { + t.Error("IsDeployed should be true with host annotation") + } +} + +func TestResolveImage_latest(t *testing.T) { + cases := []struct { + name string + version string + image string + want string + }{ + { + name: "version replaces latest", + version: "v1.2.3", + image: "quay.io/jumpstarter-dev/jumpstarter:latest", + want: "quay.io/jumpstarter-dev/jumpstarter:1.2.3", + }, + { + name: "dev version keeps latest", + version: "dev", + image: "quay.io/jumpstarter-dev/jumpstarter:latest", + want: "quay.io/jumpstarter-dev/jumpstarter:latest", + }, + { + name: "empty version keeps latest", + version: "", + image: "quay.io/jumpstarter-dev/jumpstarter:latest", + want: "quay.io/jumpstarter-dev/jumpstarter:latest", + }, + { + name: "git describe version keeps latest", + version: "1.2.3-4-gabcdef", + image: "quay.io/jumpstarter-dev/jumpstarter:latest", + want: "quay.io/jumpstarter-dev/jumpstarter:latest", + }, + { + name: "non-latest tag preserved", + version: "v1.2.3", + image: "quay.io/jumpstarter-dev/jumpstarter:custom", + want: "quay.io/jumpstarter-dev/jumpstarter:custom", + }, + { + name: "version without v prefix", + version: "1.2.3", + image: "quay.io/jumpstarter-dev/jumpstarter:latest", + want: "quay.io/jumpstarter-dev/jumpstarter:1.2.3", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := resolveImage(tc.version, tc.image) + if got != tc.want { + t.Errorf("resolveImage(%q, %q) = %q, want %q", + tc.version, tc.image, got, tc.want) + } + }) + } +} + +func TestResolveImages_overrides(t *testing.T) { + p := New("v1.0.0", nil) + + t.Run("nil images uses defaults", func(t *testing.T) { + exp, rt := p.resolveImages(nil) + if exp != "quay.io/jumpstarter-dev/jumpstarter:1.0.0" { + t.Errorf("exporter image = %q", exp) + } + if rt != "quay.io/jumpstarter-dev/virtual/qemu-runtime:1.0.0" { + t.Errorf("runtime image = %q", rt) + } + }) + + t.Run("exporter override", func(t *testing.T) { + exp, rt := p.resolveImages(&virtualtargetv1alpha1.ImageOverrides{ + Exporter: &virtualtargetv1alpha1.ImageSpec{ + Image: "custom-exporter:v2", + }, + }) + if exp != "custom-exporter:v2" { + t.Errorf("exporter image = %q", exp) + } + if rt != "quay.io/jumpstarter-dev/virtual/qemu-runtime:1.0.0" { + t.Errorf("runtime image = %q", rt) + } + }) + + t.Run("runtime override", func(t *testing.T) { + exp, rt := p.resolveImages(&virtualtargetv1alpha1.ImageOverrides{ + Runtime: &virtualtargetv1alpha1.ImageSpec{ + Image: "custom-runtime:v3", + }, + }) + if exp != "quay.io/jumpstarter-dev/jumpstarter:1.0.0" { + t.Errorf("exporter image = %q", exp) + } + if rt != "custom-runtime:v3" { + t.Errorf("runtime image = %q", rt) + } + }) +} + +func TestBuildExportMap_basic(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + { + Name: "qemu", + Type: qemuDriverType, + Config: mustJSON(map[string]any{"arch": "x86_64"}), + }, + { + Name: "ssh", + Ref: "qemu.ssh", + }, + } + + exportMap, err := buildExportMap(drivers) + if err != nil { + t.Fatal(err) + } + + if len(exportMap) != 2 { + t.Fatalf("export map len = %d, want 2", len(exportMap)) + } + + qemu := exportMap["qemu"] + if qemu.Type != qemuDriverType { + t.Errorf("qemu.type = %q", qemu.Type) + } + + sshDriver := exportMap["ssh"] + if sshDriver.Ref != "qemu.ssh" { + t.Errorf("ssh.ref = %q", sshDriver.Ref) + } +} + +func TestBuildExportMap_duplicateKey(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + {Name: "qemu", Type: qemuDriverType}, + {Name: "qemu", Type: tcpDriverType}, + } + + _, err := buildExportMap(drivers) + if err == nil { + t.Fatal("expected error for duplicate key") + } +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/quadlet.go b/controller/internal/exporterset/provisioners/qemu-ssh/quadlet.go new file mode 100644 index 000000000..c799d398d --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/quadlet.go @@ -0,0 +1,218 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "fmt" + "strings" +) + +const ( + // QuadletDir is the systemd directory for Podman quadlet + // .container files. + QuadletDir = "/etc/containers/systemd" + + // ExporterConfigDir is where exporter config YAML files are + // placed on remote hosts. + ExporterConfigDir = "/etc/jumpstarter/exporters" + + // sharedVolumeSuffix is appended to the Podman volume name. + sharedVolumeSuffix = "-shared" + + // sharedMountPath is the container mount for the shared volume + // (Unix sockets: QMP, serial, launcher). + sharedMountPath = "/shared" + + // launcherSocketPath is the Unix socket used by jumpstarter-exec. + launcherSocketPath = "/shared/launcher.sock" +) + +// QuadletConfig holds the parameters needed to generate quadlet +// .container files for a single exporter instance. +type QuadletConfig struct { + // Name is the exporter instance name (used in container and + // service names). + Name string + + // Namespace is the Kubernetes namespace (for log context). + Namespace string + + // ExporterImage is the exporter container image. + ExporterImage string + + // RuntimeImage is the QEMU runtime container image. + RuntimeImage string + + // KVM enables /dev/kvm device passthrough to the runtime + // container. + KVM bool + + // ExtraDevices lists additional host devices to pass through to + // the runtime container (e.g. /dev/vhost-net). + ExtraDevices []string +} + +// validateQuadletValue rejects values that contain newlines, +// carriage returns, or NUL bytes. Such values could inject arbitrary +// directives into root-owned systemd unit files. +func validateQuadletValue(field, value string) error { + if strings.ContainsAny(value, "\r\n\x00") { + return fmt.Errorf("quadlet field %s contains forbidden characters: %q", field, value) + } + if strings.HasSuffix(value, `\`) { + return fmt.Errorf("quadlet field %s has trailing backslash (systemd line continuation): %q", field, value) + } + return nil +} + +// validateQuadletConfig checks all values that will be interpolated +// into the generated quadlet files. +func validateQuadletConfig(cfg QuadletConfig) error { + for field, val := range map[string]string{ + "RuntimeImage": cfg.RuntimeImage, + "ExporterImage": cfg.ExporterImage, + "Name": cfg.Name, + "Namespace": cfg.Namespace, + } { + if err := validateQuadletValue(field, val); err != nil { + return err + } + } + for i, dev := range cfg.ExtraDevices { + if err := validateQuadletValue(fmt.Sprintf("ExtraDevices[%d]", i), dev); err != nil { + return err + } + if !strings.HasPrefix(dev, "/dev/") { + return fmt.Errorf("ExtraDevices[%d] must be a /dev/ path, got %q", i, dev) + } + } + return nil +} + +// RuntimeContainerFile generates the Podman quadlet .container file +// for the QEMU runtime sidecar. +func RuntimeContainerFile(cfg QuadletConfig) (string, error) { + if err := validateQuadletConfig(cfg); err != nil { + return "", err + } + + volumeName := podmanVolumeName(cfg.Name) + + var b strings.Builder + + b.WriteString("[Unit]\n") + fmt.Fprintf(&b, "Description=Jumpstarter QEMU Runtime for %s\n", cfg.Name) + b.WriteString("\n") + + b.WriteString("[Container]\n") + fmt.Fprintf(&b, "ContainerName=%s-runtime\n", cfg.Name) + fmt.Fprintf(&b, "Image=%s\n", cfg.RuntimeImage) + fmt.Fprintf(&b, "Volume=%s:%s:z\n", volumeName, sharedMountPath) + fmt.Fprintf(&b, + "Environment=JUMPSTARTER_EXEC_LOG_FIELDS=component=exporter,exporter=%s,namespace=%s\n", + cfg.Name, cfg.Namespace, + ) + + if cfg.KVM { + b.WriteString("AddDevice=/dev/kvm\n") + } + for _, dev := range cfg.ExtraDevices { + fmt.Fprintf(&b, "AddDevice=%s\n", dev) + } + + b.WriteString("\n") + + b.WriteString("[Service]\n") + b.WriteString("Restart=always\n") + b.WriteString("\n") + + b.WriteString("[Install]\n") + b.WriteString("WantedBy=default.target\n") + + return b.String(), nil +} + +// ExporterContainerFile generates the Podman quadlet .container file +// for the Jumpstarter exporter. +func ExporterContainerFile(cfg QuadletConfig) (string, error) { + if err := validateQuadletConfig(cfg); err != nil { + return "", err + } + volumeName := podmanVolumeName(cfg.Name) + runtimeService := cfg.Name + "-runtime" + configFile := ExporterConfigDir + "/" + cfg.Name + ".yaml" + + var b strings.Builder + + b.WriteString("[Unit]\n") + fmt.Fprintf(&b, "Description=Jumpstarter Exporter for %s\n", cfg.Name) + fmt.Fprintf(&b, "Requires=%s.service\n", runtimeService) + fmt.Fprintf(&b, "After=%s.service\n", runtimeService) + b.WriteString("\n") + + b.WriteString("[Container]\n") + fmt.Fprintf(&b, "ContainerName=%s-exporter\n", cfg.Name) + fmt.Fprintf(&b, "Image=%s\n", cfg.ExporterImage) + fmt.Fprintf(&b, "Volume=%s:%s:z\n", volumeName, sharedMountPath) + fmt.Fprintf(&b, "Volume=%s:%s:ro\n", ExporterConfigDir, ExporterConfigDir) + fmt.Fprintf(&b, "Environment=JUMPSTARTER_LAUNCHER_SOCKET=%s\n", launcherSocketPath) + fmt.Fprintf(&b, "Exec=jmp run --exporter-config %s\n", configFile) + b.WriteString("\n") + + b.WriteString("[Service]\n") + b.WriteString("Restart=on-failure\n") + b.WriteString("\n") + + b.WriteString("[Install]\n") + b.WriteString("WantedBy=default.target\n") + + return b.String(), nil +} + +// RuntimeContainerFileName returns the quadlet filename for the +// runtime container. +func RuntimeContainerFileName(name string) string { + return name + "-runtime.container" +} + +// ExporterContainerFileName returns the quadlet filename for the +// exporter container. +func ExporterContainerFileName(name string) string { + return name + "-exporter.container" +} + +// RuntimeServiceName returns the systemd service name for the +// runtime container. +func RuntimeServiceName(name string) string { + return name + "-runtime" +} + +// ExporterServiceName returns the systemd service name for the +// exporter container. +func ExporterServiceName(name string) string { + return name + "-exporter" +} + +// PodmanVolumeName returns the Podman volume name for shared +// communication between exporter and runtime containers. +func PodmanVolumeName(name string) string { + return podmanVolumeName(name) +} + +func podmanVolumeName(name string) string { + return "jumpstarter-" + name + sharedVolumeSuffix +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/quadlet_test.go b/controller/internal/exporterset/provisioners/qemu-ssh/quadlet_test.go new file mode 100644 index 000000000..df4d40e67 --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/quadlet_test.go @@ -0,0 +1,177 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "strings" + "testing" +) + +func baseConfig() QuadletConfig { + return QuadletConfig{ + Name: "rpi4-virtual-abc12", + Namespace: "jumpstarter", + ExporterImage: "quay.io/jumpstarter-dev/jumpstarter:latest", + RuntimeImage: "quay.io/jumpstarter-dev/virtual/qemu-runtime:latest", + } +} + +func TestRuntimeContainerFile_basic(t *testing.T) { + cfg := baseConfig() + got, err := RuntimeContainerFile(cfg) + if err != nil { + t.Fatal(err) + } + + mustContain(t, got, "[Unit]") + mustContain(t, got, "Description=Jumpstarter QEMU Runtime for rpi4-virtual-abc12") + mustContain(t, got, "[Container]") + mustContain(t, got, "ContainerName=rpi4-virtual-abc12-runtime") + mustContain(t, got, "Image=quay.io/jumpstarter-dev/virtual/qemu-runtime:latest") + mustContain(t, got, "Volume=jumpstarter-rpi4-virtual-abc12-shared:/shared:z") + mustContain(t, got, "JUMPSTARTER_EXEC_LOG_FIELDS=component=exporter,exporter=rpi4-virtual-abc12,namespace=jumpstarter") + mustContain(t, got, "[Service]") + mustContain(t, got, "Restart=always") + mustContain(t, got, "[Install]") + mustContain(t, got, "WantedBy=default.target") + mustNotContain(t, got, "AddDevice") +} + +func TestRuntimeContainerFile_withKVM(t *testing.T) { + cfg := baseConfig() + cfg.KVM = true + got, err := RuntimeContainerFile(cfg) + if err != nil { + t.Fatal(err) + } + + mustContain(t, got, "AddDevice=/dev/kvm") +} + +func TestRuntimeContainerFile_withExtraDevices(t *testing.T) { + cfg := baseConfig() + cfg.ExtraDevices = []string{"/dev/vhost-net", "/dev/net/tun"} + got, err := RuntimeContainerFile(cfg) + if err != nil { + t.Fatal(err) + } + + mustContain(t, got, "AddDevice=/dev/vhost-net") + mustContain(t, got, "AddDevice=/dev/net/tun") +} + +func TestRuntimeContainerFile_rejectsNewlineInDevice(t *testing.T) { + cfg := baseConfig() + cfg.ExtraDevices = []string{"/dev/kvm\nPrivileged=true"} + _, err := RuntimeContainerFile(cfg) + if err == nil { + t.Fatal("expected error for device path with newline injection") + } +} + +func TestRuntimeContainerFile_rejectsTrailingBackslash(t *testing.T) { + cfg := baseConfig() + cfg.ExtraDevices = []string{`/dev/kvm\`} + _, err := RuntimeContainerFile(cfg) + if err == nil { + t.Fatal("expected error for device path with trailing backslash") + } +} + +func TestExporterContainerFile_basic(t *testing.T) { + cfg := baseConfig() + got, err := ExporterContainerFile(cfg) + if err != nil { + t.Fatal(err) + } + + mustContain(t, got, "[Unit]") + mustContain(t, got, "Description=Jumpstarter Exporter for rpi4-virtual-abc12") + mustContain(t, got, "Requires=rpi4-virtual-abc12-runtime.service") + mustContain(t, got, "After=rpi4-virtual-abc12-runtime.service") + mustContain(t, got, "[Container]") + mustContain(t, got, "ContainerName=rpi4-virtual-abc12-exporter") + mustContain(t, got, "Image=quay.io/jumpstarter-dev/jumpstarter:latest") + mustContain(t, got, "Volume=jumpstarter-rpi4-virtual-abc12-shared:/shared:z") + mustContain(t, got, "Volume=/etc/jumpstarter/exporters:/etc/jumpstarter/exporters:ro") + mustContain(t, got, "Environment=JUMPSTARTER_LAUNCHER_SOCKET=/shared/launcher.sock") + mustContain(t, got, "Exec=jmp run --exporter-config /etc/jumpstarter/exporters/rpi4-virtual-abc12.yaml") + mustContain(t, got, "[Service]") + mustContain(t, got, "Restart=on-failure") + mustContain(t, got, "[Install]") + mustContain(t, got, "WantedBy=default.target") +} + +func TestQuadletConfig_rejectsNewlines(t *testing.T) { + cfg := baseConfig() + cfg.RuntimeImage = "image:latest\nExec=evil" + _, err := RuntimeContainerFile(cfg) + if err == nil { + t.Fatal("expected error for newline in RuntimeImage") + } +} + +func TestQuadletConfig_rejectsInvalidDevice(t *testing.T) { + cfg := baseConfig() + cfg.ExtraDevices = []string{"/tmp/not-a-device"} + _, err := RuntimeContainerFile(cfg) + if err == nil { + t.Fatal("expected error for non-/dev/ device path") + } +} + +func TestFileNames(t *testing.T) { + name := "demo-set-xyz" + + if got := RuntimeContainerFileName(name); got != "demo-set-xyz-runtime.container" { + t.Errorf("RuntimeContainerFileName = %q", got) + } + if got := ExporterContainerFileName(name); got != "demo-set-xyz-exporter.container" { + t.Errorf("ExporterContainerFileName = %q", got) + } +} + +func TestServiceNames(t *testing.T) { + name := "demo-set-xyz" + + if got := RuntimeServiceName(name); got != "demo-set-xyz-runtime" { + t.Errorf("RuntimeServiceName = %q", got) + } + if got := ExporterServiceName(name); got != "demo-set-xyz-exporter" { + t.Errorf("ExporterServiceName = %q", got) + } +} + +func TestPodmanVolumeName(t *testing.T) { + if got := PodmanVolumeName("rpi4-abc"); got != "jumpstarter-rpi4-abc-shared" { + t.Errorf("PodmanVolumeName = %q", got) + } +} + +func mustContain(t *testing.T, got, want string) { + t.Helper() + if !strings.Contains(got, want) { + t.Errorf("output missing %q\ngot:\n%s", want, got) + } +} + +func mustNotContain(t *testing.T, got, unwanted string) { + t.Helper() + if strings.Contains(got, unwanted) { + t.Errorf("output should not contain %q\ngot:\n%s", unwanted, got) + } +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/ssh_host.go b/controller/internal/exporterset/provisioners/qemu-ssh/ssh_host.go new file mode 100644 index 000000000..b99f0a052 --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/ssh_host.go @@ -0,0 +1,359 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package qemussh implements the qemu-ssh.jumpstarter.dev provisioner +// for ExporterSets. It deploys exporter + QEMU runtime containers on +// remote lab hosts via SSH, using Podman quadlets for container +// orchestration. +package qemussh + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sync" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// RemoteHost abstracts remote host management. SSH is the first +// implementation; macOS launchd or agent-based hosts can follow. +type RemoteHost interface { + // RunCommand executes a command and returns the result. + RunCommand(ctx context.Context, cmd string) (CommandResult, error) + + // ReconcileFile writes content to path only if it differs from + // the existing file. Returns whether the file was changed and a + // sanitized diff string for logging. + ReconcileFile(ctx context.Context, path, content string) (changed bool, diff string, err error) + + // RemoveFile deletes a file if it exists. No error if absent. + RemoveFile(ctx context.Context, path string) error + + // MkdirAll creates a directory and all parents. + MkdirAll(ctx context.Context, path string) error + + // Close releases the connection. + Close() error +} + +// CommandResult holds the output from a remote command execution. +type CommandResult struct { + Stdout string + Stderr string + ExitCode int +} + +// SSHHost implements RemoteHost via SSH/SFTP. +type SSHHost struct { + sshClient *ssh.Client + sftpClient *sftp.Client + hostName string + mu sync.Mutex +} + +// SSHConnectConfig holds the parameters needed to establish an SSH connection. +type SSHConnectConfig struct { + Host string + Port int + User string + PrivateKey []byte +} + +// Connect establishes an SSH + SFTP connection to a remote host. +func Connect(cfg SSHConnectConfig) (*SSHHost, error) { + if cfg.Port == 0 { + cfg.Port = 22 + } + + signer, err := ssh.ParsePrivateKey(cfg.PrivateKey) + if err != nil { + return nil, fmt.Errorf("parse SSH private key: %w", err) + } + + sshConfig := &ssh.ClientConfig{ + User: cfg.User, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // lab hosts; TODO: make configurable + Timeout: defaultConnectTimeout, + } + + addr := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) + + sshClient, err := ssh.Dial("tcp", addr, sshConfig) + if err != nil { + return nil, fmt.Errorf("SSH dial %s: %w", addr, err) + } + + sftpClient, err := sftp.NewClient(sshClient) + if err != nil { + _ = sshClient.Close() + return nil, fmt.Errorf("SFTP client for %s: %w", addr, err) + } + + return &SSHHost{ + sshClient: sshClient, + sftpClient: sftpClient, + hostName: cfg.Host, + }, nil +} + +const ( + // defaultCommandTimeout is the maximum duration for a single SSH command. + defaultCommandTimeout = 2 * time.Minute + + // defaultConnectTimeout is the timeout for the SSH handshake. + defaultConnectTimeout = 30 * time.Second +) + +// RunCommand executes a command on the remote host. The context is +// used for cancellation: if it expires, the SSH session is closed +// and the command is interrupted. +func (h *SSHHost) RunCommand(ctx context.Context, command string) (CommandResult, error) { + h.mu.Lock() + defer h.mu.Unlock() + + session, err := h.sshClient.NewSession() + if err != nil { + return CommandResult{}, fmt.Errorf("SSH session on %s: %w", h.hostName, err) + } + defer session.Close() //nolint:errcheck + + // If no deadline is set on the context, apply a default timeout. + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, defaultCommandTimeout) + defer cancel() + } + + // Close the session when the context expires to unblock Run. + // The done channel ensures the goroutine exits when the command + // completes, even if the context has a long or no deadline. + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + _ = session.Close() + case <-done: + } + }() + + stdout, err := session.StdoutPipe() + if err != nil { + return CommandResult{}, fmt.Errorf("stdout pipe on %s: %w", h.hostName, err) + } + + stderr, err := session.StderrPipe() + if err != nil { + return CommandResult{}, fmt.Errorf("stderr pipe on %s: %w", h.hostName, err) + } + + var stdoutBytes, stderrBytes []byte + var stdoutErr, stderrErr error + var wg sync.WaitGroup + + wg.Add(2) + go func() { + defer wg.Done() + stdoutBytes, stdoutErr = io.ReadAll(stdout) + }() + go func() { + defer wg.Done() + stderrBytes, stderrErr = io.ReadAll(stderr) + }() + + runErr := session.Run(command) + wg.Wait() + + if ctx.Err() != nil { + return CommandResult{}, fmt.Errorf("command on %s timed out: %w", h.hostName, ctx.Err()) + } + + if stdoutErr != nil { + return CommandResult{}, fmt.Errorf("read stdout on %s: %w", h.hostName, stdoutErr) + } + if stderrErr != nil { + return CommandResult{}, fmt.Errorf("read stderr on %s: %w", h.hostName, stderrErr) + } + + exitCode := 0 + if runErr != nil { + if exitErr, ok := runErr.(*ssh.ExitError); ok { + exitCode = exitErr.ExitStatus() + } else { + return CommandResult{}, fmt.Errorf("run command on %s: %w", h.hostName, runErr) + } + } + + return CommandResult{ + Stdout: string(stdoutBytes), + Stderr: string(stderrBytes), + ExitCode: exitCode, + }, nil +} + +// ReconcileFile writes content to path only if the file doesn't exist +// or its content differs. Returns whether the file changed and a +// sanitized diff for logging. +func (h *SSHHost) ReconcileFile(ctx context.Context, path, content string) (bool, string, error) { + if err := ctx.Err(); err != nil { + return false, "", fmt.Errorf("reconcile %s on %s: %w", path, h.hostName, err) + } + + h.mu.Lock() + defer h.mu.Unlock() + + existing, err := h.readFile(path) + if err != nil { + if !isNotExist(err) { + return false, "", fmt.Errorf("read %s on %s: %w", path, h.hostName, err) + } + // File doesn't exist — create it. + if err := h.writeFile(path, content); err != nil { + return false, "", fmt.Errorf("create %s on %s: %w", path, h.hostName, err) + } + return true, fmt.Sprintf("created %s", path), nil + } + + if existing == content { + return false, "", nil + } + + diff := SanitizeDiff(cmp.Diff(existing, content)) + + if err := h.writeFile(path, content); err != nil { + return false, "", fmt.Errorf("update %s on %s: %w", path, h.hostName, err) + } + + return true, diff, nil +} + +// RemoveFile deletes a file. No error if the file doesn't exist. +func (h *SSHHost) RemoveFile(ctx context.Context, path string) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("remove %s on %s: %w", path, h.hostName, err) + } + + h.mu.Lock() + defer h.mu.Unlock() + + err := h.sftpClient.Remove(path) + if err != nil && !isNotExist(err) { + return fmt.Errorf("remove %s on %s: %w", path, h.hostName, err) + } + return nil +} + +// MkdirAll creates a directory and all parents on the remote host. +func (h *SSHHost) MkdirAll(ctx context.Context, path string) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("mkdir %s on %s: %w", path, h.hostName, err) + } + + h.mu.Lock() + defer h.mu.Unlock() + + return h.sftpClient.MkdirAll(path) +} + +// Close releases the SSH and SFTP connections. +func (h *SSHHost) Close() error { + var sftpErr, sshErr error + if h.sftpClient != nil { + sftpErr = h.sftpClient.Close() + } + if h.sshClient != nil { + sshErr = h.sshClient.Close() + } + if sshErr != nil { + return sshErr + } + return sftpErr +} + +// readFile reads a remote file via SFTP. Returns an error if it +// doesn't exist. +func (h *SSHHost) readFile(path string) (string, error) { + f, err := h.sftpClient.Open(path) + if err != nil { + return "", err + } + defer f.Close() //nolint:errcheck + + data, err := io.ReadAll(f) + if err != nil { + return "", err + } + return string(data), nil +} + +// writeFile creates or overwrites a remote file, creating parent +// directories if necessary. +func (h *SSHHost) writeFile(path, content string) error { + parentDir := filepath.Dir(path) + if parentDir != "/" && parentDir != "." { + if err := h.sftpClient.MkdirAll(parentDir); err != nil { + return fmt.Errorf("mkdir %s: %w", parentDir, err) + } + } + + f, err := h.sftpClient.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC) + if err != nil { + return err + } + + if err := f.Chmod(0600); err != nil { + _ = f.Close() + return fmt.Errorf("chmod %s: %w", path, err) + } + + if _, err := f.Write([]byte(content)); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +// isNotExist checks whether an SFTP error indicates "file not found". +func isNotExist(err error) bool { + var statusErr *sftp.StatusError + if errors.As(err, &statusErr) { + return statusErr.FxCode() == sftp.ErrSSHFxNoSuchFile + } + return false +} + +// sensitivePatterns matches credential-like fields for sanitization. +// The value group matches to the end of the line so multi-word +// secrets like "password: my secret" are fully redacted. +var sensitivePatterns = regexp.MustCompile( + `(?im)(token|password|key|secret|credential)([^\S\n]*[:=][^\S\n]*)(.+)`, +) + +// SanitizeDiff redacts sensitive values from diff output. +func SanitizeDiff(diff string) string { + return sensitivePatterns.ReplaceAllString(diff, "${1}${2}[REDACTED]") +} diff --git a/controller/internal/exporterset/provisioners/qemu-ssh/ssh_host_test.go b/controller/internal/exporterset/provisioners/qemu-ssh/ssh_host_test.go new file mode 100644 index 000000000..1a87aabc1 --- /dev/null +++ b/controller/internal/exporterset/provisioners/qemu-ssh/ssh_host_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemussh + +import ( + "testing" +) + +func TestSanitizeDiff_redactsTokens(t *testing.T) { + cases := []struct { + name string + input string + want string + }{ + { + name: "token field", + input: `token: eyJhbGciOiJSUzI1NiJ9.abc`, + want: `token: [REDACTED]`, + }, + { + name: "password field", + input: `password: s3cret123`, + want: `password: [REDACTED]`, + }, + { + name: "key equals", + input: `SECRET_KEY=abcdef12345`, + want: `SECRET_KEY=[REDACTED]`, + }, + { + name: "mixed case credential", + input: `Credential: some-value`, + want: `Credential: [REDACTED]`, + }, + { + name: "non-sensitive unchanged", + input: `endpoint: https://example.com`, + want: `endpoint: https://example.com`, + }, + { + name: "multiline with token", + input: "endpoint: https://example.com\ntoken: abc123\nname: test", + want: "endpoint: https://example.com\ntoken: [REDACTED]\nname: test", + }, + { + name: "multi-word secret fully redacted", + input: `password: my secret value`, + want: `password: [REDACTED]`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := SanitizeDiff(tc.input) + if got != tc.want { + t.Errorf("SanitizeDiff(%q) = %q, want %q", tc.input, got, tc.want) + } + }) + } +} + +func TestSSHConnectConfig_defaultPort(t *testing.T) { + cfg := SSHConnectConfig{ + Host: "example.com", + User: "jumpstarter", + } + if cfg.Port != 0 { + t.Errorf("Port = %d, want 0 (defaulted in Connect())", cfg.Port) + } +} diff --git a/controller/internal/exporterset/reconciler.go b/controller/internal/exporterset/reconciler.go index 92bf02af2..e1d40ebdc 100644 --- a/controller/internal/exporterset/reconciler.go +++ b/controller/internal/exporterset/reconciler.go @@ -400,6 +400,10 @@ func (r *ExporterSetReconciler) scaleUp( // has credentials but no Pod yet. Config Secrets are always synced so token // rotation takes effect without a Pod restart (Kubernetes refreshes Secret-backed // volume mounts automatically). +// +// For off-cluster provisioners (those implementing Deployer), this method +// calls Deploy instead of creating a Pod — see ensureExporterDeployments. +// // Returns true if any Exporter is still waiting for its credential Secret. func (r *ExporterSetReconciler) ensureExporterPods( ctx context.Context, @@ -408,6 +412,24 @@ func (r *ExporterSetReconciler) ensureExporterPods( mergedParameters map[string]any, ownedExporters []jumpstarterdevv1alpha1.Exporter, podsByExporter map[string][]corev1.Pod, +) (waiting bool, err error) { + // Off-cluster provisioners manage instances via SSH/API instead of Pods. + if deployer, ok := r.Provisioner.(Deployer); ok { + return r.ensureExporterDeployments(ctx, es, vtc, mergedParameters, ownedExporters, deployer) + } + + return r.ensureExporterPodsInCluster(ctx, es, vtc, mergedParameters, ownedExporters, podsByExporter) +} + +// ensureExporterPodsInCluster is the in-cluster Pod path (unchanged from +// original ensureExporterPods logic). +func (r *ExporterSetReconciler) ensureExporterPodsInCluster( + ctx context.Context, + es *virtualtargetv1alpha1.ExporterSet, + vtc *virtualtargetv1alpha1.VirtualTargetClass, + mergedParameters map[string]any, + ownedExporters []jumpstarterdevv1alpha1.Exporter, + podsByExporter map[string][]corev1.Pod, ) (waiting bool, err error) { logger := log.FromContext(ctx) @@ -459,6 +481,67 @@ func (r *ExporterSetReconciler) ensureExporterPods( return waiting, nil } +// ensureExporterDeployments is the off-cluster path: calls Deployer.Deploy +// for exporters that have credentials but haven't been deployed yet. +func (r *ExporterSetReconciler) ensureExporterDeployments( + ctx context.Context, + es *virtualtargetv1alpha1.ExporterSet, + vtc *virtualtargetv1alpha1.VirtualTargetClass, + mergedParameters map[string]any, + ownedExporters []jumpstarterdevv1alpha1.Exporter, + deployer Deployer, +) (waiting bool, err error) { + logger := log.FromContext(ctx) + + var caBundle string + var caRead bool + + images := mergeImages(vtc.Spec.Images, es.Spec.Images) + + for i := range ownedExporters { + exp := &ownedExporters[i] + + if !exp.IsEnabled() { + continue + } + + if exp.Status.Credential == nil || exp.Status.Endpoint == "" { + logger.V(1).Info("waiting for credential", "exporter", exp.Name) + waiting = true + continue + } + + if !caRead { + caBundle, err = r.readCABundle(ctx, vtc) + if err != nil { + return false, err + } + caRead = true + } + + deployed, err := deployer.IsDeployed(ctx, exp) + if err != nil { + return waiting, fmt.Errorf("check deployment for %s: %w", exp.Name, err) + } + if deployed { + continue + } + + if err := deployer.Deploy(ctx, es, vtc, mergedParameters, images, exp, caBundle); err != nil { + return waiting, fmt.Errorf("deploy %s: %w", exp.Name, err) + } + + logger.Info("deployed exporter on remote host", "exporter", exp.Name) + + if r.Recorder != nil { + r.Recorder.Eventf(es, corev1.EventTypeNormal, "Deployed", + "Deployed Exporter %s on remote host", exp.Name) + } + } + + return waiting, nil +} + // syncConfigSecret creates or updates the per-exporter config Secret so the // exporter sidecar always has a fresh token and correct configuration. func (r *ExporterSetReconciler) syncConfigSecret( @@ -720,15 +803,19 @@ func (r *ExporterSetReconciler) cleanupDisabledExporters( return deleted, nil } -// cleanupTerminalExporters deletes unleased exporters whose Pod has reached a -// terminal phase (Succeeded or Failed). This is the ExitAndReplace recycle -// path: exitOnLeaseEnd completes the Pod, then the controller deletes the -// Exporter (cascading the Pod) so scale-up can refill minAvailableReplicas. -// Without this, Offline/Succeeded instances inflate replicas, block warm-buffer -// refill at maxReplicas, and leave Completed Pods behind. +// cleanupTerminalExporters deletes unleased exporters whose lifecycle has +// ended. This is the ExitAndReplace recycle path: the exporter completes its +// work, then the controller deletes the Exporter CR so scale-up can refill +// minAvailableReplicas. Without this, Offline/Succeeded instances inflate +// replicas, block warm-buffer refill at maxReplicas, and leave stale +// resources behind. // -// InPlaceReuse skips this path: a Failed/Succeeded Pod must not permanently -// delete the Exporter CR (lease-end reuse keeps the instance). +// For in-cluster provisioners the terminal signal is a Pod in +// Succeeded/Failed phase. For off-cluster provisioners (Deployer), it is +// an exporter that was deployed but has gone offline — meaning the remote +// container exited after the lease ended. +// +// InPlaceReuse skips this path entirely. // podsByExporter comes from a single List in Reconcile. func (r *ExporterSetReconciler) cleanupTerminalExporters( ctx context.Context, @@ -741,6 +828,7 @@ func (r *ExporterSetReconciler) cleanupTerminalExporters( } logger := log.FromContext(ctx) + deployer, isOffCluster := r.Provisioner.(Deployer) deleted := false for i := range exporters { @@ -749,8 +837,26 @@ func (r *ExporterSetReconciler) cleanupTerminalExporters( continue } - pods := podsByExporter[exp.Name] - if !allPodsTerminal(pods) { + terminal := false + + if isOffCluster { + // Off-cluster: terminal means "deployed but went offline". The + // Online condition is set to True when the exporter registers + // heartbeats, then flipped to False when they stop. If the + // condition doesn't exist at all the exporter never registered + // so we leave it alone (still starting up). + deployed, err := deployer.IsDeployed(ctx, exp) + if err != nil { + return deleted, fmt.Errorf("check deployment for %s: %w", exp.Name, err) + } + terminal = deployed && isExporterOffline(exp) + } else { + // In-cluster: terminal means all Pods reached Succeeded/Failed. + pods := podsByExporter[exp.Name] + terminal = allPodsTerminal(pods) + } + + if !terminal { continue } @@ -762,12 +868,12 @@ func (r *ExporterSetReconciler) cleanupTerminalExporters( return deleted, fmt.Errorf("unable to delete Exporter %s: %w", exp.Name, err) } - logger.Info("deleted Exporter after terminal Pod (ExitAndReplace)", - "exporter", exp.Name, "pods", len(pods)) + logger.Info("deleted terminal Exporter (ExitAndReplace)", + "exporter", exp.Name, "offCluster", isOffCluster) if r.Recorder != nil { r.Recorder.Eventf(es, corev1.EventTypeNormal, "Recycle", - "Deleted Exporter %s after terminal Pod", exp.Name) + "Deleted Exporter %s after terminal lifecycle", exp.Name) } deleted = true } @@ -775,6 +881,18 @@ func (r *ExporterSetReconciler) cleanupTerminalExporters( return deleted, nil } +// isExporterOffline reports whether the exporter's Online condition has been +// explicitly set to False. Returns false when the condition doesn't exist +// (exporter never registered) to avoid cleaning up exporters that are still +// starting. +func isExporterOffline(exp *jumpstarterdevv1alpha1.Exporter) bool { + cond := meta.FindStatusCondition( + exp.Status.Conditions, + string(jumpstarterdevv1alpha1.ExporterConditionTypeOnline), + ) + return cond != nil && cond.Status == metav1.ConditionFalse +} + func allPodsTerminal(pods []corev1.Pod) bool { if len(pods) == 0 { return false diff --git a/controller/internal/exporterset/reconciler_test.go b/controller/internal/exporterset/reconciler_test.go index 6902e06b6..6ae93da3f 100644 --- a/controller/internal/exporterset/reconciler_test.go +++ b/controller/internal/exporterset/reconciler_test.go @@ -2018,3 +2018,44 @@ func TestMergeImages_esOverridesVtc(t *testing.T) { t.Errorf("runtime should be overridden by es, got %v", got.Runtime) } } + +func TestIsExporterOffline_noCondition(t *testing.T) { + exp := &jumpstarterdevv1alpha1.Exporter{ + Status: jumpstarterdevv1alpha1.ExporterStatus{}, + } + if isExporterOffline(exp) { + t.Fatal("expected false when Online condition doesn't exist (exporter never registered)") + } +} + +func TestIsExporterOffline_online(t *testing.T) { + exp := &jumpstarterdevv1alpha1.Exporter{ + Status: jumpstarterdevv1alpha1.ExporterStatus{ + Conditions: []metav1.Condition{ + { + Type: string(jumpstarterdevv1alpha1.ExporterConditionTypeOnline), + Status: metav1.ConditionTrue, + }, + }, + }, + } + if isExporterOffline(exp) { + t.Fatal("expected false when exporter is online") + } +} + +func TestIsExporterOffline_offline(t *testing.T) { + exp := &jumpstarterdevv1alpha1.Exporter{ + Status: jumpstarterdevv1alpha1.ExporterStatus{ + Conditions: []metav1.Condition{ + { + Type: string(jumpstarterdevv1alpha1.ExporterConditionTypeOnline), + Status: metav1.ConditionFalse, + }, + }, + }, + } + if !isExporterOffline(exp) { + t.Fatal("expected true when exporter is offline") + } +} diff --git a/docs/source/getting-started/guides/setup/index.md b/docs/source/getting-started/guides/setup/index.md index aaf19be90..ff01ec87c 100644 --- a/docs/source/getting-started/guides/setup/index.md +++ b/docs/source/getting-started/guides/setup/index.md @@ -8,6 +8,8 @@ Step-by-step instructions for each operation mode. over TCP, without a {term}`controller` - [Distributed Mode](distributed-mode.md): Configuring Jumpstarter for team environments with shared resources +- [Off-Cluster QEMU](off-cluster-qemu.md): Running virtual targets on remote + lab hosts via SSH ```{toctree} :maxdepth: 1 @@ -15,4 +17,5 @@ Step-by-step instructions for each operation mode. local-mode.md direct-mode.md distributed-mode.md +off-cluster-qemu.md ``` diff --git a/docs/source/getting-started/guides/setup/off-cluster-qemu.md b/docs/source/getting-started/guides/setup/off-cluster-qemu.md new file mode 100644 index 000000000..8dcf6e4ea --- /dev/null +++ b/docs/source/getting-started/guides/setup/off-cluster-qemu.md @@ -0,0 +1,244 @@ +# Off-Cluster QEMU Provisioner + +The `qemu-ssh.jumpstarter.dev` provisioner runs QEMU virtual targets on remote +lab hosts outside the Kubernetes cluster. It connects to hosts via SSH and +deploys containers using Podman quadlets, giving you the same container-based +exporter + runtime pattern as in-cluster QEMU but on dedicated hardware with +direct KVM access. + +**When to use this provisioner:** + +- Your lab hosts have KVM-capable hardware or GPU passthrough not available + in the cluster +- You need bare-metal performance for emulation (e.g., automotive SoC targets) +- You want to scale virtual target pools across multiple lab machines while + keeping orchestration centralized in Kubernetes + +## Prerequisites + +### Remote lab hosts + +Each remote host must have: + +- **Podman** installed (for running containers) +- **systemd** (for quadlet-based container lifecycle) +- **SSH access** with key-based authentication +- (Optional) **KVM** support (`/dev/kvm` present) for hardware-accelerated + emulation + +### Kubernetes cluster + +The cluster must have: + +- Jumpstarter operator installed +- The `qemu-ssh.jumpstarter.dev` provisioner enabled in the Jumpstarter CR + +## Step 1: Enable the provisioner + +Add `qemu-ssh.jumpstarter.dev` to the `exporterSets.provisioners` list in your +Jumpstarter CR: + +```yaml +apiVersion: operator.jumpstarter.dev/v1alpha1 +kind: Jumpstarter +metadata: + name: jumpstarter + namespace: jumpstarter +spec: + exporterSets: + image: quay.io/jumpstarter-dev/exporter-set-controller:latest + provisioners: + - name: qemu.jumpstarter.dev + enabled: true + - name: qemu-ssh.jumpstarter.dev + enabled: true +``` + +The operator creates a Deployment for the `qemu-ssh` controller automatically. + +## Step 2: Create SSH credentials + +Create a Kubernetes Secret containing the SSH private key used to connect to +your lab hosts: + +```bash +kubectl create secret generic lab-ssh-key \ + --from-file=ssh-privatekey=$HOME/.ssh/id_ed25519 \ + -n jumpstarter +``` + +The Secret uses the standard `kubernetes.io/ssh-auth` type. The key must be in +the `ssh-privatekey` field. The SSH username is configured in the +VirtualTargetClass parameters (not in the Secret). + +## Step 3: Create a VirtualTargetClass + +The `VirtualTargetClass` defines the pool profile — which host to use, SSH +configuration, and default resource allocations: + +```yaml +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: VirtualTargetClass +metadata: + name: qemu-ssh-aarch64 + namespace: jumpstarter +spec: + provisioner: qemu-ssh.jumpstarter.dev + credentialsSecretRef: + name: lab-ssh-key + bindingMode: Immediate + reclaimPolicy: Delete + parameters: + ssh: + user: root + port: 22 + host: + name: lab-host-01.example.com + runtime: + kvm: true + arch: aarch64 + resources: + cpu: 4 + memory: 4Gi + storage: + size: 16Gi +``` + +### Parameters reference + +| Parameter | Description | +|-----------|-------------| +| `ssh.user` | Default SSH username (fallback: `root`) | +| `ssh.port` | Default SSH port (fallback: `22`) | +| `host.name` | FQDN or IP of the remote host | +| `host.user` | Per-host SSH user override | +| `host.port` | Per-host SSH port override | +| `runtime.kvm` | Pass `/dev/kvm` to the runtime container | +| `runtime.devices` | Additional devices to pass through (list of paths) | +| `arch` | Default QEMU architecture | +| `resources.cpu` | Default vCPU count | +| `resources.memory` | Default memory (e.g., `4Gi`) | +| `storage.size` | Default disk size (e.g., `16Gi`) | + +## Step 4: Create an ExporterSet + +Each ExporterSet manages instances on one host. The `maxReplicas` field +controls how many concurrent instances run on that host. To use multiple +hosts, create additional ExporterSets that reference the same +VirtualTargetClass but override `parameters.host`: + +```yaml +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: ExporterSet +metadata: + name: aarch64-ssh-pool + namespace: jumpstarter +spec: + minReplicas: 0 + maxReplicas: 4 + minAvailableReplicas: 1 + scaleDownCooldown: 5m + recycleStrategy: ExitAndReplace + virtualTargetClassName: qemu-ssh-aarch64 + selector: + matchLabels: + board: aarch64-qemu + virtual: "true" + template: + metadata: + labels: + board: aarch64-qemu + arch: aarch64 + virtual: "true" + spec: + drivers: + - name: qemu + type: jumpstarter_driver_qemu.driver.Qemu + - name: power + type: jumpstarter_driver_power.driver.QemuPower + - name: serial + type: jumpstarter_driver_serial.driver.QemuSerial +``` + +The `tcp` driver is auto-injected by the provisioner (SSH host-forwarding on +port 2222). + +## How it works + +When the ExporterSet scales up: + +1. The controller creates an `Exporter` CR (cluster-side) +2. Once credentials are ready, it connects to the remote host via SSH +3. Writes the exporter configuration YAML to `/etc/jumpstarter/exporters/` +4. Creates Podman quadlet `.container` files under `/etc/containers/systemd/` + for both the **runtime** (QEMU) and **exporter** containers +5. Creates a shared Podman volume for inter-container communication + (QEMU sockets) +6. Reloads systemd and starts the containers +7. Annotates the Exporter CR with the host assignment + +When scaling down or cleaning up: + +1. Stops and disables the systemd services +2. Removes the quadlet files and exporter config +3. Removes the shared Podman volume +4. Reloads systemd +5. Deletes the Exporter CR + +### Container layout on the remote host + +Each exporter instance creates two containers: + +- **`-runtime`**: Runs the QEMU emulator with access to `/dev/kvm` + (if enabled) and the shared volume for communication sockets +- **`-exporter`**: Runs the Jumpstarter exporter, connecting back to + the cluster controller and mounting the shared volume + +Both containers communicate via Unix sockets on the shared Podman volume +(`/shared/launcher.sock`). + +## Leasing targets + +Users lease off-cluster targets the same way as any other target — the +placement is transparent: + +```bash +jmp lease -l board=aarch64-qemu,virtual=true +``` + +## Troubleshooting + +### Check exporter-set controller logs + +```bash +kubectl logs -l component=exporterset-controller,provisioner=qemu-ssh-jumpstarter-dev \ + -n jumpstarter -f +``` + +### Check remote host containers + +SSH into the lab host and inspect the Podman containers: + +```bash +podman ps -a --filter "name=-(runtime|exporter)$" +systemctl status *-runtime *-exporter +journalctl -u -runtime -u -exporter +``` + +### Check quadlet files + +```bash +ls /etc/containers/systemd/*.container +cat /etc/jumpstarter/exporters/*.yaml +``` + +### Common issues + +- **SSH connection failures**: Verify the SSH key in the Secret matches + `authorized_keys` on the remote hosts +- **KVM not available**: Ensure `/dev/kvm` exists on the host and the + `runtime.kvm` parameter is set +- **Containers not starting**: Check `podman logs -runtime` and + `podman logs -exporter` on the remote host +- **Capacity full**: Increase `maxReplicas` in the ExporterSet or create + additional ExporterSets for more hosts