Initial proof-of-concept plugin implementation

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "kubernetes/kubernetes"]
+	path = kubernetes/kubernetes
+	url = https://github.com/kubernetes/kubernetes.git

Makefile

@@ -0,0 +1,25 @@
+.PHONY: proto-tools
+proto-tools:
+	cd ./kubernetes/proto/tools; \
+	cat tools.go | grep "_" | awk -F'"' '{print $$2}' | xargs -tI % go install %
+
+.PHONY: submodule-update
+submodule-update:
+	git submodule update -i
+	cp ./kubernetes/kubernetes.checkout ./.git/modules/kubernetes/kubernetes/info/sparse-checkout
+	cd ./kubernetes/kubernetes; \
+	git config core.sparsecheckout true; \
+	git read-tree -mu HEAD
+
+# This uses go-plugin to generate UnmarshalVT because generating via below
+# hangs compiling with TinyGo.
+# --go-vtproto_out=./kubernetes/proto --go-vtproto_opt=Mkubernetes/proto/kubernetes.proto=./api,features=marshal+unmarshal+size
+.PHONY: update-kubernetes-proto
+update-kubernetes-proto: proto-tools
+	echo "You need to install protoc before running this."
+	echo "Regenerate the protobuf definition from the submodule ./kubernetes/kubernetes."
+	openapi2proto -spec ./kubernetes/kubernetes/api/openapi-spec/swagger.json -out ./kubernetes/proto/kubernetes.proto
+	echo "Regenerate the Go protobuf code."
+	protoc ./kubernetes/proto/kubernetes.proto \
+		--go-plugin_out=./kubernetes/proto --go-plugin_opt=Mkubernetes/proto/kubernetes.proto=./api
+

example/Makefile

@@ -0,0 +1,5 @@
+.PHONY: build
+build: main.go
+	@for f in $^; do \
+		CGO_ENABLED=0 tinygo build -o $$(echo $$f | sed -e 's/\.go/\.wasm/') -scheduler=none --no-debug --target=wasi $$f; \
+	done

example/go.mod

@@ -0,0 +1,15 @@
+module sigs.k8s.io/kube-scheduler-wasm-extension/example
+
+go 1.19
+
+require sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000
+
+require (
+	github.com/knqyf263/go-plugin v0.7.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000 // indirect
+)
+
+replace sigs.k8s.io/kube-scheduler-wasm-extension/guest => ./../guest
+
+replace sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto => ./../kubernetes/proto

example/go.sum

@@ -0,0 +1,13 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/knqyf263/go-plugin v0.7.0 h1:m1FGUbYOaQ6nI8C8uFPNdWr4e51+9pCQzrMSux4ATFg=
+github.com/knqyf263/go-plugin v0.7.0/go.mod h1:+GzKPLhdIyI9FWBUx5d/wOfK44TME2fXk+l9uNTtVy4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

example/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
+)
+
+func main() {
+	guest.Filter = api.FilterFunc(nameEqualsPosSpec)
+}
+
+// nameEqualsPosSpec schedules this node if its name equals its pod spec.
+func nameEqualsPosSpec(nodeInfo api.NodeInfo, pod api.Pod) (api.StatusCode, string) {
+	nodeName := nodeInfo.Node().Metadata.Name
+	podSpecNodeName := pod.Spec().NodeName
+
+	if len(podSpecNodeName) == 0 || podSpecNodeName == nodeName {
+		return api.StatusCodeSuccess, ""
+	} else {
+		return api.StatusCodeUnschedulable, ""
+	}
+}

guest/api/status.go

@@ -0,0 +1,30 @@
+package api
+
+// StatusCode is the Status code/type which is returned from plugins.
+type StatusCode int
+
+// These are predefined codes used in a Status.
+const (
+	// StatusCodeSuccess means that plugin ran correctly and found pod schedulable.
+	StatusCodeSuccess StatusCode = iota
+	// StatusCodeError is used for internal plugin errors, unexpected input, etc.
+	StatusCodeError
+	// StatusCodeUnschedulable is used when a plugin finds a pod unschedulable. The scheduler might attempt to
+	// run other postFilter plugins like preemption to get this pod scheduled.
+	// Use StatusCodeUnschedulableAndUnresolvable to make the scheduler skipping other postFilter plugins.
+	// The accompanying status message should explain why the pod is unschedulable.
+	StatusCodeUnschedulable
+	// StatusCodeUnschedulableAndUnresolvable is used when a plugin finds a pod unschedulable and
+	// other postFilter plugins like preemption would not change anything.
+	// Plugins should return StatusCodeUnschedulable if it is possible that the pod can get scheduled
+	// after running other postFilter plugins.
+	// The accompanying status message should explain why the pod is unschedulable.
+	StatusCodeUnschedulableAndUnresolvable
+	// StatusCodeWait is used when a Permit plugin finds a pod scheduling should wait.
+	StatusCodeWait
+	// StatusCodeSkip is used in the following scenarios:
+	// - when a Bind plugin chooses to skip binding.
+	// - when a PreFilter plugin returns StatusCodeSkip so that coupled Filter plugin/PreFilterExtensions() will be skipped.
+	// - when a PreScore plugin returns StatusCodeSkip so that coupled Score plugin will be skipped.
+	StatusCodeSkip
+)

guest/api/types.go

@@ -0,0 +1,24 @@
+package api
+
+import protoapi "sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto/api"
+
+// Filter is a WebAssembly implementation of framework.FilterPlugin.
+type Filter interface {
+	Filter(NodeInfo, Pod) (statusCode StatusCode, statusReason string)
+}
+
+// FilterFunc adapts an ordinary function to a Filter.
+type FilterFunc func(NodeInfo, Pod) (statusCode StatusCode, statusReason string)
+
+// Filter returns f(n, p).
+func (f FilterFunc) Filter(n NodeInfo, p Pod) (statusCode StatusCode, statusReason string) {
+	return f(n, p)
+}
+
+type NodeInfo interface {
+	Node() *protoapi.IoK8SApiCoreV1Node
+}
+
+type Pod interface {
+	Spec() *protoapi.IoK8SApiCoreV1PodSpec
+}

guest/filter.go

@@ -0,0 +1,56 @@
+package guest
+
+import (
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
+	"sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/imports"
+	protoapi "sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto/api"
+)
+
+// Filter should be assigned in `main` to a FilterFunc function.
+//
+// For example:
+//
+//	func main() {
+//		guest.FilterFn = api.FilterFunc(nameEqualsPosSpec)
+//	}
+var Filter api.Filter
+
+// filter is only exported to the host.
+//
+//go:export filter
+func filter() (code uint32) { //nolint
+	if Filter == nil {
+		return
+	}
+	c, reason := Filter.Filter(nodeInfo{}, pod{})
+	if reason != "" {
+		imports.StatusReason(reason)
+	}
+	return uint32(c)
+}
+
+var _ api.NodeInfo = nodeInfo{}
+
+type nodeInfo struct{}
+
+func (nodeInfo) Node() *protoapi.IoK8SApiCoreV1Node {
+	b := imports.NodeInfoNode()
+	var msg protoapi.IoK8SApiCoreV1Node
+	if err := msg.UnmarshalVT(b); err != nil {
+		panic(err)
+	}
+	return &msg
+}
+
+var _ api.Pod = pod{}
+
+type pod struct{}
+
+func (pod) Spec() *protoapi.IoK8SApiCoreV1PodSpec {
+	b := imports.PodSpec()
+	var msg protoapi.IoK8SApiCoreV1PodSpec
+	if err := msg.UnmarshalVT(b); err != nil {
+		panic(err)
+	}
+	return &msg
+}

guest/go.mod

@@ -0,0 +1,13 @@
+module sigs.k8s.io/kube-scheduler-wasm-extension/guest
+
+go 1.19
+
+replace sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto => ../kubernetes/proto
+
+require sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000
+
+require (
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/knqyf263/go-plugin v0.7.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+)

guest/go.sum

@@ -0,0 +1,14 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/knqyf263/go-plugin v0.7.0 h1:m1FGUbYOaQ6nI8C8uFPNdWr4e51+9pCQzrMSux4ATFg=
+github.com/knqyf263/go-plugin v0.7.0/go.mod h1:+GzKPLhdIyI9FWBUx5d/wOfK44TME2fXk+l9uNTtVy4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

guest/internal/imports/host.go

@@ -0,0 +1,24 @@
+package imports
+
+import "runtime"
+
+// StatusReason overwrites the status reason
+func StatusReason(reason string) {
+	ptr, size := stringToPtr(reason)
+	_statusReason(ptr, size)
+	runtime.KeepAlive(reason) // keep reason alive until ptr is no longer needed.
+}
+
+func NodeInfoNode() []byte {
+	// Wrap to avoid TinyGo 0.27: cannot use an exported function as value
+	return getBytes(func(ptr uint32, limit bufLimit) (len uint32) {
+		return _nodeInfoNode(ptr, limit)
+	})
+}
+
+func PodSpec() []byte {
+	// Wrap to avoid TinyGo 0.27: cannot use an exported function as value
+	return getBytes(func(ptr uint32, limit bufLimit) (len uint32) {
+		return _podSpec(ptr, limit)
+	})
+}

guest/internal/imports/imports.go

@@ -0,0 +1,15 @@
+//go:build tinygo.wasm
+
+package imports
+
+//go:wasm-module k8s.io/scheduler
+//go:export status_reason
+func _statusReason(ptr, size uint32)
+
+//go:wasm-module k8s.io/api
+//go:export nodeInfo/node
+func _nodeInfoNode(ptr uint32, limit bufLimit) (len uint32)
+
+//go:wasm-module k8s.io/api
+//go:export pod/spec
+func _podSpec(ptr uint32, limit bufLimit) (len uint32)

guest/internal/imports/imports_stub.go

@@ -0,0 +1,16 @@
+//go:build !tinygo.wasm
+
+package imports
+
+// reason is stubbed for compilation outside TinyGo.
+func _statusReason(ptr, size uint32) {}
+
+// nodeInfoNodeName is stubbed for compilation outside TinyGo.
+func _nodeInfoNode(ptr uint32, limit bufLimit) (len uint32) {
+	return 0
+}
+
+// podSpec is stubbed for compilation outside TinyGo.
+func _podSpec(ptr uint32, limit bufLimit) (len uint32) {
+	return 0
+}

guest/internal/imports/mem.go

@@ -0,0 +1,43 @@
+package imports
+
+import "unsafe"
+
+// BufLimit is the possibly zero maximum length of a result value to write in
+// bytes. If the actual value is larger than this, nothing is written to
+// memory.
+type bufLimit = uint32
+
+var (
+	// readBuf is sharable because there is no parallelism in wasm.
+	readBuf = make([]byte, readBufLimit)
+	// ReadBufPtr is used to avoid duplicate host function calls.
+	readBufPtr = uintptr(unsafe.Pointer(&readBuf[0]))
+	// ReadBufLimit is constant memory overhead for reading fields.
+	readBufLimit = uint32(2048)
+)
+
+// stringToPtr returns a pointer and size pair for the given string in a way
+// compatible with WebAssembly numeric types.
+// The returned pointer aliases the string hence the string must be kept alive
+// until ptr is no longer needed.
+func stringToPtr(s string) (uint32, uint32) {
+	ptr := unsafe.Pointer(unsafe.StringData(s))
+	return uint32(uintptr(ptr)), uint32(len(s))
+}
+
+func getBytes(fn func(ptr uint32, limit bufLimit) (len uint32)) (result []byte) {
+	size := fn(uint32(readBufPtr), readBufLimit)
+	if size == 0 {
+		return
+	}
+	if size > 0 && size <= readBufLimit {
+		// copy to avoid passing a mutable buffer
+		result = make([]byte, size)
+		copy(result, readBuf)
+		return
+	}
+	buf := make([]byte, size)
+	ptr := uintptr(unsafe.Pointer(&buf[0]))
+	_ = fn(uint32(ptr), size)
+	return buf
+}

kubernetes/kubernetes.checkout

@@ -0,0 +1 @@
+api/openapi-spec/swagger.json