Refactor plugins code for sharing

Signed-off-by: Ivan Sim <[email protected]>

.gitignore

@@ -2,5 +2,5 @@ bin
 kubelet.log
 kubelet-v1.31.0
 kubelet/run
-
+flatcar
 dist/

Makefile

@@ -8,11 +8,20 @@ GOARCH ?= amd64
 MAJOR_VERSION_CRAND ?= 1
 MINOR_VERSION_CRAND ?= 8
 
-build: crand
+FLATCAR_DIR := ./flatcar
+
+build: clean crand kvm
+
+clean:
+	rm -rf bin
 
 crand: tidy
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o ./bin/dp-crand cmd/crand/main.go
 
+.PHONY: kvm
+kvm: tidy
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o ./bin/dp-kvm cmd/kvm/main.go
+
 lint: tidy
 	golangci-lint run ./...
 
@@ -22,8 +31,11 @@ tidy:
 test: tidy
 	go test -v -race -cover ./...
 
-run: build
-	sudo ./bin/device-plugin-crand
+run-crand: crand
+	sudo ./bin/dp-crand
+
+run-kvm: kvm
+	sudo ./bin/dp-kvm
 
 .PHONY: kubelet
 kubelet:
@@ -38,13 +50,36 @@ kubelet:
 .PHONY: cdi
 cdi:
 	sudo mkdir -p /etc/cdi
-	sudo cp cdi/pflex.yaml /etc/cdi/pflex.yaml
-	sudo rm -rf /dev/pflex[0-3]
-	sudo mknod -m 666 /dev/pflex0 c 1 8
-	sudo mknod -m 666 /dev/pflex1 c 1 8
-	sudo mknod -m 666 /dev/pflex2 c 1 8
+	sed \
+		-e 's/$${DEVICE_MAJOR_VERSION}/$(MAJOR_VERSION_CRAND)/g' \
+		-e 's/$${DEVICE_MINOR_VERSION}/$(MINOR_VERSION_CRAND)/g' \
+		./cdi/github.com.ihcsim.crand.yaml | sudo tee /etc/cdi/github.com.ihcsim.crand.yaml
+	sudo rm -rf /dev/crand[0-3]
+	sudo mknod -m 666 /dev/crand0 c $(MAJOR_VERSION_CRAND) $(MINOR_VERSION_CRAND)
+	sudo mknod -m 666 /dev/crand1 c $(MAJOR_VERSION_CRAND) $(MINOR_VERSION_CRAND)
+	sudo mknod -m 666 /dev/crand2 c $(MAJOR_VERSION_CRAND) $(MINOR_VERSION_CRAND)
 
 .PHONY: deploy
 deploy:
 	mkdir -p kubelet/run/{pods,logs}
 	cp yaml/busybox.yaml kubelet/run/pods
+
+$(FLATCAR_DIR):
+	mkdir -p $(FLATCAR_DIR)
+	wget https://stable.release.flatcar-linux.net/amd64-usr/current/flatcar_production_qemu_image.img -P $(FLATCAR_DIR)
+	qemu-img create -f qcow2 -F qcow2 -b flatcar_production_qemu_image.img $(FLATCAR_DIR)/flatcar-01.qcow2
+	$(MAKE) butane2ign
+
+butane2ign:
+	sed -e "s/\$${SSH_PUB_KEY}/$$(cat ~/.ssh/id_ecdsa_gh.pub)/g" ./yaml/butane.yaml | docker run --rm -i quay.io/coreos/butane:release | jq . > ./flatcar/provision.ign
+
+flatcar-start: $(FLATCAR_DIR)
+	sudo virt-install \
+    --connect qemu:///system \
+    --osinfo generic \
+    --import \
+    --name flatcar-linux1 \
+    --ram 1024 --vcpus 1 \
+    --disk path=./flatcar/flatcar-linux1.qcow2,format=qcow2,bus=virtio,size=5 \
+    --vnc --noautoconsole \
+    --qemu-commandline='-fw_cfg name=opt/org.flatcar-linux/config,file=$(shell pwd)/flatcar/provision.ign'

README.md

@@ -27,6 +27,29 @@ To enable CDI support with containerd v1.6+, update the
 
 Restart containerd for the changes to take effect.
 
+Set up the sample devices and CDI configuration files:
+
+```sh
+# sudo required
+make cdi
+```
+
+The CDI configuration files can be found in the local `/etc/cdi` directory.
+
+## Flatcar
+
+Download and start a Flatcar Linux VM with `virsh`:
+
+```sh
+make flatcar-start
+```
+
+If the host uses AppArmor, allow qemu to access the config files:
+
+```sh
+echo "  `pwd`/flatcar/provision.ign r," >> /etc/apparmor.d/abstractions/libvirt-qemu
+```
+
 ## Testing With Kubelet
 
 For ease of testing purposes, this repo comes with the kubelet v1.31.0 binary. 

pkg/plugins/crand/device.go

@@ -17,7 +17,7 @@ func (p *DevicePlugin) ListAndWatch(empty *v1beta1.Empty, stream v1beta1.DeviceP
 	defer tick.Stop()
 
 	for range tick.C {
-		_, changeSet, err := p.discoverDevices()
+		changeSet, err := p.discoverDevices()
 		if err != nil {
 			return err
 		}
@@ -75,15 +75,3 @@ func (p *DevicePlugin) GetDevicePluginOptions(context.Context, *v1beta1.Empty) (
 	p.log.Debug().Msg("calling DevicePlugin.GetDevicePluginOptions()")
 	return &v1beta1.DevicePluginOptions{}, nil
 }
-
-// Device represents a device managed by this plugin.
-type Device struct {
-	ID     string
-	Health DeviceHealth
-}
-
-// DeviceState maintains the last seen of a device at a given timestamp.
-type DeviceState struct {
-	lastSeenTimestamp int64
-	*Device
-}

pkg/plugins/crand/discovery.go

@@ -6,74 +6,76 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ihcsim/kubelet-plugin/pkg/plugins"
 	"github.com/rs/zerolog/log"
 )
 
-func (p *DevicePlugin) discoverDevices() (map[string]*Device, []*Device, error) {
+func (p *DevicePlugin) discoverDevices() ([]*plugins.Device, error) {
 	var (
-		fullSet   = map[string]*Device{}
-		changeSet = []*Device{}
+		// fullSet keeps track of all available devices. it's used to remove any stale
+		// cache entries later.
+		fullSet = map[string]*plugins.Device{}
+
+		// changeSet identifies which devices have changed since the last discovery.
+		changeSet = []*plugins.Device{}
 	)
 
 	f, err := os.Open(hostDevicePath)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	entries, err := f.ReadDir(0)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	for _, entry := range entries {
 		if entry.IsDir() {
 			continue
 		}
 
-		if strings.Contains(entry.Name(), "crand") {
-			id := entry.Name()
-			device := &Device{
-				ID:     id,
-				Health: Healthy,
-			}
+		// skip any non-crand devices
+		if !strings.Contains(entry.Name(), "crand") {
+			continue
+		}
 
-			fullSet[id] = device
-			lastSeenState, exists := p.cache[id]
+		id := entry.Name()
+		device := &plugins.Device{
+			ID:     id,
+			Health: plugins.Healthy,
+		}
 
-			log := p.log.With().
-				Str("device", id).
-				Str("health", device.Health.String()).
-				Str("path", filepath.Join(hostDevicePath, entry.Name())).
-				Logger()
+		fullSet[id] = device
+		lastSeenState, exists := p.cache[id]
 
-			if exists && lastSeenState.Health == device.Health {
-				continue
-			}
-			changeSet = append(changeSet, device)
-
-			// add new device's state to cache
-			if !exists {
-				log.Info().Msg("found new device")
-				p.cache[id] = &DeviceState{
-					Device:            device,
-					lastSeenTimestamp: time.Now().Unix(),
-				}
-				continue
-			}
+		// no change in device's state
+		if exists && lastSeenState.Health == device.Health {
+			continue
+		}
 
-			// update existing device's state in cache, if changed
+		log := p.log.With().
+			Str("device", id).
+			Str("health", device.Health.String()).
+			Str("path", filepath.Join(hostDevicePath, entry.Name())).
+			Logger()
+
+		if !exists {
+			log.Info().Msg("found new device")
+		} else {
 			if lastSeenState.Health != device.Health {
 				log.Info().
 					Str("before", lastSeenState.Health.String()).
-					Time("last seen", time.Unix(lastSeenState.lastSeenTimestamp, 0)).
+					Time("last seen", time.Unix(lastSeenState.LastSeenTimestamp, 0)).
 					Msg("device health changed")
-				p.cache[id] = &DeviceState{
-					lastSeenTimestamp: time.Now().Unix(),
-					Device:            device,
-				}
-				changeSet = append(changeSet, device)
 			}
 		}
+
+		p.cache[id] = &plugins.DeviceState{
+			Device:            device,
+			LastSeenTimestamp: time.Now().Unix(),
+		}
+		changeSet = append(changeSet, device)
 	}
 
 	// remove devices that are no longer present from cache
@@ -84,5 +86,5 @@ func (p *DevicePlugin) discoverDevices() (map[string]*Device, []*Device, error)
 		}
 	}
 
-	return fullSet, changeSet, nil
+	return changeSet, nil
 }

pkg/plugins/crand/plugin.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/fsnotify/fsnotify"
+	"github.com/ihcsim/kubelet-plugin/pkg/plugins"
 	"github.com/rs/zerolog"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -27,7 +28,7 @@ var (
 type DevicePlugin struct {
 	// cache is used to store the last-seen state of devices on the host.
 	// it's updated by the discoverDevices() method.
-	cache map[string]*DeviceState
+	cache map[string]*plugins.DeviceState
 
 	gserver *grpc.Server
 	log     *zerolog.Logger
@@ -36,7 +37,7 @@ type DevicePlugin struct {
 func NewPlugin(log *zerolog.Logger) *DevicePlugin {
 	gserver := grpc.NewServer()
 	plugin := &DevicePlugin{
-		cache:   map[string]*DeviceState{},
+		cache:   map[string]*plugins.DeviceState{},
 		gserver: gserver,
 		log:     log,
 	}

pkg/plugins/device.go

@@ -0,0 +1,26 @@
+package plugins
+
+import "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+
+const (
+	Healthy   DeviceHealth = v1beta1.Healthy
+	Unhealthy DeviceHealth = v1beta1.Unhealthy
+)
+
+// Device represents a device managed by this plugin.
+type Device struct {
+	ID     string
+	Health DeviceHealth
+}
+
+// DeviceState maintains the last seen of a device at a given timestamp.
+type DeviceState struct {
+	LastSeenTimestamp int64
+	*Device
+}
+
+type DeviceHealth string
+
+func (h DeviceHealth) String() string {
+	return string(h)
+}