[WIP] bootstrap userdata changes

Makefile

@@ -86,9 +86,9 @@ RELEASE_REGISTRY := gcr.io/cluster-api-provider-vsphere/release
 RELEASE_CONTROLLER_IMG := $(RELEASE_REGISTRY)/$(IMAGE_NAME)
 
 # Development Docker variables
-DEV_REGISTRY ?= gcr.io/$(shell gcloud config get-value project)
+DEV_REGISTRY ?= gcr.io/spectro-images-public/dev/cluster-api-vsphere
 DEV_CONTROLLER_IMG ?= $(DEV_REGISTRY)/vsphere-$(IMAGE_NAME)
-DEV_TAG ?= dev
+DEV_TAG ?= flatcar
 DEV_MANIFEST_IMG := $(DEV_CONTROLLER_IMG)-$(ARCH)
 
 # Set build time variables including git version details

go.mod

@@ -3,7 +3,9 @@ module sigs.k8s.io/cluster-api-provider-vsphere
 go 1.15
 
 require (
+	github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect
 	github.com/antihax/optional v1.0.0
+	github.com/coreos/ignition v0.35.0
 	github.com/go-logr/logr v0.1.0
 	github.com/google/go-cmp v0.4.1
 	github.com/google/uuid v1.1.1
@@ -12,6 +14,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0
 	github.com/vmware/govmomi v0.23.1
+	go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	gopkg.in/gcfg.v1 v1.2.3
 	gopkg.in/warnings.v0 v0.1.2 // indirect

pkg/services/govmomi/bootstrap/data.go

@@ -0,0 +1,32 @@
+package bootstrap
+
+type VMBootstrapData struct {
+	value []byte
+	format Format
+}
+
+type Format string
+
+const (
+	// CloudConfig make the bootstrap data to be of cloud-config format
+	CloudConfig Format = "cloud-config"
+
+	// Ignition make the bootstrap data to be of Ignition format.
+	Ignition Format = "ignition"
+)
+
+func (vbd *VMBootstrapData) GetValue() []byte {
+	return vbd.value
+}
+
+func (vbd *VMBootstrapData) SetValue(value []byte) {
+	vbd.value = value
+}
+
+func (vbd *VMBootstrapData) SetFormat(format Format) {
+	vbd.format = format
+}
+
+func (vbd *VMBootstrapData) GetFormat() Format {
+	return vbd.format
+}

pkg/services/govmomi/constants.go

@@ -24,6 +24,9 @@ const (
 const (
 	guestInfoKeyMetadata    = "guestinfo.metadata"
 	guestInfoKeyMetadataEnc = "guestinfo.metadata.encoding"
+
+	ignitionKey         = "guestinfo.ignition.config.data"
+
 	guestInfoKeyUserdata    = "guestinfo.userdata"
 	guestInfoKeyUserdataEnc = "guestinfo.userdata.encoding"
 )

pkg/services/govmomi/create.go

@@ -18,13 +18,14 @@ package govmomi
 
 import (
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/context"
+	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/govmomi/bootstrap"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/govmomi/esxi"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/govmomi/vcenter"
 )
 
-func createVM(ctx *context.VMContext, bootstrapData []byte) error {
+func createVM(ctx *context.VMContext, data bootstrap.VMBootstrapData) error {
 	if ctx.Session.IsVC() {
-		return vcenter.Clone(ctx, bootstrapData)
+		return vcenter.Clone(ctx, data)
 	}
-	return esxi.Clone(ctx, bootstrapData)
+	return esxi.Clone(ctx, data)
 }

pkg/services/govmomi/esxi/clone.go

@@ -18,11 +18,11 @@ package esxi
 
 import (
 	"github.com/pkg/errors"
-
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/context"
+	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/govmomi/bootstrap"
 )
 
 // Clone kicks off a clone operation on ESXi to create a new virtual machine.
-func Clone(ctx *context.VMContext, bootstrapData []byte) error {
+func Clone(ctx *context.VMContext, data bootstrap.VMBootstrapData) error {
 	return errors.New("temporarily disabled esxi support")
 }

pkg/services/govmomi/extra/config.go

@@ -20,11 +20,40 @@ import (
 	"encoding/base64"
 
 	"github.com/vmware/govmomi/vim25/types"
+	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/govmomi/bootstrap"
 )
 
 // Config is data used with a VM's guestInfo RPC interface.
 type Config []types.BaseOptionValue
 
+
+const (
+	userdataKey         = "guestinfo.userdata"
+	userdataEncodingKey = "guestinfo.userdata.encoding"
+	ignitionKey         = "guestinfo.ignition.config.data"
+	ignitionEncodingKey = "guestinfo.ignition.config.data.encoding"
+)
+
+func getGuestInfoKeyByFormat(format bootstrap.Format) string {
+	switch format {
+	case bootstrap.CloudConfig:
+		return userdataKey
+	case bootstrap.Ignition:
+		return ignitionKey
+	}
+	return userdataKey
+}
+
+func getGuestInfoEncodingKey(format bootstrap.Format) string {
+	switch format {
+	case bootstrap.CloudConfig:
+		return userdataEncodingKey
+	case bootstrap.Ignition:
+		return ignitionEncodingKey
+	}
+	return userdataEncodingKey
+}
+
 // SetCustomVMXKeys sets the custom VMX keys as
 // OptionValues in extraConfig
 func (e *Config) SetCustomVMXKeys(customKeys map[string]string) error {
@@ -39,14 +68,14 @@ func (e *Config) SetCustomVMXKeys(customKeys map[string]string) error {
 
 // SetCloudInitUserData sets the cloud init user data at the key
 // "guestinfo.userdata" as a base64-encoded string.
-func (e *Config) SetCloudInitUserData(data []byte) error {
+func (e *Config) SetCloudInitUserData(data bootstrap.VMBootstrapData) error {
 	*e = append(*e,
 		&types.OptionValue{
-			Key:   "guestinfo.userdata",
-			Value: e.encode(data),
+			Key:   getGuestInfoKeyByFormat(data.GetFormat()),
+			Value: e.encode(data.GetValue()),
 		},
 		&types.OptionValue{
-			Key:   "guestinfo.userdata.encoding",
+			Key:   getGuestInfoEncodingKey(data.GetFormat()),
 			Value: "base64",
 		},
 	)

pkg/services/govmomi/service.go

@@ -19,6 +19,7 @@ package govmomi
 import (
 	"encoding/base64"
 	"fmt"
+	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/govmomi/bootstrap"
 
 	"github.com/pkg/errors"
 
@@ -122,10 +123,25 @@ func (vms *VMService) ReconcileVM(ctx *context.VMContext) (vm infrav1.VirtualMac
 		return vm, err
 	}
 
-	if ok, err := vms.reconcileMetadata(vmCtx); err != nil || !ok {
+	// Get the bootstrap data.
+	bootstrapData, err := vms.getBootstrapData(ctx)
+	if err != nil {
+		conditions.MarkFalse(ctx.VSphereVM, infrav1.VMProvisionedCondition, infrav1.CloningFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
 		return vm, err
 	}
 
+	if bootstrapData.GetFormat() == bootstrap.Ignition {
+		if ok, err := vms.reconcileIgnitionMetadata(vmCtx, bootstrapData); err != nil || !ok {
+			return vm, err
+		}
+	} else {
+		if ok, err := vms.reconcileMetadata(vmCtx); err != nil || !ok {
+			return vm, err
+		}
+	}
+
+
+
 	if ok, err := vms.reconcilePowerState(vmCtx); err != nil || !ok {
 		return vm, err
 	}
@@ -245,6 +261,39 @@ func (vms *VMService) reconcileMetadata(ctx *virtualMachineContext) (bool, error
 	return false, nil
 }
 
+func (vms *VMService) reconcileIgnitionMetadata(ctx *virtualMachineContext, bootstrapData bootstrap.VMBootstrapData) (bool, error) {
+	newMetadata, err := util.GetMachineMetadataIgnition(bootstrapData, ctx.VSphereVM.Name, *ctx.VSphereVM, ctx.State.Network...)
+	if err != nil {
+		return false, err
+	}
+	oldMetadata, err := vms.getMetadataIgnition(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// If the metadata is the same then return early.
+	if string(newMetadata) == oldMetadata {
+		ctx.Logger.Info("same config skipping")
+		return true, nil
+	}
+
+	ctx.Logger.Info("updating metadata")
+
+	newBootstrapData := bootstrap.VMBootstrapData{}
+	newBootstrapData.SetFormat(bootstrapData.GetFormat())
+	newBootstrapData.SetValue(newMetadata)
+	taskRef, err := vms.setIgnitionMetadata(ctx, newBootstrapData)
+	if err != nil {
+		return false, errors.Wrapf(err, "unable to set metadata on vm %s", ctx)
+	}
+
+	ctx.VSphereVM.Status.TaskRef = taskRef
+	ctx.Logger.Info("wait for VM metadata to be updated")
+	return false, nil
+}
+
+
+
 func (vms *VMService) reconcilePowerState(ctx *virtualMachineContext) (bool, error) {
 	powerState, err := vms.getPowerState(ctx)
 	if err != nil {
@@ -347,6 +396,50 @@ func (vms *VMService) getMetadata(ctx *virtualMachineContext) (string, error) {
 	return string(metadataBuf), nil
 }
 
+func (vms *VMService) getMetadataIgnition(ctx *virtualMachineContext) (string, error) {
+	var (
+		obj mo.VirtualMachine
+
+		pc    = property.DefaultCollector(ctx.Session.Client.Client)
+		props = []string{"config.extraConfig"}
+	)
+
+	if err := pc.RetrieveOne(ctx, ctx.Ref, props, &obj); err != nil {
+		return "", errors.Wrapf(err, "unable to fetch props %v for vm %s", props, ctx)
+	}
+	if obj.Config == nil {
+		return "", nil
+	}
+
+	var metadataBase64 string
+	for _, ec := range obj.Config.ExtraConfig {
+		if optVal := ec.GetOptionValue(); optVal != nil {
+			// TODO(akutz) Using a switch instead of if in case we ever
+			//             want to check the metadata encoding as well.
+			//             Since the image stamped images always use
+			//             base64, it should be okay to not check.
+			// nolint
+			switch optVal.Key {
+			case ignitionKey:
+				if v, ok := optVal.Value.(string); ok {
+					metadataBase64 = v
+				}
+			}
+		}
+	}
+
+	if metadataBase64 == "" {
+		return "", nil
+	}
+
+	metadataBuf, err := base64.StdEncoding.DecodeString(metadataBase64)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to decode metadata for %s", ctx)
+	}
+
+	return string(metadataBuf), nil
+}
+
 func (vms *VMService) setMetadata(ctx *virtualMachineContext, metadata []byte) (string, error) {
 	var extraConfig extra.Config
 	if err := extraConfig.SetCloudInitMetadata(metadata); err != nil {
@@ -363,6 +456,23 @@ func (vms *VMService) setMetadata(ctx *virtualMachineContext, metadata []byte) (
 	return task.Reference().Value, nil
 }
 
+
+func (vms *VMService) setIgnitionMetadata(ctx *virtualMachineContext, data bootstrap.VMBootstrapData) (string, error) {
+	var extraConfig extra.Config
+	if err := extraConfig.SetCloudInitUserData(data); err != nil {
+		return "", errors.Wrapf(err, "unable to set metadata on vm %s", ctx)
+	}
+
+	task, err := ctx.Obj.Reconfigure(ctx, types.VirtualMachineConfigSpec{
+		ExtraConfig: extraConfig,
+	})
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to set metadata on vm %s", ctx)
+	}
+
+	return task.Reference().Value, nil
+}
+
 func (vms *VMService) getNetworkStatus(ctx *virtualMachineContext) ([]infrav1.NetworkStatus, error) {
 	allNetStatus, err := net.GetNetworkStatus(ctx, ctx.Session.Client.Client, ctx.Ref)
 	if err != nil {
@@ -381,10 +491,23 @@ func (vms *VMService) getNetworkStatus(ctx *virtualMachineContext) ([]infrav1.Ne
 	return apiNetStatus, nil
 }
 
-func (vms *VMService) getBootstrapData(ctx *context.VMContext) ([]byte, error) {
+
+
+func convertStringToFormat(input string) bootstrap.Format {
+	switch input {
+	case string(bootstrap.CloudConfig), "":
+		return bootstrap.CloudConfig
+	case string(bootstrap.Ignition):
+		return bootstrap.Ignition
+	}
+	return bootstrap.CloudConfig
+}
+
+func (vms *VMService) getBootstrapData(ctx *context.VMContext) (bootstrap.VMBootstrapData, error) {
+	bootstrapData := bootstrap.VMBootstrapData{}
 	if ctx.VSphereVM.Spec.BootstrapRef == nil {
 		ctx.Logger.Info("VM has no bootstrap data")
-		return nil, nil
+		return bootstrapData, nil
 	}
 
 	secret := &corev1.Secret{}
@@ -393,13 +516,17 @@ func (vms *VMService) getBootstrapData(ctx *context.VMContext) ([]byte, error) {
 		Name:      ctx.VSphereVM.Spec.BootstrapRef.Name,
 	}
 	if err := ctx.Client.Get(ctx, secretKey, secret); err != nil {
-		return nil, errors.Wrapf(err, "failed to retrieve bootstrap data secret for %s", ctx)
+		return bootstrapData, errors.Wrapf(err, "failed to retrieve bootstrap data secret for %s", ctx)
 	}
 
 	value, ok := secret.Data["value"]
 	if !ok {
-		return nil, errors.New("error retrieving bootstrap data: secret value key is missing")
+		return bootstrapData, errors.New("error retrieving bootstrap data: secret value key is missing")
 	}
 
-	return value, nil
+	format := secret.Data["format"]
+	bootstrapData.SetFormat(convertStringToFormat(string(format)))
+	bootstrapData.SetValue(value)
+
+	return bootstrapData, nil
 }