diff --git a/charts/vm-import-controller/templates/rbac.yaml b/charts/vm-import-controller/templates/rbac.yaml index 46ec034..ef09639 100644 --- a/charts/vm-import-controller/templates/rbac.yaml +++ b/charts/vm-import-controller/templates/rbac.yaml @@ -26,7 +26,14 @@ rules: resources: - customresourcedefinitions verbs: - - "*" + - "*" +- apiGroups: + - "k8s.cni.cncf.io" + resources: + - "network-attachment-definitions" + verbs: + - "list" + - "watch" - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index e42a949..b95c870 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gophercloud/gophercloud v1.11.0 github.com/harvester/harvester v1.3.0 + github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0 github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.32.0 github.com/ory/dockertest/v3 v3.9.1 @@ -65,7 +66,6 @@ require ( github.com/imdario/mergo v0.3.15 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0 // indirect github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index c63c294..c764835 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -5,6 +5,7 @@ import ( "time" harvester "github.com/harvester/harvester/pkg/generated/controllers/harvesterhci.io" + cniv1 "github.com/harvester/harvester/pkg/generated/controllers/k8s.cni.cncf.io" "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io" "github.com/rancher/lasso/pkg/cache" "github.com/rancher/lasso/pkg/client" @@ -86,13 +87,20 @@ func Register(ctx context.Context, restConfig *rest.Config) error { scCache := storageFactory.Storage().V1().StorageClass().Cache() + cniFactory, err := cniv1.NewFactoryFromConfigWithOptions(restConfig, &core.FactoryOptions{ + SharedControllerFactory: scf, + }) + if err != nil { + return err + } + sc.RegisterVmwareController(ctx, migrationFactory.Migration().V1beta1().VmwareSource(), coreFactory.Core().V1().Secret()) sc.RegisterOpenstackController(ctx, migrationFactory.Migration().V1beta1().OpenstackSource(), coreFactory.Core().V1().Secret()) sc.RegisterVMImportController(ctx, migrationFactory.Migration().V1beta1().VmwareSource(), migrationFactory.Migration().V1beta1().OpenstackSource(), coreFactory.Core().V1().Secret(), migrationFactory.Migration().V1beta1().VirtualMachineImport(), harvesterFactory.Harvesterhci().V1beta1().VirtualMachineImage(), kubevirtFactory.Kubevirt().V1().VirtualMachine(), - coreFactory.Core().V1().PersistentVolumeClaim(), scCache) + coreFactory.Core().V1().PersistentVolumeClaim(), scCache, cniFactory.K8s().V1().NetworkAttachmentDefinition().Cache()) - return start.All(ctx, 1, migrationFactory, coreFactory, harvesterFactory, kubevirtFactory, storageFactory) + return start.All(ctx, 1, migrationFactory, coreFactory, harvesterFactory, kubevirtFactory, storageFactory, cniFactory) } diff --git a/pkg/controllers/migration/openstack.go b/pkg/controllers/migration/openstack.go index a798813..2b6c139 100644 --- a/pkg/controllers/migration/openstack.go +++ b/pkg/controllers/migration/openstack.go @@ -33,12 +33,16 @@ func RegisterOpenstackController(ctx context.Context, os migrationController.Ope os.OnChange(ctx, "openstack-migration-change", oHandler.OnSourceChange) } -func (h *openstackHandler) OnSourceChange(key string, o *migration.OpenstackSource) (*migration.OpenstackSource, error) { +func (h *openstackHandler) OnSourceChange(_ string, o *migration.OpenstackSource) (*migration.OpenstackSource, error) { if o == nil || o.DeletionTimestamp != nil { return o, nil } - logrus.Infof("reconcilling openstack soure :%s", key) + logrus.WithFields(logrus.Fields{ + "kind": o.Kind, + "name": o.Name, + "namespace": o.Namespace, + }).Info("Reconciling migration source") if o.Status.Status != migration.ClusterReady { // process migration logic secretObj, err := h.secret.Get(o.Spec.Credentials.Namespace, o.Spec.Credentials.Name, metav1.GetOptions{}) @@ -59,7 +63,7 @@ func (h *openstackHandler) OnSourceChange(key string, o *migration.OpenstackSour "name": o.Name, "namespace": o.Namespace, "err": err, - }).Error("failed to verfiy client for openstack migration") + }).Error("failed to verify client for openstack migration") conds := []common.Condition{ { Type: migration.ClusterErrorCondition, diff --git a/pkg/controllers/migration/virtualmachine.go b/pkg/controllers/migration/virtualmachine.go index e9652fd..5fa48b5 100644 --- a/pkg/controllers/migration/virtualmachine.go +++ b/pkg/controllers/migration/virtualmachine.go @@ -19,6 +19,7 @@ import ( harvesterv1beta1 "github.com/harvester/harvester/pkg/apis/harvesterhci.io/v1beta1" harvester "github.com/harvester/harvester/pkg/generated/controllers/harvesterhci.io/v1beta1" + ctlcniv1 "github.com/harvester/harvester/pkg/generated/controllers/k8s.cni.cncf.io/v1" kubevirtv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" "github.com/harvester/harvester/pkg/ref" coreControllers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" @@ -46,13 +47,16 @@ type VirtualMachineOperations interface { // Any image format conversion will be performed by the VM Operation ExportVirtualMachine(vm *migration.VirtualMachineImport) error - // PowerOffVirtualMachine is responsible for the powering off the virtualmachine + // PowerOffVirtualMachine is responsible for the powering off the virtual machine PowerOffVirtualMachine(vm *migration.VirtualMachineImport) error // IsPoweredOff will check the status of VM Power and return true if machine is powered off IsPoweredOff(vm *migration.VirtualMachineImport) (bool, error) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*kubevirt.VirtualMachine, error) + + // PreFlightChecks checks the cluster specific configurations. + PreFlightChecks(vm *migration.VirtualMachineImport) error } type virtualMachineHandler struct { @@ -65,10 +69,11 @@ type virtualMachineHandler struct { kubevirt kubevirtv1.VirtualMachineController pvc coreControllers.PersistentVolumeClaimController sc storageControllers.StorageClassCache + nadCache ctlcniv1.NetworkAttachmentDefinitionCache } func RegisterVMImportController(ctx context.Context, vmware migrationController.VmwareSourceController, openstack migrationController.OpenstackSourceController, - secret coreControllers.SecretController, importVM migrationController.VirtualMachineImportController, vmi harvester.VirtualMachineImageController, kubevirt kubevirtv1.VirtualMachineController, pvc coreControllers.PersistentVolumeClaimController, scCache storageControllers.StorageClassCache) { + secret coreControllers.SecretController, importVM migrationController.VirtualMachineImportController, vmi harvester.VirtualMachineImageController, kubevirt kubevirtv1.VirtualMachineController, pvc coreControllers.PersistentVolumeClaimController, scCache storageControllers.StorageClassCache, nadCache ctlcniv1.NetworkAttachmentDefinitionCache) { vmHandler := &virtualMachineHandler{ ctx: ctx, @@ -80,6 +85,7 @@ func RegisterVMImportController(ctx context.Context, vmware migrationController. kubevirt: kubevirt, pvc: pvc, sc: scCache, + nadCache: nadCache, } relatedresource.Watch(ctx, "virtualmachineimage-change", vmHandler.ReconcileVMI, importVM, vmi) @@ -237,6 +243,55 @@ func (h *virtualMachineHandler) preFlightChecks(vm *migration.VirtualMachineImpo return fmt.Errorf("source network %s appears multiple times in vm spec", network.SourceNetwork) } + // Validate the destination network configuration. + for _, nm := range vm.Spec.Mapping { + // The destination network supports the following format: + // - + // - / + // See `MultusNetwork.NetworkName` for more details. + parts := strings.Split(nm.DestinationNetwork, "/") + switch len(parts) { + case 1: + // If namespace is not specified, `VirtualMachineImport` namespace is assumed. + parts = append([]string{vm.Namespace}, parts[0]) + fallthrough + case 2: + _, err := h.nadCache.Get(parts[0], parts[1]) + if err != nil { + logrus.WithFields(logrus.Fields{ + "name": vm.Name, + "namespace": vm.Namespace, + "spec.sourcecluster.kind": vm.Spec.SourceCluster.Kind, + }).Errorf("Failed to get destination network '%s/%s': %v", + parts[0], parts[1], err) + return err + } + default: + logrus.WithFields(logrus.Fields{ + "name": vm.Name, + "namespace": vm.Namespace, + "spec.sourcecluster.kind": vm.Spec.SourceCluster.Kind, + }).Errorf("Invalid destination network '%s'", nm.DestinationNetwork) + return fmt.Errorf("invalid destination network '%s'", nm.DestinationNetwork) + } + } + + // Validate the source network as part of the source cluster preflight + // checks. + vmo, err := h.generateVMO(vm) + if err != nil { + return fmt.Errorf("error generating VMO in preFlightChecks: %v", err) + } + err = vmo.PreFlightChecks(vm) + if err != nil { + logrus.WithFields(logrus.Fields{ + "name": vm.Name, + "namespace": vm.Namespace, + "spec.sourcecluster.kind": vm.Spec.SourceCluster.Kind, + }).Errorf("Failed to perform source cluster specific preflight checks: %v", err) + return err + } + return nil } diff --git a/pkg/controllers/migration/vmware.go b/pkg/controllers/migration/vmware.go index 9583d1e..07642af 100644 --- a/pkg/controllers/migration/vmware.go +++ b/pkg/controllers/migration/vmware.go @@ -33,12 +33,16 @@ func RegisterVmwareController(ctx context.Context, vc migrationController.Vmware vc.OnChange(ctx, "vmware-migration-change", vHandler.OnSourceChange) } -func (h *vmwareHandler) OnSourceChange(key string, v *migration.VmwareSource) (*migration.VmwareSource, error) { +func (h *vmwareHandler) OnSourceChange(_ string, v *migration.VmwareSource) (*migration.VmwareSource, error) { if v == nil || v.DeletionTimestamp != nil { return v, nil } - logrus.Infof("reoncilling vmware migration %s", key) + logrus.WithFields(logrus.Fields{ + "kind": v.Kind, + "name": v.Name, + "namespace": v.Namespace, + }).Info("Reconciling migration source") if v.Status.Status != migration.ClusterReady { secretObj, err := h.secret.Get(v.Spec.Credentials.Namespace, v.Spec.Credentials.Name, metav1.GetOptions{}) if err != nil { @@ -57,7 +61,7 @@ func (h *vmwareHandler) OnSourceChange(key string, v *migration.VmwareSource) (* "name": v.Name, "namespace": v.Namespace, "err": err, - }).Error("failed to verfiy client for vmware migration") + }).Error("failed to verify client for vmware migration") // unable to find specific datacenter conds := []common.Condition{ { diff --git a/pkg/source/openstack/client.go b/pkg/source/openstack/client.go index 8ed1f6f..8b66bc6 100644 --- a/pkg/source/openstack/client.go +++ b/pkg/source/openstack/client.go @@ -22,6 +22,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" "github.com/gophercloud/gophercloud/openstack/imageservice/v2/imagedata" "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" + "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -176,6 +177,16 @@ func (c *Client) Verify() error { return nil } +func (c *Client) PreFlightChecks(vm *migration.VirtualMachineImport) (err error) { + for _, nm := range vm.Spec.Mapping { + _, err := networks.Get(c.computeClient, nm.SourceNetwork).Extract() + if err != nil { + return fmt.Errorf("error getting source network '%s': %v", nm.SourceNetwork, err) + } + } + return nil +} + func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) error { vmObj, err := c.findVM(vm.Spec.VirtualMachineName) if err != nil { diff --git a/pkg/source/vmware/client.go b/pkg/source/vmware/client.go index e652a64..2d28f21 100644 --- a/pkg/source/vmware/client.go +++ b/pkg/source/vmware/client.go @@ -123,6 +123,23 @@ func (c *Client) Verify() error { return nil } +func (c *Client) PreFlightChecks(vm *migration.VirtualMachineImport) (err error) { + f := find.NewFinder(c.Client.Client, true) + dc := c.dc + if !strings.HasPrefix(c.dc, "/") { + dc = fmt.Sprintf("/%s", c.dc) + } + for _, nm := range vm.Spec.Mapping { + // The path looks like `//network/`. + path := filepath.Join(dc, "/network", nm.SourceNetwork) + _, err := f.Network(c.ctx, path) + if err != nil { + return fmt.Errorf("error getting source network '%s': %v", nm.SourceNetwork, err) + } + } + return nil +} + func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) (err error) { var ( tmpPath string diff --git a/tests/integration/suite_test.go b/tests/integration/suite_test.go index fac1b53..1e0bb99 100644 --- a/tests/integration/suite_test.go +++ b/tests/integration/suite_test.go @@ -20,6 +20,8 @@ import ( log "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + k8scnicncfiov1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + migration "github.com/harvester/vm-import-controller/pkg/apis/migration.harvesterhci.io/v1beta1" "github.com/harvester/vm-import-controller/pkg/controllers" "github.com/harvester/vm-import-controller/pkg/server" @@ -81,8 +83,9 @@ var _ = BeforeSuite(func() { testEnv = &envtest.Environment{} if !useExisting { - crds, err := setup.GenerateKubeVirtCRD() + crds, err := setup.GenerateCRD() Expect(err).ToNot(HaveOccurred()) + testEnv.CRDInstallOptions = envtest.CRDInstallOptions{ CRDs: crds, } @@ -107,6 +110,9 @@ var _ = BeforeSuite(func() { err = kubevirtv1.AddToScheme(scheme) Expect(err).NotTo(HaveOccurred()) + err = k8scnicncfiov1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/tests/setup/harvester_mock.go b/tests/setup/harvester_mock.go index e7b7bb7..2dcaf03 100644 --- a/tests/setup/harvester_mock.go +++ b/tests/setup/harvester_mock.go @@ -4,6 +4,7 @@ import ( "context" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components" @@ -11,8 +12,8 @@ import ( "github.com/harvester/harvester/pkg/util/crd" ) -// InstallCRD will install the core harvester CRD's -// copied from harvester/harvester/pkg/data/crd.go +// InstallCRD will install the core Harvester CRD's +// partly copied from harvester/harvester/pkg/data/crd.go func InstallCRD(ctx context.Context, cfg *rest.Config) error { factory, err := crd.NewFactoryFromClient(ctx, cfg) if err != nil { @@ -37,10 +38,23 @@ func InstallCRD(ctx context.Context, cfg *rest.Config) error { BatchWait() } +// GenerateCRD will generate other required CRD's +func GenerateCRD() ([]*extv1.CustomResourceDefinition, error) { + kubeVirtCrds, err := generateKubeVirtCRD() + if err != nil { + return nil, err + } + k8sCniCncfIoCrds, err := generateK8sCniCncfIoCRD() + if err != nil { + return nil, err + } + return append(kubeVirtCrds, k8sCniCncfIoCrds...), nil +} + type kubeVirtCRDGenerator func() (*extv1.CustomResourceDefinition, error) -// GenerateKubeVirtCRD's will generate kubevirt CRDs -func GenerateKubeVirtCRD() ([]*extv1.CustomResourceDefinition, error) { +// generateKubeVirtCRD will generate kubevirt CRDs +func generateKubeVirtCRD() ([]*extv1.CustomResourceDefinition, error) { v := []kubeVirtCRDGenerator{ components.NewVirtualMachineCrd, components.NewVirtualMachineInstanceCrd, @@ -65,3 +79,52 @@ func GenerateKubeVirtCRD() ([]*extv1.CustomResourceDefinition, error) { return results, nil } + +// generateK8sCniCncfIoCRD will generate k8s.cni.cncf.io CRDs +func generateK8sCniCncfIoCRD() ([]*extv1.CustomResourceDefinition, error) { + var results []*extv1.CustomResourceDefinition + results = append(results, + // See https://github.com/k8snetworkplumbingwg/network-attachment-definition-client/blob/v1.3.0/artifacts/networks-crd.yaml + &extv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: extv1.SchemeGroupVersion.String(), + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "network-attachment-definitions.k8s.cni.cncf.io", + }, + Spec: extv1.CustomResourceDefinitionSpec{ + Group: "k8s.cni.cncf.io", + Scope: "Namespaced", + Names: extv1.CustomResourceDefinitionNames{ + Plural: "network-attachment-definitions", + Singular: "network-attachment-definition", + Kind: "NetworkAttachmentDefinition", + ShortNames: []string{"net-attach-def"}, + }, + Versions: []extv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Schema: &extv1.CustomResourceValidation{ + OpenAPIV3Schema: &extv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]extv1.JSONSchemaProps{ + "spec": { + Type: "object", + Properties: map[string]extv1.JSONSchemaProps{ + "config": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }) + return results, nil +}