Additional validation to skip import jobs without disks or invalid vm names

bumped up harvester/harvester to v1.1.0-rc2

changed vmware client to check VM state before attempting poweroff, add closure for ovf client to fix Operation timed out errors

fine tuned integration tests

changed logging message
This commit is contained in:
Gaurav Mehta 2022-10-05 06:11:40 +00:00
parent 57376e3d34
commit 88b5df8ae6
8 changed files with 302 additions and 1429 deletions

100
go.mod
View File

@ -3,13 +3,14 @@ module github.com/harvester/vm-import-controller
go 1.18
require (
github.com/google/uuid v1.3.0
github.com/gophercloud/gophercloud v0.7.0
github.com/harvester/harvester v1.0.2
github.com/harvester/harvester v1.1.0-rc2
github.com/onsi/ginkgo/v2 v2.1.4
github.com/onsi/gomega v1.19.0
github.com/ory/dockertest/v3 v3.9.1
github.com/rancher/lasso v0.0.0-20210709145333-6c6cd7fd6607
github.com/rancher/wrangler v1.0.0
github.com/rancher/lasso v0.0.0-20220519004610-700f167d8324
github.com/rancher/wrangler v1.0.1-0.20220520195731-8eeded9bae2a
github.com/sirupsen/logrus v1.9.0
github.com/stretchr/testify v1.7.1
github.com/vmware/govmomi v0.29.0
@ -18,8 +19,8 @@ require (
k8s.io/apiextensions-apiserver v0.24.2
k8s.io/apimachinery v0.24.2
k8s.io/client-go v12.0.0+incompatible
kubevirt.io/api v0.0.0-20220430221853-33880526e414
kubevirt.io/kubevirt v0.49.0
kubevirt.io/api v0.54.0
kubevirt.io/kubevirt v0.54.0
sigs.k8s.io/controller-runtime v0.12.2
)
@ -29,14 +30,16 @@ require (
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/containerd/containerd v1.6.6 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/coreos/prometheus-operator v0.38.1-0.20200424145508-7e176fda06cc // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.14+incompatible // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.7+incompatible // indirect
github.com/docker/docker v20.10.12+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/emicklei/go-restful v2.15.0+incompatible // indirect
@ -49,27 +52,25 @@ require (
github.com/go-logr/zapr v1.2.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/spec v0.20.3 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200331171230-d50e42f2b669 // indirect
github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 // indirect
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mitchellh/mapstructure v1.4.2 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
@ -82,6 +83,11 @@ require (
github.com/pborman/uuid v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@ -89,7 +95,6 @@ require (
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
@ -100,12 +105,14 @@ require (
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gotest.tools v2.2.0+incompatible // indirect
k8s.io/code-generator v0.24.2 // indirect
k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect
k8s.io/klog/v2 v2.60.1 // indirect
@ -114,7 +121,8 @@ require (
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
kubevirt.io/client-go v0.49.0 // indirect
kubevirt.io/containerized-data-importer-api v1.47.0 // indirect
kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 // indirect
kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
@ -127,35 +135,35 @@ replace (
github.com/operator-framework/operator-lifecycle-manager => github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190128024246-5eb7ae5bdb7a
github.com/rancher/rancher/pkg/apis => github.com/rancher/rancher/pkg/apis v0.0.0-20211208233239-77392a65423d
github.com/rancher/rancher/pkg/client => github.com/rancher/rancher/pkg/client v0.0.0-20211208233239-77392a65423d
k8s.io/api => k8s.io/api v0.20.2 // Dropped to v0.20.2 to handle kubevirt deps for installing CRDs
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.5
k8s.io/apimachinery => k8s.io/apimachinery v0.21.5
k8s.io/apiserver => k8s.io/apiserver v0.21.5
k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.5
k8s.io/client-go => k8s.io/client-go v0.20.2 // Dropped to v0.20.2 to handle kubevirt deps for installing CRDs
k8s.io/cloud-provider => k8s.io/cloud-provider v0.21.5
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.5
k8s.io/code-generator => k8s.io/code-generator v0.21.5
k8s.io/component-base => k8s.io/component-base v0.21.5
k8s.io/component-helpers => k8s.io/component-helpers v0.21.5
k8s.io/controller-manager => k8s.io/controller-manager v0.21.5
k8s.io/cri-api => k8s.io/cri-api v0.21.5
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.21.5
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.21.5
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.21.5
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7
k8s.io/kube-proxy => k8s.io/kube-proxy v0.21.5
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.21.5
k8s.io/kubectl => k8s.io/kubectl v0.21.5
k8s.io/kubelet => k8s.io/kubelet v0.21.5
k8s.io/kubernetes => k8s.io/kubernetes v1.21.5
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.21.5
k8s.io/metrics => k8s.io/metrics v0.21.5
k8s.io/mount-utils => k8s.io/mount-utils v0.21.5
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.21.5
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.5
kubevirt.io/api => github.com/kubevirt/api v0.49.0
kubevirt.io/client-go => github.com/kubevirt/client-go v0.49.0
kubevirt.io/containerized-data-importer-api => kubevirt.io/containerized-data-importer-api v1.41.0
k8s.io/api => k8s.io/api v0.23.7 // Dropped to v0.20.2 to handle kubevirt deps for installing CRDs
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.7
k8s.io/apimachinery => k8s.io/apimachinery v0.23.7
k8s.io/apiserver => k8s.io/apiserver v0.23.7
k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.7
k8s.io/client-go => k8s.io/client-go v0.23.7 // Dropped to v0.20.2 to handle kubevirt deps for installing CRDs
k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.7
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.7
k8s.io/code-generator => k8s.io/code-generator v0.23.7
k8s.io/component-base => k8s.io/component-base v0.23.7
k8s.io/component-helpers => k8s.io/component-helpers v0.23.7
k8s.io/controller-manager => k8s.io/controller-manager v0.23.7
k8s.io/cri-api => k8s.io/cri-api v0.23.7
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.7
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.7
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.7
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf
k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.7
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.7
k8s.io/kubectl => k8s.io/kubectl v0.23.7
k8s.io/kubelet => k8s.io/kubelet v0.23.7
k8s.io/kubernetes => k8s.io/kubernetes v1.23.7
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.7
k8s.io/metrics => k8s.io/metrics v0.23.7
k8s.io/mount-utils => k8s.io/mount-utils v0.23.7
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.7
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.7
kubevirt.io/api => github.com/kubevirt/api v0.54.0
kubevirt.io/client-go => github.com/kubevirt/client-go v0.54.0
kubevirt.io/containerized-data-importer-api => kubevirt.io/containerized-data-importer-api v1.47.0
sigs.k8s.io/structured-merge-diff => sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2
)

1477
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -66,10 +66,12 @@ const (
DiskImagesFailed ImportStatus = "diskImageFailed"
VirtualMachineCreated ImportStatus = "virtualMachineCreated"
VirtualMachineRunning ImportStatus = "virtualMachineRunning"
VirtualMachineInvalid ImportStatus = "virtualMachineInvalid"
VirtualMachinePoweringOff condition.Cond = "VMPoweringOff"
VirtualMachinePoweredOff condition.Cond = "VMPoweredOff"
VirtualMachineExported condition.Cond = "VMExported"
VirtualMachineImageSubmitted condition.Cond = "VirtualMachineImageSubmitted"
VirtualMachineImageReady condition.Cond = "VirtualMachineImageReady"
VirtualMachineImageFailed condition.Cond = "VirtualMachineImageFailed"
NotValidDNS1123Label string = "not a valid DNS1123 label"
)

View File

@ -10,6 +10,7 @@ import (
"time"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/validation"
kubevirt "kubevirt.io/api/core/v1"
harvesterv1beta1 "github.com/harvester/harvester/pkg/apis/harvesterhci.io/v1beta1"
@ -78,18 +79,25 @@ func RegisterVMImportController(ctx context.Context, vmware migrationController.
importVM.OnChange(ctx, "virtualmachine-import-job-change", vmHandler.OnVirtualMachineChange)
}
func (h *virtualMachineHandler) OnVirtualMachineChange(key string, vm *migration.VirtualMachineImport) (*migration.VirtualMachineImport, error) {
func (h *virtualMachineHandler) OnVirtualMachineChange(key string, vmObj *migration.VirtualMachineImport) (*migration.VirtualMachineImport, error) {
if vm == nil || vm.DeletionTimestamp != nil {
return vm, nil
if vmObj == nil || vmObj.DeletionTimestamp != nil {
return nil, nil
}
vm := vmObj.DeepCopy()
switch vm.Status.Status {
case "": // run preflight checks and make vm ready for import
err := h.preFlightChecks(vm)
if err != nil {
if err.Error() == migration.NotValidDNS1123Label {
logrus.Errorf("vm migration target %s in VM %s in namespace %s is not RFC 1123 compliant", vm.Spec.VirtualMachineName, vm.Name, vm.Namespace)
vm.Status.Status = migration.VirtualMachineInvalid
h.importVM.UpdateStatus(vm)
} else {
return vm, err
}
}
vm.Status.Status = migration.SourceReady
return h.importVM.UpdateStatus(vm)
case migration.SourceReady: //vm migration is valid and ready. trigger migration specific import
@ -103,6 +111,14 @@ func (h *virtualMachineHandler) OnVirtualMachineChange(key string, vm *migration
return h.importVM.UpdateStatus(vm)
case migration.DisksExported: // prepare and add routes for disks to be used for VirtualMachineImage CRD
orgStatus := vm.Status.DeepCopy()
// If VM has no disks associated ignore the VM
if len(orgStatus.DiskImportStatus) == 0 {
logrus.Errorf("Imported VM %s in namespace %s, has no disks, being marked as invalid and will be ignored", vm.Name, vm.Namespace)
vm.Status.Status = migration.VirtualMachineInvalid
return h.importVM.UpdateStatus(vm)
}
err := h.createVirtualMachineImages(vm)
if err != nil {
// check if any disks have been updated. We need to save this info to eventually reconcile the VMI creation
@ -192,6 +208,9 @@ func (h *virtualMachineHandler) OnVirtualMachineChange(key string, vm *migration
case migration.VirtualMachineRunning:
logrus.Infof("vm %s in namespace %v imported successfully", vm.Name, vm.Namespace)
return vm, h.tidyUpObjects(vm)
case migration.VirtualMachineInvalid:
logrus.Infof("vm %s in namespace %v has an invalid spec", vm.Name, vm.Namespace)
return vm, nil
}
return vm, nil
@ -199,6 +218,11 @@ func (h *virtualMachineHandler) OnVirtualMachineChange(key string, vm *migration
// preFlightChecks is used to validate that the associate sources and VM migration references are valid
func (h *virtualMachineHandler) preFlightChecks(vm *migration.VirtualMachineImport) error {
if errs := validation.IsDNS1123Label(vm.Spec.VirtualMachineName); len(errs) != 0 {
return fmt.Errorf(migration.NotValidDNS1123Label)
}
if vm.Spec.SourceCluster.APIVersion != "migration.harvesterhci.io/v1beta1" {
return fmt.Errorf("expected migration cluster apiversion to be migration.harvesterhci.io/v1beta1 but got %s", vm.Spec.SourceCluster.APIVersion)
}

View File

@ -146,6 +146,7 @@ func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) error
u := lease.StartUpdater(c.ctx, info)
defer u.Done()
defer lease.Complete(c.ctx)
for _, i := range info.Items {
// ignore iso and nvram disks
@ -198,10 +199,19 @@ func (c *Client) PowerOffVirtualMachine(vm *migration.VirtualMachineImport) erro
return fmt.Errorf("error finding vm in PowerOffVirtualMachine: %v", err)
}
ok, err := c.IsPoweredOff(vm)
if err != nil {
return err
}
if !ok {
_, err = vmObj.PowerOff(c.ctx)
return err
}
return nil
}
func (c *Client) IsPoweredOff(vm *migration.VirtualMachineImport) (bool, error) {
vmObj, err := c.findVM(vm.Spec.Folder, vm.Spec.VirtualMachineName)
if err != nil {

View File

@ -0,0 +1,100 @@
package integration
import (
"fmt"
migration "github.com/harvester/vm-import-controller/pkg/apis/migration.harvesterhci.io/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
var _ = Describe("perform valid dns names", func() {
var creds *corev1.Secret
var vcsim *migration.VmwareSource
var vm *migration.VirtualMachineImport
BeforeEach(func() {
creds = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "vcsim-creds",
Namespace: "default",
},
StringData: map[string]string{
"username": "user",
"password": "pass",
},
}
vcsim = &migration.VmwareSource{
ObjectMeta: metav1.ObjectMeta{
Name: "local-vm-validation",
Namespace: "default",
},
Spec: migration.VmwareSourceSpec{
EndpointAddress: "",
Datacenter: "DC0",
Credentials: corev1.SecretReference{
Name: creds.Name,
Namespace: creds.Namespace,
},
},
}
vm = &migration.VirtualMachineImport{
ObjectMeta: metav1.ObjectMeta{
Name: "vm-validation",
Namespace: "default",
},
Spec: migration.VirtualMachineImportSpec{
SourceCluster: corev1.ObjectReference{
Name: vcsim.Name,
Namespace: vcsim.Namespace,
Kind: vcsim.Kind,
APIVersion: vcsim.APIVersion,
},
VirtualMachineName: "someRandomName",
Folder: "",
},
}
err := k8sClient.Create(ctx, creds)
Expect(err).ToNot(HaveOccurred())
vcsim.Spec.EndpointAddress = fmt.Sprintf("https://localhost:%s/sdk", vcsimPort)
err = k8sClient.Create(ctx, vcsim)
Expect(err).ToNot(HaveOccurred())
err = k8sClient.Create(ctx, vm)
Expect(err).ToNot(HaveOccurred())
})
It("check virtualmachine import has failed", func() {
By("checking state of VM Import", func() {
Eventually(func() error {
vmObj := &migration.VirtualMachineImport{}
err := k8sClient.Get(ctx, types.NamespacedName{Name: vm.Name,
Namespace: vm.Namespace}, vmObj)
if err != nil {
return err
}
if vmObj.Status.Status == migration.VirtualMachineInvalid {
return nil
}
return fmt.Errorf("waiiting for vm obj to be marked invalid")
}, "30s", "5s").ShouldNot(HaveOccurred())
})
})
AfterEach(func() {
err := k8sClient.Delete(ctx, creds)
Expect(err).ToNot(HaveOccurred())
err = k8sClient.Delete(ctx, vcsim)
Expect(err).ToNot(HaveOccurred())
err = k8sClient.Delete(ctx, vm)
Expect(err).ToNot(HaveOccurred())
})
})

View File

@ -116,6 +116,7 @@ var _ = Describe("test openstack export/import integration", func() {
}, "300s", "10s").ShouldNot(HaveOccurred())
})
// can take upto 5 mins for the VM to be marked as running
By("checking that the virtualmachineimage ownership has been removed", func() {
Eventually(func() error {
v := &migration.VirtualMachineImport{}
@ -138,7 +139,7 @@ var _ = Describe("test openstack export/import integration", func() {
}
return nil
}, "300s", "10s").ShouldNot(HaveOccurred())
}, "600s", "30s").ShouldNot(HaveOccurred())
})
})

View File

@ -147,6 +147,7 @@ var _ = Describe("test vmware export/import integration", func() {
}, "300s", "10s").ShouldNot(HaveOccurred())
})
// can take upto 5 mins for the VM to be marked as running
By("checking that the virtualmachineimage ownership has been removed", func() {
Eventually(func() error {
v := &migration.VirtualMachineImport{}
@ -169,7 +170,7 @@ var _ = Describe("test vmware export/import integration", func() {
}
return nil
}, "300s", "10s").ShouldNot(HaveOccurred())
}, "600s", "30s").ShouldNot(HaveOccurred())
})
})