added debug logging and also change behaviour of controller when vm export from source fails to stop rather than keep trying

added more debug info

close lease earlier

stage vmware migration improvements

improve openstack migration to check uefi/tpm/secureboot from associated image

improve firmware lookup for openstack, and add ability to specify custom storage classes

fix storage class caching issue

drop need for sync

fix imports
This commit is contained in:
Gaurav Mehta 2024-05-02 11:39:35 +10:00
parent f9d329d1ab
commit bb1cef97a5
22 changed files with 948 additions and 442 deletions

213
go.mod
View File

@ -3,129 +3,126 @@ module github.com/harvester/vm-import-controller
go 1.22
require (
github.com/google/uuid v1.3.0
github.com/gophercloud/gophercloud v0.7.0
github.com/harvester/harvester v1.1.0-rc2
github.com/onsi/ginkgo/v2 v2.1.4
github.com/onsi/gomega v1.19.0
github.com/google/uuid v1.6.0
github.com/gophercloud/gophercloud v1.11.0
github.com/harvester/harvester v1.3.0
github.com/onsi/ginkgo/v2 v2.17.1
github.com/onsi/gomega v1.32.0
github.com/ory/dockertest/v3 v3.9.1
github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc
github.com/rancher/wrangler v1.1.0
github.com/sirupsen/logrus v1.9.0
github.com/stretchr/testify v1.8.1
github.com/vmware/govmomi v0.29.0
golang.org/x/sync v0.3.0
k8s.io/api v0.25.4
k8s.io/apiextensions-apiserver v0.25.4
k8s.io/apimachinery v0.25.4
github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29
github.com/rancher/wrangler v1.1.1
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
github.com/vmware/govmomi v0.37.2
golang.org/x/sync v0.6.0
k8s.io/api v0.30.0
k8s.io/apiextensions-apiserver v0.30.0
k8s.io/apimachinery v0.30.0
k8s.io/client-go v12.0.0+incompatible
kubevirt.io/api v0.54.0
kubevirt.io/kubevirt v0.54.0
sigs.k8s.io/controller-runtime v0.12.2
kubevirt.io/api v1.1.0
kubevirt.io/kubevirt v1.1.0
sigs.k8s.io/controller-runtime v0.17.5
)
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/coreos/prometheus-operator v0.38.1-0.20200424145508-7e176fda06cc // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.14+incompatible // indirect
github.com/docker/docker v20.10.12+incompatible // indirect
github.com/coreos/prometheus-operator v0.38.3 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/docker/cli v20.10.20+incompatible // indirect
github.com/docker/docker v20.10.27+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/emicklei/go-restful v2.15.0+incompatible // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-kit/kit v0.10.0 // indirect
github.com/go-logfmt/logfmt v0.5.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/zapr v1.2.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/zapr v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/glog v1.1.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/imdario/mergo v0.3.15 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200331171230-d50e42f2b669 // indirect
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0 // indirect
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/opencontainers/runc v1.1.2 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/opencontainers/runc v1.1.9 // indirect
github.com/openshift/api v0.0.0 // indirect
github.com/openshift/client-go v0.0.0 // indirect
github.com/openshift/custom-resource-status v1.1.2 // indirect
github.com/pborman/uuid v1.2.0 // indirect
github.com/pborman/uuid v1.2.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.18.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.17.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/mod v0.15.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.13.0 // indirect
golang.org/x/tools v0.18.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
google.golang.org/grpc v1.47.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/code-generator v0.25.4 // indirect
k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect
k8s.io/klog/v2 v2.70.1 // indirect
k8s.io/kube-aggregator v0.25.4 // indirect
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
kubevirt.io/client-go v0.49.0 // indirect
kubevirt.io/containerized-data-importer-api v1.50.0 // indirect
k8s.io/code-generator v0.27.1 // indirect
k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/kube-aggregator v0.26.4 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
kubevirt.io/client-go v1.1.0 // indirect
kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect
kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
replace (
github.com/containerd/containerd => github.com/containerd/containerd v1.6.18
github.com/docker/distribution => github.com/docker/distribution v2.8.0+incompatible
github.com/docker/distribution => github.com/docfker/distribution v2.8.0+incompatible
github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0+incompatible
github.com/openshift/api => github.com/openshift/api v0.0.0-20191219222812-2987a591a72c
github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20200521150516-05eb9880269c
@ -134,36 +131,36 @@ replace (
github.com/rancher/rancher/pkg/client => github.com/rancher/rancher/pkg/client v0.0.0-20211208233239-77392a65423d
golang.org/x/net => golang.org/x/net v0.17.0
golang.org/x/text => golang.org/x/text v0.3.8
k8s.io/api => k8s.io/api v0.23.7 // Dropped to v0.20.2 to handle kubevirt deps for installing CRDs
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.7
k8s.io/apimachinery => k8s.io/apimachinery v0.23.7
k8s.io/apiserver => k8s.io/apiserver v0.23.7
k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.7
k8s.io/client-go => k8s.io/client-go v0.23.7 // Dropped to v0.20.2 to handle kubevirt deps for installing CRDs
k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.7
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.7
k8s.io/code-generator => k8s.io/code-generator v0.23.7
k8s.io/component-base => k8s.io/component-base v0.23.7
k8s.io/component-helpers => k8s.io/component-helpers v0.23.7
k8s.io/controller-manager => k8s.io/controller-manager v0.23.7
k8s.io/cri-api => k8s.io/cri-api v0.23.7
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.7
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.7
k8s.io/api => k8s.io/api v0.26.4 //Dropped to v0.26.4 to handle kubevirt.io/client-go requirements
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.27.1
k8s.io/apimachinery => k8s.io/apimachinery v0.27.1
k8s.io/apiserver => k8s.io/apiserver v0.27.1
k8s.io/cli-runtime => k8s.io/cli-runtime v0.27.1
k8s.io/client-go => k8s.io/client-go v0.26.4 //Dropped to v0.26.4 to handle kubevirt.io/client-go requirements
k8s.io/cloud-provider => k8s.io/cloud-provider v0.27.1
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.27.1
k8s.io/code-generator => k8s.io/code-generator v0.27.1
k8s.io/component-base => k8s.io/component-base v0.27.1
k8s.io/component-helpers => k8s.io/component-helpers v0.27.1
k8s.io/controller-manager => k8s.io/controller-manager v0.27.1
k8s.io/cri-api => k8s.io/cri-api v0.27.1
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.27.1
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.27.1
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.7
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf
k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.7
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.7
k8s.io/kubectl => k8s.io/kubectl v0.23.7
k8s.io/kubelet => k8s.io/kubelet v0.23.7
k8s.io/kubernetes => k8s.io/kubernetes v1.23.7
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.7
k8s.io/metrics => k8s.io/metrics v0.23.7
k8s.io/mount-utils => k8s.io/mount-utils v0.23.7
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.7
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.7
kubevirt.io/api => github.com/kubevirt/api v0.55.1
kubevirt.io/client-go => github.com/kubevirt/client-go v0.55.1
kubevirt.io/containerized-data-importer-api => kubevirt.io/containerized-data-importer-api v1.47.0
kubevirt.io/kubevirt => kubevirt.io/kubevirt v0.55.1
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f
k8s.io/kube-proxy => k8s.io/kube-proxy v0.27.1
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.27.1
k8s.io/kubectl => k8s.io/kubectl v0.27.1
k8s.io/kubelet => k8s.io/kubelet v0.27.1
k8s.io/kubernetes => k8s.io/kubernetes v1.27.1
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.27.1
k8s.io/metrics => k8s.io/metrics v0.27.1
k8s.io/mount-utils => k8s.io/mount-utils v0.27.1
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.27.1
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.27.1
kubevirt.io/api => github.com/kubevirt/api v1.1.1
kubevirt.io/client-go => github.com/kubevirt/client-go v1.1.1
kubevirt.io/containerized-data-importer-api => kubevirt.io/containerized-data-importer-api v1.57.0-alpha1
kubevirt.io/kubevirt => kubevirt.io/kubevirt v1.1.1
sigs.k8s.io/structured-merge-diff => sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2
)

799
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -2,9 +2,11 @@ package main
import (
"log"
"os"
harvesterv1beta1 "github.com/harvester/harvester/pkg/apis/harvesterhci.io/v1beta1"
"github.com/rancher/wrangler/pkg/signals"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/clientcmd"
@ -27,6 +29,10 @@ func init() {
if err = kubevirtv1.AddToScheme(scheme); err != nil {
log.Fatalf("failed to add kubevirtv1 scheme, %v", err)
}
debug := os.Getenv("DEBUG")
if debug == "true" || debug == "TRUE" {
logrus.SetLevel(logrus.DebugLevel)
}
}
func main() {

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -4,6 +4,7 @@ import (
"github.com/rancher/wrangler/pkg/condition"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubevirtv1 "kubevirt.io/api/core/v1"
"github.com/harvester/vm-import-controller/pkg/apis/common"
)
@ -24,6 +25,7 @@ type VirtualMachineImportSpec struct {
VirtualMachineName string `json:"virtualMachineName"`
Folder string `json:"folder,omitempty"`
Mapping []NetworkMapping `json:"networkMapping,omitempty"` //If empty new VirtualMachineImport will be mapped to Management Network
StorageClass string `json:"storageClass,omitempty"`
}
// VirtualMachineImportStatus tracks the status of the VirtualMachineImport export from migration and import into the Harvester cluster
@ -50,6 +52,7 @@ type DiskInfo struct {
DiskRoute string `json:"diskRoute,omitempty"`
VirtualMachineImage string `json:"VirtualMachineImage,omitempty"`
DiskConditions []common.Condition `json:"diskConditions,omitempty"`
BusType kubevirtv1.DiskBus `json:"busType" default:"virtio"`
}
type NetworkMapping struct {
@ -75,4 +78,6 @@ const (
VirtualMachineImageReady condition.Cond = "VirtualMachineImageReady"
VirtualMachineImageFailed condition.Cond = "VirtualMachineImageFailed"
NotValidDNS1123Label string = "not a valid DNS1123 label"
VirtualMachineExportFailed condition.Cond = "VMExportFailed"
VirtualMachineMigrationFailed ImportStatus = "VMMigrationFailed"
)

View File

@ -2,7 +2,7 @@
// +build !ignore_autogenerated
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -10,6 +10,7 @@ import (
"github.com/rancher/lasso/pkg/client"
"github.com/rancher/lasso/pkg/controller"
"github.com/rancher/wrangler/pkg/generated/controllers/core"
"github.com/rancher/wrangler/pkg/generated/controllers/storage"
"github.com/rancher/wrangler/pkg/start"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/workqueue"
@ -46,10 +47,6 @@ func Register(ctx context.Context, restConfig *rest.Config) error {
DefaultWorkers: 5,
})
if err != nil {
return err
}
migrationFactory, err := migration.NewFactoryFromConfigWithOptions(restConfig, &migration.FactoryOptions{
SharedControllerFactory: scf,
})
@ -74,18 +71,28 @@ func Register(ctx context.Context, restConfig *rest.Config) error {
kubevirtFactory, err := kubevirt.NewFactoryFromConfigWithOptions(restConfig, &kubevirt.FactoryOptions{
SharedControllerFactory: scf,
SharedCacheFactory: cacheFactory,
})
if err != nil {
return err
}
storageFactory, err := storage.NewFactoryFromConfigWithOptions(restConfig, &core.FactoryOptions{
SharedControllerFactory: scf,
})
if err != nil {
return err
}
scCache := storageFactory.Storage().V1().StorageClass().Cache()
sc.RegisterVmareController(ctx, migrationFactory.Migration().V1beta1().VmwareSource(), coreFactory.Core().V1().Secret())
sc.RegisterOpenstackController(ctx, migrationFactory.Migration().V1beta1().OpenstackSource(), coreFactory.Core().V1().Secret())
sc.RegisterVMImportController(ctx, migrationFactory.Migration().V1beta1().VmwareSource(), migrationFactory.Migration().V1beta1().OpenstackSource(),
coreFactory.Core().V1().Secret(), migrationFactory.Migration().V1beta1().VirtualMachineImport(),
harvesterFactory.Harvesterhci().V1beta1().VirtualMachineImage(), kubevirtFactory.Kubevirt().V1().VirtualMachine(),
coreFactory.Core().V1().PersistentVolumeClaim())
coreFactory.Core().V1().PersistentVolumeClaim(), scCache)
return start.All(ctx, 1, migrationFactory)
return start.All(ctx, 1, migrationFactory, coreFactory, harvesterFactory, kubevirtFactory, storageFactory)
}

View File

@ -117,6 +117,11 @@ func (h *virtualMachineHandler) runVirtualMachineExport(vm *migration.VirtualMac
if util.ConditionExists(vm.Status.ImportConditions, migration.VirtualMachineExported, v1.ConditionTrue) {
vm.Status.Status = migration.DisksExported
}
if util.ConditionExists(vm.Status.ImportConditions, migration.VirtualMachineExportFailed, v1.ConditionTrue) {
vm.Status.Status = migration.VirtualMachineMigrationFailed
}
return h.importVM.UpdateStatus(vm)
}

View File

@ -9,6 +9,14 @@ import (
"strings"
"time"
"github.com/harvester/vm-import-controller/pkg/apis/common"
migration "github.com/harvester/vm-import-controller/pkg/apis/migration.harvesterhci.io/v1beta1"
migrationController "github.com/harvester/vm-import-controller/pkg/generated/controllers/migration.harvesterhci.io/v1beta1"
"github.com/harvester/vm-import-controller/pkg/server"
"github.com/harvester/vm-import-controller/pkg/source/openstack"
"github.com/harvester/vm-import-controller/pkg/source/vmware"
"github.com/harvester/vm-import-controller/pkg/util"
harvesterv1beta1 "github.com/harvester/harvester/pkg/apis/harvesterhci.io/v1beta1"
harvester "github.com/harvester/harvester/pkg/generated/controllers/harvesterhci.io/v1beta1"
kubevirtv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1"
@ -25,13 +33,7 @@ import (
"k8s.io/apimachinery/pkg/util/validation"
kubevirt "kubevirt.io/api/core/v1"
"github.com/harvester/vm-import-controller/pkg/apis/common"
migration "github.com/harvester/vm-import-controller/pkg/apis/migration.harvesterhci.io/v1beta1"
migrationController "github.com/harvester/vm-import-controller/pkg/generated/controllers/migration.harvesterhci.io/v1beta1"
"github.com/harvester/vm-import-controller/pkg/server"
"github.com/harvester/vm-import-controller/pkg/source/openstack"
"github.com/harvester/vm-import-controller/pkg/source/vmware"
"github.com/harvester/vm-import-controller/pkg/util"
storageControllers "github.com/rancher/wrangler/pkg/generated/controllers/storage/v1"
)
const (
@ -62,10 +64,11 @@ type virtualMachineHandler struct {
vmi harvester.VirtualMachineImageController
kubevirt kubevirtv1.VirtualMachineController
pvc coreControllers.PersistentVolumeClaimController
sc storageControllers.StorageClassCache
}
func RegisterVMImportController(ctx context.Context, vmware migrationController.VmwareSourceController, openstack migrationController.OpenstackSourceController,
secret coreControllers.SecretController, importVM migrationController.VirtualMachineImportController, vmi harvester.VirtualMachineImageController, kubevirt kubevirtv1.VirtualMachineController, pvc coreControllers.PersistentVolumeClaimController) {
secret coreControllers.SecretController, importVM migrationController.VirtualMachineImportController, vmi harvester.VirtualMachineImageController, kubevirt kubevirtv1.VirtualMachineController, pvc coreControllers.PersistentVolumeClaimController, scCache storageControllers.StorageClassCache) {
vmHandler := &virtualMachineHandler{
ctx: ctx,
@ -76,6 +79,7 @@ func RegisterVMImportController(ctx context.Context, vmware migrationController.
vmi: vmi,
kubevirt: kubevirt,
pvc: pvc,
sc: scCache,
}
relatedresource.Watch(ctx, "virtualmachineimage-change", vmHandler.ReconcileVMI, importVM, vmi)
@ -128,6 +132,9 @@ func (h *virtualMachineHandler) OnVirtualMachineChange(_ string, vmObj *migratio
case migration.VirtualMachineInvalid:
logrus.Infof("vm %s in namespace %v has an invalid spec", vm.Name, vm.Namespace)
return vm, nil
case migration.VirtualMachineMigrationFailed:
logrus.Infof("vm migration failed for %s in namespace %s", vm.Name, vm.Namespace)
return vm, nil
}
return vm, nil
@ -161,6 +168,15 @@ func (h *virtualMachineHandler) preFlightChecks(vm *migration.VirtualMachineImpo
return fmt.Errorf("migration not yet ready. current status is %s", ss.ClusterStatus())
}
// verify specified storage class exists. Empty storage class means default storage class
if vm.Spec.StorageClass != "" {
_, err := h.sc.Get(vm.Spec.StorageClass)
if err != nil {
logrus.Errorf("error looking up storageclass %s: %v", vm.Spec.StorageClass, err)
return err
}
}
return nil
}
@ -214,10 +230,23 @@ func (h *virtualMachineHandler) triggerExport(vm *migration.VirtualMachineImport
if util.ConditionExists(vm.Status.ImportConditions, migration.VirtualMachinePoweredOff, v1.ConditionTrue) &&
util.ConditionExists(vm.Status.ImportConditions, migration.VirtualMachinePoweringOff, v1.ConditionTrue) &&
!util.ConditionExists(vm.Status.ImportConditions, migration.VirtualMachineExported, v1.ConditionTrue) {
!util.ConditionExists(vm.Status.ImportConditions, migration.VirtualMachineExported, v1.ConditionTrue) &&
!util.ConditionExists(vm.Status.ImportConditions, migration.VirtualMachineExportFailed, v1.ConditionTrue) {
err := vmo.ExportVirtualMachine(vm)
if err != nil {
return fmt.Errorf("error exporting virtual machine: %v", err)
// avoid retrying if vm export fails
conds := []common.Condition{
{
Type: migration.VirtualMachineExportFailed,
Status: v1.ConditionTrue,
LastUpdateTime: metav1.Now().Format(time.RFC3339),
LastTransitionTime: metav1.Now().Format(time.RFC3339),
Message: fmt.Sprintf("error exporting VM: %v", err),
},
}
vm.Status.ImportConditions = util.MergeConditions(vm.Status.ImportConditions, conds)
logrus.Errorf("error exporting virtualmachine %s for virtualmachineimport %s-%s: %v", vm.Spec.VirtualMachineName, vm.Namespace, vm.Name, err)
return nil
}
conds := []common.Condition{
{
@ -388,7 +417,7 @@ func (h *virtualMachineHandler) createVirtualMachine(vm *migration.VirtualMachin
BootOrder: &[]uint{uint(diskOrder)}[0],
DiskDevice: kubevirt.DiskDevice{
Disk: &kubevirt.DiskTarget{
Bus: "virtio",
Bus: v.BusType,
},
},
})
@ -612,5 +641,13 @@ func (h *virtualMachineHandler) checkAndCreateVirtualMachineImage(vm *migration.
SourceType: "download",
},
}
if vm.Spec.StorageClass != "" {
// update storage class annotations
vmi.Annotations = map[string]string{
"harvesterhci.io/storageClassName": vm.Spec.StorageClass,
}
}
return h.vmi.Create(vmi)
}

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -19,6 +19,7 @@ limitations under the License.
package migration
import (
"github.com/rancher/lasso/pkg/controller"
"github.com/rancher/wrangler/pkg/generic"
"k8s.io/client-go/rest"
)
@ -65,3 +66,7 @@ func NewFactoryFromConfigWithOptionsOrDie(config *rest.Config, opts *FactoryOpti
func (c *Factory) Migration() Interface {
return New(c.ControllerFactory())
}
func (c *Factory) WithAgent(userAgent string) Interface {
return New(controller.NewSharedControllerFactoryWithAgent(userAgent, c.ControllerFactory()))
}

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,5 +1,5 @@
/*
Copyright 2022 Rancher Labs, Inc.
Copyright 2024 Rancher Labs, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

View File

@ -1,27 +1,56 @@
package qemu
import (
"fmt"
"io"
"os/exec"
"syscall"
"github.com/sirupsen/logrus"
)
const defaultCommand = "qemu-wrapper.sh"
func ConvertVMDKtoRAW(source, target string) error {
logrus.Infof("converting image %s to %s", source, target)
args := []string{"convert", "-f", "vmdk", "-O", "raw", source, target}
cmd := exec.Command(defaultCommand, args...)
cmd.SysProcAttr = &syscall.SysProcAttr{}
return cmd.Run()
return runCommand(defaultCommand, args...)
}
func ConvertQCOW2toRAW(source, target string) error {
args := []string{"convert", "-f", "qcow2", "-O", "raw", source, target}
cmd := exec.Command(defaultCommand, args...)
return cmd.Run()
return runCommand(defaultCommand, args...)
}
func createVMDK(path string, size string) error {
args := []string{"create", "-f", "vmdk", path, size}
cmd := exec.Command(defaultCommand, args...)
return cmd.Run()
return runCommand(defaultCommand, args...)
}
func runCommand(command string, args ...string) error {
cmd := exec.Command(command, args...)
cmd.SysProcAttr = &syscall.SysProcAttr{}
stderr, err := cmd.StderrPipe()
if err != nil {
return fmt.Errorf("error creating stderr pipe: %v", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("error creating stdout pipe: %v", err)
}
if err := cmd.Start(); err != nil {
return fmt.Errorf("error in command start: %v", err)
}
errOut, _ := io.ReadAll(stderr)
out, err := io.ReadAll(stdout)
err = cmd.Wait()
if err != nil {
return fmt.Errorf("error in command: %s, %s", command, errOut)
}
logrus.Debugf("image command complete: %v", string(out))
return nil
}

View File

@ -38,6 +38,7 @@ const (
NotServerFound = "noServerFound"
defaultInterval = 10 * time.Second
defaultCount = 30
pollingTimeout = 2 * 60 * 60 // in seconds
)
type Client struct {
@ -49,6 +50,10 @@ type Client struct {
imageClient *gophercloud.ServiceClient
}
type ExtendedVolume struct {
VolumeImageMetadata map[string]string `json:"volume_image_metadata,omitempty"`
}
// NewClient will generate a GopherCloud client
func NewClient(ctx context.Context, endpoint string, region string, secret *corev1.Secret) (*Client, error) {
username, ok := secret.Data["username"]
@ -120,6 +125,7 @@ func NewClient(ctx context.Context, endpoint string, region string, secret *core
if err != nil {
return nil, fmt.Errorf("error generating image client: %v", err)
}
return &Client{
ctx: ctx,
pClient: client,
@ -185,16 +191,8 @@ func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) error
return err
}
for i := 0; i < defaultCount; i++ {
snapObj, err := snapshots.Get(c.storageClient, snapInfo.ID).Extract()
if err != nil {
return err
}
if snapObj.Status == "available" {
break
}
time.Sleep(defaultInterval)
if err := snapshots.WaitForStatus(c.storageClient, snapInfo.ID, "available", pollingTimeout); err != nil {
return fmt.Errorf("timeout waiting for snapshot %v to become available: %v", snapInfo.ID, err)
}
volObj, err := volumes.Create(c.storageClient, volumes.CreateOpts{
@ -207,15 +205,8 @@ func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) error
logrus.Info(volObj)
for i := 0; i < defaultCount; i++ {
tmpVolObj, err := volumes.Get(c.storageClient, volObj.ID).Extract()
if err != nil {
return err
}
if tmpVolObj.Status == "available" {
break
}
time.Sleep(defaultInterval)
if err := volumes.WaitForStatus(c.storageClient, volObj.ID, "available", pollingTimeout); err != nil {
return fmt.Errorf("timeout waiting for volumes %v to become available: %v", volObj.ID, err)
}
logrus.Info("attempting to create new image from volume")
@ -285,6 +276,7 @@ func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) error
Name: fmt.Sprintf("%s-%d.img", vmObj.Name, i),
DiskSize: int64(volObj.Size),
DiskLocalPath: server.TempDir(),
BusType: kubevirt.DiskBusVirtio,
})
}
return os.RemoveAll(tmpDir)
@ -325,6 +317,8 @@ func (c *Client) IsPoweredOff(vm *migration.VirtualMachineImport) (bool, error)
}
func (c *Client) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*kubevirt.VirtualMachine, error) {
var boolFalse = false
var boolTrue = true
vmObj, err := c.findVM(vm.Spec.VirtualMachineName)
if err != nil {
return nil, fmt.Errorf("error finding vm in generatevirtualmachine: %v", err)
@ -335,6 +329,11 @@ func (c *Client) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*ku
return nil, fmt.Errorf("error looking up flavor: %v", err)
}
uefi, tpm, secureboot, err := c.ImageFirmwareSettings(vmObj)
if err != nil {
return nil, fmt.Errorf("error getting firware settings: %v", err)
}
var networks []networkInfo
for network, values := range vmObj.Addresses {
valArr, ok := values.([]interface{})
@ -429,6 +428,26 @@ func (c *Client) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*ku
})
}
if uefi {
firmware := &kubevirt.Firmware{
Bootloader: &kubevirt.Bootloader{
EFI: &kubevirt.EFI{
SecureBoot: &boolFalse,
},
},
}
if secureboot {
firmware.Bootloader.EFI.SecureBoot = &boolTrue
}
vmSpec.Template.Spec.Domain.Firmware = firmware
if tpm {
vmSpec.Template.Spec.Domain.Features.SMM = &kubevirt.FeatureState{
Enabled: &boolTrue,
}
vmSpec.Template.Spec.Domain.Devices.TPM = &kubevirt.TPMDevice{}
}
}
vmSpec.Template.Spec.Networks = networkConfig
vmSpec.Template.Spec.Domain.Devices.Interfaces = interfaces
newVM.Spec = vmSpec
@ -528,11 +547,24 @@ func (c *Client) checkOrGetUUID(input string) (string, error) {
return "", fmt.Errorf("error extracting servers in checkorgetuuid:%v", err)
}
if len(allServers) > 1 {
return "", fmt.Errorf(NotUniqueName)
if len(allServers) == 0 {
return allServers[0].ID, nil
}
return allServers[0].ID, nil
// api could return multiple servers matching the pattern of name
// eg server names test and testvm will match name search "test"
// in which case we need to filter on actual name
var filteredServers []servers.Server
for _, v := range allServers {
if v.Name == input {
filteredServers = append(filteredServers, v)
}
}
if len(filteredServers) > 1 {
return "", fmt.Errorf(NotUniqueName)
}
return filteredServers[0].ID, nil
}
func (c *Client) findVM(name string) (*servers.Server, error) {
@ -563,3 +595,41 @@ func mapNetworkCards(networkCards []networkInfo, mapping []migration.NetworkMapp
return retNetwork
}
func (c *Client) ImageFirmwareSettings(instance *servers.Server) (bool, bool, bool, error) {
var imageID string
var uefiType, tpmEnabled, secureBoot bool
for _, v := range instance.AttachedVolumes {
resp := volumes.Get(c.storageClient, v.ID)
var volInfo volumes.Volume
if err := resp.ExtractIntoStructPtr(&volInfo, "volume"); err != nil {
return uefiType, tpmEnabled, secureBoot, fmt.Errorf("error extracting volume info for volume %s: %v", v.ID, err)
}
if volInfo.Bootable == "true" {
var volStatus ExtendedVolume
if err := resp.ExtractIntoStructPtr(&volStatus, "volume"); err != nil {
return uefiType, tpmEnabled, secureBoot, fmt.Errorf("error extracting volume status for volume %s: %v", v.ID, err)
}
imageID = volStatus.VolumeImageMetadata["image_id"]
}
}
imageInfo, err := images.Get(c.imageClient, imageID).Extract()
if err != nil {
return uefiType, tpmEnabled, secureBoot, fmt.Errorf("error getting image details for image %s: %v", imageID, err)
}
firmwareType, ok := imageInfo.Properties["hw_firmware_type"]
if ok && firmwareType.(string) == "uefi" {
uefiType = true
}
logrus.Info(imageInfo.Properties)
if _, ok := imageInfo.Properties["hw_tpm_model"]; ok {
tpmEnabled = true
}
if val, ok := imageInfo.Properties["os_secure_boot"]; ok && (val == "required" || val == "optional") {
secureBoot = true
}
return uefiType, tpmEnabled, secureBoot, nil
}

View File

@ -152,17 +152,7 @@ func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) (err e
}
u := lease.StartUpdater(c.ctx, info)
defer u.Done()
defer func() {
leaseErr := lease.Complete(c.ctx)
if leaseErr != nil {
if err == nil {
err = leaseErr
} else {
err = fmt.Errorf("err: %w, leaseErr: %v", err, leaseErr)
}
}
}()
defer os.RemoveAll(tmpPath)
for _, i := range info.Items {
// ignore iso and nvram disks
@ -179,10 +169,20 @@ func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) (err e
vm.Status.DiskImportStatus = append(vm.Status.DiskImportStatus, migration.DiskInfo{
Name: i.Path,
DiskSize: i.Size,
BusType: adapterType(i.DeviceId),
})
}
}
u.Done()
// complete lease since disks have been downloaded
// and all subsequence processing is local
// we ignore the error since we have the disks and can continue conversion
err = lease.Complete(c.ctx)
if err != nil {
logrus.Errorf("error marking lease complete: %v", err)
}
// disk info will name of disks including the format suffix ".vmdk"
// once the disks are converted this needs to be updated to ".img"
// spec for how download_url is generated
@ -199,14 +199,13 @@ func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) (err e
destFile := filepath.Join(server.TempDir(), rawDiskName)
err = qemu.ConvertVMDKtoRAW(sourceFile, destFile)
if err != nil {
return err
return fmt.Errorf("error during conversion of vmdk to raw disk %v", err)
}
// update fields to reflect final location of raw image file
vm.Status.DiskImportStatus[i].DiskLocalPath = server.TempDir()
vm.Status.DiskImportStatus[i].Name = rawDiskName
}
return os.RemoveAll(tmpPath)
return nil
}
func (c *Client) PowerOffVirtualMachine(vm *migration.VirtualMachineImport) error {
@ -247,6 +246,9 @@ func (c *Client) IsPoweredOff(vm *migration.VirtualMachineImport) (bool, error)
}
func (c *Client) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*kubevirt.VirtualMachine, error) {
var boolFalse = false
var boolTrue = true
vmObj, err := c.findVM(vm.Spec.Folder, vm.Spec.VirtualMachineName)
if err != nil {
return nil, fmt.Errorf("error quering vm in GenerateVirtualMachine: %v", err)
@ -292,6 +294,11 @@ func (c *Client) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*ku
corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", o.Config.Hardware.NumCPU)),
},
},
Features: &kubevirt.Features{
ACPI: kubevirt.FeatureState{
Enabled: &boolTrue,
},
},
},
},
},
@ -338,6 +345,27 @@ func (c *Client) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*ku
})
}
// setup bios/efi, secureboot and tpm settings
if o.Config.Firmware == "efi" {
firmware := &kubevirt.Firmware{
Bootloader: &kubevirt.Bootloader{
EFI: &kubevirt.EFI{
SecureBoot: &boolFalse,
},
},
}
if *o.Config.BootOptions.EfiSecureBootEnabled {
firmware.Bootloader.EFI.SecureBoot = &boolTrue
}
vmSpec.Template.Spec.Domain.Firmware = firmware
if *o.Summary.Config.TpmPresent {
vmSpec.Template.Spec.Domain.Features.SMM = &kubevirt.FeatureState{
Enabled: &boolTrue,
}
vmSpec.Template.Spec.Domain.Devices.TPM = &kubevirt.TPMDevice{}
}
}
vmSpec.Template.Spec.Networks = networkConfig
vmSpec.Template.Spec.Domain.Devices.Interfaces = interfaces
newVM.Spec = vmSpec
@ -414,3 +442,15 @@ func mapNetworkCards(networkCards []networkInfo, mapping []migration.NetworkMapp
return retNetwork
}
// adapterType tries to identify the disk bus type from vmware
// to attempt and set correct bus types in kubevirt
func adapterType(deviceID string) kubevirt.DiskBus {
if strings.Contains(deviceID, "AHCI") {
return kubevirt.DiskBusSATA
}
if strings.Contains(deviceID, "SCSI") {
return kubevirt.DiskBusSCSI
}
return kubevirt.DiskBusVirtio
}

View File

@ -51,8 +51,6 @@ func GenerateKubeVirtCRD() ([]*extv1.CustomResourceDefinition, error) {
components.NewVirtualMachineSnapshotCrd,
components.NewVirtualMachineSnapshotContentCrd,
components.NewVirtualMachineRestoreCrd,
components.NewVirtualMachineFlavorCrd,
components.NewVirtualMachineClusterFlavorCrd,
components.NewMigrationPolicyCrd,
}