mirror of
https://github.com/harvester/vm-import-controller.git
synced 2025-06-03 01:44:51 +00:00
566 lines
16 KiB
Go
566 lines
16 KiB
Go
package openstack
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/gophercloud/gophercloud"
|
|
"github.com/gophercloud/gophercloud/openstack"
|
|
"github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions"
|
|
"github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
|
|
"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots"
|
|
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/startstop"
|
|
"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
|
|
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
|
|
"github.com/gophercloud/gophercloud/openstack/imageservice/v2/imagedata"
|
|
"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
|
|
"github.com/sirupsen/logrus"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
kubevirt "kubevirt.io/api/core/v1"
|
|
|
|
migration "github.com/harvester/vm-import-controller/pkg/apis/migration.harvesterhci.io/v1beta1"
|
|
"github.com/harvester/vm-import-controller/pkg/qemu"
|
|
"github.com/harvester/vm-import-controller/pkg/server"
|
|
)
|
|
|
|
const (
|
|
NotUniqueName = "notUniqueName"
|
|
NotServerFound = "noServerFound"
|
|
defaultInterval = 10 * time.Second
|
|
defaultCount = 30
|
|
)
|
|
|
|
type Client struct {
|
|
ctx context.Context
|
|
pClient *gophercloud.ProviderClient
|
|
opts gophercloud.EndpointOpts
|
|
storageClient *gophercloud.ServiceClient
|
|
computeClient *gophercloud.ServiceClient
|
|
imageClient *gophercloud.ServiceClient
|
|
}
|
|
|
|
// NewClient will generate a GopherCloud client
|
|
func NewClient(ctx context.Context, endpoint string, region string, secret *corev1.Secret) (*Client, error) {
|
|
username, ok := secret.Data["username"]
|
|
if !ok {
|
|
return nil, fmt.Errorf("no username provided in secret %s", secret.Name)
|
|
}
|
|
|
|
password, ok := secret.Data["password"]
|
|
if !ok {
|
|
return nil, fmt.Errorf("no password provided in secret %s", secret.Name)
|
|
}
|
|
|
|
projectName, ok := secret.Data["project_name"]
|
|
if !ok {
|
|
return nil, fmt.Errorf("no project_name provided in secret %s", secret.Name)
|
|
}
|
|
|
|
domainName, ok := secret.Data["domain_name"]
|
|
if !ok {
|
|
return nil, fmt.Errorf("no domain_name provided in secret %s", secret.Name)
|
|
}
|
|
|
|
config := &tls.Config{}
|
|
|
|
customCA, ok := secret.Data["ca_cert"]
|
|
if ok {
|
|
caCertPool := x509.NewCertPool()
|
|
caCertPool.AppendCertsFromPEM(customCA)
|
|
config.RootCAs = caCertPool
|
|
} else {
|
|
config.InsecureSkipVerify = true
|
|
}
|
|
|
|
tr := &http.Transport{TLSClientConfig: config}
|
|
|
|
authOpts := gophercloud.AuthOptions{
|
|
IdentityEndpoint: endpoint,
|
|
Username: string(username),
|
|
Password: string(password),
|
|
TenantName: string(projectName),
|
|
DomainName: string(domainName),
|
|
}
|
|
|
|
endPointOpts := gophercloud.EndpointOpts{
|
|
Region: region,
|
|
}
|
|
|
|
client, err := openstack.NewClient(endpoint)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error generating new client: %v", err)
|
|
}
|
|
client.HTTPClient.Transport = tr
|
|
err = openstack.Authenticate(client, authOpts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error authenticated client: %v", err)
|
|
}
|
|
|
|
storageClient, err := openstack.NewBlockStorageV3(client, endPointOpts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error generating storage client: %v", err)
|
|
}
|
|
|
|
computeClient, err := openstack.NewComputeV2(client, endPointOpts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error generating compute client: %v", err)
|
|
}
|
|
|
|
imageClient, err := openstack.NewImageServiceV2(client, endPointOpts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error generating image client: %v", err)
|
|
}
|
|
return &Client{
|
|
ctx: ctx,
|
|
pClient: client,
|
|
opts: endPointOpts,
|
|
storageClient: storageClient,
|
|
computeClient: computeClient,
|
|
imageClient: imageClient,
|
|
}, nil
|
|
}
|
|
|
|
func (c *Client) Verify() error {
|
|
computeClient, err := openstack.NewComputeV2(c.pClient, c.opts)
|
|
if err != nil {
|
|
return fmt.Errorf("error generating compute client during verify phase :%v", err)
|
|
}
|
|
|
|
pg := servers.List(computeClient, servers.ListOpts{})
|
|
allPg, err := pg.AllPages()
|
|
if err != nil {
|
|
return fmt.Errorf("error generating all pages :%v", err)
|
|
}
|
|
|
|
ok, err := allPg.IsEmpty()
|
|
if err != nil {
|
|
return fmt.Errorf("error checking if pages were empty: %v", err)
|
|
}
|
|
|
|
if ok {
|
|
return nil
|
|
}
|
|
|
|
allServers, err := servers.ExtractServers(allPg)
|
|
if err != nil {
|
|
return fmt.Errorf("error extracting servers :%v", err)
|
|
}
|
|
|
|
logrus.Infof("found %d servers", len(allServers))
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) ExportVirtualMachine(vm *migration.VirtualMachineImport) error {
|
|
vmObj, err := c.findVM(vm.Spec.VirtualMachineName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
tmpDir, err := os.MkdirTemp("/tmp", "openstack-image-")
|
|
if err != nil {
|
|
return fmt.Errorf("error creating tmp image directory: %v", err)
|
|
}
|
|
|
|
for i, v := range vmObj.AttachedVolumes {
|
|
// create snapshot for volume
|
|
snapInfo, err := snapshots.Create(c.storageClient, snapshots.CreateOpts{
|
|
Name: fmt.Sprintf("import-controller-%v-%d", vm.Spec.VirtualMachineName, i),
|
|
VolumeID: v.ID,
|
|
Force: true,
|
|
}).Extract()
|
|
|
|
// snapshot creation is async, so call returns a 202 error when successful.
|
|
// this is ignored
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for i := 0; i < defaultCount; i++ {
|
|
snapObj, err := snapshots.Get(c.storageClient, snapInfo.ID).Extract()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if snapObj.Status == "available" {
|
|
break
|
|
}
|
|
time.Sleep(defaultInterval)
|
|
}
|
|
|
|
volObj, err := volumes.Create(c.storageClient, volumes.CreateOpts{
|
|
SnapshotID: snapInfo.ID,
|
|
Size: snapInfo.Size,
|
|
}).Extract()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Info(volObj)
|
|
|
|
for i := 0; i < defaultCount; i++ {
|
|
tmpVolObj, err := volumes.Get(c.storageClient, volObj.ID).Extract()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if tmpVolObj.Status == "available" {
|
|
break
|
|
}
|
|
time.Sleep(defaultInterval)
|
|
}
|
|
|
|
logrus.Info("attempting to create new image from volume")
|
|
|
|
volImage, err := volumeactions.UploadImage(c.storageClient, volObj.ID, volumeactions.UploadImageOpts{
|
|
ImageName: fmt.Sprintf("import-controller-%s-%d", vm.Spec.VirtualMachineName, i),
|
|
DiskFormat: "qcow2",
|
|
}).Extract()
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// wait for image to be ready
|
|
for i := 0; i < defaultCount; i++ {
|
|
imgObj, err := images.Get(c.imageClient, volImage.ImageID).Extract()
|
|
if err != nil {
|
|
return fmt.Errorf("error checking status of volume image: %v", err)
|
|
}
|
|
if imgObj.Status == "active" {
|
|
break
|
|
}
|
|
time.Sleep(defaultInterval)
|
|
}
|
|
|
|
contents, err := imagedata.Download(c.imageClient, volImage.ImageID).Extract()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
imageContents, err := io.ReadAll(contents)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
qcowFileName := filepath.Join(tmpDir, fmt.Sprintf("%s-%d", vm.Spec.VirtualMachineName, i))
|
|
imgFile, err := os.Create(qcowFileName)
|
|
if err != nil {
|
|
return fmt.Errorf("error creating disk file: %v", err)
|
|
}
|
|
|
|
_, err = imgFile.Write(imageContents)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
imgFile.Close()
|
|
|
|
// downloaded image is qcow2. Convert to raw file
|
|
rawFileName := filepath.Join(server.TempDir(), fmt.Sprintf("%s-%d.img", vmObj.Name, i))
|
|
err = qemu.ConvertQCOW2toRAW(qcowFileName, rawFileName)
|
|
if err != nil {
|
|
return fmt.Errorf("error converting qcow2 to raw file: %v", err)
|
|
}
|
|
|
|
if err := volumes.Delete(c.storageClient, volObj.ID, volumes.DeleteOpts{}).ExtractErr(); err != nil {
|
|
return fmt.Errorf("error deleting volume %s: %v", volObj.ID, err)
|
|
}
|
|
|
|
if err := snapshots.Delete(c.storageClient, snapInfo.ID).ExtractErr(); err != nil {
|
|
return fmt.Errorf("error deleting snapshot %s: %v", snapInfo.ID, err)
|
|
}
|
|
|
|
if err := images.Delete(c.imageClient, volImage.ImageID).ExtractErr(); err != nil {
|
|
return fmt.Errorf("error deleting image %s: %v", volImage.ImageID, err)
|
|
}
|
|
|
|
vm.Status.DiskImportStatus = append(vm.Status.DiskImportStatus, migration.DiskInfo{
|
|
Name: fmt.Sprintf("%s-%d.img", vmObj.Name, i),
|
|
DiskSize: int64(volObj.Size),
|
|
DiskLocalPath: server.TempDir(),
|
|
})
|
|
}
|
|
return os.RemoveAll(tmpDir)
|
|
}
|
|
|
|
func (c *Client) PowerOffVirtualMachine(vm *migration.VirtualMachineImport) error {
|
|
computeClient, err := openstack.NewComputeV2(c.pClient, c.opts)
|
|
if err != nil {
|
|
return fmt.Errorf("error generating compute client during poweroffvirtualmachine: %v", err)
|
|
}
|
|
uuid, err := c.checkOrGetUUID(vm.Spec.VirtualMachineName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ok, err := c.IsPoweredOff(vm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !ok {
|
|
return startstop.Stop(computeClient, uuid).ExtractErr()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) IsPoweredOff(vm *migration.VirtualMachineImport) (bool, error) {
|
|
|
|
s, err := c.findVM(vm.Spec.VirtualMachineName)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if s.Status == "SHUTOFF" {
|
|
return true, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
func (c *Client) GenerateVirtualMachine(vm *migration.VirtualMachineImport) (*kubevirt.VirtualMachine, error) {
|
|
vmObj, err := c.findVM(vm.Spec.VirtualMachineName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error finding vm in generatevirtualmachine: %v", err)
|
|
}
|
|
|
|
flavorObj, err := flavors.Get(c.computeClient, vmObj.Flavor["id"].(string)).Extract()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error looking up flavor: %v", err)
|
|
}
|
|
|
|
var networks []networkInfo
|
|
for network, values := range vmObj.Addresses {
|
|
valArr, ok := values.([]interface{})
|
|
if !ok {
|
|
return nil, fmt.Errorf("error asserting interface []interface")
|
|
}
|
|
for _, v := range valArr {
|
|
valMap, ok := v.(map[string]interface{})
|
|
if !ok {
|
|
return nil, fmt.Errorf("error asserting network array element into map[string]string")
|
|
}
|
|
networks = append(networks, networkInfo{
|
|
NetworkName: network,
|
|
MAC: valMap["OS-EXT-IPS-MAC:mac_addr"].(string),
|
|
})
|
|
}
|
|
}
|
|
newVM := &kubevirt.VirtualMachine{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: vm.Spec.VirtualMachineName,
|
|
Namespace: vm.Namespace,
|
|
},
|
|
}
|
|
|
|
vmSpec := kubevirt.VirtualMachineSpec{
|
|
RunStrategy: &[]kubevirt.VirtualMachineRunStrategy{kubevirt.RunStrategyRerunOnFailure}[0],
|
|
Template: &kubevirt.VirtualMachineInstanceTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"harvesterhci.io/vmName": vm.Spec.VirtualMachineName,
|
|
},
|
|
},
|
|
Spec: kubevirt.VirtualMachineInstanceSpec{
|
|
Domain: kubevirt.DomainSpec{
|
|
CPU: &kubevirt.CPU{
|
|
Cores: uint32(flavorObj.VCPUs),
|
|
Sockets: uint32(1),
|
|
Threads: 1,
|
|
},
|
|
Memory: &kubevirt.Memory{
|
|
Guest: &[]resource.Quantity{resource.MustParse(fmt.Sprintf("%dM", flavorObj.RAM))}[0],
|
|
},
|
|
Resources: kubevirt.ResourceRequirements{
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dM", flavorObj.RAM)),
|
|
corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%d", flavorObj.VCPUs)),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
mappedNetwork := mapNetworkCards(networks, vm.Spec.Mapping)
|
|
networkConfig := make([]kubevirt.Network, 0, len(mappedNetwork))
|
|
for i, v := range mappedNetwork {
|
|
networkConfig = append(networkConfig, kubevirt.Network{
|
|
NetworkSource: kubevirt.NetworkSource{
|
|
Multus: &kubevirt.MultusNetwork{
|
|
NetworkName: v.MappedNetwork,
|
|
},
|
|
},
|
|
Name: fmt.Sprintf("migrated-%d", i),
|
|
})
|
|
}
|
|
|
|
interfaces := make([]kubevirt.Interface, 0, len(mappedNetwork))
|
|
for i, v := range mappedNetwork {
|
|
interfaces = append(interfaces, kubevirt.Interface{
|
|
Name: fmt.Sprintf("migrated-%d", i),
|
|
MacAddress: v.MAC,
|
|
Model: "virtio",
|
|
InterfaceBindingMethod: kubevirt.InterfaceBindingMethod{
|
|
Bridge: &kubevirt.InterfaceBridge{},
|
|
},
|
|
})
|
|
}
|
|
// if there is no network, attach to Pod network. Essential for VM to be booted up
|
|
if len(networkConfig) == 0 {
|
|
networkConfig = append(networkConfig, kubevirt.Network{
|
|
Name: "pod-network",
|
|
NetworkSource: kubevirt.NetworkSource{
|
|
Pod: &kubevirt.PodNetwork{},
|
|
},
|
|
})
|
|
interfaces = append(interfaces, kubevirt.Interface{
|
|
Name: "pod-network",
|
|
Model: "virtio",
|
|
InterfaceBindingMethod: kubevirt.InterfaceBindingMethod{
|
|
Masquerade: &kubevirt.InterfaceMasquerade{},
|
|
},
|
|
})
|
|
}
|
|
|
|
vmSpec.Template.Spec.Networks = networkConfig
|
|
vmSpec.Template.Spec.Domain.Devices.Interfaces = interfaces
|
|
newVM.Spec = vmSpec
|
|
// disk attachment needs query by core controller for storage classes, so will be added by the migration controller
|
|
return newVM, nil
|
|
}
|
|
|
|
// SetupOpenStackSecretFromEnv is a helper function to ease with testing
|
|
func SetupOpenstackSecretFromEnv(name string) (*corev1.Secret, error) {
|
|
s := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
|
|
username, ok := os.LookupEnv("OS_USERNAME")
|
|
if !ok {
|
|
return nil, fmt.Errorf("no env variable OS_USERNAME specified")
|
|
}
|
|
|
|
password, ok := os.LookupEnv("OS_PASSWORD")
|
|
if !ok {
|
|
return nil, fmt.Errorf("no env variable OS_PASSWORD specified")
|
|
}
|
|
|
|
tenant, ok := os.LookupEnv("OS_PROJECT_NAME")
|
|
if !ok {
|
|
return nil, fmt.Errorf("no env variable OS_PROJECT_NAME specified")
|
|
}
|
|
|
|
domain, ok := os.LookupEnv("OS_USER_DOMAIN_NAME")
|
|
if !ok {
|
|
return nil, fmt.Errorf("no env variable OS_DOMAIN_NAME specified")
|
|
}
|
|
|
|
// generate common secret
|
|
data := map[string][]byte{
|
|
"username": []byte(username),
|
|
"password": []byte(password),
|
|
"project_name": []byte(tenant),
|
|
"domain_name": []byte(domain),
|
|
}
|
|
s.Data = data
|
|
return s, nil
|
|
}
|
|
|
|
// SetupOpenstackSourceFromEnv is a helper function to ease with testing
|
|
func SetupOpenstackSourceFromEnv() (string, string, error) {
|
|
var endpoint, region string
|
|
var ok bool
|
|
endpoint, ok = os.LookupEnv("OS_AUTH_URL")
|
|
if !ok {
|
|
return endpoint, region, fmt.Errorf("no env variable OS_AUTH_URL specified")
|
|
}
|
|
|
|
region, ok = os.LookupEnv("OS_REGION_NAME")
|
|
if !ok {
|
|
return endpoint, region, fmt.Errorf("no env variable OS_AUTH_URL specified")
|
|
}
|
|
|
|
return endpoint, region, nil
|
|
}
|
|
|
|
// checkOrGetUUID will check if input is a valid uuid. If not, it assume that the given input
|
|
// is a servername and will try and find a uuid for this server.
|
|
// openstack allows multiple server names to have the same name, in which case an error will be returned
|
|
func (c *Client) checkOrGetUUID(input string) (string, error) {
|
|
parsedUUID, err := uuid.Parse(input)
|
|
if err == nil {
|
|
return parsedUUID.String(), nil
|
|
}
|
|
|
|
// assume this is a name and find server based on name
|
|
/*computeClient, err := openstack.NewComputeV2(c.pClient, c.opts)
|
|
if err != nil {
|
|
return "", fmt.Errorf("error generating compute client during checkorGetUUID: %v", err)
|
|
}*/
|
|
|
|
pg := servers.List(c.computeClient, servers.ListOpts{Name: input})
|
|
allPg, err := pg.AllPages()
|
|
if err != nil {
|
|
return "", fmt.Errorf("error generating all pages in checkorgetuuid :%v", err)
|
|
}
|
|
|
|
ok, err := allPg.IsEmpty()
|
|
if err != nil {
|
|
return "", fmt.Errorf("error checking if pages were empty in checkorgetuuid: %v", err)
|
|
}
|
|
|
|
if ok {
|
|
return "", fmt.Errorf(NotServerFound)
|
|
}
|
|
|
|
allServers, err := servers.ExtractServers(allPg)
|
|
if err != nil {
|
|
return "", fmt.Errorf("error extracting servers in checkorgetuuid:%v", err)
|
|
}
|
|
|
|
if len(allServers) > 1 {
|
|
return "", fmt.Errorf(NotUniqueName)
|
|
}
|
|
|
|
return allServers[0].ID, nil
|
|
}
|
|
|
|
func (c *Client) findVM(name string) (*servers.Server, error) {
|
|
parsedUUID, err := c.checkOrGetUUID(name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return servers.Get(c.computeClient, parsedUUID).Extract()
|
|
|
|
}
|
|
|
|
type networkInfo struct {
|
|
NetworkName string
|
|
MAC string
|
|
MappedNetwork string
|
|
}
|
|
|
|
func mapNetworkCards(networkCards []networkInfo, mapping []migration.NetworkMapping) []networkInfo {
|
|
var retNetwork []networkInfo
|
|
for _, nc := range networkCards {
|
|
for _, m := range mapping {
|
|
if m.SourceNetwork == nc.NetworkName {
|
|
nc.MappedNetwork = m.DestinationNetwork
|
|
retNetwork = append(retNetwork, nc)
|
|
}
|
|
}
|
|
}
|
|
|
|
return retNetwork
|
|
}
|