packaging changes. includes helm chart, drone.yml and minor fixes for

openstack client

.drone.yml

@@ -0,0 +1,81 @@
+---
+kind: pipeline
+name: harvester-vm-import-controller
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: fetch
+  image: alpine/git
+  commands:
+  - git fetch --tags
+  when:
+    instance:
+      - drone-publish.rancher.io
+    ref:
+      - refs/head/master
+      - refs/tags/*
+    event:
+      - tag
+
+- name: build
+  image: rancher/dapper:v0.4.1
+  commands:
+  - dapper ci
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+- name: docker-publish-branch
+  image: plugins/docker
+  settings:
+    build_args:
+    - ARCH=amd64
+    - VERSION=${DRONE_BRANCH}-${DRONE_COMMIT_SHA:0:8}-head
+    context: package/
+    custom_dns: 1.1.1.1
+    dockerfile: package/Dockerfile
+    password:
+      from_secret: docker_password
+    repo: "rancher/harvester-vm-import-controller"
+    tag: ${DRONE_BRANCH}-head-amd64
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+      include:
+      - "refs/heads/master"
+      - "refs/heads/v*"
+    event:
+    - push
+
+- name: docker-publish
+  image: plugins/docker
+  settings:
+    build_args:
+    - ARCH=amd64
+    - "VERSION=${DRONE_TAG}"
+    context: package/
+    custom_dns: 1.1.1.1
+    dockerfile: package/Dockerfile
+    password:
+      from_secret: docker_password
+    repo: "rancher/harvester-vm-import-controller"
+    tag: "${DRONE_TAG}-amd64"
+    username:
+      from_secret: docker_username
+  when:
+    instance:
+    - drone-publish.rancher.io
+    ref:
+    - refs/head/master
+    - refs/tags/*
+    event:
+    - tag
+
+volumes:
+- name: docker
+  host:
+    path: /var/run/docker.sock

Dockerfile.dapper

@@ -0,0 +1,21 @@
+FROM golang:1.18
+
+ARG DAPPER_HOST_ARCH
+ENV ARCH $DAPPER_HOST_ARCH
+
+RUN export K8S_VERSION=1.24.2 && \
+    curl -sSLo envtest-bins.tar.gz "https://go.kubebuilder.io/test-tools/${K8S_VERSION}/$(go env GOOS)/$(go env GOARCH)" && \
+    mkdir /usr/local/kubebuilder && \
+    tar -C /usr/local/kubebuilder --strip-components=1 -zvxf envtest-bins.tar.gz
+RUN apt update && \
+    apt install -y bash git gcc docker.io vim less file curl wget ca-certificates qemu-utils
+
+ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS
+ENV DAPPER_SOURCE /go/src/github.com/harvester/vm-import-controller
+ENV DAPPER_OUTPUT ./bin
+ENV DAPPER_DOCKER_SOCKET true
+ENV DAPPER_RUN_ARGS "--network=host"
+WORKDIR ${DAPPER_SOURCE}
+
+ENTRYPOINT ["./scripts/entry"]
+CMD ["ci"]

Makefile

@@ -0,0 +1,15 @@
+TARGETS := $(shell ls scripts)
+
+.dapper:
+	@echo Downloading dapper
+	@curl -sL https://releases.rancher.com/dapper/latest/dapper-$$(uname -s)-$$(uname -m) > .dapper.tmp
+	@@chmod +x .dapper.tmp
+	@./.dapper.tmp -v
+	@mv .dapper.tmp .dapper
+
+$(TARGETS): .dapper
+	./.dapper $@
+
+.DEFAULT_GOAL := default
+
+.PHONY: $(TARGETS)

README.md

@@ -80,6 +80,7 @@ stringData:
   "password": "password"
   "project_name": "admin"
   "domain_name": "default"
+  "ca_cert": "pem-encoded-ca-cert"
 ```
 
 Openstack source reconcile process, attempts to list VM's in the project, and marks the source as ready

charts/vm-import-controller/.helmignore

@@ -0,0 +1,24 @@
+
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/vm-import-controller/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: vm-import-controller
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

charts/vm-import-controller/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "vm-import-controller.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "vm-import-controller.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "vm-import-controller.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "vm-import-controller.labels" -}}
+helm.sh/chart: {{ include "vm-import-controller.chart" . }}
+{{ include "vm-import-controller.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "vm-import-controller.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "vm-import-controller.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "vm-import-controller.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "vm-import-controller.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

charts/vm-import-controller/templates/deployment.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "vm-import-controller.fullname" . }}
+  labels:
+    {{- include "vm-import-controller.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "vm-import-controller.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "vm-import-controller.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "vm-import-controller.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/vm-import-controller/templates/rbac.yaml

@@ -0,0 +1,51 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name:  {{ include "vm-import-controller.fullname" . }}-role
+rules:
+- apiGroups:
+  - source.harvesterhci.io
+  - importjob.harvesterhci.io
+  resources:
+  - "*"
+  verbs:
+  - "*"
+- apiGroups:
+    - kubevirt.io
+  resources:
+    - virtualmachines
+  verbs:
+    - "*"
+- apiGroups:
+    - harvesterhci.io
+  resources:
+    - virtualmachineimages
+  verbs:
+    - "*"
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - "*"  
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  - persistentvolumeclaims
+  verbs:
+  - "*"
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "vm-import-controller.fullname" . }}-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "vm-import-controller.fullname" . }}-role
+subjects:
+- kind: ServiceAccount
+  name: {{ include "vm-import-controller.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}

charts/vm-import-controller/templates/service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "vm-import-controller.fullname" . }}
+spec:
+  selector:
+    {{- include "vm-import-controller.selectorLabels" . | nindent 4 }}
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080

charts/vm-import-controller/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "vm-import-controller.serviceAccountName" . }}
+  labels:
+    {{- include "vm-import-controller.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/vm-import-controller/values.yaml

@@ -0,0 +1,43 @@
+# Default values for bot.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: rancher/vm-import-controller
+  pullPolicy: Always
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: dev
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

package/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.15.0
+RUN apk add qemu-img
+COPY bin/vm-import-controller /usr/bin/vm-import-controller
+USER 1000
+CMD ["vm-import-controller"]

pkg/source/openstack/client.go

@@ -2,6 +2,8 @@ package openstack
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
@@ -10,6 +12,7 @@ import (
 	"github.com/harvester/vm-import-controller/pkg/server"
 	"io/ioutil"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
@@ -67,6 +70,19 @@ func NewClient(ctx context.Context, endpoint string, region string, secret *core
 		return nil, fmt.Errorf("no domain_name provided in secret %s", secret.Name)
 	}
 
+	config := &tls.Config{}
+
+	customCA, ok := secret.Data["ca_cert"]
+	if ok {
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(customCA)
+		config.RootCAs = caCertPool
+	} else {
+		config.InsecureSkipVerify = true
+	}
+
+	tr := &http.Transport{TLSClientConfig: config}
+
 	authOpts := gophercloud.AuthOptions{
 		IdentityEndpoint: endpoint,
 		Username:         string(username),
@@ -78,7 +94,13 @@ func NewClient(ctx context.Context, endpoint string, region string, secret *core
 	endPointOpts := gophercloud.EndpointOpts{
 		Region: region,
 	}
-	client, err := openstack.AuthenticatedClient(authOpts)
+
+	client, err := openstack.NewClient(endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("error generating new client: %v", err)
+	}
+	client.HTTPClient.Transport = tr
+	err = openstack.Authenticate(client, authOpts)
 	if err != nil {
 		return nil, fmt.Errorf("error authenticated client: %v", err)
 	}
@@ -202,6 +224,9 @@ func (c *Client) ExportVirtualMachine(vm *importjob.VirtualMachine) error {
 			DiskFormat: "qcow2",
 		}).Extract()
 
+		if err != nil {
+			return err
+		}
 		// wait for image to be ready
 		for i := 0; i < defaultCount; i++ {
 			imgObj, err := images.Get(c.imageClient, volImage.ImageID).Extract()

pkg/source/openstack/client_test.go

@@ -17,6 +17,13 @@ var (
 
 func TestMain(t *testing.M) {
 	var err error
+
+	// skip tests, needed for current builds
+	_, ok := os.LookupEnv("USE_EXISTING")
+	if !ok {
+		return
+	}
+
 	s, err := SetupOpenstackSecretFromEnv("devstack")
 	if err != nil {
 		logrus.Fatal(err)

pkg/source/vmware/client_test.go

@@ -149,6 +149,12 @@ func Test_IsPoweredOff(t *testing.T) {
 
 // Test_ExportVirtualMachine needs to reference a real vcenter as the vcsim doesnt support ovf export functionality
 func Test_ExportVirtualMachine(t *testing.T) {
+	// skip as vscim doesnt implement the same
+	_, ok := os.LookupEnv("USE_EXISTING")
+	if !ok {
+		return
+	}
+
 	ctx := context.TODO()
 	assert := require.New(t)
 	govc_url := os.Getenv("GOVC_URL")

scripts/build

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/..
+CGO_ENABLED=0 go build -o bin/vm-import-controller .
+mkdir -p bin

scripts/ci

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)
+
+./build
+./test

scripts/default

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)
+
+./build
+./package

scripts/entry

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+mkdir -p bin
+if [ -e ./scripts/$1 ]; then
+    ./scripts/"$@"
+else
+    exec "$@"
+fi
+
+chown -R $DAPPER_UID:$DAPPER_GID .

scripts/package

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/version
+cd $(dirname $0)/..
+
+IMAGE=${REPO}/vm-import-controller:${TAG}
+DOCKERFILE=package/Dockerfile
+
+docker build -f ${DOCKERFILE} -t ${IMAGE}  .
+echo Built ${IMAGE}

scripts/test

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+cd $(dirname $0)/..
+go test -coverprofile /tmp/cover.out ./...

scripts/version

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+    DIRTY="-dirty"
+fi
+
+COMMIT=$(git rev-parse --short HEAD)
+GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)}
+
+if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then
+    VERSION=$GIT_TAG
+else
+    VERSION="${COMMIT}${DIRTY}"
+fi
+
+if [ -z "$ARCH" ]; then
+    ARCH=$(go env GOHOSTARCH)
+fi
+
+if [ -z "$OS" ]; then
+    OS=$(go env GOHOSTOS)
+fi
+
+SUFFIX="-${ARCH}"
+
+HELM_TAG=${TAG:-${VERSION}}
+HELM_VERSION=${HELM_TAG/v/}
+TAG=${TAG:-${VERSION}${SUFFIX}}
+REPO=${REPO:-rancher}
+
+if echo $TAG | grep -q dirty; then
+    TAG=dev
+    HELM_TAG=dev
+    HELM_VERSION=0.0.0-dev
+fi

tests/integration/source_test.go

@@ -249,6 +249,9 @@ var _ = Describe("verify openstack is ready", func() {
 
 	const secretName = "devstack"
 	BeforeEach(func() {
+		if !useExisting {
+			return
+		}
 		var err error
 		creds, err = openstack.SetupOpenstackSecretFromEnv(secretName)
 		Expect(err).ToNot(HaveOccurred())
@@ -276,6 +279,10 @@ var _ = Describe("verify openstack is ready", func() {
 	})
 
 	It("check openstack source is ready", func() {
+		if !useExisting {
+			Skip("skipping openstack integration tests as not using an existing environment")
+		}
+
 		Eventually(func() error {
 			oObj := &source.Openstack{}
 			err := k8sClient.Get(ctx, types.NamespacedName{Name: o.Name, Namespace: o.Namespace}, oObj)
@@ -305,6 +312,9 @@ var _ = Describe("verify openstack is ready", func() {
 		}, "30s", "5s").ShouldNot(HaveOccurred())
 	})
 	AfterEach(func() {
+		if !useExisting {
+			return
+		}
 		err := k8sClient.Delete(ctx, creds)
 		Expect(err).ToNot(HaveOccurred())
 		err = k8sClient.Delete(ctx, o)

tests/integration/suite_test.go

@@ -111,7 +111,7 @@ var _ = BeforeSuite(func() {
 	pool, err = dockertest.NewPool("")
 	Expect(err).NotTo(HaveOccurred())
 	runOpts := &dockertest.RunOptions{
-		Name:       "vcsim",
+		Name:       "vcsim-integration",
 		Repository: "vmware/vcsim",
 		Tag:        "v0.29.0",
 	}
@@ -127,8 +127,10 @@ var _ = BeforeSuite(func() {
 
 var _ = AfterSuite(func() {
 	By("tearing down the test environment")
-	err := pool.Purge(vcsimMock)
-	Expect(err).NotTo(HaveOccurred())
+	if vcsimMock != nil {
+		err := pool.Purge(vcsimMock)
+		Expect(err).NotTo(HaveOccurred())
+	}
 	egctx.Done()
 	cancel()
 	testEnv.Stop()