diff --git a/Makefile b/Makefile index 976b7d02..fe2b4621 100644 --- a/Makefile +++ b/Makefile @@ -77,11 +77,10 @@ test-with-kind: fixture intel-sgx-admissionwebhook intel-fpga-admissionwebhook i kind load docker-image $(REG)intel-fpga-admissionwebhook:$(TAG) kind load docker-image $(REG)intel-deviceplugin-operator:$(TAG) kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml - # Test SGX Admission Webhook & FPGA Admission Webhook + # Test SGX Admission Webhook, FPGA Admission Webhook and Device Plugin Operator Manager's Webhook $(GO) test -v ./test/e2e -args -kubeconfig ~/.kube/config -ginkgo.focus "SGX Admission" $(GO) test -v ./test/e2e -args -kubeconfig ~/.kube/config -ginkgo.focus "FPGA Admission" - # Deploy Operator - kubectl apply -k deployments/operator/default/ + $(GO) test -v ./test/e2e -args -kubeconfig ~/.kube/config -ginkgo.focus "Operator" envtest: @$(GO) test ./test/envtest diff --git a/cmd/fpga_admissionwebhook/main.go b/cmd/fpga_admissionwebhook/main.go index 3effb017..eab2af9c 100644 --- a/cmd/fpga_admissionwebhook/main.go +++ b/cmd/fpga_admissionwebhook/main.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -56,11 +56,16 @@ func main() { ctrl.SetLogger(klogr.New()) + webHook := &webhook.Server{ + Port: 9443, + TLSMinVersion: "1.3", + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, - Port: 9443, Logger: ctrl.Log.WithName("FpgaAdmissionWebhook"), + WebhookServer: webHook, LeaderElection: enableLeaderElection, LeaderElectionID: "f2c6a4df.intel.com", }) diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 7300a918..22c4b82d 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -126,11 +126,16 @@ func main() { "sgx": sgx.SetupReconciler, } + webHook := &webhook.Server{ + Port: 9443, + TLSMinVersion: "1.3", + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, - Port: 9443, Logger: ctrl.Log.WithName("intel-device-plugins-manager"), + WebhookServer: webHook, LeaderElection: enableLeaderElection, LeaderElectionID: "d1c7b6d5.intel.com", }) diff --git a/cmd/sgx_admissionwebhook/main.go b/cmd/sgx_admissionwebhook/main.go index 38937beb..424f0a76 100644 --- a/cmd/sgx_admissionwebhook/main.go +++ b/cmd/sgx_admissionwebhook/main.go @@ -1,4 +1,4 @@ -// Copyright 2021 Intel Corporation. All Rights Reserved. +// Copyright 2021-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -49,11 +49,16 @@ func main() { ctrl.SetLogger(klogr.New()) + webHook := &webhook.Server{ + Port: 9443, + TLSMinVersion: "1.3", + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, - Port: 9443, Logger: ctrl.Log.WithName("SgxAdmissionWebhook"), + WebhookServer: webHook, LeaderElection: enableLeaderElection, LeaderElectionID: "a9b71ad3.intel.com", }) diff --git a/test/e2e/deviceplugins_suite_test.go b/test/e2e/deviceplugins_suite_test.go index bedf1624..76af307a 100644 --- a/test/e2e/deviceplugins_suite_test.go +++ b/test/e2e/deviceplugins_suite_test.go @@ -28,6 +28,7 @@ import ( _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/fpgaadmissionwebhook" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/gpu" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/iaa" + _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/operator" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/qat" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgx" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgxadmissionwebhook" diff --git a/test/e2e/operator/operator.go b/test/e2e/operator/operator.go new file mode 100644 index 00000000..996dd4aa --- /dev/null +++ b/test/e2e/operator/operator.go @@ -0,0 +1,55 @@ +// Copyright 2022 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package inteldevicepluginsoperator implements E2E tests for Intel Device Plugins Operator. +package inteldevicepluginsoperator + +import ( + "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +const ( + kustomizationYaml = "deployments/operator/default/kustomization.yaml" +) + +func init() { + ginkgo.Describe("Device Plugins Operator", describe) +} + +func describe() { + f := framework.NewDefaultFramework("inteldevicepluginsoperator") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + var webhook v1.Pod + + ginkgo.BeforeEach(func() { + kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml) + if err != nil { + framework.Failf("unable to locate %q: %v", kustomizationYaml, err) + } + webhook = utils.DeployWebhook(f, kustomizationPath) + }) + + ginkgo.It("checks the operator webhook pod is safely configured", func() { + err := utils.TestContainersRunAsNonRoot([]v1.Pod{webhook}) + gomega.Expect(err).To(gomega.BeNil()) + err = utils.TestWebhookServerTLS(f, "https://inteldeviceplugins-webhook-service") + gomega.Expect(err).To(gomega.BeNil()) + }) +} diff --git a/test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go b/test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go index 538b002c..e5fe8a2f 100644 --- a/test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go +++ b/test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go @@ -1,4 +1,4 @@ -// Copyright 2021 Intel Corporation. All Rights Reserved. +// Copyright 2021-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -56,6 +56,8 @@ func describe() { ginkgo.It("checks the webhook pod is safely configured", func() { err := utils.TestContainersRunAsNonRoot([]v1.Pod{webhook}) gomega.Expect(err).To(gomega.BeNil()) + err = utils.TestWebhookServerTLS(f, "https://intelsgxwebhook-webhook-service") + gomega.Expect(err).To(gomega.BeNil()) }) ginkgo.It("mutates created pods when no quote generation is needed", func() { ginkgo.By("submitting the pod") diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index beb7c704..b28dc280 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2022 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -152,7 +152,7 @@ func DeployWebhook(f *framework.Framework, kustomizationPath string) v1.Pod { framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", tmpDir) podList, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name, - labels.Set{"control-plane": "controller-manager"}.AsSelector(), 1 /* one replica */, 30*time.Second) + labels.Set{"control-plane": "controller-manager"}.AsSelector(), 1 /* one replica */, 60*time.Second) if err != nil { framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name) kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) @@ -210,3 +210,45 @@ func TestPodsFileSystemInfo(pods []v1.Pod) error { return nil } + +func TestWebhookServerTLS(f *framework.Framework, serviceName string) error { + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testssl-tester", + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Args: []string{ + "--openssl=/usr/bin/openssl", + "-p", + "-f", + serviceName}, + Name: "testssl-container", + Image: "drwetter/testssl.sh", + ImagePullPolicy: "IfNotPresent", + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } + + _, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), podSpec, metav1.CreateOptions{}) + framework.ExpectNoError(err, "pod Create API error") + + // TODO: check testssl.sh exit codes + err = e2epod.WaitForPodSuccessInNamespaceTimeout(f.ClientSet, "testssl-tester", f.Namespace.Name, 180*time.Second) + if err != nil { + return errors.Wrap(err, "testssl.sh run did not succeed") + } + + output, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, "testssl-tester", "testssl-container") + if err != nil { + return errors.Wrap(err, "failed to get output for testssl.sh run") + } + + framework.Logf("testssl.sh output:\n %s", output) + + return nil +}