diff --git a/Makefile b/Makefile index 4f24e5ef..d112dfad 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ test-with-kind: @$(KIND) create cluster --name "intel-device-plugins" --kubeconfig $(e2e_tmp_dir)/kubeconfig --image "kindest/node:v1.19.0" @$(KIND) load image-archive --name "intel-device-plugins" $(e2e_tmp_dir)/$(WEBHOOK_IMAGE_FILE) $(KUBECTL) --kubeconfig=$(e2e_tmp_dir)/kubeconfig apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml - @$(GO) test -v ./test/e2e -args -kubeconfig $(e2e_tmp_dir)/kubeconfig -kubectl-path $(KUBECTL) -ginkgo.focus "Webhook" || rc=1; \ + @$(GO) test -v ./test/e2e -args -kubeconfig $(e2e_tmp_dir)/kubeconfig -kubectl-path $(KUBECTL) -ginkgo.focus "FPGA Admission" || rc=1; \ $(KIND) delete cluster --name "intel-device-plugins"; \ rm -rf $(e2e_tmp_dir); \ exit $$rc diff --git a/test/e2e/deviceplugins_suite_test.go b/test/e2e/deviceplugins_suite_test.go index 729eeda0..deee74e7 100644 --- a/test/e2e/deviceplugins_suite_test.go +++ b/test/e2e/deviceplugins_suite_test.go @@ -27,6 +27,7 @@ import ( _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/gpu" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/qat" _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgx" + _ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgxadmissionwebhook" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/logs" diff --git a/test/e2e/fpgaadmissionwebhook/fpgaadmissionwebhook.go b/test/e2e/fpgaadmissionwebhook/fpgaadmissionwebhook.go index 7967834e..51d818b9 100644 --- a/test/e2e/fpgaadmissionwebhook/fpgaadmissionwebhook.go +++ b/test/e2e/fpgaadmissionwebhook/fpgaadmissionwebhook.go @@ -73,7 +73,7 @@ func checkPodMutation(f *framework.Framework, mappingsNamespace string, source, } ginkgo.By("deploying webhook") - utils.DeployFpgaWebhook(f, kustomizationPath) + _ = utils.DeployWebhook(f, kustomizationPath) ginkgo.By("deploying mappings") framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-n", mappingsNamespace, "-f", filepath.Dir(kustomizationPath)+"/../mappings-collection.yaml") diff --git a/test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go b/test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go new file mode 100644 index 00000000..efeb15cf --- /dev/null +++ b/test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go @@ -0,0 +1,195 @@ +// Copyright 2021 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sgxadmissionwebhook implements E2E tests for SGX admission webhook. +package sgxadmissionwebhook + +import ( + "context" + "reflect" + + "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/kubectl" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +const ( + kustomizationYaml = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml" +) + +func init() { + ginkgo.Describe("SGX Admission Webhook", describe) +} + +func describe() { + f := framework.NewDefaultFramework("sgxwebhook") + var webhook v1.Pod + + ginkgo.BeforeEach(func() { + kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml) + if err != nil { + framework.Failf("unable to locate %q: %v", kustomizationYaml, err) + } + webhook = utils.DeployWebhook(f, kustomizationPath) + }) + + ginkgo.It("checks the webhook pod is safely configured", func() { + err := utils.TestContainersRunAsNonRoot([]v1.Pod{webhook}) + gomega.Expect(err).To(gomega.BeNil()) + }) + ginkgo.It("mutates created pods when no quote generation is needed", func() { + ginkgo.By("submitting the pod") + pod := submitPod(f, []string{"test"}, "") + + ginkgo.By("checking the container resources have been mutated") + checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"}) + + ginkgo.By("checking the pod total EPC size annotation is correctly set") + gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi")) + }) + ginkgo.It("mutates created pods when the container contains the quote generation libraries", func() { + ginkgo.By("submitting the pod") + pod := submitPod(f, []string{"test"}, "test") + + ginkgo.By("checking the container resources have been mutated") + checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave", "sgx.intel.com/provision"}, []v1.ResourceName{}) + + ginkgo.By("checking the pod total EPC size annotation is correctly set") + gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi")) + }) + ginkgo.It("mutates created pods when the container uses aesmd from a side-car container to generate quotes", func() { + ginkgo.By("submitting the pod") + pod := submitPod(f, []string{"test", "aesmd"}, "aesmd") + ginkgo.By("checking the container resources have been mutated") + checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"}) + checkMutatedResources(f, pod.Spec.Containers[1].Resources, []v1.ResourceName{"sgx.intel.com/enclave", "sgx.intel.com/provision"}, []v1.ResourceName{}) + ginkgo.By("checking the container volumes have been mutated") + checkMutatedVolumes(f, pod, "aesmd-socket", v1.EmptyDirVolumeSource{}) + ginkgo.By("checking the container envvars have been mutated") + gomega.Expect(pod.Spec.Containers[0].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR")) + gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1")) + ginkgo.By("checking the pod total EPC size annotation is correctly set") + gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("2Mi")) + }) + ginkgo.It("mutates created pods where one container uses host/daemonset aesmd to generate quotes", func() { + ginkgo.By("submitting the pod") + pod := submitPod(f, []string{"test"}, "aesmd") + ginkgo.By("checking the container resources have been mutated") + checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"}) + ginkgo.By("checking the container volumes have been mutated") + checkMutatedVolumes(f, pod, "aesmd-socket", v1.HostPathVolumeSource{}) + ginkgo.By("checking the container envvars have been mutated") + gomega.Expect(pod.Spec.Containers[0].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR")) + gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1")) + ginkgo.By("checking the pod total EPC size annotation is correctly set") + gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi")) + }) + ginkgo.It("mutates created pods where three containers use host/daemonset aesmd to generate quotes", func() { + ginkgo.By("submitting the pod") + pod := submitPod(f, []string{"test1", "test2", "test3"}, "aesmd") + ginkgo.By("checking the container resources have been mutated") + checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"}) + checkMutatedResources(f, pod.Spec.Containers[1].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"}) + checkMutatedResources(f, pod.Spec.Containers[2].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"}) + ginkgo.By("checking the container volumes have been mutated") + checkMutatedVolumes(f, pod, "aesmd-socket", v1.HostPathVolumeSource{}) + ginkgo.By("checking the container envvars have been mutated") + gomega.Expect(pod.Spec.Containers[0].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR")) + gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1")) + gomega.Expect(pod.Spec.Containers[1].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR")) + gomega.Expect(pod.Spec.Containers[1].Env[0].Value).To(gomega.Equal("1")) + gomega.Expect(pod.Spec.Containers[2].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR")) + gomega.Expect(pod.Spec.Containers[2].Env[0].Value).To(gomega.Equal("1")) + ginkgo.By("checking the pod total EPC size annotation is correctly set") + gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("3Mi")) + }) +} + +func checkMutatedVolumes(f *framework.Framework, pod *v1.Pod, volumeName string, volumeType interface{}) { + switch reflect.TypeOf(volumeType).String() { + case "v1.HostPathVolumeSource": + gomega.Expect(pod.Spec.Volumes[0].HostPath).NotTo(gomega.BeNil()) + gomega.Expect(pod.Spec.Volumes[0].Name).To(gomega.Equal(volumeName)) + case "v1.EmptyDirVolumeSource": + gomega.Expect(pod.Spec.Volumes[0].EmptyDir).NotTo(gomega.BeNil()) + gomega.Expect(pod.Spec.Volumes[0].Name).To(gomega.Equal(volumeName)) + } + + for _, c := range pod.Spec.Containers { + gomega.Expect(c.VolumeMounts[0].Name).To(gomega.Equal(volumeName)) + } +} + +func checkMutatedResources(f *framework.Framework, r v1.ResourceRequirements, expectedResources, forbiddenResources []v1.ResourceName) { + for _, res := range expectedResources { + q, ok := r.Limits[res] + if !ok { + framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name) + kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) + framework.Fail("the pod has missing resources") + } + gomega.Expect(q.String()).To(gomega.Equal("1")) + } + for _, res := range forbiddenResources { + _, ok := r.Limits[res] + if ok { + framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name) + kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) + framework.Fail("the pod has extra resources") + } + } +} + +func submitPod(f *framework.Framework, containerNames []string, quoteProvider string) *v1.Pod { + containers := make([]v1.Container, 0) + + for _, c := range containerNames { + containers = append(containers, v1.Container{ + Name: c, + Image: imageutils.GetPauseImageName(), + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{"sgx.intel.com/epc": resource.MustParse("1Mi")}, + Limits: v1.ResourceList{"sgx.intel.com/epc": resource.MustParse("1Mi")}, + }, + }) + } + + disabled := false + + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "webhook-tester-pod", + Annotations: map[string]string{ + "sgx.intel.com/quote-provider": quoteProvider, + }, + }, + Spec: v1.PodSpec{ + AutomountServiceAccountToken: &disabled, + Containers: containers, + }, + } + + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), + podSpec, metav1.CreateOptions{}) + + framework.ExpectNoError(err, "pod Create API error") + + return pod +} diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index 0737800f..7499dcf9 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -126,14 +126,14 @@ func CreateKustomizationOverlay(namespace, base, overlay string) error { return os.WriteFile(overlay+"/kustomization.yaml", []byte(content), 0600) } -// DeployFpgaWebhook deploys FPGA admission webhook to a framework-specific namespace. -func DeployFpgaWebhook(f *framework.Framework, kustomizationPath string) { +// DeployWebhook deploys an admission webhook to a framework-specific namespace. +func DeployWebhook(f *framework.Framework, kustomizationPath string) v1.Pod { if _, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, "cert-manager", labels.Set{"app.kubernetes.io/name": "cert-manager"}.AsSelector(), 1 /* one replica */, 10*time.Second); err != nil { framework.Failf("unable to detect running cert-manager: %v", err) } - tmpDir, err := os.MkdirTemp("", "fpgawebhooke2etest-"+f.Namespace.Name) + tmpDir, err := os.MkdirTemp("", "webhooke2etest-"+f.Namespace.Name) if err != nil { framework.Failf("unable to create temp directory: %v", err) } @@ -145,10 +145,31 @@ func DeployFpgaWebhook(f *framework.Framework, kustomizationPath string) { } framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", tmpDir) - if _, err = e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name, - labels.Set{"control-plane": "controller-manager"}.AsSelector(), 1 /* one replica */, 10*time.Second); err != nil { + podList, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name, + labels.Set{"control-plane": "controller-manager"}.AsSelector(), 1 /* one replica */, 10*time.Second) + if err != nil { framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name) kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) framework.Failf("unable to wait for all pods to be running and ready: %v", err) } + return podList.Items[0] +} + +// TestContainersRunAsNonRoot checks that all containers within the Pods run +// with non-root UID/GID. +func TestContainersRunAsNonRoot(pods []v1.Pod) error { + for _, p := range pods { + for _, c := range append(p.Spec.InitContainers, p.Spec.Containers...) { + if !*c.SecurityContext.RunAsNonRoot { + return fmt.Errorf("%s (container: %s): RunAsNonRoot is not true", p.Name, c.Name) + } + if *c.SecurityContext.RunAsGroup == 0 { + return fmt.Errorf("%s (container: %s): RunAsGroup is root (0)", p.Name, c.Name) + } + if *c.SecurityContext.RunAsUser == 0 { + return fmt.Errorf("%s (container: %s): RunAsUser is root (0)", p.Name, c.Name) + } + } + } + return nil }