// Copyright 2020 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package utils contails utilities useful in the context of E2E tests. package utils import ( "context" "fmt" "os" "path/filepath" "strings" "time" "github.com/onsi/gomega" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/kubectl" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" ) const ( poll = time.Second ) // WaitForNodesWithResource waits for nodes to have positive allocatable resource. func WaitForNodesWithResource(c clientset.Interface, res v1.ResourceName, timeout time.Duration) error { framework.Logf("Waiting up to %s for any positive allocatable resource %q", timeout, res) start := time.Now() err := wait.Poll(poll, timeout, func() (bool, error) { for t := time.Now(); time.Since(t) < timeout; time.Sleep(poll) { nodelist, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err } resNum := 0 for _, item := range nodelist.Items { if q, ok := item.Status.Allocatable[res]; ok { resNum = resNum + int(q.Value()) } } framework.Logf("Found %d of %q. Elapsed: %s", resNum, res, time.Since(start)) if resNum > 0 { return true, nil } } return false, errors.New("unable to list nodes") }) return err } // WaitForPodFailure waits for a pod to fail. // This function used to be a part of k8s e2e framework, but was deleted in // https://github.com/kubernetes/kubernetes/pull/86732. func WaitForPodFailure(f *framework.Framework, name string, timeout time.Duration) { gomega.Expect(e2epod.WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout, func(pod *v1.Pod) (bool, error) { switch pod.Status.Phase { case v1.PodFailed: return true, nil case v1.PodSucceeded: return true, errors.Errorf("pod %q successed with reason: %q, message: %q", name, pod.Status.Reason, pod.Status.Message) default: return false, nil } }, )).To(gomega.Succeed(), "wait for pod %q to fail", name) } // LocateRepoFile locates a file inside this repository. func LocateRepoFile(repopath string) (string, error) { root := os.Getenv("PLUGINS_REPO_DIR") if root != "" { path := filepath.Join(root, repopath) if _, err := os.Stat(path); !os.IsNotExist(err) { return path, nil } } currentDir, err := os.Getwd() if err != nil { return "", err } path := filepath.Join(currentDir, repopath) if _, err := os.Stat(path); !os.IsNotExist(err) { return path, nil } path = filepath.Join(currentDir, "../../"+repopath) if _, err := os.Stat(path); !os.IsNotExist(err) { return path, err } return "", errors.New("no file found, try to define PLUGINS_REPO_DIR pointing to the root of the repository") } // CreateKustomizationOverlay creates an overlay with overridden namespace. func CreateKustomizationOverlay(namespace, base, overlay string) error { relPath := "" for range strings.Split(overlay[1:], "/") { relPath = relPath + "../" } relPath = relPath + base[1:] content := fmt.Sprintf("namespace: %s\nbases:\n - %s", namespace, relPath) return os.WriteFile(overlay+"/kustomization.yaml", []byte(content), 0600) } // DeployWebhook deploys an admission webhook to a framework-specific namespace. func DeployWebhook(f *framework.Framework, kustomizationPath string) v1.Pod { if _, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, "cert-manager", labels.Set{"app.kubernetes.io/name": "cert-manager"}.AsSelector(), 1 /* one replica */, 10*time.Second); err != nil { framework.Failf("unable to detect running cert-manager: %v", err) } tmpDir, err := os.MkdirTemp("", "webhooke2etest-"+f.Namespace.Name) if err != nil { framework.Failf("unable to create temp directory: %v", err) } defer os.RemoveAll(tmpDir) err = CreateKustomizationOverlay(f.Namespace.Name, filepath.Dir(kustomizationPath), tmpDir) if err != nil { framework.Failf("unable to kustomization overlay: %v", err) } framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", tmpDir) podList, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name, labels.Set{"control-plane": "controller-manager"}.AsSelector(), 1 /* one replica */, 30*time.Second) if err != nil { framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name) kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf) framework.Failf("unable to wait for all pods to be running and ready: %v", err) } // Wait for the webhook to initialize time.Sleep(2 * time.Second) return podList.Items[0] } // TestContainersRunAsNonRoot checks that all containers within the Pods run // with non-root UID/GID. func TestContainersRunAsNonRoot(pods []v1.Pod) error { for _, p := range pods { for _, c := range append(p.Spec.InitContainers, p.Spec.Containers...) { if c.SecurityContext.RunAsNonRoot == nil || !*c.SecurityContext.RunAsNonRoot { return errors.Errorf("%s (container: %s): RunAsNonRoot is not true", p.Name, c.Name) } if c.SecurityContext.RunAsGroup == nil || *c.SecurityContext.RunAsGroup == 0 { return errors.Errorf("%s (container: %s): RunAsGroup is root (0)", p.Name, c.Name) } if c.SecurityContext.RunAsUser == nil || *c.SecurityContext.RunAsUser == 0 { return errors.Errorf("%s (container: %s): RunAsUser is root (0)", p.Name, c.Name) } } } return nil } func printVolumeMounts(vm []v1.VolumeMount) { for _, v := range vm { if !v.ReadOnly { framework.Logf("Available RW volume mounts: %v", v) } } } // TestPodsFileSystemInfo checks that all containers within the Pods run // with ReadOnlyRootFileSystem. It also prints RW volume mounts. func TestPodsFileSystemInfo(pods []v1.Pod) error { for _, p := range pods { for _, c := range append(p.Spec.InitContainers, p.Spec.Containers...) { if c.SecurityContext.ReadOnlyRootFilesystem == nil || !*c.SecurityContext.ReadOnlyRootFilesystem { return errors.Errorf("%s (container: %s): Writable root filesystem", p.Name, c.Name) } printVolumeMounts(c.VolumeMounts) } } return nil }