diff --git a/cmd/operator/main.go b/cmd/operator/main.go index b4902264..2fca03da 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -27,6 +27,7 @@ import ( "k8s.io/klog/v2/textlogger" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -97,12 +98,45 @@ func contains(arr []string, val string) bool { return false } +func createTLSCfgs(enableHTTP2 bool) []func(*tls.Config) { + tlsCfgFuncs := []func(*tls.Config){ + func(cfg *tls.Config) { + cfg.MinVersion = tls.VersionTLS12 + cfg.MaxVersion = tls.VersionTLS12 + cfg.CipherSuites = []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + } + }, + } + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(cfg *tls.Config) { + setupLog.Info("disabling http/2") + + cfg.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsCfgFuncs = append(tlsCfgFuncs, disableHTTP2) + } + + return tlsCfgFuncs +} + func main() { var ( metricsAddr string probeAddr string devicePluginNamespace string enableLeaderElection bool + enableHTTP2 bool + secureMetrics bool pm *patcher.Manager ) @@ -115,6 +149,10 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", false, + "Enable role based authentication/authorization for the metrics endpoint") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "Enable HTTP/2 for the metrics and webhook servers") flag.Var(&devices, "devices", "Device(s) to set up.") flag.Parse() @@ -134,27 +172,33 @@ func main() { "sgx": sgx.SetupReconciler, } - tlsCfgFunc := func(cfg *tls.Config) { - cfg.MinVersion = tls.VersionTLS12 - cfg.MaxVersion = tls.VersionTLS12 - cfg.CipherSuites = []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - } + tlsCfgFuncs := createTLSCfgs(enableHTTP2) + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: tlsCfgFuncs, + }) + + // Metrics endpoint is enabled in 'deployments/operator/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsCfgFuncs, } - webhookOptions := webhook.Options{ - Port: 9443, - TLSOpts: []func(*tls.Config){ - tlsCfgFunc, - }, + if secureMetrics { + // More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: metricsAddr}, + Metrics: metricsServerOptions, Logger: ctrl.Log.WithName("intel-device-plugins-manager"), - WebhookServer: webhook.NewServer(webhookOptions), + WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "d1c7b6d5.intel.com", diff --git a/deployments/operator/default/kustomization.yaml b/deployments/operator/default/kustomization.yaml index 5f7daddf..404936bb 100644 --- a/deployments/operator/default/kustomization.yaml +++ b/deployments/operator/default/kustomization.yaml @@ -18,17 +18,19 @@ resources: - ../manager - ../webhook - ../certmanager +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml + patches: - # Protect the /metrics endpoint by putting it behind auth. - # If you want your controller-manager to expose the /metrics - # endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml target: - name: controller-manager - # Enable webhook + kind: Deployment - path: manager_webhook_patch.yaml target: + kind: Deployment name: controller-manager # Enable certmanager integration - path: webhookcainjection_patch_mutate.yaml diff --git a/deployments/operator/default/manager_auth_proxy_patch.yaml b/deployments/operator/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 1e0fd03d..00000000 --- a/deployments/operator/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - image: quay.io/brancz/kube-rbac-proxy:v0.18.1 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" - - "--v=10" - ports: - - containerPort: 8443 - name: https - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - - name: manager - args: - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/deployments/operator/default/manager_metrics_patch.yaml b/deployments/operator/default/manager_metrics_patch.yaml new file mode 100644 index 00000000..a89e27f5 --- /dev/null +++ b/deployments/operator/default/manager_metrics_patch.yaml @@ -0,0 +1,7 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: "--metrics-bind-address=:8443" +- op: add + path: /spec/template/spec/containers/0/args/0 + value: "--metrics-secure" diff --git a/deployments/operator/rbac/auth_proxy_service.yaml b/deployments/operator/default/metrics_service.yaml similarity index 79% rename from deployments/operator/rbac/auth_proxy_service.yaml rename to deployments/operator/default/metrics_service.yaml index b7169849..ef414742 100644 --- a/deployments/operator/rbac/auth_proxy_service.yaml +++ b/deployments/operator/default/metrics_service.yaml @@ -9,7 +9,7 @@ spec: ports: - name: https port: 8443 - targetPort: https + protocol: TCP + targetPort: 8443 selector: control-plane: controller-manager - manager: intel-deviceplugin-operator diff --git a/deployments/operator/manager/manager.yaml b/deployments/operator/manager/manager.yaml index 6cd574cb..eb8a9a1c 100644 --- a/deployments/operator/manager/manager.yaml +++ b/deployments/operator/manager/manager.yaml @@ -30,6 +30,9 @@ spec: - image: docker.io/intel/intel-deviceplugin-operator:devel imagePullPolicy: IfNotPresent name: manager + args: + - "--health-probe-bind-address=:8081" + - "--leader-elect" livenessProbe: httpGet: path: /healthz diff --git a/deployments/operator/rbac/kustomization.yaml b/deployments/operator/rbac/kustomization.yaml index 7c85fb87..ddec56bf 100644 --- a/deployments/operator/rbac/kustomization.yaml +++ b/deployments/operator/rbac/kustomization.yaml @@ -4,10 +4,12 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml - gpu_manager_role.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/deployments/operator/rbac/auth_proxy_role.yaml b/deployments/operator/rbac/metrics_auth_role.yaml similarity index 50% rename from deployments/operator/rbac/auth_proxy_role.yaml rename to deployments/operator/rbac/metrics_auth_role.yaml index 618f5e41..32d2e4ec 100644 --- a/deployments/operator/rbac/auth_proxy_role.yaml +++ b/deployments/operator/rbac/metrics_auth_role.yaml @@ -1,13 +1,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: proxy-role + name: metrics-auth-role rules: -- apiGroups: ["authentication.k8s.io"] +- apiGroups: + - authentication.k8s.io resources: - tokenreviews - verbs: ["create"] -- apiGroups: ["authorization.k8s.io"] + verbs: + - create +- apiGroups: + - authorization.k8s.io resources: - subjectaccessreviews - verbs: ["create"] + verbs: + - create diff --git a/deployments/operator/rbac/auth_proxy_role_binding.yaml b/deployments/operator/rbac/metrics_auth_role_binding.yaml similarity index 78% rename from deployments/operator/rbac/auth_proxy_role_binding.yaml rename to deployments/operator/rbac/metrics_auth_role_binding.yaml index 48ed1e4b..484ff4e0 100644 --- a/deployments/operator/rbac/auth_proxy_role_binding.yaml +++ b/deployments/operator/rbac/metrics_auth_role_binding.yaml @@ -1,11 +1,11 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: proxy-rolebinding + name: metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-auth-role subjects: - kind: ServiceAccount name: default diff --git a/deployments/operator/rbac/auth_proxy_client_clusterrole.yaml b/deployments/operator/rbac/metrics_reader_role.yaml similarity index 66% rename from deployments/operator/rbac/auth_proxy_client_clusterrole.yaml rename to deployments/operator/rbac/metrics_reader_role.yaml index bd4af137..51a75db4 100644 --- a/deployments/operator/rbac/auth_proxy_client_clusterrole.yaml +++ b/deployments/operator/rbac/metrics_reader_role.yaml @@ -3,5 +3,7 @@ kind: ClusterRole metadata: name: metrics-reader rules: -- nonResourceURLs: ["/metrics"] - verbs: ["get"] +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/deployments/operator/scorecard/bases/config.yaml b/deployments/operator/scorecard/bases/config.yaml deleted file mode 100644 index c7704784..00000000 --- a/deployments/operator/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/deployments/operator/scorecard/kustomization.yaml b/deployments/operator/scorecard/kustomization.yaml deleted file mode 100644 index 61ceb4d7..00000000 --- a/deployments/operator/scorecard/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -resources: -- bases/config.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -patches: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - kind: Configuration - name: config - version: v1alpha3 -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - kind: Configuration - name: config - version: v1alpha3 diff --git a/deployments/operator/scorecard/patches/basic.config.yaml b/deployments/operator/scorecard/patches/basic.config.yaml deleted file mode 100644 index e7fa3050..00000000 --- a/deployments/operator/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:master - labels: - suite: basic - test: basic-check-spec-test diff --git a/deployments/operator/scorecard/patches/olm.config.yaml b/deployments/operator/scorecard/patches/olm.config.yaml deleted file mode 100644 index e564c42f..00000000 --- a/deployments/operator/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:master - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:master - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:master - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:master - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:master - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/go.mod b/go.mod index a0317a8d..546038c3 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( k8s.io/kubernetes v1.31.0 k8s.io/pod-security-admission v0.0.0 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 - sigs.k8s.io/controller-runtime v0.19.0 + sigs.k8s.io/controller-runtime v0.19.1 sigs.k8s.io/yaml v1.4.0 tags.cncf.io/container-device-interface v0.8.0 tags.cncf.io/container-device-interface/specs-go v0.8.0 diff --git a/go.sum b/go.sum index 8fb86dd4..9ac9ca55 100644 --- a/go.sum +++ b/go.sum @@ -371,8 +371,8 @@ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= -sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk= +sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=