operator: drop rbac-proxy in favor of controller-runtime's authz/authn

rbac-proxy will be deprecated in 2025

Signed-off-by: Tuomas Katila <tuomas.katila@intel.com>

cmd/operator/main.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/klog/v2/textlogger"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -97,12 +98,45 @@ func contains(arr []string, val string) bool {
 	return false
 }
 
+func createTLSCfgs(enableHTTP2 bool) []func(*tls.Config) {
+	tlsCfgFuncs := []func(*tls.Config){
+		func(cfg *tls.Config) {
+			cfg.MinVersion = tls.VersionTLS12
+			cfg.MaxVersion = tls.VersionTLS12
+			cfg.CipherSuites = []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			}
+		},
+	}
+
+	// if the enable-http2 flag is false (the default), http/2 should be disabled
+	// due to its vulnerabilities. More specifically, disabling http/2 will
+	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
+	// Rapid Reset CVEs. For more information see:
+	// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+	// - https://github.com/advisories/GHSA-4374-p667-p6c8
+	disableHTTP2 := func(cfg *tls.Config) {
+		setupLog.Info("disabling http/2")
+
+		cfg.NextProtos = []string{"http/1.1"}
+	}
+
+	if !enableHTTP2 {
+		tlsCfgFuncs = append(tlsCfgFuncs, disableHTTP2)
+	}
+
+	return tlsCfgFuncs
+}
+
 func main() {
 	var (
 		metricsAddr           string
 		probeAddr             string
 		devicePluginNamespace string
 		enableLeaderElection  bool
+		enableHTTP2           bool
+		secureMetrics         bool
 		pm                    *patcher.Manager
 	)
 
@@ -115,6 +149,10 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", false,
+		"Enable role based authentication/authorization for the metrics endpoint")
+	flag.BoolVar(&enableHTTP2, "enable-http2", false,
+		"Enable HTTP/2 for the metrics and webhook servers")
 	flag.Var(&devices, "devices", "Device(s) to set up.")
 	flag.Parse()
 
@@ -134,27 +172,33 @@ func main() {
 		"sgx":  sgx.SetupReconciler,
 	}
 
-	tlsCfgFunc := func(cfg *tls.Config) {
+	tlsCfgFuncs := createTLSCfgs(enableHTTP2)
-		cfg.MinVersion = tls.VersionTLS12
+
-		cfg.MaxVersion = tls.VersionTLS12
+	webhookServer := webhook.NewServer(webhook.Options{
-		cfg.CipherSuites = []uint16{
+		TLSOpts: tlsCfgFuncs,
-			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+	})
-			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+
-		}
+	// Metrics endpoint is enabled in 'deployments/operator/default/kustomization.yaml'. The Metrics options configure the server.
+	// More info:
+	// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server
+	// - https://book.kubebuilder.io/reference/metrics.html
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsCfgFuncs,
 	}
 
-	webhookOptions := webhook.Options{
+	if secureMetrics {
-		Port: 9443,
+		// More info:
-		TLSOpts: []func(*tls.Config){
+		// https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization
-			tlsCfgFunc,
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
-		},
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
-		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		Logger:                 ctrl.Log.WithName("intel-device-plugins-manager"),
-		WebhookServer:          webhook.NewServer(webhookOptions),
+		WebhookServer:          webhookServer,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "d1c7b6d5.intel.com",

deployments/operator/default/kustomization.yaml

@@ -18,17 +18,19 @@ resources:
 - ../manager
 - ../webhook
 - ../certmanager
+# [METRICS] Expose the controller manager metrics service.
+- metrics_service.yaml
+
 
 patches:
-  # Protect the /metrics endpoint by putting it behind auth.
+# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
-  # If you want your controller-manager to expose the /metrics
+# More info: https://book.kubebuilder.io/reference/metrics
-  # endpoint w/o any authn/z, please comment the following line.
+- path: manager_metrics_patch.yaml
-- path: manager_auth_proxy_patch.yaml
   target:
-    name: controller-manager
+    kind: Deployment
-  # Enable webhook
 - path: manager_webhook_patch.yaml
   target:
+    kind: Deployment
     name: controller-manager
   # Enable certmanager integration
 - path: webhookcainjection_patch_mutate.yaml

deployments/operator/default/manager_auth_proxy_patch.yaml

@@ -1,32 +0,0 @@
-# This patch inject a sidecar container which is a HTTP proxy for the
-# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.18.1
-        args:
-        - "--secure-listen-address=0.0.0.0:8443"
-        - "--upstream=http://127.0.0.1:8080/"
-        - "--logtostderr=true"
-        - "--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
-        - "--v=10"
-        ports:
-        - containerPort: 8443
-          name: https
-        securityContext:
-          runAsNonRoot: true
-          runAsUser: 1000
-          runAsGroup: 1000
-          readOnlyRootFilesystem: true
-          allowPrivilegeEscalation: false
-      - name: manager
-        args:
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"

deployments/operator/default/manager_metrics_patch.yaml

@@ -0,0 +1,7 @@
+# This patch adds the args to allow exposing the metrics endpoint using HTTPS
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: "--metrics-bind-address=:8443"
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: "--metrics-secure"

deployments/operator/default/metrics_service.yaml

@@ -9,7 +9,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: https
+    protocol: TCP
+    targetPort: 8443
   selector:
     control-plane: controller-manager
-    manager: intel-deviceplugin-operator

deployments/operator/manager/manager.yaml

@@ -30,6 +30,9 @@ spec:
       - image: docker.io/intel/intel-deviceplugin-operator:devel
         imagePullPolicy: IfNotPresent
         name: manager
+        args:
+          - "--health-probe-bind-address=:8081"
+          - "--leader-elect"
         livenessProbe:
           httpGet:
             path: /healthz

deployments/operator/rbac/kustomization.yaml

@@ -4,10 +4,12 @@ resources:
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
 - gpu_manager_role.yaml
-# Comment the following 4 lines if you want to disable
+# The following RBAC configurations are used to protect
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
+# the metrics endpoint with authn/authz. These configurations
-# which protects your /metrics endpoint.
+# ensure that only authorized users and service accounts
-- auth_proxy_service.yaml
+# can access the metrics endpoint. Comment the following
-- auth_proxy_role.yaml
+# permissions if you want to disable this protection.
-- auth_proxy_role_binding.yaml
+# More info: https://book.kubebuilder.io/reference/metrics.html
-- auth_proxy_client_clusterrole.yaml
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml

deployments/operator/rbac/metrics_auth_role.yaml

@@ -1,13 +1,17 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
-- apiGroups: ["authentication.k8s.io"]
+- apiGroups:
+  - authentication.k8s.io
   resources:
   - tokenreviews
-  verbs: ["create"]
+  verbs:
-- apiGroups: ["authorization.k8s.io"]
+  - create
+- apiGroups:
+  - authorization.k8s.io
   resources:
   - subjectaccessreviews
-  verbs: ["create"]
+  verbs:
+  - create

deployments/operator/rbac/metrics_auth_role_binding.yaml

@@ -1,11 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-auth-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-auth-role
 subjects:
 - kind: ServiceAccount
   name: default

deployments/operator/rbac/metrics_reader_role.yaml

@@ -3,5 +3,7 @@ kind: ClusterRole
 metadata:
   name: metrics-reader
 rules:
-- nonResourceURLs: ["/metrics"]
+- nonResourceURLs:
-  verbs: ["get"]
+  - "/metrics"
+  verbs:
+  - get

deployments/operator/scorecard/bases/config.yaml

@@ -1,7 +0,0 @@
-apiVersion: scorecard.operatorframework.io/v1alpha3
-kind: Configuration
-metadata:
-  name: config
-stages:
-- parallel: true
-  tests: []

deployments/operator/scorecard/kustomization.yaml

@@ -1,17 +0,0 @@
-resources:
-- bases/config.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-patches:
-- path: patches/basic.config.yaml
-  target:
-    group: scorecard.operatorframework.io
-    kind: Configuration
-    name: config
-    version: v1alpha3
-- path: patches/olm.config.yaml
-  target:
-    group: scorecard.operatorframework.io
-    kind: Configuration
-    name: config
-    version: v1alpha3

deployments/operator/scorecard/patches/basic.config.yaml

@@ -1,10 +0,0 @@
-- op: add
-  path: /stages/0/tests/-
-  value:
-    entrypoint:
-    - scorecard-test
-    - basic-check-spec
-    image: quay.io/operator-framework/scorecard-test:master
-    labels:
-      suite: basic
-      test: basic-check-spec-test

deployments/operator/scorecard/patches/olm.config.yaml

@@ -1,50 +0,0 @@
-- op: add
-  path: /stages/0/tests/-
-  value:
-    entrypoint:
-    - scorecard-test
-    - olm-bundle-validation
-    image: quay.io/operator-framework/scorecard-test:master
-    labels:
-      suite: olm
-      test: olm-bundle-validation-test
-- op: add
-  path: /stages/0/tests/-
-  value:
-    entrypoint:
-    - scorecard-test
-    - olm-crds-have-validation
-    image: quay.io/operator-framework/scorecard-test:master
-    labels:
-      suite: olm
-      test: olm-crds-have-validation-test
-- op: add
-  path: /stages/0/tests/-
-  value:
-    entrypoint:
-    - scorecard-test
-    - olm-crds-have-resources
-    image: quay.io/operator-framework/scorecard-test:master
-    labels:
-      suite: olm
-      test: olm-crds-have-resources-test
-- op: add
-  path: /stages/0/tests/-
-  value:
-    entrypoint:
-    - scorecard-test
-    - olm-spec-descriptors
-    image: quay.io/operator-framework/scorecard-test:master
-    labels:
-      suite: olm
-      test: olm-spec-descriptors-test
-- op: add
-  path: /stages/0/tests/-
-  value:
-    entrypoint:
-    - scorecard-test
-    - olm-status-descriptors
-    image: quay.io/operator-framework/scorecard-test:master
-    labels:
-      suite: olm
-      test: olm-status-descriptors-test

go.mod

@@ -27,7 +27,7 @@ require (
 	k8s.io/kubernetes v1.31.0
 	k8s.io/pod-security-admission v0.0.0
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
-	sigs.k8s.io/controller-runtime v0.19.0
+	sigs.k8s.io/controller-runtime v0.19.1
 	sigs.k8s.io/yaml v1.4.0
 	tags.cncf.io/container-device-interface v0.8.0
 	tags.cncf.io/container-device-interface/specs-go v0.8.0

go.sum

@@ -371,8 +371,8 @@ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
-sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=
+sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk=
-sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
+sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=