Remove kube-rbac-proxy (#1099)

* Remove kubeproxy Signed-off-by: Ruben Vargas <[email protected]> * Update .chloggen/remove_metrics_proxy.yaml Co-authored-by: Andreas Gerstmayr <[email protected]> * Update remove_metrics_proxy.yaml --------- Signed-off-by: Ruben Vargas <[email protected]> Co-authored-by: Andreas Gerstmayr <[email protected]>
grafana · Jan 23, 2025 · b4747c7 · b4747c7
1 parent c2c2f7b
commit b4747c7
Show file tree

Hide file tree

Showing 25 changed files with 157 additions and 144 deletions.
diff --git a/.chloggen/remove_metrics_proxy.yaml b/.chloggen/remove_metrics_proxy.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Remove kube-rbac-proxy
+
+# One or more tracking issues related to the change
+issues: [1094]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: The image won't be available and won't be mantained, switched to use WithAuthenticationAndAuthorization
diff --git a/api/config/v1alpha1/projectconfig_types.go b/api/config/v1alpha1/projectconfig_types.go
@@ -250,6 +250,8 @@ type ControllerMetrics struct {
 	// It can be set to "0" to disable the metrics serving.
 	// +optional
 	BindAddress string `json:"bindAddress,omitempty"`
+
+	Secure bool `json:"secure,omitempty"`
 }
 
 // ControllerHealth defines the health configs.

diff --git a/bundle/community/manifests/tempo-operator-manager-config_v1_configmap.yaml b/bundle/community/manifests/tempo-operator-manager-config_v1_configmap.yaml
@@ -7,7 +7,8 @@ data:
     health:
       healthProbeBindAddress: :8081
     metrics:
-      bindAddress: 127.0.0.1:8080
+      secure: true
+      bindAddress: ":8443"
     webhook:
       port: 9443
     leaderElection:

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.2
-    createdAt: "2024-12-12T05:29:14Z"
+    createdAt: "2025-01-16T09:58:41Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -1516,6 +1516,9 @@ spec:
                 - containerPort: 9443
                   name: webhook-server
                   protocol: TCP
+                - containerPort: 8443
+                  name: https
+                  protocol: TCP
                 readinessProbe:
                   httpGet:
                     path: /readyz
@@ -1538,29 +1541,6 @@ spec:
                 - mountPath: /controller_manager_config.yaml
                   name: manager-config
                   subPath: controller_manager_config.yaml
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  limits:
-                    cpu: 500m
-                    memory: 128Mi
-                  requests:
-                    cpu: 5m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: tempo-operator-controller-manager

diff --git a/bundle/openshift/manifests/tempo-operator-manager-config_v1_configmap.yaml b/bundle/openshift/manifests/tempo-operator-manager-config_v1_configmap.yaml
@@ -7,7 +7,8 @@ data:
     health:
       healthProbeBindAddress: :8081
     metrics:
-      bindAddress: 127.0.0.1:8080
+      secure: true
+      bindAddress: ":8443"
     webhook:
       port: 9443
     leaderElection:

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.2
-    createdAt: "2024-12-12T05:29:13Z"
+    createdAt: "2025-01-16T09:58:39Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -1501,6 +1501,8 @@ spec:
                 - --zap-log-level=info
                 - start
                 - --config=controller_manager_config.yaml
+                - --metrics-tls-private-key-file=/var/run/tls/server/tls.key
+                - --metrics-tls-cert-file=/var/run/tls/server/tls.crt
                 env:
                 - name: RELATED_IMAGE_TEMPO
                   value: docker.io/grafana/tempo:2.6.1
@@ -1526,6 +1528,9 @@ spec:
                 - containerPort: 9443
                   name: webhook-server
                   protocol: TCP
+                - containerPort: 8443
+                  name: https
+                  protocol: TCP
                 readinessProbe:
                   httpGet:
                     path: /readyz
@@ -1548,52 +1553,24 @@ spec:
                 - mountPath: /controller_manager_config.yaml
                   name: manager-config
                   subPath: controller_manager_config.yaml
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                - --tls-cert-file=/var/run/tls/server/tls.crt
-                - --tls-private-key-file=/var/run/tls/server/tls.key
-                - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA256
-                - --tls-min-version=VersionTLS12
-                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  limits:
-                    cpu: 500m
-                    memory: 128Mi
-                  requests:
-                    cpu: 5m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
-                volumeMounts:
                 - mountPath: /var/run/tls/server
                   name: tempo-operator-metrics-cert
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: tempo-operator-controller-manager
               terminationGracePeriodSeconds: 10
               volumes:
-              - name: tempo-operator-metrics-cert
-                secret:
-                  defaultMode: 420
-                  secretName: tempo-operator-metrics
               - name: cert
                 secret:
                   defaultMode: 420
                   secretName: webhook-server-cert
               - configMap:
                   name: tempo-operator-manager-config
                 name: manager-config
+              - name: tempo-operator-metrics-cert
+                secret:
+                  defaultMode: 420
+                  secretName: tempo-operator-metrics
       permissions:
       - rules:
         - apiGroups:

diff --git a/cmd/root/options.go b/cmd/root/options.go
@@ -12,6 +12,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	configv1alpha1 "github.com/grafana/tempo-operator/api/config/v1alpha1"
@@ -65,6 +66,14 @@ func mergeOptionsFromFile(o manager.Options, cfg *configv1alpha1.ProjectConfig)
 		o.Metrics.BindAddress = cfg.Metrics.BindAddress
 	}
 
+	o.Metrics.SecureServing = cfg.Metrics.Secure
+	if cfg.Metrics.Secure {
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
+		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
+		o.Metrics.FilterProvider = filters.WithAuthenticationAndAuthorization
+	}
 	if o.HealthProbeBindAddress == "" && cfg.Health.HealthProbeBindAddress != "" {
 		o.HealthProbeBindAddress = cfg.Health.HealthProbeBindAddress
 	}

diff --git a/cmd/start/main.go b/cmd/start/main.go
@@ -32,6 +32,8 @@ func start(c *cobra.Command, args []string) {
 	version := version.Get()
 
 	options.PprofBindAddress, _ = c.Flags().GetString("pprof-addr")
+	options.Metrics.CertName, _ = c.Flags().GetString("metrics-tls-cert-file")
+	options.Metrics.KeyName, _ = c.Flags().GetString("metrics-tls-private-key-file")
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
 	if err != nil {
@@ -160,5 +162,7 @@ func NewStartCommand() *cobra.Command {
 		Run:   start,
 	}
 	cmd.Flags().String("pprof-addr", "", "The address the pprof server binds to. Default is empty string which disables the pprof server.")
+	cmd.Flags().String("metrics-tls-cert-file", "", "TLS certificate used by metrics server")
+	cmd.Flags().String("metrics-tls-private-key-file", "", "TLS key used by metrics server")
 	return cmd
 }
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -8,10 +8,6 @@ bases:
 #- ../prometheus
 
 patchesStrategicMerge:
-# Protect the /metrics endpoint by putting it behind auth.
-# If you want your controller-manager to expose the /metrics
-# endpoint w/o any authn/z, please comment the following line.
-- manager_auth_proxy_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type

diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml
@@ -16,6 +16,10 @@ spec:
         - name: manager-config
           mountPath: /controller_manager_config.yaml
           subPath: controller_manager_config.yaml
+        ports:
+        - containerPort: 8443
+          name: https
+          protocol: TCP
       volumes:
       - name: manager-config
         configMap:

diff --git a/config/rbac/auth_proxy_service.yaml → config/default/metrics_service.yaml b/config/rbac/auth_proxy_service.yaml → config/default/metrics_service.yaml
diff --git a/config/overlays/community/controller_manager_config.yaml b/config/overlays/community/controller_manager_config.yaml
@@ -4,7 +4,8 @@ distribution: community
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: 127.0.0.1:8080
+  secure: true
+  bindAddress: ":8443"
 webhook:
   port: 9443
 leaderElection:

diff --git a/config/overlays/openshift/controller_manager_config.yaml b/config/overlays/openshift/controller_manager_config.yaml
@@ -4,7 +4,8 @@ distribution: openshift
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: 127.0.0.1:8080
+  secure: true
+  bindAddress: ":8443"
 webhook:
   port: 9443
 leaderElection:

diff --git a/config/overlays/openshift/kustomization.yaml b/config/overlays/openshift/kustomization.yaml
@@ -33,4 +33,6 @@ configMapGenerator:
 
 patches:
 - path: metrics_service_tls_patch.yaml
-- path: manager_auth_proxy_tls_patch.yaml
+- path: patch_tls_metrics_args.yaml
+  target:
+    kind: Deployment
diff --git a/config/overlays/openshift/manager_auth_proxy_tls_patch.yaml b/config/overlays/openshift/manager_auth_proxy_tls_patch.yaml
diff --git a/config/overlays/openshift/patch_tls_metrics_args.yaml b/config/overlays/openshift/patch_tls_metrics_args.yaml
@@ -0,0 +1,22 @@
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --metrics-tls-private-key-file=/var/run/tls/server/tls.key
+
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --metrics-tls-cert-file=/var/run/tls/server/tls.crt
+
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: tempo-operator-metrics-cert
+    secret:
+      defaultMode: 420
+      # secret generated by the 'service.beta.openshift.io/serving-cert-secret-name' annotation on the metrics-service
+      secretName: tempo-operator-metrics
+
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /var/run/tls/server
+    name: tempo-operator-metrics-cert
diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml
@@ -9,10 +9,7 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml
+- metrics_service.yaml
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml
diff --git a/config/rbac/auth_proxy_role.yaml → config/rbac/metrics_auth_role.yaml b/config/rbac/auth_proxy_role.yaml → config/rbac/metrics_auth_role.yaml
@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
 - apiGroups:
   - authentication.k8s.io