Do not set --prometheus.query.namespace if field is empty

.chloggen/red_metrics_namespace.yaml

@@ -0,0 +1,15 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempostack
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use default Jaeger RED metrics namespace if field is unset
+# One or more tracking issues related to the change
+issues: [1096]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Use the default Jaeger RED metrics namespace if `.spec.template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace` is not set.
+  Before Jaeger 1.62 the default namespace was empty, since [Jaeger 1.62](https://github.com/jaegertracing/jaeger/releases/tag/v1.62.0) (shipped in Tempo Operator v0.14.0) the default namespace is "traces_span_metrics".
+  Before OpenTelemetry Collector v0.109.0 the default namespace of the spanmetrics connector was empty, since [OpenTelemetry Collector v0.109.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.109.0) the default namespace is "traces_span_metrics".

apis/tempo/v1alpha1/tempostack_types.go

@@ -690,12 +690,10 @@ type JaegerQueryMonitor struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Prometheus endpoint"
 	PrometheusEndpoint string `json:"prometheusEndpoint"`
 	// REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
-	// By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
 	// +optional
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default:=traces.span.metrics
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="RED Metric Namespace"
-	REDMetricsNamespace string `json:"redMetricsNamespace"`
+	REDMetricsNamespace *string `json:"redMetricsNamespace,omitempty"`
 }
 
 // IngressSpec defines Jaeger Query Ingress options.

bundle/community/manifests/tempo-operator.clusterserviceversion.yaml

@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.1
-    createdAt: "2024-11-06T14:25:14Z"
+    createdAt: "2024-12-02T11:40:40Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -1071,9 +1071,7 @@ spec:
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
       - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
-          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
-          following the default namespace of the OpenTelemetry Collector since Version
-          0.109.0.
+          error, and duration (RED) metrics.
         displayName: RED Metric Namespace
         path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override

bundle/community/manifests/tempo.grafana.com_tempostacks.yaml

@@ -2471,10 +2471,9 @@ spec:
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
                               redMetricsNamespace:
-                                default: traces.span.metrics
-                                description: |-
-                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
-                                  By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+                                description: REDMetricsNamespace defines the a prefix
+                                  used retrieve span rate, error, and duration (RED)
+                                  metrics.
                                 type: string
                             type: object
                           resources:

bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml

@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.1
-    createdAt: "2024-11-06T14:25:12Z"
+    createdAt: "2024-12-02T11:40:38Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -1071,9 +1071,7 @@ spec:
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
       - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
-          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
-          following the default namespace of the OpenTelemetry Collector since Version
-          0.109.0.
+          error, and duration (RED) metrics.
         displayName: RED Metric Namespace
         path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override

bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml

@@ -2471,10 +2471,9 @@ spec:
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
                               redMetricsNamespace:
-                                default: traces.span.metrics
-                                description: |-
-                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
-                                  By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+                                description: REDMetricsNamespace defines the a prefix
+                                  used retrieve span rate, error, and duration (RED)
+                                  metrics.
                                 type: string
                             type: object
                           resources:

config/crd/bases/tempo.grafana.com_tempostacks.yaml

@@ -2467,10 +2467,9 @@ spec:
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
                               redMetricsNamespace:
-                                default: traces.span.metrics
-                                description: |-
-                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
-                                  By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+                                description: REDMetricsNamespace defines the a prefix
+                                  used retrieve span rate, error, and duration (RED)
+                                  metrics.
                                 type: string
                             type: object
                           resources:

config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml

@@ -1000,9 +1000,7 @@ spec:
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
       - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
-          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
-          following the default namespace of the OpenTelemetry Collector since Version
-          0.109.0.
+          error, and duration (RED) metrics.
         displayName: RED Metric Namespace
         path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override

config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml

@@ -1000,9 +1000,7 @@ spec:
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
       - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
-          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
-          following the default namespace of the OpenTelemetry Collector since Version
-          0.109.0.
+          error, and duration (RED) metrics.
         displayName: RED Metric Namespace
         path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override

docs/spec/tempo.grafana.com_tempostacks.yaml

@@ -342,7 +342,7 @@ spec:                                    # TempoStackSpec defines the desired st
         monitorTab:                      # MonitorTab defines the monitor tab configuration.
           enabled: false                 # Enabled enables the monitor tab in the Jaeger console. The PrometheusEndpoint must be configured to enable this feature.
           prometheusEndpoint: ""         # PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
-          redMetricsNamespace: "traces.span.metrics" # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+          redMetricsNamespace: ""        # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
         servicesQueryDuration: ""        # ServicesQueryDuration defines how long the services will be available in the services list
         tempoQuery:                      # TempoQuery defines options specific to the Tempoo Query component.
           resources:                     # Resources defines resources for this component, this will override the calculated resources derived from total

internal/manifests/queryfrontend/query_frontend.go

@@ -398,11 +398,6 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1.
 			// However, we do not intend to support them.
 			// --prometheus.query.normalize-calls
 			// --prometheus.query.normalize-duration
-			//
-			// NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics".
-			// We fallback to no namespace.
-			// See https://github.com/jaegertracing/jaeger/pull/6007.
-			fmt.Sprintf("--prometheus.query.namespace=%s", tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace),
 		},
 	}
 	// If the endpoint matches Prometheus on OpenShift, configure TLS and token based auth
@@ -417,6 +412,15 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1.
 			"--prometheus.tls.ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt")
 	}
 
+	if tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace != nil {
+		// NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics".
+		// Set .spec.template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace explicitly to "" to disable the namespace.
+		// See https://github.com/jaegertracing/jaeger/pull/6007.
+		container.Args = append(container.Args,
+			fmt.Sprintf("--prometheus.query.namespace=%s", *tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace),
+		)
+	}
+
 	err := mergo.Merge(&jaegerQueryContainer, container, mergo.WithAppendSlice)
 	if err != nil {
 		return corev1.Container{}, err

internal/manifests/queryfrontend/query_frontend_test.go

@@ -614,6 +614,26 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) {
 		},
 		{
 			name: "custom prometheus",
+			tempo: v1alpha1.TempoStack{
+				Spec: v1alpha1.TempoStackSpec{
+					Template: v1alpha1.TempoTemplateSpec{
+						QueryFrontend: v1alpha1.TempoQueryFrontendSpec{
+							JaegerQuery: v1alpha1.JaegerQuerySpec{
+								Enabled: true,
+								MonitorTab: v1alpha1.JaegerQueryMonitor{
+									Enabled:            true,
+									PrometheusEndpoint: "http://prometheus:9091",
+								},
+							},
+						},
+					},
+				},
+			},
+			args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true"},
+			env:  []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}},
+		},
+		{
+			name: "custom RED metrics namespace",
 			tempo: v1alpha1.TempoStack{
 				Spec: v1alpha1.TempoStackSpec{
 					Template: v1alpha1.TempoTemplateSpec{
@@ -623,7 +643,7 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) {
 								MonitorTab: v1alpha1.JaegerQueryMonitor{
 									Enabled:             true,
 									PrometheusEndpoint:  "http://prometheus:9091",
-									REDMetricsNamespace: "test",
+									REDMetricsNamespace: ptr.To("test"),
 								},
 							},
 						},
@@ -633,6 +653,27 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) {
 			args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace=test"},
 			env:  []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}},
 		},
+		{
+			name: "disable default RED metrics namespace",
+			tempo: v1alpha1.TempoStack{
+				Spec: v1alpha1.TempoStackSpec{
+					Template: v1alpha1.TempoTemplateSpec{
+						QueryFrontend: v1alpha1.TempoQueryFrontendSpec{
+							JaegerQuery: v1alpha1.JaegerQuerySpec{
+								Enabled: true,
+								MonitorTab: v1alpha1.JaegerQueryMonitor{
+									Enabled:             true,
+									PrometheusEndpoint:  "http://prometheus:9091",
+									REDMetricsNamespace: ptr.To(""),
+								},
+							},
+						},
+					},
+				},
+			},
+			args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace="},
+			env:  []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}},
+		},
 		{
 			name: "OpenShift user-workload monitoring",
 			tempo: v1alpha1.TempoStack{
@@ -658,7 +699,6 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) {
 				"--span-storage.type=grpc",
 				"--grpc-storage.server=localhost:7777",
 				"--query.bearer-token-propagation=true",
-				"--prometheus.query.namespace=",
 				"--prometheus.tls.enabled=true",
 				"--prometheus.token-file=/var/run/secrets/kubernetes.io/serviceaccount/token",
 				"--prometheus.token-override-from-context=false",

tests/e2e-openshift/red-metrics/03-install-tempo.yaml

@@ -28,6 +28,5 @@ spec:
         monitorTab:
           enabled: true
           prometheusEndpoint: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
-          redMetricsNamespace: traces_span_metrics
         ingress:
           type: route