diff --git a/api/kubelb.k8c.io/v1alpha1/common_types.go b/api/kubelb.k8c.io/v1alpha1/common_types.go index 9ea4445..65e01d1 100644 --- a/api/kubelb.k8c.io/v1alpha1/common_types.go +++ b/api/kubelb.k8c.io/v1alpha1/common_types.go @@ -86,14 +86,37 @@ type EndpointAddress struct { Hostname string `json:"hostname,omitempty" protobuf:"bytes,3,opt,name=hostname"` } +type Annotations map[string]string + +// +kubebuilder:validation:Enum=all;service;ingress;gateway;httproute;grpcroute;tcproute;udproute;tlsroute +type AnnotatedResource string + +const ( + AnnotatedResourceAll AnnotatedResource = "all" + AnnotatedResourceService AnnotatedResource = "service" + AnnotatedResourceIngress AnnotatedResource = "ingress" + AnnotatedResourceGateway AnnotatedResource = "gateway" + AnnotatedResourceHTTPRoute AnnotatedResource = "httproute" + AnnotatedResourceGRPCRoute AnnotatedResource = "grpcroute" + AnnotatedResourceTCPRoute AnnotatedResource = "tcproute" + AnnotatedResourceUDPRoute AnnotatedResource = "udproute" + AnnotatedResourceTLSRoute AnnotatedResource = "tlsroute" +) + type AnnotationSettings struct { // PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - // This will have a higher precedence than the annotations specified at the Config level. + // Tenant configuration has higher precedence than the annotations specified at the Config level. // +optional PropagatedAnnotations *map[string]string `json:"propagatedAnnotations,omitempty"` // PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - // This will have a higher precedence than the value specified at the Config level. + // Tenant configuration has higher precedence than the value specified at the Config level. // +optional PropagateAllAnnotations *bool `json:"propagateAllAnnotations,omitempty"` + + // DefaultAnnotations defines the list of annotations(key-value pairs) that will be set on the load balancing resources if not already present. A special key `all` can be used to apply the same + // set of annotations to all resources. + // Tenant configuration has higher precedence than the annotations specified at the Config level. + // +optional + DefaultAnnotations map[AnnotatedResource]Annotations `json:"defaultAnnotations,omitempty"` } diff --git a/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go b/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go index 2f64aa0..a0c8094 100644 --- a/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go +++ b/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go @@ -139,6 +139,24 @@ func (in *AnnotationSettings) DeepCopyInto(out *AnnotationSettings) { *out = new(bool) **out = **in } + if in.DefaultAnnotations != nil { + in, out := &in.DefaultAnnotations, &out.DefaultAnnotations + *out = make(map[AnnotatedResource]Annotations, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(Annotations, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AnnotationSettings. @@ -151,6 +169,27 @@ func (in *AnnotationSettings) DeepCopy() *AnnotationSettings { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Annotations) DeepCopyInto(out *Annotations) { + { + in := &in + *out = make(Annotations, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Annotations. +func (in Annotations) DeepCopy() Annotations { + if in == nil { + return nil + } + out := new(Annotations) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Config) DeepCopyInto(out *Config) { *out = *in diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml index 0dd0156..6b60da5 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml @@ -40,6 +40,16 @@ spec: spec: description: ConfigSpec defines the desired state of the Config properties: + defaultAnnotations: + additionalProperties: + additionalProperties: + type: string + type: object + description: |- + DefaultAnnotations defines the list of annotations(key-value pairs) that will be set on the load balancing resources if not already present. A special key `all` can be used to apply the same + set of annotations to all resources. + Tenant configuration has higher precedence than the annotations specified at the Config level. + type: object envoyProxy: description: EnvoyProxy defines the desired state of the Envoy Proxy properties: @@ -1149,14 +1159,14 @@ spec: propagateAllAnnotations: description: |- PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - This will have a higher precedence than the value specified at the Config level. + Tenant configuration has higher precedence than the value specified at the Config level. type: boolean propagatedAnnotations: additionalProperties: type: string description: |- PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - This will have a higher precedence than the annotations specified at the Config level. + Tenant configuration has higher precedence than the annotations specified at the Config level. type: object type: object type: object diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml index da5a800..c85445d 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml @@ -39,6 +39,16 @@ spec: spec: description: TenantSpec defines the desired state of Tenant properties: + defaultAnnotations: + additionalProperties: + additionalProperties: + type: string + type: object + description: |- + DefaultAnnotations defines the list of annotations(key-value pairs) that will be set on the load balancing resources if not already present. A special key `all` can be used to apply the same + set of annotations to all resources. + Tenant configuration has higher precedence than the annotations specified at the Config level. + type: object gatewayAPI: description: GatewayAPISettings defines the settings for the gateway API. @@ -83,14 +93,14 @@ spec: propagateAllAnnotations: description: |- PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - This will have a higher precedence than the value specified at the Config level. + Tenant configuration has higher precedence than the value specified at the Config level. type: boolean propagatedAnnotations: additionalProperties: type: string description: |- PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - This will have a higher precedence than the annotations specified at the Config level. + Tenant configuration has higher precedence than the annotations specified at the Config level. type: object type: object status: diff --git a/config/crd/bases/kubelb.k8c.io_configs.yaml b/config/crd/bases/kubelb.k8c.io_configs.yaml index 0dd0156..6b60da5 100644 --- a/config/crd/bases/kubelb.k8c.io_configs.yaml +++ b/config/crd/bases/kubelb.k8c.io_configs.yaml @@ -40,6 +40,16 @@ spec: spec: description: ConfigSpec defines the desired state of the Config properties: + defaultAnnotations: + additionalProperties: + additionalProperties: + type: string + type: object + description: |- + DefaultAnnotations defines the list of annotations(key-value pairs) that will be set on the load balancing resources if not already present. A special key `all` can be used to apply the same + set of annotations to all resources. + Tenant configuration has higher precedence than the annotations specified at the Config level. + type: object envoyProxy: description: EnvoyProxy defines the desired state of the Envoy Proxy properties: @@ -1149,14 +1159,14 @@ spec: propagateAllAnnotations: description: |- PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - This will have a higher precedence than the value specified at the Config level. + Tenant configuration has higher precedence than the value specified at the Config level. type: boolean propagatedAnnotations: additionalProperties: type: string description: |- PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - This will have a higher precedence than the annotations specified at the Config level. + Tenant configuration has higher precedence than the annotations specified at the Config level. type: object type: object type: object diff --git a/config/crd/bases/kubelb.k8c.io_tenants.yaml b/config/crd/bases/kubelb.k8c.io_tenants.yaml index da5a800..c85445d 100644 --- a/config/crd/bases/kubelb.k8c.io_tenants.yaml +++ b/config/crd/bases/kubelb.k8c.io_tenants.yaml @@ -39,6 +39,16 @@ spec: spec: description: TenantSpec defines the desired state of Tenant properties: + defaultAnnotations: + additionalProperties: + additionalProperties: + type: string + type: object + description: |- + DefaultAnnotations defines the list of annotations(key-value pairs) that will be set on the load balancing resources if not already present. A special key `all` can be used to apply the same + set of annotations to all resources. + Tenant configuration has higher precedence than the annotations specified at the Config level. + type: object gatewayAPI: description: GatewayAPISettings defines the settings for the gateway API. @@ -83,14 +93,14 @@ spec: propagateAllAnnotations: description: |- PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - This will have a higher precedence than the value specified at the Config level. + Tenant configuration has higher precedence than the value specified at the Config level. type: boolean propagatedAnnotations: additionalProperties: type: string description: |- PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - This will have a higher precedence than the annotations specified at the Config level. + Tenant configuration has higher precedence than the annotations specified at the Config level. type: object type: object status: diff --git a/internal/controllers/kubelb/loadbalancer_controller.go b/internal/controllers/kubelb/loadbalancer_controller.go index 7d7cbdb..8bb4e25 100644 --- a/internal/controllers/kubelb/loadbalancer_controller.go +++ b/internal/controllers/kubelb/loadbalancer_controller.go @@ -235,7 +235,7 @@ func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalan Name: svcName, Namespace: namespace, Labels: labels, - Annotations: kubelb.PropagateAnnotations(loadBalancer.Annotations, annotations), + Annotations: kubelb.PropagateAnnotations(loadBalancer.Annotations, annotations, kubelbv1alpha1.AnnotatedResourceService), }, } err := r.Get(ctx, types.NamespacedName{ @@ -288,7 +288,7 @@ func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalan service.Annotations = make(map[string]string) } - for k, v := range kubelb.PropagateAnnotations(loadBalancer.Annotations, annotations) { + for k, v := range kubelb.PropagateAnnotations(loadBalancer.Annotations, annotations, kubelbv1alpha1.AnnotatedResourceService) { service.Annotations[k] = v } service.Spec.Ports = ports diff --git a/internal/controllers/kubelb/shared.go b/internal/controllers/kubelb/shared.go index 1e28b35..65a1aba 100644 --- a/internal/controllers/kubelb/shared.go +++ b/internal/controllers/kubelb/shared.go @@ -72,5 +72,12 @@ func GetAnnotations(tenant *kubelbv1alpha1.Tenant, config *kubelbv1alpha1.Config } else if config.Spec.AnnotationSettings.PropagatedAnnotations != nil { annotations.PropagatedAnnotations = config.Spec.AnnotationSettings.PropagatedAnnotations } + + if tenant.Spec.AnnotationSettings.DefaultAnnotations != nil { + annotations.DefaultAnnotations = tenant.Spec.AnnotationSettings.DefaultAnnotations + } else if config.Spec.AnnotationSettings.DefaultAnnotations != nil { + annotations.DefaultAnnotations = config.Spec.AnnotationSettings.DefaultAnnotations + } + return annotations } diff --git a/internal/kubelb/utils.go b/internal/kubelb/utils.go index 191928b..6e14e80 100644 --- a/internal/kubelb/utils.go +++ b/internal/kubelb/utils.go @@ -90,13 +90,13 @@ func GetNamespace(obj client.Object) string { return namespace } -func PropagateAnnotations(loadbalancer map[string]string, annotations kubelbv1alpha1.AnnotationSettings) map[string]string { +func PropagateAnnotations(loadbalancer map[string]string, annotations kubelbv1alpha1.AnnotationSettings, resource kubelbv1alpha1.AnnotatedResource) map[string]string { if loadbalancer == nil { loadbalancer = make(map[string]string) } if annotations.PropagateAllAnnotations != nil && *annotations.PropagateAllAnnotations { - return loadbalancer + return applyDefaultAnnotations(loadbalancer, annotations, resource) } permitted := make(map[string]string) @@ -126,7 +126,33 @@ func PropagateAnnotations(loadbalancer map[string]string, annotations kubelbv1al } } } - return a + return applyDefaultAnnotations(a, annotations, resource) +} + +func applyDefaultAnnotations(loadbalancer map[string]string, annotations kubelbv1alpha1.AnnotationSettings, resource kubelbv1alpha1.AnnotatedResource) map[string]string { + if annotations.DefaultAnnotations != nil { + if _, ok := annotations.DefaultAnnotations[resource]; ok { + // Merge the default annotations with the loadbalancer annotations while giving precedence to the loadbalancer annotations. If an annotation + // already exists in the loadbalancer, it will not be overridden by the default annotations. + for k, v := range annotations.DefaultAnnotations[resource] { + if _, ok := loadbalancer[k]; !ok { + loadbalancer[k] = v + } + } + } + + // Apply the default annotations for all resources if they are set. + if _, ok := annotations.DefaultAnnotations[kubelbv1alpha1.AnnotatedResourceAll]; ok { + // Merge the default annotations with the loadbalancer annotations while giving precedence to the loadbalancer annotations. If an annotation + // already exists in the loadbalancer, it will not be overridden by the default annotations. + for k, v := range annotations.DefaultAnnotations[kubelbv1alpha1.AnnotatedResourceAll] { + if _, ok := loadbalancer[k]; !ok { + loadbalancer[k] = v + } + } + } + } + return loadbalancer } func AddKubeLBLabels(labels map[string]string, name, namespace, gvk string) map[string]string { diff --git a/internal/resources/gatewayapi/gateway/gateway.go b/internal/resources/gatewayapi/gateway/gateway.go index 19ba7b4..967b52c 100644 --- a/internal/resources/gatewayapi/gateway/gateway.go +++ b/internal/resources/gatewayapi/gateway/gateway.go @@ -63,7 +63,7 @@ func CreateOrUpdateGateway(ctx context.Context, log logr.Logger, client ctrlclie } // Process annotations. - object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations, kubelbv1alpha1.AnnotatedResourceGateway) // Process secrets. for i, listener := range object.Spec.Listeners { diff --git a/internal/resources/gatewayapi/grpcroute/grpcroute.go b/internal/resources/gatewayapi/grpcroute/grpcroute.go index c81a586..e6b1204 100644 --- a/internal/resources/gatewayapi/grpcroute/grpcroute.go +++ b/internal/resources/gatewayapi/grpcroute/grpcroute.go @@ -92,7 +92,7 @@ func CreateOrUpdateGRPCRoute(ctx context.Context, log logr.Logger, client ctrlcl } // Process annotations. - object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations, kubelbv1alpha1.AnnotatedResourceGRPCRoute) // Process labels object.Labels = kubelb.AddKubeLBLabels(object.Labels, object.Name, object.Namespace, "") diff --git a/internal/resources/gatewayapi/httproute/httproute.go b/internal/resources/gatewayapi/httproute/httproute.go index 7c16707..529f6fa 100644 --- a/internal/resources/gatewayapi/httproute/httproute.go +++ b/internal/resources/gatewayapi/httproute/httproute.go @@ -92,7 +92,7 @@ func CreateOrUpdateHTTPRoute(ctx context.Context, log logr.Logger, client ctrlcl } // Process annotations. - object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations, kubelbv1alpha1.AnnotatedResourceHTTPRoute) // Process labels object.Labels = kubelb.AddKubeLBLabels(object.Labels, object.Name, object.Namespace, "") diff --git a/internal/resources/ingress/ingress.go b/internal/resources/ingress/ingress.go index aef351c..dac3455 100644 --- a/internal/resources/ingress/ingress.go +++ b/internal/resources/ingress/ingress.go @@ -70,7 +70,7 @@ func CreateOrUpdateIngress(ctx context.Context, log logr.Logger, client ctrlclie object.Spec.IngressClassName = className // Process annotations. - object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations, kubelbv1alpha1.AnnotatedResourceIngress) // Process secrets. if object.Spec.TLS != nil { diff --git a/internal/resources/service/service.go b/internal/resources/service/service.go index 913ac7f..a06d005 100644 --- a/internal/resources/service/service.go +++ b/internal/resources/service/service.go @@ -140,7 +140,7 @@ func GenerateServiceForLBCluster(service corev1.Service, appName, namespace stri kubelb.LabelAppKubernetesName: appName, } } - service.Annotations = kubelb.PropagateAnnotations(service.Annotations, annotations) + service.Annotations = kubelb.PropagateAnnotations(service.Annotations, annotations, kubelbv1alpha1.AnnotatedResourceService) return service }