diff --git a/.gitignore b/.gitignore index 7eb3623..0959165 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,5 @@ _build .DS_Store /bin cover.out -charts/*/Chart.lock kubelb-*.tgz __debug* \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 66a6532..ce2707a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -74,8 +74,8 @@ linters-settings: issues: exclude: # TODO: These methods are more or less the same and we need to build an abstraction for this. That would lower the cyclomatic complexity. - - 'cyclomatic complexity 33 of func `\(\*RouteReconciler\)\.createOrUpdateGRPCRoute` is high' - - 'cyclomatic complexity 33 of func `\(\*RouteReconciler\)\.createOrUpdateHTTPRoute` is high' + - "cyclomatic complexity 33 of func `CreateOrUpdateGRPCRoute` is high" + - "cyclomatic complexity 33 of func `CreateOrUpdateHTTPRoute` is high" exclude-dirs: - hack - vendor diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 461bc82..380dcb0 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -67,27 +67,28 @@ postsubmits: requests: cpu: 100m memory: 1Gi -# - name: ci-push-kubelb-charts -# always_run: true -# decorate: true -# clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" -# branches: -# # Match on tags -# - ^v\d+\.\d+\.\d+.* -# reporter_config: -# slack: -# channel: dev-kubelb -# labels: -# preset-docker-push: "true" -# preset-goproxy: "true" -# spec: -# containers: -# - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 -# command: -# - make -# args: -# - release-charts -# resources: -# requests: -# cpu: 100m -# memory: 500m + + - name: ci-push-kubelb-charts + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" + branches: + # Match on tags + - ^v\d+\.\d+\.\d+.* + reporter_config: + slack: + channel: dev-kubelb + labels: + preset-docker-push: "true" + preset-goproxy: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + command: + - make + args: + - release-charts + resources: + requests: + cpu: 100m + memory: 500m diff --git a/.prow/verify.yaml b/.prow/verify.yaml index c71bb0c..ff6dbd7 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -133,7 +133,6 @@ presubmits: cpu: 4 - name: pull-kubelb-e2e-tests - optional: true always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" diff --git a/api/kubelb.k8c.io/v1alpha1/addresses_types.go b/api/kubelb.k8c.io/v1alpha1/addresses_types.go index c676799..1f612c5 100644 --- a/api/kubelb.k8c.io/v1alpha1/addresses_types.go +++ b/api/kubelb.k8c.io/v1alpha1/addresses_types.go @@ -23,7 +23,7 @@ import ( // AddressesSpec defines the desired state of Addresses type AddressesSpec struct { // Addresses contains a list of addresses. - //+kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MinItems:=1 Addresses []EndpointAddress `json:"addresses,omitempty" protobuf:"bytes,1,rep,name=addresses"` } diff --git a/api/kubelb.k8c.io/v1alpha1/common_types.go b/api/kubelb.k8c.io/v1alpha1/common_types.go index 452a474..9ea4445 100644 --- a/api/kubelb.k8c.io/v1alpha1/common_types.go +++ b/api/kubelb.k8c.io/v1alpha1/common_types.go @@ -42,7 +42,7 @@ type LoadBalancerEndpoints struct { // IP addresses which offer the related ports that are marked as ready. These endpoints // should be considered safe for load balancers and clients to utilize. - //+kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MinItems:=1 Addresses []EndpointAddress `json:"addresses,omitempty" protobuf:"bytes,1,rep,name=addresses"` // AddressesReference is a reference to the Addresses object that contains the IP addresses. @@ -53,7 +53,7 @@ type LoadBalancerEndpoints struct { // Port numbers available on the related IP addresses. // This field is ignored for routes that are using kubernetes resources as the source. // +optional - //+kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MinItems=1 Ports []EndpointPort `json:"ports,omitempty" protobuf:"bytes,3,rep,name=ports"` } @@ -85,3 +85,15 @@ type EndpointAddress struct { // +optional Hostname string `json:"hostname,omitempty" protobuf:"bytes,3,opt,name=hostname"` } + +type AnnotationSettings struct { + // PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. + // This will have a higher precedence than the annotations specified at the Config level. + // +optional + PropagatedAnnotations *map[string]string `json:"propagatedAnnotations,omitempty"` + + // PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. + // This will have a higher precedence than the value specified at the Config level. + // +optional + PropagateAllAnnotations *bool `json:"propagateAllAnnotations,omitempty"` +} diff --git a/api/kubelb.k8c.io/v1alpha1/config_types.go b/api/kubelb.k8c.io/v1alpha1/config_types.go index dc5aa5e..2fadb9c 100644 --- a/api/kubelb.k8c.io/v1alpha1/config_types.go +++ b/api/kubelb.k8c.io/v1alpha1/config_types.go @@ -31,27 +31,14 @@ const ( // ConfigSpec defines the desired state of the Config type ConfigSpec struct { + AnnotationSettings `json:",inline"` + // EnvoyProxy defines the desired state of the Envoy Proxy EnvoyProxy EnvoyProxy `json:"envoyProxy,omitempty"` - // PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the value empty to allow any value. - // Annotations specified at the namespace level will have a higher precedence than the annotations specified at the Config level. - // +optional - PropagatedAnnotations map[string]string `json:"propagatedAnnotations,omitempty"` - - // PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - // +optional - PropagateAllAnnotations bool `json:"propagateAllAnnotations,omitempty"` - - // IngressClassName is the name of the IngressClass that will be used for the routes created by KubeLB. If not specified, KubeLB will replace the IngressClassName - // with an empty value in the Ingress resource which would result in the default IngressClass being used. - // +optional - IngressClassName *string `json:"ingressClassName,omitempty"` - - // GatewayClassName is the name of the GatewayClass that will be used for the routes created by KubeLB. - // If not specified, KubeLB will replace the GatewayClassName with an empty value in the Gateway resource which would result in the default GatewayClass being used. - // +optional - GatewayClassName *string `json:"gatewayClassName,omitempty"` + LoadBalancer LoadBalancerSettings `json:"loadBalancer,omitempty"` + Ingress IngressSettings `json:"ingress,omitempty"` + GatewayAPI GatewayAPISettings `json:"gatewayAPI,omitempty"` } // EnvoyProxy defines the desired state of the EnvoyProxy @@ -97,8 +84,8 @@ type EnvoyProxy struct { Affinity *corev1.Affinity `json:"affinity,omitempty"` } -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status // Config is the object that represents the Config for the KubeLB management controller. type Config struct { @@ -108,7 +95,15 @@ type Config struct { Spec ConfigSpec `json:"spec,omitempty"` } -//+kubebuilder:object:root=true +func (c *Config) GetEnvoyProxyTopology() EnvoyProxyTopology { + return c.Spec.EnvoyProxy.Topology +} + +func (c *Config) IsGlobalTopology() bool { + return c.Spec.EnvoyProxy.Topology == EnvoyProxyTopologyGlobal +} + +// +kubebuilder:object:root=true // ConfigList contains a list of Config type ConfigList struct { diff --git a/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go b/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go index 515f021..73d12d1 100644 --- a/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go +++ b/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go @@ -72,7 +72,7 @@ type LoadBalancerSpec struct { // Sets of addresses and ports that comprise an exposed user service on a cluster. // +required - //+kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MinItems=1 Endpoints []LoadBalancerEndpoints `json:"endpoints,omitempty"` // The list of ports that are exposed by the load balancer service. @@ -102,8 +102,8 @@ type LoadBalancerSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:shortName=lb -// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-name",name="OriginName",type="string" -// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-ns",name="OriginNamespace",type="string" +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb\\.k8c\\.io/origin-name",name="OriginName",type="string" +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb\\.k8c\\.io/origin-ns",name="OriginNamespace",type="string" // +genclient // LoadBalancer is the Schema for the loadbalancers API diff --git a/api/kubelb.k8c.io/v1alpha1/route_types.go b/api/kubelb.k8c.io/v1alpha1/route_types.go index 1c18c7e..3be1084 100644 --- a/api/kubelb.k8c.io/v1alpha1/route_types.go +++ b/api/kubelb.k8c.io/v1alpha1/route_types.go @@ -28,7 +28,7 @@ import ( type RouteSpec struct { // Sets of addresses and ports that comprise an exposed user service on a cluster. // +required - //+kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MinItems=1 Endpoints []LoadBalancerEndpoints `json:"endpoints,omitempty"` // Source contains the information about the source of the route. This is used when the route is created from external sources. @@ -58,13 +58,6 @@ type KubernetesSource struct { // Services contains the list of services that are used as the source for the Route. // +kubebuilder:pruning:PreserveUnknownFields Services []UpstreamService `json:"services,omitempty"` - - // ReferenceGrants contains the list of ReferenceGrants that are used as the source for the Route. - // ReferenceGrant identifies kinds of resources in other namespaces that are - // trusted to reference the specified kinds of resources in the same namespace - // as the policy. - // +kubebuilder:pruning:PreserveUnknownFields - ReferenceGrants []UpstreamReferenceGrant `json:"referenceGrants,omitempty"` } // TODO(waleed): Evaluate if this is really worth it, semantically it makes sense but it adds a lot of boilerplate. Alternatively, @@ -90,11 +83,11 @@ type UpstreamReferenceGrant struct { gwapiv1alpha2.ReferenceGrant `json:",inline"` } -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status -// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-name",name="OriginName",type="string" -// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-ns",name="OriginNamespace",type="string" -// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-resource-kind",name="OriginResource",type="string" +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb\\.k8c\\.io/origin-name",name="OriginName",type="string" +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb\\.k8c\\.io/origin-ns",name="OriginNamespace",type="string" +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb\\.k8c\\.io/origin-resource-kind",name="OriginResource",type="string" // Route is the object that represents a route in the cluster. type Route struct { @@ -116,8 +109,6 @@ type RouteResourcesStatus struct { Services map[string]RouteServiceStatus `json:"services,omitempty"` - ReferenceGrants map[string]ResourceState `json:"referenceGrants,omitempty"` - Route ResourceState `json:"route,omitempty"` } @@ -158,7 +149,7 @@ func (t ConditionType) String() string { return string(t) } -//+kubebuilder:object:root=true +// +kubebuilder:object:root=true // RouteList contains a list of Routes type RouteList struct { diff --git a/api/kubelb.k8c.io/v1alpha1/tenant_types.go b/api/kubelb.k8c.io/v1alpha1/tenant_types.go index 62c18e6..e4b1c7a 100644 --- a/api/kubelb.k8c.io/v1alpha1/tenant_types.go +++ b/api/kubelb.k8c.io/v1alpha1/tenant_types.go @@ -20,43 +20,47 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - // TenantSpec defines the desired state of Tenant type TenantSpec struct { + AnnotationSettings `json:",inline"` + LoadBalancer LoadBalancerSettings `json:"loadBalancer,omitempty"` - // Ingress IngressSettings `json:"ingress,omitempty"` - // GatewayAPI GatewayAPISettings `json:"gatewayAPI,omitempty"` + Ingress IngressSettings `json:"ingress,omitempty"` + GatewayAPI GatewayAPISettings `json:"gatewayAPI,omitempty"` } +// LoadBalancerSettings defines the settings for the load balancers. type LoadBalancerSettings struct { // Class is the class of the load balancer to use. + // This has higher precedence than the value specified in the Config. // +optional Class *string `json:"class,omitempty"` - // PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - // This will have a higher precedence than the annotations specified at the Config level. - // +optional - PropagatedAnnotations *map[string]string `json:"propagatedAnnotations,omitempty"` + // Disable is a flag that can be used to disable L4 load balancing for a tenant. + Disable bool `json:"disable,omitempty"` +} - // PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - // This will have a higher precedence than the value specified at the Config level. +// IngressSettings defines the settings for the ingress. +type IngressSettings struct { + // Class is the class of the ingress to use. + // This has higher precedence than the value specified in the Config. // +optional - PropagateAllAnnotations *bool `json:"propagateAllAnnotations,omitempty"` + Class *string `json:"class,omitempty"` + + // Disable is a flag that can be used to disable Ingress for a tenant. + Disable bool `json:"disable,omitempty"` } -// type IngressSettings struct { -// // Class is the class of the ingress to use. -// // +optional -// Class *string `json:"class,omitempty"` -// } - -// type GatewayAPISettings struct { -// // Class is the class of the gateway API to use. This can be used to -// // +optional -// Class *string `json:"class,omitempty"` -// } +// GatewayAPISettings defines the settings for the gateway API. +type GatewayAPISettings struct { + // Class is the class of the gateway API to use. This can be used to specify a specific gateway API implementation. + // This has higher precedence than the value specified in the Config. + // +optional + Class *string `json:"class,omitempty"` + + // Disable is a flag that can be used to disable Gateway API for a tenant. + Disable bool `json:"disable,omitempty"` +} // TenantStatus defines the observed state of Tenant type TenantStatus struct { diff --git a/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go b/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go index 90385ad..eb051d4 100644 --- a/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go +++ b/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go @@ -120,6 +120,37 @@ func (in *AddressesStatus) DeepCopy() *AddressesStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AnnotationSettings) DeepCopyInto(out *AnnotationSettings) { + *out = *in + if in.PropagatedAnnotations != nil { + in, out := &in.PropagatedAnnotations, &out.PropagatedAnnotations + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.PropagateAllAnnotations != nil { + in, out := &in.PropagateAllAnnotations, &out.PropagateAllAnnotations + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AnnotationSettings. +func (in *AnnotationSettings) DeepCopy() *AnnotationSettings { + if in == nil { + return nil + } + out := new(AnnotationSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Config) DeepCopyInto(out *Config) { *out = *in @@ -181,24 +212,11 @@ func (in *ConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = *in + in.AnnotationSettings.DeepCopyInto(&out.AnnotationSettings) in.EnvoyProxy.DeepCopyInto(&out.EnvoyProxy) - if in.PropagatedAnnotations != nil { - in, out := &in.PropagatedAnnotations, &out.PropagatedAnnotations - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.IngressClassName != nil { - in, out := &in.IngressClassName, &out.IngressClassName - *out = new(string) - **out = **in - } - if in.GatewayClassName != nil { - in, out := &in.GatewayClassName, &out.GatewayClassName - *out = new(string) - **out = **in - } + in.LoadBalancer.DeepCopyInto(&out.LoadBalancer) + in.Ingress.DeepCopyInto(&out.Ingress) + in.GatewayAPI.DeepCopyInto(&out.GatewayAPI) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -280,6 +298,46 @@ func (in *EnvoyProxy) DeepCopy() *EnvoyProxy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayAPISettings) DeepCopyInto(out *GatewayAPISettings) { + *out = *in + if in.Class != nil { + in, out := &in.Class, &out.Class + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayAPISettings. +func (in *GatewayAPISettings) DeepCopy() *GatewayAPISettings { + if in == nil { + return nil + } + out := new(GatewayAPISettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressSettings) DeepCopyInto(out *IngressSettings) { + *out = *in + if in.Class != nil { + in, out := &in.Class, &out.Class + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSettings. +func (in *IngressSettings) DeepCopy() *IngressSettings { + if in == nil { + return nil + } + out := new(IngressSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesSource) DeepCopyInto(out *KubernetesSource) { *out = *in @@ -291,13 +349,6 @@ func (in *KubernetesSource) DeepCopyInto(out *KubernetesSource) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.ReferenceGrants != nil { - in, out := &in.ReferenceGrants, &out.ReferenceGrants - *out = make([]UpstreamReferenceGrant, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSource. @@ -422,22 +473,6 @@ func (in *LoadBalancerSettings) DeepCopyInto(out *LoadBalancerSettings) { *out = new(string) **out = **in } - if in.PropagatedAnnotations != nil { - in, out := &in.PropagatedAnnotations, &out.PropagatedAnnotations - *out = new(map[string]string) - if **in != nil { - in, out := *in, *out - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - } - if in.PropagateAllAnnotations != nil { - in, out := &in.PropagateAllAnnotations, &out.PropagateAllAnnotations - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerSettings. @@ -586,13 +621,6 @@ func (in *RouteResourcesStatus) DeepCopyInto(out *RouteResourcesStatus) { (*out)[key] = *val.DeepCopy() } } - if in.ReferenceGrants != nil { - in, out := &in.ReferenceGrants, &out.ReferenceGrants - *out = make(map[string]ResourceState, len(*in)) - for key, val := range *in { - (*out)[key] = *val.DeepCopy() - } - } in.Route.DeepCopyInto(&out.Route) } @@ -788,7 +816,10 @@ func (in *TenantList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { *out = *in + in.AnnotationSettings.DeepCopyInto(&out.AnnotationSettings) in.LoadBalancer.DeepCopyInto(&out.LoadBalancer) + in.Ingress.DeepCopyInto(&out.Ingress) + in.GatewayAPI.DeepCopyInto(&out.GatewayAPI) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantSpec. diff --git a/charts/kubelb-ccm/templates/clusterrole.yaml b/charts/kubelb-ccm/templates/clusterrole.yaml index 0c2f892..135eade 100644 --- a/charts/kubelb-ccm/templates/clusterrole.yaml +++ b/charts/kubelb-ccm/templates/clusterrole.yaml @@ -33,6 +33,7 @@ rules: - get - patch - update +{{- if not .Values.kubelb.disableGatewayAPI }} {{- if not .Values.kubelb.disableGatewayController }} - apiGroups: - gateway.networking.k8s.io @@ -63,6 +64,7 @@ rules: - list - watch {{- end }} +{{- end }} {{- if not .Values.kubelb.disableIngressController }} - apiGroups: - networking.k8s.io diff --git a/charts/kubelb-ccm/templates/deployment.yaml b/charts/kubelb-ccm/templates/deployment.yaml index 857ad57..872be58 100644 --- a/charts/kubelb-ccm/templates/deployment.yaml +++ b/charts/kubelb-ccm/templates/deployment.yaml @@ -34,7 +34,6 @@ spec: - args: - --secure-listen-address=0.0.0.0:8443 - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - --v=0 image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 imagePullPolicy: {{ .Values.image.pullPolicy }} diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml index da7fb93..759c0ad 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml @@ -1105,27 +1105,58 @@ spec: If set to true, Replicas will be ignored. type: boolean type: object - gatewayClassName: - description: |- - GatewayClassName is the name of the GatewayClass that will be used for the routes created by KubeLB. - If not specified, KubeLB will replace the GatewayClassName with an empty value in the Gateway resource which would result in the default GatewayClass being used. - type: string - ingressClassName: - description: |- - IngressClassName is the name of the IngressClass that will be used for the routes created by KubeLB. If not specified, KubeLB will replace the IngressClassName - with an empty value in the Ingress resource which would result in the default IngressClass being used. - type: string + gatewayAPI: + description: GatewayAPISettings defines the settings for the gateway + API. + properties: + class: + description: |- + Class is the class of the gateway API to use. This can be used to specify a specific gateway API implementation. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable Gateway + API for a tenant. + type: boolean + type: object + ingress: + description: IngressSettings defines the settings for the ingress. + properties: + class: + description: |- + Class is the class of the ingress to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable Ingress + for a tenant. + type: boolean + type: object + loadBalancer: + description: LoadBalancerSettings defines the settings for the load + balancers. + properties: + class: + description: |- + Class is the class of the load balancer to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable L4 + load balancing for a tenant. + type: boolean + type: object propagateAllAnnotations: - description: PropagateAllAnnotations defines whether all annotations - will be propagated to the LoadBalancer service. If set to true, - PropagatedAnnotations will be ignored. + description: |- + PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. + This will have a higher precedence than the value specified at the Config level. type: boolean propagatedAnnotations: additionalProperties: type: string description: |- - PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the value empty to allow any value. - Annotations specified at the namespace level will have a higher precedence than the annotations specified at the Config level. + PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. + This will have a higher precedence than the annotations specified at the Config level. type: object type: object type: object diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml index 85d50cb..38ae808 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml @@ -17,10 +17,10 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-name name: OriginName type: string - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-ns name: OriginNamespace type: string name: v1alpha1 diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml index 9df3edb..7eb04e5 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml @@ -15,13 +15,13 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-name name: OriginName type: string - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-ns name: OriginNamespace type: string - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-resource-kind + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-resource-kind name: OriginResource type: string name: v1alpha1 @@ -178,166 +178,6 @@ spec: Kubernetes contains the information about the Kubernetes source. This field is automatically populated by the KubeLB CCM and in most cases, users should not set this field manually. properties: - referenceGrants: - description: |- - ReferenceGrants contains the list of ReferenceGrants that are used as the source for the Route. - ReferenceGrant identifies kinds of resources in other namespaces that are - trusted to reference the specified kinds of resources in the same namespace - as the policy. - items: - description: |- - UpstreamReferenceGrant is a wrapper over the sigs.k8s.io/gateway-api/apis/v1alpha2.ReferenceGrant object. - This is required as kubebuilder:validation:EmbeddedResource marker adds the x-kubernetes-embedded-resource to the array instead of - the elements within it. Which results in a broken CRD; validation error. Without this marker, the embedded resource is not properly - serialized to the CRD. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of ReferenceGrant. - properties: - from: - description: |- - From describes the trusted namespaces and kinds that can reference the - resources described in "To". Each entry in this list MUST be considered - to be an additional place that references can be valid from, or to put - this another way, entries MUST be combined using OR. - - - Support: Core - items: - description: ReferenceGrantFrom describes trusted - namespaces and kinds. - properties: - group: - description: |- - Group is the group of the referent. - When empty, the Kubernetes core API group is inferred. - - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: |- - Kind is the kind of the referent. Although implementations may support - additional resources, the following types are part of the "Core" - support level for this field. - - - When used to permit a SecretObjectReference: - - - * Gateway - - - When used to permit a BackendObjectReference: - - - * GRPCRoute - * HTTPRoute - * TCPRoute - * TLSRoute - * UDPRoute - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - namespace: - description: |- - Namespace is the namespace of the referent. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - namespace - type: object - maxItems: 16 - minItems: 1 - type: array - to: - description: |- - To describes the resources that may be referenced by the resources - described in "From". Each entry in this list MUST be considered to be an - additional place that references can be valid to, or to put this another - way, entries MUST be combined using OR. - - - Support: Core - items: - description: |- - ReferenceGrantTo describes what Kinds are allowed as targets of the - references. - properties: - group: - description: |- - Group is the group of the referent. - When empty, the Kubernetes core API group is inferred. - - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: |- - Kind is the kind of the referent. Although implementations may support - additional resources, the following types are part of the "Core" - support level for this field: - - - * Secret when used to permit a SecretObjectReference - * Service when used to permit a BackendObjectReference - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. When unspecified, this policy - refers to all resources of the specified Group and Kind in the local - namespace. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - type: object - maxItems: 16 - minItems: 1 - type: array - required: - - from - - to - type: object - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - type: array - x-kubernetes-preserve-unknown-fields: true resource: type: object x-kubernetes-embedded-resource: true @@ -897,103 +737,6 @@ spec: description: Resources contains the list of resources that are created/processed as a result of the Route. properties: - referenceGrants: - additionalProperties: - properties: - apiVersion: - description: APIVersion is the API version of the resource. - type: string - conditions: - items: - description: "Condition contains details for one aspect - of the current state of this API Resource.\n---\nThis - struct is intended for direct use as an array at the - field path .status.conditions. For example,\n\n\n\ttype - FooStatus struct{\n\t // Represents the observations - of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t - \ // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t - \ // +listType=map\n\t // +listMapKey=type\n\t - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, - False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - generatedName: - description: GeneratedName is the generated name of the - resource. - type: string - kind: - description: Kind is the kind of the resource. - type: string - name: - description: Name is the name of the resource. - type: string - namespace: - description: Namespace is the namespace of the resource. - type: string - status: - description: Status is the actual status of the resource. - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - type: object route: properties: apiVersion: diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml index bf00a4a..2de5283 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml @@ -39,23 +39,58 @@ spec: spec: description: TenantSpec defines the desired state of Tenant properties: - loadBalancer: + gatewayAPI: + description: GatewayAPISettings defines the settings for the gateway + API. properties: class: - description: Class is the class of the load balancer to use. + description: |- + Class is the class of the gateway API to use. This can be used to specify a specific gateway API implementation. + This has higher precedence than the value specified in the Config. type: string - propagateAllAnnotations: + disable: + description: Disable is a flag that can be used to disable Gateway + API for a tenant. + type: boolean + type: object + ingress: + description: IngressSettings defines the settings for the ingress. + properties: + class: description: |- - PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - This will have a higher precedence than the value specified at the Config level. + Class is the class of the ingress to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable Ingress + for a tenant. type: boolean - propagatedAnnotations: - additionalProperties: - type: string + type: object + loadBalancer: + description: LoadBalancerSettings defines the settings for the load + balancers. + properties: + class: description: |- - PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - This will have a higher precedence than the annotations specified at the Config level. - type: object + Class is the class of the load balancer to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable L4 + load balancing for a tenant. + type: boolean + type: object + propagateAllAnnotations: + description: |- + PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. + This will have a higher precedence than the value specified at the Config level. + type: boolean + propagatedAnnotations: + additionalProperties: + type: string + description: |- + PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. + This will have a higher precedence than the annotations specified at the Config level. type: object type: object status: diff --git a/charts/kubelb-manager/templates/deployment.yaml b/charts/kubelb-manager/templates/deployment.yaml index 535afe7..62248f2 100644 --- a/charts/kubelb-manager/templates/deployment.yaml +++ b/charts/kubelb-manager/templates/deployment.yaml @@ -34,7 +34,6 @@ spec: - args: - --secure-listen-address=0.0.0.0:8443 - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - --v=0 image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 imagePullPolicy: {{ .Values.image.pullPolicy }} diff --git a/cmd/ccm/main.go b/cmd/ccm/main.go index e82b812..83d14c6 100644 --- a/cmd/ccm/main.go +++ b/cmd/ccm/main.go @@ -285,7 +285,7 @@ func main() { os.Exit(1) // second signal. Exit directly. }() - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") diff --git a/cmd/kubelb/main.go b/cmd/kubelb/main.go index d5d2ef8..6105a5f 100644 --- a/cmd/kubelb/main.go +++ b/cmd/kubelb/main.go @@ -145,7 +145,7 @@ func main() { ctx := ctrl.SetupSignalHandler() // Load the Config for controller - err = config.LoadConfig(ctx, mgr.GetAPIReader(), opt.namespace) + conf, err := config.GetConfig(ctx, mgr.GetAPIReader(), opt.namespace) if err != nil { setupLog.Error(err, "unable to load controller config") os.Exit(1) @@ -162,7 +162,7 @@ func main() { Cache: mgr.GetCache(), Scheme: mgr.GetScheme(), Namespace: opt.namespace, - EnvoyProxyTopology: kubelb.EnvoyProxyTopology(config.GetEnvoyProxyTopology()), + EnvoyProxyTopology: kubelb.EnvoyProxyTopology(conf.GetEnvoyProxyTopology()), PortAllocator: portAllocator, }).SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LoadBalancer") @@ -181,10 +181,11 @@ func main() { if err = (&kubelb.EnvoyCPReconciler{ Client: envoyMgr.GetClient(), EnvoyCache: envoyServer.Cache, - EnvoyProxyTopology: kubelb.EnvoyProxyTopology(config.GetEnvoyProxyTopology()), + EnvoyProxyTopology: kubelb.EnvoyProxyTopology(conf.GetEnvoyProxyTopology()), PortAllocator: portAllocator, Namespace: opt.namespace, EnvoyBootstrap: envoyServer.GenerateBootstrap(), + DisableGatewayAPI: opt.disableGatewayAPI, }).SetupWithManager(ctx, envoyMgr); err != nil { setupLog.Error(err, "unable to create envoy control-plane controller", "controller", "LoadBalancer") os.Exit(1) @@ -195,9 +196,10 @@ func main() { Scheme: mgr.GetScheme(), Log: ctrl.Log.WithName("controllers").WithName(kubelb.RouteControllerName), Recorder: mgr.GetEventRecorderFor(kubelb.RouteControllerName), - EnvoyProxyTopology: kubelb.EnvoyProxyTopology(config.GetEnvoyProxyTopology()), + EnvoyProxyTopology: kubelb.EnvoyProxyTopology(conf.GetEnvoyProxyTopology()), PortAllocator: portAllocator, Namespace: opt.namespace, + DisableGatewayAPI: opt.disableGatewayAPI, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", kubelb.RouteControllerName) os.Exit(1) diff --git a/config/crd/bases/kubelb.k8c.io_configs.yaml b/config/crd/bases/kubelb.k8c.io_configs.yaml index da7fb93..759c0ad 100644 --- a/config/crd/bases/kubelb.k8c.io_configs.yaml +++ b/config/crd/bases/kubelb.k8c.io_configs.yaml @@ -1105,27 +1105,58 @@ spec: If set to true, Replicas will be ignored. type: boolean type: object - gatewayClassName: - description: |- - GatewayClassName is the name of the GatewayClass that will be used for the routes created by KubeLB. - If not specified, KubeLB will replace the GatewayClassName with an empty value in the Gateway resource which would result in the default GatewayClass being used. - type: string - ingressClassName: - description: |- - IngressClassName is the name of the IngressClass that will be used for the routes created by KubeLB. If not specified, KubeLB will replace the IngressClassName - with an empty value in the Ingress resource which would result in the default IngressClass being used. - type: string + gatewayAPI: + description: GatewayAPISettings defines the settings for the gateway + API. + properties: + class: + description: |- + Class is the class of the gateway API to use. This can be used to specify a specific gateway API implementation. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable Gateway + API for a tenant. + type: boolean + type: object + ingress: + description: IngressSettings defines the settings for the ingress. + properties: + class: + description: |- + Class is the class of the ingress to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable Ingress + for a tenant. + type: boolean + type: object + loadBalancer: + description: LoadBalancerSettings defines the settings for the load + balancers. + properties: + class: + description: |- + Class is the class of the load balancer to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable L4 + load balancing for a tenant. + type: boolean + type: object propagateAllAnnotations: - description: PropagateAllAnnotations defines whether all annotations - will be propagated to the LoadBalancer service. If set to true, - PropagatedAnnotations will be ignored. + description: |- + PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. + This will have a higher precedence than the value specified at the Config level. type: boolean propagatedAnnotations: additionalProperties: type: string description: |- - PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the value empty to allow any value. - Annotations specified at the namespace level will have a higher precedence than the annotations specified at the Config level. + PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. + This will have a higher precedence than the annotations specified at the Config level. type: object type: object type: object diff --git a/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml b/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml index 85d50cb..38ae808 100644 --- a/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml +++ b/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml @@ -17,10 +17,10 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-name name: OriginName type: string - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-ns name: OriginNamespace type: string name: v1alpha1 diff --git a/config/crd/bases/kubelb.k8c.io_routes.yaml b/config/crd/bases/kubelb.k8c.io_routes.yaml index 9df3edb..7eb04e5 100644 --- a/config/crd/bases/kubelb.k8c.io_routes.yaml +++ b/config/crd/bases/kubelb.k8c.io_routes.yaml @@ -15,13 +15,13 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-name name: OriginName type: string - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-ns name: OriginNamespace type: string - - jsonPath: .metadata.labels.kubelb.k8c.io/origin-resource-kind + - jsonPath: .metadata.labels.kubelb\.k8c\.io/origin-resource-kind name: OriginResource type: string name: v1alpha1 @@ -178,166 +178,6 @@ spec: Kubernetes contains the information about the Kubernetes source. This field is automatically populated by the KubeLB CCM and in most cases, users should not set this field manually. properties: - referenceGrants: - description: |- - ReferenceGrants contains the list of ReferenceGrants that are used as the source for the Route. - ReferenceGrant identifies kinds of resources in other namespaces that are - trusted to reference the specified kinds of resources in the same namespace - as the policy. - items: - description: |- - UpstreamReferenceGrant is a wrapper over the sigs.k8s.io/gateway-api/apis/v1alpha2.ReferenceGrant object. - This is required as kubebuilder:validation:EmbeddedResource marker adds the x-kubernetes-embedded-resource to the array instead of - the elements within it. Which results in a broken CRD; validation error. Without this marker, the embedded resource is not properly - serialized to the CRD. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of ReferenceGrant. - properties: - from: - description: |- - From describes the trusted namespaces and kinds that can reference the - resources described in "To". Each entry in this list MUST be considered - to be an additional place that references can be valid from, or to put - this another way, entries MUST be combined using OR. - - - Support: Core - items: - description: ReferenceGrantFrom describes trusted - namespaces and kinds. - properties: - group: - description: |- - Group is the group of the referent. - When empty, the Kubernetes core API group is inferred. - - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: |- - Kind is the kind of the referent. Although implementations may support - additional resources, the following types are part of the "Core" - support level for this field. - - - When used to permit a SecretObjectReference: - - - * Gateway - - - When used to permit a BackendObjectReference: - - - * GRPCRoute - * HTTPRoute - * TCPRoute - * TLSRoute - * UDPRoute - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - namespace: - description: |- - Namespace is the namespace of the referent. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - namespace - type: object - maxItems: 16 - minItems: 1 - type: array - to: - description: |- - To describes the resources that may be referenced by the resources - described in "From". Each entry in this list MUST be considered to be an - additional place that references can be valid to, or to put this another - way, entries MUST be combined using OR. - - - Support: Core - items: - description: |- - ReferenceGrantTo describes what Kinds are allowed as targets of the - references. - properties: - group: - description: |- - Group is the group of the referent. - When empty, the Kubernetes core API group is inferred. - - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: |- - Kind is the kind of the referent. Although implementations may support - additional resources, the following types are part of the "Core" - support level for this field: - - - * Secret when used to permit a SecretObjectReference - * Service when used to permit a BackendObjectReference - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. When unspecified, this policy - refers to all resources of the specified Group and Kind in the local - namespace. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - type: object - maxItems: 16 - minItems: 1 - type: array - required: - - from - - to - type: object - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - type: array - x-kubernetes-preserve-unknown-fields: true resource: type: object x-kubernetes-embedded-resource: true @@ -897,103 +737,6 @@ spec: description: Resources contains the list of resources that are created/processed as a result of the Route. properties: - referenceGrants: - additionalProperties: - properties: - apiVersion: - description: APIVersion is the API version of the resource. - type: string - conditions: - items: - description: "Condition contains details for one aspect - of the current state of this API Resource.\n---\nThis - struct is intended for direct use as an array at the - field path .status.conditions. For example,\n\n\n\ttype - FooStatus struct{\n\t // Represents the observations - of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t - \ // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t - \ // +listType=map\n\t // +listMapKey=type\n\t - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, - False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - generatedName: - description: GeneratedName is the generated name of the - resource. - type: string - kind: - description: Kind is the kind of the resource. - type: string - name: - description: Name is the name of the resource. - type: string - namespace: - description: Namespace is the namespace of the resource. - type: string - status: - description: Status is the actual status of the resource. - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - type: object route: properties: apiVersion: diff --git a/config/crd/bases/kubelb.k8c.io_tenants.yaml b/config/crd/bases/kubelb.k8c.io_tenants.yaml index bf00a4a..2de5283 100644 --- a/config/crd/bases/kubelb.k8c.io_tenants.yaml +++ b/config/crd/bases/kubelb.k8c.io_tenants.yaml @@ -39,23 +39,58 @@ spec: spec: description: TenantSpec defines the desired state of Tenant properties: - loadBalancer: + gatewayAPI: + description: GatewayAPISettings defines the settings for the gateway + API. properties: class: - description: Class is the class of the load balancer to use. + description: |- + Class is the class of the gateway API to use. This can be used to specify a specific gateway API implementation. + This has higher precedence than the value specified in the Config. type: string - propagateAllAnnotations: + disable: + description: Disable is a flag that can be used to disable Gateway + API for a tenant. + type: boolean + type: object + ingress: + description: IngressSettings defines the settings for the ingress. + properties: + class: description: |- - PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. - This will have a higher precedence than the value specified at the Config level. + Class is the class of the ingress to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable Ingress + for a tenant. type: boolean - propagatedAnnotations: - additionalProperties: - type: string + type: object + loadBalancer: + description: LoadBalancerSettings defines the settings for the load + balancers. + properties: + class: description: |- - PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. - This will have a higher precedence than the annotations specified at the Config level. - type: object + Class is the class of the load balancer to use. + This has higher precedence than the value specified in the Config. + type: string + disable: + description: Disable is a flag that can be used to disable L4 + load balancing for a tenant. + type: boolean + type: object + propagateAllAnnotations: + description: |- + PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored. + This will have a higher precedence than the value specified at the Config level. + type: boolean + propagatedAnnotations: + additionalProperties: + type: string + description: |- + PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value. + This will have a higher precedence than the annotations specified at the Config level. type: object type: object status: diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 65520ce..08b898e 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -35,7 +35,6 @@ spec: args: - "--secure-listen-address=0.0.0.0:8443" - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - "--v=0" ports: - containerPort: 8443 diff --git a/internal/config/config.go b/internal/config/config.go index afb5a83..3f526d6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -29,32 +29,13 @@ const ( DefaultConfigResourceName string = "default" ) -var config v1alpha1.Config - -func LoadConfig(ctx context.Context, apiReader client.Reader, namespace string) error { +func GetConfig(ctx context.Context, apiReader client.Reader, namespace string) (v1alpha1.Config, error) { conf := v1alpha1.Config{} if err := apiReader.Get(ctx, client.ObjectKey{ Namespace: namespace, Name: DefaultConfigResourceName, }, &conf); err != nil { - return err + return conf, err } - config = conf - return nil -} - -func GetConfig() v1alpha1.Config { - return config -} - -func SetConfig(conf v1alpha1.Config) { - config = conf -} - -func GetEnvoyProxyTopology() v1alpha1.EnvoyProxyTopology { - return config.Spec.EnvoyProxy.Topology -} - -func IsGlobalTopology() bool { - return GetEnvoyProxyTopology() == v1alpha1.EnvoyProxyTopologyGlobal + return conf, nil } diff --git a/internal/controllers/kubelb/envoy_cp_controller.go b/internal/controllers/kubelb/envoy_cp_controller.go index 8dac416..ef65590 100644 --- a/internal/controllers/kubelb/envoy_cp_controller.go +++ b/internal/controllers/kubelb/envoy_cp_controller.go @@ -25,7 +25,6 @@ import ( envoyresource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" - "k8c.io/kubelb/internal/config" utils "k8c.io/kubelb/internal/controllers" envoycp "k8c.io/kubelb/internal/envoy" "k8c.io/kubelb/internal/kubelb" @@ -40,6 +39,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -55,6 +55,8 @@ type EnvoyCPReconciler struct { PortAllocator *portlookup.PortAllocator Namespace string EnvoyBootstrap string + DisableGatewayAPI bool + Config *kubelbv1alpha1.Config } // +kubebuilder:rbac:groups=kubelb.k8c.io,resources=loadbalancers,verbs=get;list;watch @@ -64,6 +66,13 @@ func (r *EnvoyCPReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct log.V(2).Info("reconciling LoadBalancer") + // Retrieve updated config. + config, err := GetConfig(ctx, r.Client, r.Namespace) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to retrieve config: %w", err) + } + r.Config = config + return ctrl.Result{}, r.reconcile(ctx, req) } @@ -164,14 +173,14 @@ func (r *EnvoyCPReconciler) ListLoadBalancersAndRoutes(ctx context.Context, req lbs := make([]kubelbv1alpha1.LoadBalancer, 0, len(loadBalancers.Items)) for _, lb := range loadBalancers.Items { - if lb.DeletionTimestamp.IsZero() { + if lb.DeletionTimestamp.IsZero() && controllerutil.ContainsFinalizer(&lb, envoyProxyCleanupFinalizer) { lbs = append(lbs, lb) } } routeList := make([]kubelbv1alpha1.Route, 0, len(routes.Items)) for _, route := range routes.Items { - if route.DeletionTimestamp.IsZero() { + if route.DeletionTimestamp.IsZero() && controllerutil.ContainsFinalizer(&route, CleanupFinalizer) { routeList = append(routeList, route) } } @@ -192,7 +201,7 @@ func (r *EnvoyCPReconciler) cleanupEnvoyProxy(ctx context.Context, appName strin Namespace: namespace, } var envoyProxy ctrlruntimeclient.Object - if config.GetConfig().Spec.EnvoyProxy.UseDaemonset { + if r.Config.Spec.EnvoyProxy.UseDaemonset { envoyProxy = &appsv1.DaemonSet{ ObjectMeta: objMeta, } @@ -218,7 +227,7 @@ func (r *EnvoyCPReconciler) ensureEnvoyProxy(ctx context.Context, namespace, app Name: fmt.Sprintf(envoyResourcePattern, appName), Namespace: namespace, } - if config.GetConfig().Spec.EnvoyProxy.UseDaemonset { + if r.Config.Spec.EnvoyProxy.UseDaemonset { envoyProxy = &appsv1.DaemonSet{ ObjectMeta: objMeta, } @@ -238,7 +247,7 @@ func (r *EnvoyCPReconciler) ensureEnvoyProxy(ctx context.Context, namespace, app } original := envoyProxy.DeepCopyObject() - if config.GetConfig().Spec.EnvoyProxy.UseDaemonset { + if r.Config.Spec.EnvoyProxy.UseDaemonset { daemonset := envoyProxy.(*appsv1.DaemonSet) daemonset.Spec.Selector = &metav1.LabelSelector{ MatchLabels: map[string]string{kubelb.LabelAppKubernetesName: appName}, @@ -247,7 +256,7 @@ func (r *EnvoyCPReconciler) ensureEnvoyProxy(ctx context.Context, namespace, app envoyProxy = daemonset } else { deployment := envoyProxy.(*appsv1.Deployment) - var replicas = config.GetConfig().Spec.EnvoyProxy.Replicas + var replicas = r.Config.Spec.EnvoyProxy.Replicas deployment.Spec.Replicas = &replicas deployment.Spec.Selector = &metav1.LabelSelector{ MatchLabels: map[string]string{kubelb.LabelAppKubernetesName: appName}, @@ -275,7 +284,7 @@ func (r *EnvoyCPReconciler) ensureEnvoyProxy(ctx context.Context, namespace, app } func (r *EnvoyCPReconciler) getEnvoyProxyPodSpec(namespace, appName, snapshotName string) corev1.PodTemplateSpec { - envoyProxy := config.GetConfig().Spec.EnvoyProxy + envoyProxy := r.Config.Spec.EnvoyProxy template := corev1.PodTemplateSpec{ ObjectMeta: v1.ObjectMeta{ Name: appName, diff --git a/internal/controllers/kubelb/loadbalancer_controller.go b/internal/controllers/kubelb/loadbalancer_controller.go index 2d625f1..4f6938e 100644 --- a/internal/controllers/kubelb/loadbalancer_controller.go +++ b/internal/controllers/kubelb/loadbalancer_controller.go @@ -22,7 +22,6 @@ import ( "reflect" kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" - "k8c.io/kubelb/internal/config" utils "k8c.io/kubelb/internal/controllers" "k8c.io/kubelb/internal/kubelb" portlookup "k8c.io/kubelb/internal/port-lookup" @@ -129,7 +128,6 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request resourceNamespace = req.Namespace case EnvoyProxyTopologyGlobal: // List all loadbalancers. We don't care about the namespace here. - // TODO: ideally we should only process the load balancer that is being reconciled. err = r.List(ctx, &loadBalancers) if err != nil { log.Error(err, "unable to fetch LoadBalancer list") @@ -146,6 +144,38 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request return reconcile.Result{}, nil } + // Before proceeding further we need to make sure that the resource is reconcilable. + tenant, config, err := GetTenantAndConfig(ctx, r.Client, r.Namespace, RemoveTenantPrefix(loadBalancer.Namespace)) + if err != nil { + log.Error(err, "unable to fetch Tenant and Config, cannot proceed") + return reconcile.Result{}, err + } + + shouldReconcile, disabled, err := r.shouldReconcile(ctx, &loadBalancer, tenant, config) + if err != nil { + log.Error(err, "unable to determine if the LoadBalancer should be reconciled") + return reconcile.Result{}, err + } + + // If the resource is disabled, we need to clean up the resources + if controllerutil.ContainsFinalizer(&loadBalancer, envoyProxyCleanupFinalizer) && disabled { + log.V(3).Info("Removing load balancer as load balancing is disabled") + return reconcile.Result{}, r.handleEnvoyProxyCleanup(ctx, loadBalancer, resourceNamespace) + } + + if !shouldReconcile { + return reconcile.Result{}, nil + } + + var className *string + if tenant.Spec.LoadBalancer.Class != nil { + className = tenant.Spec.LoadBalancer.Class + } else if config.Spec.LoadBalancer.Class != nil { + className = config.Spec.LoadBalancer.Class + } + + annotations := GetAnnotations(tenant, config) + // Add finalizer if it doesn't exist if !controllerutil.ContainsFinalizer(&loadBalancer, envoyProxyCleanupFinalizer) { if ok := controllerutil.AddFinalizer(&loadBalancer, envoyProxyCleanupFinalizer); !ok { @@ -166,7 +196,7 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request } _, appName := envoySnapshotAndAppName(r.EnvoyProxyTopology, req) - err = r.reconcileService(ctx, &loadBalancer, appName, resourceNamespace, r.PortAllocator) + err = r.reconcileService(ctx, &loadBalancer, appName, resourceNamespace, r.PortAllocator, className, annotations) if err != nil { log.Error(err, "Unable to reconcile service") return ctrl.Result{}, err @@ -175,17 +205,12 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } -func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalancer *kubelbv1alpha1.LoadBalancer, appName, namespace string, portAllocator *portlookup.PortAllocator) error { +func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalancer *kubelbv1alpha1.LoadBalancer, appName, namespace string, portAllocator *portlookup.PortAllocator, className *string, + annotations kubelbv1alpha1.AnnotationSettings) error { log := ctrl.LoggerFrom(ctx).WithValues("reconcile", "service") log.V(2).Info("verify service") - tenant, err := getTenantForNamespace(ctx, r.Client, namespace) - if err != nil { - // This should never happen as the namespace should always have a tenant owner. We simply log this and continue. - log.V(5).Info("Tenant not found for namespace", "namespace", namespace) - } - labels := map[string]string{ kubelb.LabelAppKubernetesName: appName, // This helps us to identify the LoadBalancer that this service belongs to. @@ -206,10 +231,10 @@ func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalan Name: svcName, Namespace: namespace, Labels: labels, - Annotations: propagateAnnotations(tenant, loadBalancer.Annotations), + Annotations: kubelb.PropagateAnnotations(loadBalancer.Annotations, annotations), }, } - err = r.Get(ctx, types.NamespacedName{ + err := r.Get(ctx, types.NamespacedName{ Name: svcName, Namespace: namespace, }, service) @@ -255,13 +280,19 @@ func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalan ports = append(ports, allocatedPort) } - for k, v := range propagateAnnotations(tenant, loadBalancer.Annotations) { + for k, v := range kubelb.PropagateAnnotations(loadBalancer.Annotations, annotations) { service.Annotations[k] = v } service.Spec.Ports = ports service.Spec.Selector = map[string]string{kubelb.LabelAppKubernetesName: appName} service.Spec.Type = loadBalancer.Spec.Type + + // Set the LoadBalancerClassName if it is specified in the configuration. + if className != nil { + service.Spec.LoadBalancerClass = className + } + return nil }) @@ -367,6 +398,20 @@ func (r *LoadBalancerReconciler) handleEnvoyProxyCleanup(ctx context.Context, lb return nil } +func (r *LoadBalancerReconciler) shouldReconcile(ctx context.Context, _ *kubelbv1alpha1.LoadBalancer, tenant *kubelbv1alpha1.Tenant, config *kubelbv1alpha1.Config) (bool, bool, error) { + log := ctrl.LoggerFrom(ctx) + + // 1. Ensure that L4 loadbalancing is enabled. + if config.Spec.LoadBalancer.Disable { + log.Error(fmt.Errorf("L4 loadbalancing is disabled at the global level"), "cannot proceed") + return false, true, nil + } else if tenant.Spec.LoadBalancer.Disable { + log.Error(fmt.Errorf("L4 loadbalancing is disabled at the tenant level"), "cannot proceed") + return false, true, nil + } + return true, false, nil +} + func (r *LoadBalancerReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&kubelbv1alpha1.LoadBalancer{}). @@ -376,6 +421,10 @@ func (r *LoadBalancerReconciler) SetupWithManager(ctx context.Context, mgr ctrl. handler.EnqueueRequestsFromMapFunc(r.enqueueLoadBalancersForConfig()), builder.WithPredicates(filterServicesPredicate()), ). + Watches( + &kubelbv1alpha1.Tenant{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueLoadBalancersForTenant()), + ). Watches( &corev1.Service{}, handler.EnqueueRequestsFromMapFunc(r.enqueueLoadBalancers()), @@ -442,66 +491,15 @@ func filterServicesPredicate() predicate.TypedPredicate[client.Object] { } } -func propagateAnnotations(tenant *kubelbv1alpha1.Tenant, loadbalancer map[string]string) map[string]string { - permitted := make(map[string]string) - if tenant != nil { - if tenant.Spec.LoadBalancer.PropagateAllAnnotations != nil && *tenant.Spec.LoadBalancer.PropagateAllAnnotations { - return loadbalancer - } - if tenant.Spec.LoadBalancer.PropagatedAnnotations != nil { - permitted = *tenant.Spec.LoadBalancer.PropagatedAnnotations - } - } - - if permitted == nil { - if config.GetConfig().Spec.PropagateAllAnnotations { - return loadbalancer - } - permitted = config.GetConfig().Spec.PropagatedAnnotations - } - - a := make(map[string]string) - permittedMap := make(map[string][]string) - for k, v := range permitted { - if _, found := permittedMap[k]; !found { - permittedMap[k] = []string{v} - } - } - - for k, v := range loadbalancer { - if valuesFilter, ok := permittedMap[k]; ok { - if len(valuesFilter) == 0 { - a[k] = v - } else { - for _, vf := range valuesFilter { - if v == vf { - a[k] = v - break - } - } - } - } - } - return a -} - // enqueueLoadBalancersForConfig is a handler.MapFunc to be used to enqeue requests for reconciliation // for LoadBalancers if some change is made to the controller config. func (r *LoadBalancerReconciler) enqueueLoadBalancersForConfig() handler.MapFunc { return func(ctx context.Context, _ ctrlruntimeclient.Object) []ctrl.Request { result := []reconcile.Request{} - // Reload the Config for the controller. - conf := &kubelbv1alpha1.Config{} - err := r.Get(ctx, types.NamespacedName{Name: config.DefaultConfigResourceName, Namespace: r.Namespace}, conf) - if err != nil { - return result - } - config.SetConfig(*conf) - // List all loadbalancers. We don't care about the namespace here. loadBalancers := &kubelbv1alpha1.LoadBalancerList{} - err = r.List(ctx, loadBalancers) + err := r.List(ctx, loadBalancers) if err != nil { return result } @@ -519,22 +517,30 @@ func (r *LoadBalancerReconciler) enqueueLoadBalancersForConfig() handler.MapFunc } } -func getTenantForNamespace(ctx context.Context, client ctrlruntimeclient.Client, namespace string) (*kubelbv1alpha1.Tenant, error) { - ns := &corev1.Namespace{} - err := client.Get(ctx, types.NamespacedName{Name: namespace}, ns) - if err != nil { - return nil, err - } +// enqueueLoadBalancersForTenant is a handler.MapFunc to be used to enqeue requests for reconciliation +// for e changLoadBalancers if some is made to the tenant config. +func (r *LoadBalancerReconciler) enqueueLoadBalancersForTenant() handler.MapFunc { + return func(ctx context.Context, o ctrlruntimeclient.Object) []ctrl.Request { + result := []reconcile.Request{} - for _, owner := range ns.GetOwnerReferences() { - if owner.Kind == "Tenant" { - tenant := &kubelbv1alpha1.Tenant{} - err := client.Get(ctx, types.NamespacedName{Name: owner.Name}, tenant) - if err != nil { - return nil, err - } - return tenant, nil + namespace := fmt.Sprintf(tenantNamespacePattern, o.GetName()) + + // List all loadbalancers in tenant namespace + loadBalancers := &kubelbv1alpha1.LoadBalancerList{} + err := r.List(ctx, loadBalancers, ctrlruntimeclient.InNamespace(namespace)) + if err != nil { + return result + } + + for _, lb := range loadBalancers.Items { + result = append(result, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: lb.Name, + Namespace: lb.Namespace, + }, + }) } + + return result } - return nil, fmt.Errorf("no tenant found for namespace %s", namespace) } diff --git a/internal/controllers/kubelb/route_controller.go b/internal/controllers/kubelb/route_controller.go index 76c6148..e70eea6 100644 --- a/internal/controllers/kubelb/route_controller.go +++ b/internal/controllers/kubelb/route_controller.go @@ -26,18 +26,21 @@ import ( "github.com/go-logr/logr" kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" - "k8c.io/kubelb/internal/config" "k8c.io/kubelb/internal/kubelb" portlookup "k8c.io/kubelb/internal/port-lookup" + gatewayHelpers "k8c.io/kubelb/internal/resources/gatewayapi/gateway" + grpcrouteHelpers "k8c.io/kubelb/internal/resources/gatewayapi/grpcroute" + httprouteHelpers "k8c.io/kubelb/internal/resources/gatewayapi/httproute" + ingressHelpers "k8c.io/kubelb/internal/resources/ingress" serviceHelpers "k8c.io/kubelb/internal/resources/service" "k8c.io/kubelb/internal/resources/unstructured" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/api/equality" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" @@ -45,6 +48,7 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -65,6 +69,7 @@ type RouteReconciler struct { Namespace string PortAllocator *portlookup.PortAllocator EnvoyProxyTopology EnvoyProxyTopology + DisableGatewayAPI bool } // +kubebuilder:rbac:groups=kubelb.k8c.io,resources=routes,verbs=get;list;watch;create;update;patch;delete @@ -91,15 +96,42 @@ func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return reconcile.Result{}, err } + // Before proceeding further we need to make sure that the resource is reconcilable. + tenant, config, err := GetTenantAndConfig(ctx, r.Client, r.Namespace, RemoveTenantPrefix(resource.Namespace)) + if err != nil { + log.Error(err, "unable to fetch Tenant and Config, cannot proceed") + return reconcile.Result{}, err + } + + resourceNamespace := resource.Namespace + if config.Spec.EnvoyProxy.Topology == kubelbv1alpha1.EnvoyProxyTopologyGlobal { + resourceNamespace = r.Namespace + } + // Resource is marked for deletion if resource.DeletionTimestamp != nil { if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { - return r.cleanup(ctx, resource) + return r.cleanup(ctx, resource, resourceNamespace) } // Finalizer doesn't exist so clean up is already done return reconcile.Result{}, nil } + shouldReconcile, disabled, err := r.shouldReconcile(ctx, resource, tenant, config) + if err != nil { + log.Error(err, "unable to determine if the Route should be reconciled") + return reconcile.Result{}, err + } + + // If the resource is disabled, we need to clean up the resources + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) && disabled { + return r.cleanup(ctx, resource, resourceNamespace) + } + + if !shouldReconcile { + return reconcile.Result{}, nil + } + // Add finalizer if it doesn't exist if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { @@ -111,7 +143,7 @@ func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } } - err := r.reconcile(ctx, log, resource) + err = r.reconcile(ctx, log, resource, resourceNamespace, config, tenant) if err != nil { log.Error(err, "reconciling failed") } @@ -119,20 +151,17 @@ func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return reconcile.Result{}, err } -func (r *RouteReconciler) reconcile(ctx context.Context, log logr.Logger, route *kubelbv1alpha1.Route) error { - resourceNamespace := route.Namespace - if config.IsGlobalTopology() { - resourceNamespace = r.Namespace - } +func (r *RouteReconciler) reconcile(ctx context.Context, log logr.Logger, route *kubelbv1alpha1.Route, resourceNamespace string, config *kubelbv1alpha1.Config, tenant *kubelbv1alpha1.Tenant) error { + annotations := GetAnnotations(tenant, config) // Create or update services based on the route. - err := r.manageServices(ctx, log, route, resourceNamespace) + err := r.manageServices(ctx, log, route, resourceNamespace, annotations) if err != nil { return fmt.Errorf("failed to create or update services: %w", err) } // Create or update the route object. - err = r.manageRoutes(ctx, log, route, resourceNamespace) + err = r.manageRoutes(ctx, log, route, resourceNamespace, config, tenant, annotations) if err != nil { return fmt.Errorf("failed to create or update route: %w", err) } @@ -140,7 +169,7 @@ func (r *RouteReconciler) reconcile(ctx context.Context, log logr.Logger, route return nil } -func (r *RouteReconciler) cleanup(ctx context.Context, route *kubelbv1alpha1.Route) (ctrl.Result, error) { +func (r *RouteReconciler) cleanup(ctx context.Context, route *kubelbv1alpha1.Route, ns string) (ctrl.Result, error) { // Route will be removed automatically because of owner reference. We need to take care of removing // the services while ensuring that the services are not being used by other routes. @@ -148,11 +177,6 @@ func (r *RouteReconciler) cleanup(ctx context.Context, route *kubelbv1alpha1.Rou return reconcile.Result{}, nil } - ns := route.Namespace - if config.IsGlobalTopology() { - ns = r.Namespace - } - for _, value := range route.Status.Resources.Services { log := r.Log.WithValues("name", value.Name, "namespace", value.Namespace) log.V(1).Info("Deleting service", "name", value.GeneratedName, "namespace", ns) @@ -182,7 +206,7 @@ func (r *RouteReconciler) cleanup(ctx context.Context, route *kubelbv1alpha1.Rou return reconcile.Result{}, nil } -func (r *RouteReconciler) manageServices(ctx context.Context, log logr.Logger, route *kubelbv1alpha1.Route, resourceNamespace string) error { +func (r *RouteReconciler) manageServices(ctx context.Context, log logr.Logger, route *kubelbv1alpha1.Route, resourceNamespace string, annotations kubelbv1alpha1.AnnotationSettings) error { if route.Spec.Source.Kubernetes == nil { return nil } @@ -202,7 +226,7 @@ func (r *RouteReconciler) manageServices(ctx context.Context, log logr.Logger, r services := []corev1.Service{} for _, service := range route.Spec.Source.Kubernetes.Services { // Transform the service into desired state. - svc := serviceHelpers.GenerateServiceForLBCluster(service.Service, appName, route.Namespace, resourceNamespace, r.PortAllocator, r.EnvoyProxyTopology.IsGlobalTopology()) + svc := serviceHelpers.GenerateServiceForLBCluster(service.Service, appName, route.Namespace, resourceNamespace, r.PortAllocator, r.EnvoyProxyTopology.IsGlobalTopology(), annotations) services = append(services, svc) } @@ -281,7 +305,7 @@ func (r *RouteReconciler) UpdateRouteStatus(ctx context.Context, route *kubelbv1 }) } -func (r *RouteReconciler) manageRoutes(ctx context.Context, log logr.Logger, route *kubelbv1alpha1.Route, resourceNamespace string) error { +func (r *RouteReconciler) manageRoutes(ctx context.Context, log logr.Logger, route *kubelbv1alpha1.Route, resourceNamespace string, config *kubelbv1alpha1.Config, tenant *kubelbv1alpha1.Tenant, annotations kubelbv1alpha1.AnnotationSettings) error { if route.Spec.Source.Kubernetes == nil { return nil } @@ -318,7 +342,7 @@ func (r *RouteReconciler) manageRoutes(ctx context.Context, log logr.Logger, rou // Determine the type of the resource and call the appropriate method switch v := resource.(type) { case *v1.Ingress: // v1 "k8s.io/api/networking/v1" - err = r.createOrUpdateIngress(ctx, log, v, referencedServices, resourceNamespace) + err = ingressHelpers.CreateOrUpdateIngress(ctx, log, r.Client, v, referencedServices, resourceNamespace, config, tenant, annotations) if err == nil { // Retrieve updated object to get the status. key := client.ObjectKey{Namespace: v.Namespace, Name: v.Name} @@ -332,7 +356,7 @@ func (r *RouteReconciler) manageRoutes(ctx context.Context, log logr.Logger, rou } case *gwapiv1.Gateway: // v1 "sigs.k8s.io/gateway-api/apis/v1" - err = r.createOrUpdateGateway(ctx, log, v, resourceNamespace) + err = gatewayHelpers.CreateOrUpdateGateway(ctx, log, r.Client, v, resourceNamespace, config, tenant, annotations, config.IsGlobalTopology()) if err == nil { // Retrieve updated object to get the status. key := client.ObjectKey{Namespace: v.Namespace, Name: v.Name} @@ -346,7 +370,7 @@ func (r *RouteReconciler) manageRoutes(ctx context.Context, log logr.Logger, rou } case *gwapiv1.HTTPRoute: // v1 "sigs.k8s.io/gateway-api/apis/v1" - err = r.createOrUpdateHTTPRoute(ctx, log, v, referencedServices, resourceNamespace) + err = httprouteHelpers.CreateOrUpdateHTTPRoute(ctx, log, r.Client, v, referencedServices, resourceNamespace, tenant, annotations, config.IsGlobalTopology()) if err == nil { // Retrieve updated object to get the status. key := client.ObjectKey{Namespace: v.Namespace, Name: v.Name} @@ -360,7 +384,7 @@ func (r *RouteReconciler) manageRoutes(ctx context.Context, log logr.Logger, rou } case *gwapiv1.GRPCRoute: // v1 "sigs.k8s.io/gateway-api/apis/v1" - err = r.createOrUpdateGRPCRoute(ctx, log, v, referencedServices, resourceNamespace) + err = grpcrouteHelpers.CreateOrUpdateGRPCRoute(ctx, log, r.Client, v, referencedServices, resourceNamespace, tenant, annotations, config.IsGlobalTopology()) if err == nil { // Retrieve updated object to get the status. key := client.ObjectKey{Namespace: v.Namespace, Name: v.Name} @@ -380,309 +404,71 @@ func (r *RouteReconciler) manageRoutes(ctx context.Context, log logr.Logger, rou return r.UpdateRouteStatus(ctx, route, *routeStatus) } -func (r *RouteReconciler) createOrUpdateGateway(ctx context.Context, log logr.Logger, gateway *gwapiv1.Gateway, namespace string) error { - // Check if Gateway with the same name but different namespace already exists. If it does, log an error as we don't support - // multiple Gateway objects. - gateways := &gwapiv1.GatewayList{} - if err := r.Client.List(ctx, gateways, client.InNamespace(namespace)); err != nil { - return fmt.Errorf("failed to list Gateways: %w", err) - } - - found := false - for _, existingGateway := range gateways.Items { - if existingGateway.Name == gateway.Name { - found = true - break - } - } - - if !found && len(gateways.Items) >= 1 { - return fmt.Errorf("multiple Gateway objects are not supported") - } - - // Create/Update the Gateway object in the cluster. - gateway.Namespace = namespace - gateway.SetUID("") // Reset UID to generate a new UID for the Gateway object - - // Set the GatewayClassName if it is specified in the configuration. - if config.GetConfig().Spec.GatewayClassName != nil { - gateway.Spec.GatewayClassName = gwapiv1.ObjectName(*config.GetConfig().Spec.GatewayClassName) - } - - log.V(4).Info("Creating/Updating Gateway", "name", gateway.Name, "namespace", gateway.Namespace) - // Check if it already exists. - gatewayKey := client.ObjectKey{Namespace: gateway.Namespace, Name: gateway.Name} - existingGateway := &gwapiv1.Gateway{} - if err := r.Client.Get(ctx, gatewayKey, existingGateway); err != nil { - if !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to get Gateway: %w", err) - } - err := r.Client.Create(ctx, gateway) - if err != nil { - return fmt.Errorf("failed to create Gateway: %w", err) - } - return nil - } - - // Update the Gateway object if it is different from the existing one. - if equality.Semantic.DeepEqual(existingGateway.Spec, gateway.Spec) && - equality.Semantic.DeepEqual(existingGateway.Labels, gateway.Labels) && - equality.Semantic.DeepEqual(existingGateway.Annotations, gateway.Annotations) { - return nil - } +func (r *RouteReconciler) shouldReconcile(ctx context.Context, route *kubelbv1alpha1.Route, tenant *kubelbv1alpha1.Tenant, config *kubelbv1alpha1.Config) (bool, bool, error) { + log := ctrl.LoggerFrom(ctx) - // Required to update the object. - gateway.ResourceVersion = existingGateway.ResourceVersion - gateway.UID = existingGateway.UID - - if err := r.Client.Update(ctx, gateway); err != nil { - return fmt.Errorf("failed to update Gateway: %w", err) + // First step is to determine the route type. + if route.Spec.Source.Kubernetes == nil { + // There is no source defined. + return false, false, nil } - return nil -} - -// createOrUpdateHTTPRoute creates or updates the HTTPRoute object in the cluster. -func (r *RouteReconciler) createOrUpdateHTTPRoute(ctx context.Context, log logr.Logger, object *gwapiv1.HTTPRoute, referencedServices []metav1.ObjectMeta, namespace string) error { - // Name of the services referenced by the Object have to be updated to match the services created against the Route in the LB cluster. - for i, rule := range object.Spec.Rules { - for j, filter := range rule.Filters { - if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { - ref := filter.RequestMirror.BackendRef - for _, service := range referencedServices { - if string(ref.Name) == service.Name { - ns := ref.Namespace - // Corresponding service found, update the name. - if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) - // Set the namespace to nil since all the services are created in the same namespace as the Route. - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil - } - } - } - } + //nolint:gosimple + var resource client.Object + resource = &route.Spec.Source.Kubernetes.Route + switch resource := resource.(type) { + case *v1.Ingress: + // Ensure that Ingress is enabled + if config.Spec.Ingress.Disable { + log.Error(fmt.Errorf("Ingress is disabled at the global level"), "cannot proceed") + return false, true, nil + } else if tenant.Spec.Ingress.Disable { + log.Error(fmt.Errorf("Ingress is disabled at the tenant level"), "cannot proceed") + return false, true, nil } - for j, ref := range rule.BackendRefs { - if ref.Kind == nil || *ref.Kind == kubelb.ServiceKind { - for _, service := range referencedServices { - if string(ref.Name) == service.Name { - ns := ref.Namespace - // Corresponding service found, update the name. - if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { - object.Spec.Rules[i].BackendRefs[j].Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) - // Set the namespace to nil since all the services are created in the same namespace as the Route. - object.Spec.Rules[i].BackendRefs[j].Namespace = nil - } - } - } - } - // Collect services from the filters. - if ref.Filters != nil { - for _, filter := range ref.Filters { - if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { - ref := filter.RequestMirror.BackendRef - for _, service := range referencedServices { - if string(ref.Name) == service.Name { - ns := ref.Namespace - // Corresponding service found, update the name. - if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) - // Set the namespace to nil since all the services are created in the same namespace as the Route. - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil - } - } - } - } - } - } + case *gwapiv1.Gateway: + // Ensure that Gateway is enabled + if isGatewayAPIDisabled(log, r.DisableGatewayAPI, *config, *tenant) { + return false, true, nil } - } - object.Name = kubelb.GenerateName(false, string(object.UID), object.Name, object.Namespace) - object.Namespace = namespace - object.SetUID("") // Reset UID to generate a new UID for the object - - log.V(4).Info("Creating/Updating HTTPRoute", "name", object.Name, "namespace", object.Namespace) - // Check if it already exists. - key := client.ObjectKey{Namespace: object.Namespace, Name: object.Name} - existingObject := &gwapiv1.HTTPRoute{} - if err := r.Client.Get(ctx, key, existingObject); err != nil { - if !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to get HTTPRoute: %w", err) - } - err := r.Client.Create(ctx, object) - if err != nil { - return fmt.Errorf("failed to create HTTPRoute: %w", err) + if resource.Name != "kubelb" { + return false, false, nil } - return nil - } - - // Update the Ingress object if it is different from the existing one. - if equality.Semantic.DeepEqual(existingObject.Spec, object.Spec) && - equality.Semantic.DeepEqual(existingObject.Labels, object.Labels) && - equality.Semantic.DeepEqual(existingObject.Annotations, object.Annotations) { - return nil - } - // Required to update the object. - object.ResourceVersion = existingObject.ResourceVersion - object.UID = existingObject.UID - - if err := r.Client.Update(ctx, object); err != nil { - return fmt.Errorf("failed to update HTTPRoute: %w", err) - } - return nil -} - -// createOrUpdateGRPCRoute creates or updates the GRPCRoute object in the cluster. -func (r *RouteReconciler) createOrUpdateGRPCRoute(ctx context.Context, log logr.Logger, object *gwapiv1.GRPCRoute, referencedServices []metav1.ObjectMeta, namespace string) error { - // Name of the services referenced by the Object have to be updated to match the services created against the Route in the LB cluster. - for i, rule := range object.Spec.Rules { - for j, filter := range rule.Filters { - if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { - ref := filter.RequestMirror.BackendRef - for _, service := range referencedServices { - if string(ref.Name) == service.Name { - ns := ref.Namespace - // Corresponding service found, update the name. - if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) - // Set the namespace to nil since all the services are created in the same namespace as the Route. - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil - } - } - } - } - } - - for j, ref := range rule.BackendRefs { - if ref.Kind == nil || *ref.Kind == kubelb.ServiceKind { - for _, service := range referencedServices { - if string(ref.Name) == service.Name { - ns := ref.Namespace - // Corresponding service found, update the name. - if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { - object.Spec.Rules[i].BackendRefs[j].Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) - // Set the namespace to nil since all the services are created in the same namespace as the Route. - object.Spec.Rules[i].BackendRefs[j].Namespace = nil - } - } - } - } - // Collect services from the filters. - if ref.Filters != nil { - for _, filter := range ref.Filters { - if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { - ref := filter.RequestMirror.BackendRef - for _, service := range referencedServices { - if string(ref.Name) == service.Name { - ns := ref.Namespace - // Corresponding service found, update the name. - if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) - // Set the namespace to nil since all the services are created in the same namespace as the Route. - object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil - } - } - } - } - } - } + case *gwapiv1.GRPCRoute: + // Ensure that Gateway is enabled + if isGatewayAPIDisabled(log, r.DisableGatewayAPI, *config, *tenant) { + return false, true, nil } - } - - object.Name = kubelb.GenerateName(false, string(object.UID), object.Name, object.Namespace) - object.Namespace = namespace - object.SetUID("") // Reset UID to generate a new UID for the object - log.V(4).Info("Creating/Updating GRPCRoute", "name", object.Name, "namespace", object.Namespace) - // Check if it already exists. - key := client.ObjectKey{Namespace: object.Namespace, Name: object.Name} - existingObject := &gwapiv1.GRPCRoute{} - if err := r.Client.Get(ctx, key, existingObject); err != nil { - if !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to get GRPCRoute: %w", err) - } - err := r.Client.Create(ctx, object) - if err != nil { - return fmt.Errorf("failed to create GRPCRoute: %w", err) + case *gwapiv1.HTTPRoute: + // Ensure that Gateway is enabled + if isGatewayAPIDisabled(log, r.DisableGatewayAPI, *config, *tenant) { + return false, true, nil } - return nil - } - - // Update the Ingress object if it is different from the existing one. - if equality.Semantic.DeepEqual(existingObject.Spec, object.Spec) && - equality.Semantic.DeepEqual(existingObject.Labels, object.Labels) && - equality.Semantic.DeepEqual(existingObject.Annotations, object.Annotations) { - return nil - } - // Required to update the object. - object.ResourceVersion = existingObject.ResourceVersion - object.UID = existingObject.UID - - if err := r.Client.Update(ctx, object); err != nil { - return fmt.Errorf("failed to update GRPCRoute: %w", err) + default: + log.Error(fmt.Errorf("Resource %v is not supported", resource.GetObjectKind().GroupVersionKind().GroupKind().String()), "cannot proceed") + return false, false, nil } - return nil + return true, false, nil } -// createOrUpdateIngress creates or updates the Ingress object in the cluster. -func (r *RouteReconciler) createOrUpdateIngress(ctx context.Context, log logr.Logger, object *v1.Ingress, referencedServices []metav1.ObjectMeta, namespace string) error { - // Name of the services referenced by the Ingress have to be updated to match the services created against the Route in the LB cluster. - for i, rule := range object.Spec.Rules { - for j, path := range rule.HTTP.Paths { - for _, service := range referencedServices { - if path.Backend.Service.Name == service.Name { - object.Spec.Rules[i].HTTP.Paths[j].Backend.Service.Name = kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace) - } - } - } +func isGatewayAPIDisabled(log logr.Logger, disableGatewayAPI bool, config kubelbv1alpha1.Config, tenant kubelbv1alpha1.Tenant) bool { + if disableGatewayAPI { + log.Error(fmt.Errorf("Gateway API is disabled at the global level"), "cannot proceed") + return true } - - if object.Spec.DefaultBackend != nil && object.Spec.DefaultBackend.Service != nil { - for _, service := range referencedServices { - if object.Spec.DefaultBackend.Service.Name == service.Name { - object.Spec.DefaultBackend.Service.Name = kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace) - } - } + // Ensure that Gateway API is enabled + if config.Spec.GatewayAPI.Disable { + log.Error(fmt.Errorf("Gateway API is disabled at the global level"), "cannot proceed") + return true + } else if tenant.Spec.GatewayAPI.Disable { + log.Error(fmt.Errorf("Gateway API is disabled at the tenant level"), "cannot proceed") + return true } - - object.Spec.IngressClassName = config.GetConfig().Spec.IngressClassName - object.Name = kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(object.UID), object.Name, object.Namespace) - object.Namespace = namespace - object.SetUID("") // Reset UID to generate a new UID for the object - - log.V(4).Info("Creating/Updating Ingress", "name", object.Name, "namespace", object.Namespace) - // Check if it already exists. - key := client.ObjectKey{Namespace: object.Namespace, Name: object.Name} - existingObject := &v1.Ingress{} - if err := r.Client.Get(ctx, key, existingObject); err != nil { - if !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to get Ingress: %w", err) - } - err := r.Client.Create(ctx, object) - if err != nil { - return fmt.Errorf("failed to create Ingress: %w", err) - } - return nil - } - - // Update the Ingress object if it is different from the existing one. - if equality.Semantic.DeepEqual(existingObject.Spec, object.Spec) && - equality.Semantic.DeepEqual(existingObject.Labels, object.Labels) && - equality.Semantic.DeepEqual(existingObject.Annotations, object.Annotations) { - return nil - } - - // Required to update the object. - object.ResourceVersion = existingObject.ResourceVersion - object.UID = existingObject.UID - - if err := r.Client.Update(ctx, object); err != nil { - return fmt.Errorf("failed to update Ingress: %w", err) - } - return nil + return false } func updateServiceStatus(routeStatus *kubelbv1alpha1.RouteStatus, svc *corev1.Service, err error) { @@ -789,5 +575,66 @@ func (r *RouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&kubelbv1alpha1.Route{}). WithEventFilter(predicate.GenerationChangedPredicate{}). + Watches( + &kubelbv1alpha1.Config{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueRoutesForConfig()), + ). + Watches( + &kubelbv1alpha1.Tenant{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueRoutesForTenant()), + ). Complete(r) } + +// enqueueRoutesForConfig is a handler.MapFunc to be used to enqeue requests for reconciliation +// for Routes if some change is made to the controller config. +func (r *RouteReconciler) enqueueRoutesForConfig() handler.MapFunc { + return func(ctx context.Context, _ ctrlruntimeclient.Object) []ctrl.Request { + result := []reconcile.Request{} + + // List all routes. We don't care about the namespace here. + routes := &kubelbv1alpha1.RouteList{} + err := r.List(ctx, routes) + if err != nil { + return result + } + + for _, route := range routes.Items { + result = append(result, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: route.Name, + Namespace: route.Namespace, + }, + }) + } + return result + } +} + +// enqueueRoutesForTenant is a handler.MapFunc to be used to enqeue requests for reconciliation +// for Routes if some change is made to the tenant config. +func (r *RouteReconciler) enqueueRoutesForTenant() handler.MapFunc { + return func(ctx context.Context, o ctrlruntimeclient.Object) []ctrl.Request { + result := []reconcile.Request{} + + namespace := fmt.Sprintf(tenantNamespacePattern, o.GetName()) + + // List all routes in tenant namespace + routes := &kubelbv1alpha1.RouteList{} + err := r.List(ctx, routes, ctrlruntimeclient.InNamespace(namespace)) + if err != nil { + return result + } + + for _, lb := range routes.Items { + result = append(result, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: lb.Name, + Namespace: lb.Namespace, + }, + }) + } + + return result + } +} diff --git a/internal/controllers/kubelb/shared.go b/internal/controllers/kubelb/shared.go new file mode 100644 index 0000000..da9ee3b --- /dev/null +++ b/internal/controllers/kubelb/shared.go @@ -0,0 +1,75 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelb + +import ( + "context" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + configpkg "k8c.io/kubelb/internal/config" + + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetTenantAndConfig(ctx context.Context, client ctrlclient.Client, configNamespace, tenantName string) (*kubelbv1alpha1.Tenant, *kubelbv1alpha1.Config, error) { + tenant, err := GetTenant(ctx, client, tenantName) + if err != nil { + return nil, nil, err + } + config, err := GetConfig(ctx, client, configNamespace) + if err != nil { + return nil, nil, err + } + + return tenant, config, nil +} + +func GetTenant(ctx context.Context, client ctrlclient.Client, tenantName string) (*kubelbv1alpha1.Tenant, error) { + tenant := &kubelbv1alpha1.Tenant{} + if err := client.Get(ctx, ctrlclient.ObjectKey{ + Name: tenantName, + }, tenant); err != nil { + return nil, err + } + return tenant, nil +} + +func GetConfig(ctx context.Context, client ctrlclient.Client, configNamespace string) (*kubelbv1alpha1.Config, error) { + config := &kubelbv1alpha1.Config{} + if err := client.Get(ctx, ctrlclient.ObjectKey{ + Namespace: configNamespace, + Name: configpkg.DefaultConfigResourceName, + }, config); err != nil { + return nil, err + } + return config, nil +} + +func GetAnnotations(tenant *kubelbv1alpha1.Tenant, config *kubelbv1alpha1.Config) kubelbv1alpha1.AnnotationSettings { + var annotations kubelbv1alpha1.AnnotationSettings + if tenant.Spec.AnnotationSettings.PropagatedAnnotations != nil { + annotations.PropagatedAnnotations = tenant.Spec.AnnotationSettings.PropagatedAnnotations + } else if config.Spec.AnnotationSettings.PropagatedAnnotations != nil { + annotations.PropagatedAnnotations = config.Spec.AnnotationSettings.PropagatedAnnotations + } + if tenant.Spec.AnnotationSettings.PropagateAllAnnotations == nil { + annotations.PropagateAllAnnotations = tenant.Spec.AnnotationSettings.PropagateAllAnnotations + } else if config.Spec.AnnotationSettings.PropagateAllAnnotations == nil { + annotations.PropagateAllAnnotations = config.Spec.AnnotationSettings.PropagateAllAnnotations + } + return annotations +} diff --git a/internal/controllers/kubelb/suite_test.go b/internal/controllers/kubelb/suite_test.go index edbbd91..958d47f 100644 --- a/internal/controllers/kubelb/suite_test.go +++ b/internal/controllers/kubelb/suite_test.go @@ -24,7 +24,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1alpha12 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + v1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/envoy" "k8c.io/kubelb/internal/kubelb" portlookup "k8c.io/kubelb/internal/port-lookup" @@ -58,6 +58,7 @@ const ( APIVersion = "kubelb.k8c.io/v1alpha1" Kind = "LoadBalancer" LBNamespace = "tenant-uno" + Tenant = "uno" ) func TestLoadBalancerCustomResource(t *testing.T) { @@ -79,7 +80,7 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) Expect(cfg).ToNot(BeNil()) - err = v1alpha12.AddToScheme(scheme.Scheme) + err = v1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) ctrl.SetLogger(zap.New(zap.UseDevMode(false))) @@ -90,7 +91,7 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, @@ -109,8 +110,27 @@ var _ = BeforeSuite(func() { }, }, } + + tenant := &v1alpha1.Tenant{ + ObjectMeta: v1.ObjectMeta{ + Name: Tenant, + }, + } + + config := &v1alpha1.Config{ + ObjectMeta: v1.ObjectMeta{ + Name: "default", + Namespace: LBNamespace, + }, + } + + err = k8sManager.GetClient().Create(ctx, tenant) + Expect(err).ToNot(HaveOccurred()) + err = k8sManager.GetClient().Create(ctx, ns) Expect(err).ToNot(HaveOccurred()) + err = k8sManager.GetClient().Create(ctx, config) + Expect(err).ToNot(HaveOccurred()) lbr = &LoadBalancerReconciler{ Client: k8sManager.GetClient(), @@ -151,8 +171,8 @@ var _ = AfterSuite(func() { Expect(err).ToNot(HaveOccurred()) }) -func GetDefaultLoadBalancer(name string, namespace string) *v1alpha12.LoadBalancer { - return &v1alpha12.LoadBalancer{ +func GetDefaultLoadBalancer(name string, namespace string) *v1alpha1.LoadBalancer { + return &v1alpha1.LoadBalancer{ TypeMeta: v1.TypeMeta{ APIVersion: APIVersion, Kind: Kind, @@ -161,10 +181,10 @@ func GetDefaultLoadBalancer(name string, namespace string) *v1alpha12.LoadBalanc Name: name, Namespace: namespace, }, - Spec: v1alpha12.LoadBalancerSpec{ - Endpoints: []v1alpha12.LoadBalancerEndpoints{ + Spec: v1alpha1.LoadBalancerSpec{ + Endpoints: []v1alpha1.LoadBalancerEndpoints{ { - Addresses: []v1alpha12.EndpointAddress{ + Addresses: []v1alpha1.EndpointAddress{ { IP: "123.123.123.123", }, @@ -172,14 +192,14 @@ func GetDefaultLoadBalancer(name string, namespace string) *v1alpha12.LoadBalanc IP: "123.123.123.124", }, }, - Ports: []v1alpha12.EndpointPort{ + Ports: []v1alpha1.EndpointPort{ { Port: 8080, }, }, }, }, - Ports: []v1alpha12.LoadBalancerPort{ + Ports: []v1alpha1.LoadBalancerPort{ { Port: 80, }, diff --git a/internal/controllers/kubelb/tenant_controller.go b/internal/controllers/kubelb/tenant_controller.go index cf70d5d..479f180 100644 --- a/internal/controllers/kubelb/tenant_controller.go +++ b/internal/controllers/kubelb/tenant_controller.go @@ -207,9 +207,16 @@ func (r *TenantReconciler) generateKubeconfig(ctx context.Context, client ctrlru } serverURL := r.Config.Host - ca := r.Config.TLSClientConfig.CAData + ca := secret.Data[corev1.ServiceAccountRootCAKey] token := secret.Data[corev1.ServiceAccountTokenKey] + if len(token) == 0 { + return "", fmt.Errorf("failed to get ServiceAccount token") + } + if len(ca) == 0 { + return "", fmt.Errorf("failed to get CA certificate") + } + // Generate kubeconfig. data := struct { CACertificate string diff --git a/internal/controllers/kubelb/tenant_migration_controller.go b/internal/controllers/kubelb/tenant_migration_controller.go index b6f4366..a7a84bc 100644 --- a/internal/controllers/kubelb/tenant_migration_controller.go +++ b/internal/controllers/kubelb/tenant_migration_controller.go @@ -88,7 +88,7 @@ func (r *TenantMigrationReconciler) Reconcile(ctx context.Context, req ctrl.Requ func (r *TenantMigrationReconciler) reconcile(ctx context.Context, _ logr.Logger, namespace *corev1.Namespace) error { // We need to create tenant resource corresponding to the namespace. All the other aspects are handled by the tenant controller // Remove `tenant-` prefix from namespace name if it exists - tenantName := removeTenantPrefix(namespace.Name) + tenantName := RemoveTenantPrefix(namespace.Name) // Copy `kubelb.k8c.io/propagate-annotation` from namespace to the tenant resource permittedMap := make(map[string]string) @@ -103,16 +103,14 @@ func (r *TenantMigrationReconciler) reconcile(ctx context.Context, _ logr.Logger } } - lb := kubelbv1alpha1.LoadBalancerSettings{ - PropagatedAnnotations: &permittedMap, - } - tenant := &kubelbv1alpha1.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: tenantName, }, Spec: kubelbv1alpha1.TenantSpec{ - LoadBalancer: lb, + AnnotationSettings: kubelbv1alpha1.AnnotationSettings{ + PropagatedAnnotations: &permittedMap, + }, }, } @@ -172,7 +170,7 @@ func (r *TenantMigrationReconciler) shouldReconcile(ns *corev1.Namespace) bool { return reconcile } -func removeTenantPrefix(namespace string) string { +func RemoveTenantPrefix(namespace string) string { prefix := "tenant-" if strings.HasPrefix(namespace, prefix) { return strings.TrimPrefix(namespace, prefix) diff --git a/internal/kubelb/utils.go b/internal/kubelb/utils.go index c94193e..56d6150 100644 --- a/internal/kubelb/utils.go +++ b/internal/kubelb/utils.go @@ -19,10 +19,11 @@ package kubelb import ( "fmt" + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// TODO(waleed): Rename to origin-namespace const LabelOriginNamespace = "kubelb.k8c.io/origin-ns" const LabelOriginName = "kubelb.k8c.io/origin-name" const LabelOriginResourceKind = "kubelb.k8c.io/origin-resource-kind" @@ -86,3 +87,42 @@ func GetNamespace(obj client.Object) string { } return namespace } + +func PropagateAnnotations(loadbalancer map[string]string, annotations kubelbv1alpha1.AnnotationSettings) map[string]string { + if loadbalancer == nil { + loadbalancer = make(map[string]string) + } + + if annotations.PropagateAllAnnotations != nil && *annotations.PropagateAllAnnotations { + return loadbalancer + } + permitted := make(map[string]string) + + if annotations.PropagatedAnnotations != nil { + permitted = *annotations.PropagatedAnnotations + } + + a := make(map[string]string) + permittedMap := make(map[string][]string) + for k, v := range permitted { + if _, found := permittedMap[k]; !found { + permittedMap[k] = []string{v} + } + } + + for k, v := range loadbalancer { + if valuesFilter, ok := permittedMap[k]; ok { + if len(valuesFilter) == 0 { + a[k] = v + } else { + for _, vf := range valuesFilter { + if v == vf { + a[k] = v + break + } + } + } + } + } + return a +} diff --git a/internal/resources/gatewayapi/gateway/gateway.go b/internal/resources/gatewayapi/gateway/gateway.go new file mode 100644 index 0000000..d0a3996 --- /dev/null +++ b/internal/resources/gatewayapi/gateway/gateway.go @@ -0,0 +1,104 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package httproute + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + "k8c.io/kubelb/internal/kubelb" + + "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func CreateOrUpdateGateway(ctx context.Context, log logr.Logger, client ctrlclient.Client, object *gwapiv1.Gateway, namespace string, config *kubelbv1alpha1.Config, tenant *kubelbv1alpha1.Tenant, + annotations kubelbv1alpha1.AnnotationSettings, _ bool) error { + // Transformations to make it compliant with the LB cluster. + // Update the GatewayClass name to match the GatewayClass object in the LB cluster. + var gatewayClassName *string + if tenant.Spec.GatewayAPI.Class != nil { + gatewayClassName = tenant.Spec.GatewayAPI.Class + } else if config.Spec.GatewayAPI.Class != nil { + gatewayClassName = config.Spec.GatewayAPI.Class + } + + // Check if Gateway with the same name but different namespace already exists. If it does, log an error as we don't support + // multiple Gateway objects. + gateways := &gwapiv1.GatewayList{} + if err := client.List(ctx, gateways, ctrlclient.InNamespace(namespace)); err != nil { + return fmt.Errorf("failed to list Gateways: %w", err) + } + + found := false + for _, existingGateway := range gateways.Items { + if existingGateway.Name == object.Name { + found = true + break + } + } + + if !found && len(gateways.Items) >= 1 { + return fmt.Errorf("multiple Gateway objects are not supported") + } + + // Process annotations. + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + object.Namespace = namespace + object.SetUID("") // Reset UID to generate a new UID for the Gateway object + + // Set the GatewayClassName if it is specified in the configuration. + if gatewayClassName != nil { + object.Spec.GatewayClassName = gwapiv1.ObjectName(*gatewayClassName) + } + + log.V(4).Info("Creating/Updating Gateway", "name", object.Name, "namespace", object.Namespace) + // Check if it already exists. + gatewayKey := ctrlclient.ObjectKey{Namespace: object.Namespace, Name: object.Name} + existingGateway := &gwapiv1.Gateway{} + if err := client.Get(ctx, gatewayKey, existingGateway); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to get Gateway: %w", err) + } + err := client.Create(ctx, object) + if err != nil { + return fmt.Errorf("failed to create Gateway: %w", err) + } + return nil + } + + // Update the Gateway object if it is different from the existing one. + if equality.Semantic.DeepEqual(existingGateway.Spec, object.Spec) && + equality.Semantic.DeepEqual(existingGateway.Labels, object.Labels) && + equality.Semantic.DeepEqual(existingGateway.Annotations, object.Annotations) { + return nil + } + + // Required to update the object. + object.ResourceVersion = existingGateway.ResourceVersion + object.UID = existingGateway.UID + + if err := client.Update(ctx, object); err != nil { + return fmt.Errorf("failed to update Gateway: %w", err) + } + return nil +} diff --git a/internal/resources/gatewayapi/grpcroute/grpcroute.go b/internal/resources/gatewayapi/grpcroute/grpcroute.go index d28811a..9485ac3 100644 --- a/internal/resources/gatewayapi/grpcroute/grpcroute.go +++ b/internal/resources/gatewayapi/grpcroute/grpcroute.go @@ -17,12 +17,119 @@ limitations under the License. package grpcroute import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" + "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" ) +// createOrUpdateGRPCRoute creates or updates the GRPCRoute object in the cluster. +func CreateOrUpdateGRPCRoute(ctx context.Context, log logr.Logger, client ctrlclient.Client, object *gwapiv1.GRPCRoute, referencedServices []metav1.ObjectMeta, namespace string, + _ *kubelbv1alpha1.Tenant, annotations kubelbv1alpha1.AnnotationSettings, globalTopology bool) error { + // Name of the services referenced by the Object have to be updated to match the services created against the Route in the LB cluster. + for i, rule := range object.Spec.Rules { + for j, filter := range rule.Filters { + if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { + ref := filter.RequestMirror.BackendRef + for _, service := range referencedServices { + if string(ref.Name) == service.Name { + ns := ref.Namespace + // Corresponding service found, update the name. + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace)) + // Set the namespace to nil since all the services are created in the same namespace as the Route. + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil + } + } + } + } + } + + for j, ref := range rule.BackendRefs { + if ref.Kind == nil || *ref.Kind == kubelb.ServiceKind { + for _, service := range referencedServices { + if string(ref.Name) == service.Name { + ns := ref.Namespace + // Corresponding service found, update the name. + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { + object.Spec.Rules[i].BackendRefs[j].Name = gwapiv1.ObjectName(kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace)) + // Set the namespace to nil since all the services are created in the same namespace as the Route. + object.Spec.Rules[i].BackendRefs[j].Namespace = nil + } + } + } + } + // Collect services from the filters. + if ref.Filters != nil { + for _, filter := range ref.Filters { + if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { + ref := filter.RequestMirror.BackendRef + for _, service := range referencedServices { + if string(ref.Name) == service.Name { + ns := ref.Namespace + // Corresponding service found, update the name. + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace)) + // Set the namespace to nil since all the services are created in the same namespace as the Route. + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil + } + } + } + } + } + } + } + } + + // Process annotations. + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + + object.Name = kubelb.GenerateName(globalTopology, string(object.UID), object.Name, object.Namespace) + object.Namespace = namespace + object.SetUID("") // Reset UID to generate a new UID for the object + + log.V(4).Info("Creating/Updating GRPCRoute", "name", object.Name, "namespace", object.Namespace) + // Check if it already exists. + key := ctrlclient.ObjectKey{Namespace: object.Namespace, Name: object.Name} + existingObject := &gwapiv1.GRPCRoute{} + if err := client.Get(ctx, key, existingObject); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to get GRPCRoute: %w", err) + } + err := client.Create(ctx, object) + if err != nil { + return fmt.Errorf("failed to create GRPCRoute: %w", err) + } + return nil + } + + // Update the Ingress object if it is different from the existing one. + if equality.Semantic.DeepEqual(existingObject.Spec, object.Spec) && + equality.Semantic.DeepEqual(existingObject.Labels, object.Labels) && + equality.Semantic.DeepEqual(existingObject.Annotations, object.Annotations) { + return nil + } + + // Required to update the object. + object.ResourceVersion = existingObject.ResourceVersion + object.UID = existingObject.UID + + if err := client.Update(ctx, object); err != nil { + return fmt.Errorf("failed to update GRPCRoute: %w", err) + } + return nil +} + // GetServicesFromGRPCRoute returns a list of services referenced by the given GRPCRoute. func GetServicesFromGRPCRoute(grpcroute *gwapiv1.GRPCRoute) []types.NamespacedName { serviceReferences := make([]types.NamespacedName, 0) diff --git a/internal/resources/gatewayapi/httproute/httproute.go b/internal/resources/gatewayapi/httproute/httproute.go index 34a39db..d1e3012 100644 --- a/internal/resources/gatewayapi/httproute/httproute.go +++ b/internal/resources/gatewayapi/httproute/httproute.go @@ -17,12 +17,119 @@ limitations under the License. package httproute import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" + "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" ) +// createOrUpdateHTTPRoute creates or updates the HTTPRoute object in the cluster. +func CreateOrUpdateHTTPRoute(ctx context.Context, log logr.Logger, client ctrlclient.Client, object *gwapiv1.HTTPRoute, referencedServices []metav1.ObjectMeta, namespace string, + _ *kubelbv1alpha1.Tenant, annotations kubelbv1alpha1.AnnotationSettings, globalTopology bool) error { + // Name of the services referenced by the Object have to be updated to match the services created against the Route in the LB cluster. + for i, rule := range object.Spec.Rules { + for j, filter := range rule.Filters { + if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { + ref := filter.RequestMirror.BackendRef + for _, service := range referencedServices { + if string(ref.Name) == service.Name { + ns := ref.Namespace + // Corresponding service found, update the name. + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace)) + // Set the namespace to nil since all the services are created in the same namespace as the Route. + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil + } + } + } + } + } + + for j, ref := range rule.BackendRefs { + if ref.Kind == nil || *ref.Kind == kubelb.ServiceKind { + for _, service := range referencedServices { + if string(ref.Name) == service.Name { + ns := ref.Namespace + // Corresponding service found, update the name. + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { + object.Spec.Rules[i].BackendRefs[j].Name = gwapiv1.ObjectName(kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace)) + // Set the namespace to nil since all the services are created in the same namespace as the Route. + object.Spec.Rules[i].BackendRefs[j].Namespace = nil + } + } + } + } + // Collect services from the filters. + if ref.Filters != nil { + for _, filter := range ref.Filters { + if filter.RequestMirror != nil && (filter.RequestMirror.BackendRef.Kind == nil || *filter.RequestMirror.BackendRef.Kind == kubelb.ServiceKind) { + ref := filter.RequestMirror.BackendRef + for _, service := range referencedServices { + if string(ref.Name) == service.Name { + ns := ref.Namespace + // Corresponding service found, update the name. + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace)) + // Set the namespace to nil since all the services are created in the same namespace as the Route. + object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil + } + } + } + } + } + } + } + } + + // Process annotations. + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + + object.Name = kubelb.GenerateName(globalTopology, string(object.UID), object.Name, object.Namespace) + object.Namespace = namespace + object.SetUID("") // Reset UID to generate a new UID for the object + + log.V(4).Info("Creating/Updating HTTPRoute", "name", object.Name, "namespace", object.Namespace) + // Check if it already exists. + key := ctrlclient.ObjectKey{Namespace: object.Namespace, Name: object.Name} + existingObject := &gwapiv1.HTTPRoute{} + if err := client.Get(ctx, key, existingObject); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to get HTTPRoute: %w", err) + } + err := client.Create(ctx, object) + if err != nil { + return fmt.Errorf("failed to create HTTPRoute: %w", err) + } + return nil + } + + // Update the Ingress object if it is different from the existing one. + if equality.Semantic.DeepEqual(existingObject.Spec, object.Spec) && + equality.Semantic.DeepEqual(existingObject.Labels, object.Labels) && + equality.Semantic.DeepEqual(existingObject.Annotations, object.Annotations) { + return nil + } + + // Required to update the object. + object.ResourceVersion = existingObject.ResourceVersion + object.UID = existingObject.UID + + if err := client.Update(ctx, object); err != nil { + return fmt.Errorf("failed to update HTTPRoute: %w", err) + } + return nil +} + // GetServicesFromHTTPRoute returns a list of services referenced by the given HTTPRoute. func GetServicesFromHTTPRoute(httpRoute *gwapiv1.HTTPRoute) []types.NamespacedName { serviceReferences := make([]types.NamespacedName, 0) diff --git a/internal/resources/ingress/ingress.go b/internal/resources/ingress/ingress.go index 643b5a3..80011a5 100644 --- a/internal/resources/ingress/ingress.go +++ b/internal/resources/ingress/ingress.go @@ -17,10 +17,97 @@ limitations under the License. package route import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + "k8c.io/kubelb/internal/kubelb" + networkingv1 "k8s.io/api/networking/v1" + v1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) +// createOrUpdateIngress creates or updates the Ingress object in the cluster. +func CreateOrUpdateIngress(ctx context.Context, log logr.Logger, client ctrlclient.Client, object *v1.Ingress, referencedServices []metav1.ObjectMeta, namespace string, config *kubelbv1alpha1.Config, + tenant *kubelbv1alpha1.Tenant, annotations kubelbv1alpha1.AnnotationSettings) error { + globalTopology := config.IsGlobalTopology() + // Transformations to make it compliant with the LB cluster. + // Name of the services referenced by the Ingress have to be updated to match the services created against the Route in the LB cluster. + for i, rule := range object.Spec.Rules { + for j, path := range rule.HTTP.Paths { + for _, service := range referencedServices { + if path.Backend.Service.Name == service.Name { + object.Spec.Rules[i].HTTP.Paths[j].Backend.Service.Name = kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace) + } + } + } + } + + if object.Spec.DefaultBackend != nil && object.Spec.DefaultBackend.Service != nil { + for _, service := range referencedServices { + if object.Spec.DefaultBackend.Service.Name == service.Name { + object.Spec.DefaultBackend.Service.Name = kubelb.GenerateName(globalTopology, string(service.UID), service.Name, service.Namespace) + } + } + } + + // Class name depends on the chosen Ingress controller for the tenant or global cluster. + var className *string + if tenant.Spec.Ingress.Class != nil { + className = tenant.Spec.Ingress.Class + } else if config.Spec.Ingress.Class != nil { + className = config.Spec.Ingress.Class + } + + object.Spec.IngressClassName = className + + // Process annotations. + object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + + // Update name and other fields before creating/updating the object. + object.Name = kubelb.GenerateName(globalTopology, string(object.UID), object.Name, object.Namespace) + object.Namespace = namespace + object.SetUID("") // Reset UID to generate a new UID for the object + + log.V(4).Info("Creating/Updating Ingress", "name", object.Name, "namespace", object.Namespace) + // Check if it already exists. + key := ctrlclient.ObjectKey{Namespace: object.Namespace, Name: object.Name} + existingObject := &v1.Ingress{} + if err := client.Get(ctx, key, existingObject); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to get Ingress: %w", err) + } + err := client.Create(ctx, object) + if err != nil { + return fmt.Errorf("failed to create Ingress: %w", err) + } + return nil + } + + // Update the Ingress object if it is different from the existing one. + if equality.Semantic.DeepEqual(existingObject.Spec, object.Spec) && + equality.Semantic.DeepEqual(existingObject.Labels, object.Labels) && + equality.Semantic.DeepEqual(existingObject.Annotations, object.Annotations) { + return nil + } + + // Required to update the object. + object.ResourceVersion = existingObject.ResourceVersion + object.UID = existingObject.UID + + if err := client.Update(ctx, object); err != nil { + return fmt.Errorf("failed to update Ingress: %w", err) + } + return nil +} + // GetServicesFromIngress returns a list of services referenced by the given Ingress. func GetServicesFromIngress(ingress networkingv1.Ingress) []types.NamespacedName { serviceReferences := make([]types.NamespacedName, 0) diff --git a/internal/resources/route/route.go b/internal/resources/route/route.go index b0e4154..dedbf67 100644 --- a/internal/resources/route/route.go +++ b/internal/resources/route/route.go @@ -58,7 +58,6 @@ func GenerateRoute(resource unstructured.Unstructured, resources Subresources, n }, }, Spec: kubelbv1alpha1.RouteSpec{ - // TODO(waleed): Once we have everything in place, figure out how this should look like. Endpoints: []kubelbv1alpha1.LoadBalancerEndpoints{ { Name: "default", @@ -69,9 +68,8 @@ func GenerateRoute(resource unstructured.Unstructured, resources Subresources, n }, Source: kubelbv1alpha1.RouteSource{ Kubernetes: &kubelbv1alpha1.KubernetesSource{ - Route: resource, - Services: kubelbv1alpha1.ConvertServicesToUpstreamServices(resources.Services), - ReferenceGrants: kubelbv1alpha1.ConvertReferenceGrantsToUpstreamReferenceGrants(resources.ReferenceGrants), + Route: resource, + Services: kubelbv1alpha1.ConvertServicesToUpstreamServices(resources.Services), }, }, }, diff --git a/internal/resources/service/service.go b/internal/resources/service/service.go index a0dac52..737ffb7 100644 --- a/internal/resources/service/service.go +++ b/internal/resources/service/service.go @@ -22,6 +22,7 @@ import ( "github.com/go-logr/logr" + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" portlookup "k8c.io/kubelb/internal/port-lookup" "k8c.io/reconciler/pkg/equality" @@ -107,7 +108,7 @@ func NormalizeAndReplicateServices(ctx context.Context, log logr.Logger, client return services, nil } -func GenerateServiceForLBCluster(service corev1.Service, appName, namespace, resourceNamespace string, portAllocator *portlookup.PortAllocator, globalTopology bool) corev1.Service { +func GenerateServiceForLBCluster(service corev1.Service, appName, namespace, resourceNamespace string, portAllocator *portlookup.PortAllocator, globalTopology bool, annotations kubelbv1alpha1.AnnotationSettings) corev1.Service { endpointKey := fmt.Sprintf(kubelb.EnvoyEndpointRoutePattern, namespace, service.Namespace, service.Name) service.Name = kubelb.GenerateName(globalTopology, string(service.UID), GetServiceName(service), service.Namespace) @@ -134,6 +135,7 @@ func GenerateServiceForLBCluster(service corev1.Service, appName, namespace, res service.Spec.Selector = map[string]string{ kubelb.LabelAppKubernetesName: appName, } + service.Annotations = kubelb.PropagateAnnotations(service.Annotations, annotations) return service } diff --git a/internal/resources/unstructured/unstructured.go b/internal/resources/unstructured/unstructured.go index de24bc7..2621acf 100644 --- a/internal/resources/unstructured/unstructured.go +++ b/internal/resources/unstructured/unstructured.go @@ -71,6 +71,10 @@ func ConvertUnstructuredToObject(unstruct *unstructured.Unstructured) (client.Ob object = &gwapiv1.HTTPRoute{} case gwapiv1.SchemeGroupVersion.WithKind("GRPCRoute"): object = &gwapiv1.GRPCRoute{} + + default: + // Not a known type, we can't process further. + return nil, fmt.Errorf("unsupported type: %s", gvk) } err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstruct.UnstructuredContent(), object)