From 445edcb7754b6073e310f3e9bfc410e4efea175a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 12 Aug 2024 21:49:16 +0500 Subject: [PATCH] Support for synchronizing secrets from tenant to LB cluster (#42) * Support for synchronizing secrets from tenant to LB cluster Signed-off-by: Waleed Malik * Manager: Process and consume sync secrets Signed-off-by: Waleed Malik * Fixes Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- Makefile | 1 + PROJECT | 20 +- .../v1alpha1/sync_secret_types.go | 56 ++++++ .../v1alpha1/zz_generated.deepcopy.go | 80 ++++++++ charts/kubelb-ccm/README.md | 5 +- .../crds/kubelb.k8c.io_syncsecrets.yaml | 56 ++++++ charts/kubelb-ccm/templates/clusterrole.yaml | 26 +++ charts/kubelb-ccm/templates/deployment.yaml | 3 + charts/kubelb-ccm/values.yaml | 7 +- charts/kubelb-manager/README.md | 4 +- .../crds/kubelb.k8c.io_syncsecrets.yaml | 56 ++++++ .../kubelb-manager/templates/clusterrole.yaml | 12 ++ charts/kubelb-manager/values.yaml | 4 +- cmd/ccm/main.go | 37 ++++ cmd/kubelb/main.go | 12 ++ config/ccm/ccm.yaml | 63 +++--- config/ccm/kustomization.yaml | 6 +- config/ccm/rbac/role.yaml | 24 +++ .../crd/bases/kubelb.k8c.io_syncsecrets.yaml | 56 ++++++ config/crd/kustomization.yaml | 1 + config/kubelb/manager.yaml | 7 +- config/kubelb/rbac/role.yaml | 12 ++ .../controllers/ccm/gateway_controller.go | 2 +- .../ccm/secret_conversion_controller.go | 137 +++++++++++++ .../controllers/ccm/sync_secret_controller.go | 170 ++++++++++++++++ .../controllers/kubelb/route_controller.go | 15 +- .../kubelb/sync_secret_controller.go | 186 ++++++++++++++++++ .../kubelb/tenant_migration_controller.go | 2 +- internal/controllers/utils.go | 32 --- .../resources/gatewayapi/gateway/gateway.go | 14 ++ internal/resources/ingress/ingress.go | 11 ++ internal/resources/service/service.go | 4 + internal/util/kubernetes/secret.go | 38 ++++ internal/util/predicate/predicate.go | 136 +++++++++++++ 34 files changed, 1210 insertions(+), 85 deletions(-) create mode 100644 api/kubelb.k8c.io/v1alpha1/sync_secret_types.go create mode 100644 charts/kubelb-ccm/crds/kubelb.k8c.io_syncsecrets.yaml create mode 100644 charts/kubelb-manager/crds/kubelb.k8c.io_syncsecrets.yaml create mode 100644 config/crd/bases/kubelb.k8c.io_syncsecrets.yaml create mode 100644 internal/controllers/ccm/secret_conversion_controller.go create mode 100644 internal/controllers/ccm/sync_secret_controller.go create mode 100644 internal/controllers/kubelb/sync_secret_controller.go create mode 100644 internal/util/kubernetes/secret.go create mode 100644 internal/util/predicate/predicate.go diff --git a/Makefile b/Makefile index 2db5c19..0d55850 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,7 @@ manifests: generate controller-gen ## Generate WebhookConfiguration, ClusterRole $(CONTROLLER_GEN) rbac:roleName=kubelb-ccm paths="./internal/controllers/ccm/..." output:artifacts:config=config/ccm/rbac $(CONTROLLER_GEN) rbac:roleName=kubelb paths="./internal/controllers/kubelb/..." output:artifacts:config=config/kubelb/rbac $(CONTROLLER_GEN) crd webhook paths="./..." output:crd:artifacts:config=charts/kubelb-manager/crds + cp charts/kubelb-manager/crds/kubelb.k8c.io_syncsecrets.yaml charts/kubelb-ccm/crds/kubelb.k8c.io_syncsecrets.yaml .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. diff --git a/PROJECT b/PROJECT index 672b75a..66b2a38 100644 --- a/PROJECT +++ b/PROJECT @@ -35,11 +35,29 @@ resources: version: v1alpha1 - api: crdVersion: v1 - namespaced: true + namespaced: false controller: true domain: k8c.io group: kubelb.k8c.io kind: Tenant path: github.com/kubermatic/kubelb/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: k8c.io + group: kubelb.k8c.io + kind: Addresses + path: github.com/kubermatic/kubelb/api/kubelb.k8c.io/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: k8c.io + group: kubelb.k8c.io + kind: SyncSecret + path: github.com/kubermatic/kubelb/api/kubelb.k8c.io/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/kubelb.k8c.io/v1alpha1/sync_secret_types.go b/api/kubelb.k8c.io/v1alpha1/sync_secret_types.go new file mode 100644 index 0000000..80b1a4a --- /dev/null +++ b/api/kubelb.k8c.io/v1alpha1/sync_secret_types.go @@ -0,0 +1,56 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// SyncSecret is a wrapper over Kubernetes Secret object. This is used to sync secrets from tenants to the LB cluster in a controlled and secure way. +type SyncSecret struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Source: https://pkg.go.dev/k8s.io/api/core/v1#Secret + + // +optional + Data map[string][]byte `json:"data,omitempty" protobuf:"bytes,2,rep,name=data"` + + // +k8s:conversion-gen=false + // +optional + StringData map[string]string `json:"stringData,omitempty" protobuf:"bytes,4,rep,name=stringData"` + + // +optional + Type corev1.SecretType `json:"type,omitempty" protobuf:"bytes,3,opt,name=type,casttype=SecretType"` +} + +// +kubebuilder:object:root=true + +// SyncSecretList contains a list of SyncSecrets +type SyncSecretList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SyncSecret `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SyncSecret{}, &SyncSecretList{}) +} diff --git a/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go b/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go index eb051d4..961b120 100644 --- a/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go +++ b/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go @@ -754,6 +754,86 @@ func (in *ServiceStatus) DeepCopy() *ServiceStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SyncSecret) DeepCopyInto(out *SyncSecret) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make(map[string][]byte, len(*in)) + for key, val := range *in { + var outVal []byte + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]byte, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.StringData != nil { + in, out := &in.StringData, &out.StringData + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SyncSecret. +func (in *SyncSecret) DeepCopy() *SyncSecret { + if in == nil { + return nil + } + out := new(SyncSecret) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SyncSecret) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SyncSecretList) DeepCopyInto(out *SyncSecretList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SyncSecret, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SyncSecretList. +func (in *SyncSecretList) DeepCopy() *SyncSecretList { + if in == nil { + return nil + } + out := new(SyncSecretList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SyncSecretList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tenant) DeepCopyInto(out *Tenant) { *out = *in diff --git a/charts/kubelb-ccm/README.md b/charts/kubelb-ccm/README.md index 6929b75..b7ee0aa 100644 --- a/charts/kubelb-ccm/README.md +++ b/charts/kubelb-ccm/README.md @@ -54,6 +54,7 @@ helm install kubelb-ccm kubelb-ccm --namespace kubelb -f values.yaml --create-na | kubelb.disableHTTPRouteController | bool | `false` | disableHTTPRouteController specifies whether to disable the HTTPRoute Controller. | | kubelb.disableIngressController | bool | `false` | disableIngressController specifies whether to disable the Ingress Controller. | | kubelb.enableLeaderElection | bool | `true` | Enable the leader election. | +| kubelb.enableSecretSynchronizer | bool | `false` | Enable to automatically convert Secrets labelled with `kubelb.k8c.io/managed-by: kubelb` to Sync Secrets. This is used to sync secrets from tenants to the LB cluster in a controlled and secure way. | | kubelb.nodeAddressType | string | `"InternalIP"` | | | kubelb.tenantName | string | `nil` | Name of the tenant, must be unique against a load balancer cluster. | | kubelb.useGatewayClass | bool | `true` | useGatewayClass specifies whether to target resources with `kubelb` gateway class or all resources. | @@ -70,8 +71,8 @@ helm install kubelb-ccm kubelb-ccm --namespace kubelb -f values.yaml --create-na | rbac.allowProxyRole | bool | `true` | | | rbac.enabled | bool | `true` | | | replicaCount | int | `1` | | -| resources.limits.cpu | string | `"100m"` | | -| resources.limits.memory | string | `"128Mi"` | | +| resources.limits.cpu | string | `"500m"` | | +| resources.limits.memory | string | `"512Mi"` | | | resources.requests.cpu | string | `"100m"` | | | resources.requests.memory | string | `"128Mi"` | | | securityContext.allowPrivilegeEscalation | bool | `false` | | diff --git a/charts/kubelb-ccm/crds/kubelb.k8c.io_syncsecrets.yaml b/charts/kubelb-ccm/crds/kubelb.k8c.io_syncsecrets.yaml new file mode 100644 index 0000000..b8cd115 --- /dev/null +++ b/charts/kubelb-ccm/crds/kubelb.k8c.io_syncsecrets.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: syncsecrets.kubelb.k8c.io +spec: + group: kubelb.k8c.io + names: + kind: SyncSecret + listKind: SyncSecretList + plural: syncsecrets + singular: syncsecret + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SyncSecret is a wrapper over Kubernetes Secret object. This is + used to sync secrets from tenants to the LB cluster in a controlled and + secure way. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + data: + additionalProperties: + format: byte + type: string + type: object + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + stringData: + additionalProperties: + type: string + type: object + type: + type: string + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/kubelb-ccm/templates/clusterrole.yaml b/charts/kubelb-ccm/templates/clusterrole.yaml index 135eade..50e072d 100644 --- a/charts/kubelb-ccm/templates/clusterrole.yaml +++ b/charts/kubelb-ccm/templates/clusterrole.yaml @@ -75,6 +75,32 @@ rules: - list - watch {{- end }} +{{ if .Values.kubelb.enableSecretSynchronizer -}} +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +{{- end }} +- apiGroups: + - kubelb.k8c.io + resources: + - syncsecrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch --- {{- if .Values.rbac.allowProxyRole }} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kubelb-ccm/templates/deployment.yaml b/charts/kubelb-ccm/templates/deployment.yaml index 872be58..e69dfaf 100644 --- a/charts/kubelb-ccm/templates/deployment.yaml +++ b/charts/kubelb-ccm/templates/deployment.yaml @@ -72,6 +72,9 @@ spec: {{ if .Values.kubelb.disableGatewayAPI -}} - --disable-gateway-api=true {{ end -}} + {{ if .Values.kubelb.enableSecretSynchronizer -}} + - --enable-secret-synchronizer=true + {{ end -}} - --cluster-name={{ required "A valid .Values.kubelb.tenantName to specify the tenant name is required!" .Values.kubelb.tenantName }} env: - name: NAMESPACE diff --git a/charts/kubelb-ccm/values.yaml b/charts/kubelb-ccm/values.yaml index 6d9ddac..daab091 100644 --- a/charts/kubelb-ccm/values.yaml +++ b/charts/kubelb-ccm/values.yaml @@ -16,6 +16,8 @@ kubelb: # -- Enable the leader election. enableLeaderElection: true nodeAddressType: InternalIP + # -- Enable to automatically convert Secrets labelled with `kubelb.k8c.io/managed-by: kubelb` to Sync Secrets. This is used to sync secrets from tenants to the LB cluster in a controlled and secure way. + enableSecretSynchronizer: false # -- useIngressClass specifies whether to target resources with `kubelb` ingress class or all resources. useIngressClass: true # -- useGatewayClass specifies whether to target resources with `kubelb` gateway class or all resources. @@ -76,12 +78,11 @@ service: resources: limits: - cpu: 100m - memory: 128Mi + cpu: 500m + memory: 512Mi requests: cpu: 100m memory: 128Mi - autoscaling: enabled: false minReplicas: 1 diff --git a/charts/kubelb-manager/README.md b/charts/kubelb-manager/README.md index da60ba0..41c3dd4 100644 --- a/charts/kubelb-manager/README.md +++ b/charts/kubelb-manager/README.md @@ -59,8 +59,8 @@ helm install kubelb-manager kubelb-manager --namespace kubelb -f values.yaml --c | rbac.allowProxyRole | bool | `true` | | | rbac.enabled | bool | `true` | | | replicaCount | int | `1` | | -| resources.limits.cpu | string | `"100m"` | | -| resources.limits.memory | string | `"128Mi"` | | +| resources.limits.cpu | string | `"500m"` | | +| resources.limits.memory | string | `"512Mi"` | | | resources.requests.cpu | string | `"100m"` | | | resources.requests.memory | string | `"128Mi"` | | | securityContext.allowPrivilegeEscalation | bool | `false` | | diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_syncsecrets.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_syncsecrets.yaml new file mode 100644 index 0000000..b8cd115 --- /dev/null +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_syncsecrets.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: syncsecrets.kubelb.k8c.io +spec: + group: kubelb.k8c.io + names: + kind: SyncSecret + listKind: SyncSecretList + plural: syncsecrets + singular: syncsecret + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SyncSecret is a wrapper over Kubernetes Secret object. This is + used to sync secrets from tenants to the LB cluster in a controlled and + secure way. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + data: + additionalProperties: + format: byte + type: string + type: object + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + stringData: + additionalProperties: + type: string + type: object + type: + type: string + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/kubelb-manager/templates/clusterrole.yaml b/charts/kubelb-manager/templates/clusterrole.yaml index 0904842..3539093 100644 --- a/charts/kubelb-manager/templates/clusterrole.yaml +++ b/charts/kubelb-manager/templates/clusterrole.yaml @@ -213,6 +213,18 @@ rules: - patch - update - watch +- apiGroups: + - kubelb.k8c.io + resources: + - syncsecrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch --- {{- if .Values.rbac.allowProxyRole }} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kubelb-manager/values.yaml b/charts/kubelb-manager/values.yaml index 0ec1c1a..08607f6 100644 --- a/charts/kubelb-manager/values.yaml +++ b/charts/kubelb-manager/values.yaml @@ -69,8 +69,8 @@ service: resources: limits: - cpu: 100m - memory: 128Mi + cpu: 500m + memory: 512Mi requests: cpu: 100m memory: 128Mi diff --git a/cmd/ccm/main.go b/cmd/ccm/main.go index 83d14c6..0691c4f 100644 --- a/cmd/ccm/main.go +++ b/cmd/ccm/main.go @@ -79,6 +79,7 @@ func main() { var disableHTTPRouteController bool var disableGRPCRouteController bool var disableGatewayAPI bool + var enableSecretSynchronizer bool if flag.Lookup("kubeconfig") == nil { flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") @@ -104,6 +105,8 @@ func main() { flag.BoolVar(&disableHTTPRouteController, "disable-httproute-controller", false, "Disable the HTTPRoute controller.") flag.BoolVar(&disableGRPCRouteController, "disable-grpcroute-controller", false, "Disable the GRPCRoute controller.") + flag.BoolVar(&enableSecretSynchronizer, "enable-secret-synchronizer", false, "Enable to automatically convert Secrets labelled with `kubelb.k8c.io/managed-by: kubelb` to Sync Secrets. This is used to sync secrets from tenants to the LB cluster in a controlled and secure way.") + if !disableGatewayAPI { utilruntime.Must(gwapiv1alpha2.Install(scheme)) utilruntime.Must(gwapiv1.Install(scheme)) @@ -165,6 +168,16 @@ func main() { clusterName: {}, }, }, + &kubelbv1alpha1.Addresses{}: { + Namespaces: map[string]cache.Config{ + clusterName: {}, + }, + }, + &kubelbv1alpha1.SyncSecret{}: { + Namespaces: map[string]cache.Config{ + clusterName: {}, + }, + }, }, }, }) @@ -274,6 +287,30 @@ func main() { } } + if enableSecretSynchronizer { + if err = (&ccm.SecretConversionReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName(ccm.SecretConversionControllerName), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor(ccm.SecretConversionControllerName), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", ccm.SecretConversionControllerName) + os.Exit(1) + } + } + + if err = (&ccm.SyncSecretReconciler{ + Client: mgr.GetClient(), + LBClient: kubeLBMgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName(ccm.SyncSecretControllerName), + Scheme: mgr.GetScheme(), + ClusterName: clusterName, + Recorder: mgr.GetEventRecorderFor(ccm.SyncSecretControllerName), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", ccm.SyncSecretControllerName) + os.Exit(1) + } + // this is a copy and paste of SetupSignalHandler which only returns a context signals := make(chan struct{}) c := make(chan os.Signal, 2) diff --git a/cmd/kubelb/main.go b/cmd/kubelb/main.go index 6105a5f..1d71aaa 100644 --- a/cmd/kubelb/main.go +++ b/cmd/kubelb/main.go @@ -205,6 +205,18 @@ func main() { os.Exit(1) } + if err = (&kubelb.SyncSecretReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: ctrl.Log.WithName("controllers").WithName(kubelb.SyncSecretControllerName), + Recorder: mgr.GetEventRecorderFor(kubelb.SyncSecretControllerName), + EnvoyProxyTopology: kubelb.EnvoyProxyTopology(conf.GetEnvoyProxyTopology()), + Namespace: opt.namespace, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", kubelb.SyncSecretControllerName) + os.Exit(1) + } + if err = (&kubelb.TenantReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/config/ccm/ccm.yaml b/config/ccm/ccm.yaml index 8235aa0..f14e227 100644 --- a/config/ccm/ccm.yaml +++ b/config/ccm/ccm.yaml @@ -35,36 +35,39 @@ spec: seccompProfile: type: RuntimeDefault containers: - - name: kubelb-ccm - args: - - --enable-leader-election - - --node-address-type=InternalIP - image: controller:latest - securityContext: - allowPrivilegeEscalation: false - runAsUser: 65532 - capabilities: - drop: - - "ALL" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - requests: - cpu: 10m - memory: 64Mi - volumeMounts: - - mountPath: /home/nonroot/.kube - name: kubelb-cluster + - name: kubelb-ccm + args: + - --enable-leader-election + - --node-address-type=InternalIP + image: controller:latest + securityContext: + allowPrivilegeEscalation: false + runAsUser: 65532 + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /home/nonroot/.kube + name: kubelb-cluster serviceAccountName: kubelb terminationGracePeriodSeconds: 10 volumes: diff --git a/config/ccm/kustomization.yaml b/config/ccm/kustomization.yaml index c33cadb..f8ec3e6 100644 --- a/config/ccm/kustomization.yaml +++ b/config/ccm/kustomization.yaml @@ -17,6 +17,6 @@ kind: Kustomization namespace: kubelb resources: -- ccm.yaml -- rbac -- ../rbac + - ccm.yaml + - rbac + - ../rbac diff --git a/config/ccm/rbac/role.yaml b/config/ccm/rbac/role.yaml index 9edef9f..23cf58e 100644 --- a/config/ccm/rbac/role.yaml +++ b/config/ccm/rbac/role.yaml @@ -12,6 +12,18 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -75,6 +87,18 @@ rules: - get - patch - update +- apiGroups: + - kubelb.k8c.io + resources: + - syncsecrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - networking.k8s.io resources: diff --git a/config/crd/bases/kubelb.k8c.io_syncsecrets.yaml b/config/crd/bases/kubelb.k8c.io_syncsecrets.yaml new file mode 100644 index 0000000..b8cd115 --- /dev/null +++ b/config/crd/bases/kubelb.k8c.io_syncsecrets.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: syncsecrets.kubelb.k8c.io +spec: + group: kubelb.k8c.io + names: + kind: SyncSecret + listKind: SyncSecretList + plural: syncsecrets + singular: syncsecret + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SyncSecret is a wrapper over Kubernetes Secret object. This is + used to sync secrets from tenants to the LB cluster in a controlled and + secure way. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + data: + additionalProperties: + format: byte + type: string + type: object + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + stringData: + additionalProperties: + type: string + type: object + type: + type: string + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 9cc6ac1..12e669c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,7 @@ resources: - bases/kubelb.k8c.io_routes.yaml - bases/kubelb.k8c.io_addresses.yaml - bases/kubelb.k8c.io_tenants.yaml + - bases/kubelb.k8c.io_syncsecrets.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/kubelb/manager.yaml b/config/kubelb/manager.yaml index 5a7710a..045de70 100644 --- a/config/kubelb/manager.yaml +++ b/config/kubelb/manager.yaml @@ -91,8 +91,11 @@ spec: initialDelaySeconds: 5 periodSeconds: 10 resources: + limits: + cpu: 500m + memory: 512Mi requests: - cpu: 10m - memory: 64Mi + cpu: 100m + memory: 128Mi serviceAccountName: kubelb terminationGracePeriodSeconds: 10 diff --git a/config/kubelb/rbac/role.yaml b/config/kubelb/rbac/role.yaml index c599a58..ee0a60e 100644 --- a/config/kubelb/rbac/role.yaml +++ b/config/kubelb/rbac/role.yaml @@ -210,6 +210,18 @@ rules: - get - patch - update +- apiGroups: + - kubelb.k8c.io + resources: + - syncsecrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - kubelb.k8c.io resources: diff --git a/internal/controllers/ccm/gateway_controller.go b/internal/controllers/ccm/gateway_controller.go index cc38d70..c1a3e33 100644 --- a/internal/controllers/ccm/gateway_controller.go +++ b/internal/controllers/ccm/gateway_controller.go @@ -49,7 +49,7 @@ const ( ParentGatewayName = "kubelb" ) -// GatewayReconciler reconciles an Ingress Object +// GatewayReconciler reconciles a Gateway Object type GatewayReconciler struct { ctrlclient.Client diff --git a/internal/controllers/ccm/secret_conversion_controller.go b/internal/controllers/ccm/secret_conversion_controller.go new file mode 100644 index 0000000..72c3c9a --- /dev/null +++ b/internal/controllers/ccm/secret_conversion_controller.go @@ -0,0 +1,137 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ccm + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + "k8c.io/kubelb/internal/kubelb" + predicateutil "k8c.io/kubelb/internal/util/predicate" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + SecretConversionControllerName = "secret-conversion-controller" +) + +// SecretConversionReconciler reconciles an Ingress Object +type SecretConversionReconciler struct { + ctrlclient.Client + + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kubelb.k8c.io,resources=syncsecrets,verbs=get;list;watch;create;update;patch;delete + +func (r *SecretConversionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("name", req.NamespacedName) + + log.Info("Reconciling Secrets") + + resource := &corev1.Secret{} + if err := r.Get(ctx, req.NamespacedName, resource); err != nil { + if kerrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + // Resource is marked for deletion + if resource.DeletionTimestamp != nil { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + return r.cleanup(ctx, resource) + } + // Finalizer doesn't exist so clean up is already done + return reconcile.Result{}, nil + } + + // Add finalizer if it doesn't exist + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the Secret") + return ctrl.Result{Requeue: true}, nil + } + + if err := r.Update(ctx, resource); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) + } + } + + err := r.reconcile(ctx, log, resource) + if err != nil { + log.Error(err, "reconciling failed") + } + + return reconcile.Result{}, err +} + +func (r *SecretConversionReconciler) reconcile(ctx context.Context, _ logr.Logger, secret *corev1.Secret) error { + syncSecret := &kubelbv1alpha1.SyncSecret{ + ObjectMeta: metav1.ObjectMeta{Name: secret.Name, Namespace: secret.Namespace}, + } + syncSecret.Labels = secret.Labels + if syncSecret.Labels == nil { + syncSecret.Labels = make(map[string]string) + } + syncSecret.Labels[kubelb.LabelOriginNamespace] = secret.Namespace + syncSecret.Labels[kubelb.LabelOriginName] = secret.Name + syncSecret.Data = secret.Data + syncSecret.StringData = secret.StringData + syncSecret.Type = secret.Type + return CreateOrUpdateSyncSecret(ctx, r.Client, syncSecret) +} + +func (r *SecretConversionReconciler) cleanup(ctx context.Context, object *corev1.Secret) (ctrl.Result, error) { + resource := &kubelbv1alpha1.SyncSecret{ + ObjectMeta: metav1.ObjectMeta{Name: object.Name, Namespace: object.Namespace}, + } + + err := r.Delete(ctx, resource) + if err != nil && !kerrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete Sync Secret %s against %s: %w", resource.Name, object.Name, err) + } + + controllerutil.RemoveFinalizer(object, CleanupFinalizer) + if err := r.Update(ctx, object); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) + } + + return reconcile.Result{}, nil +} + +func (r *SecretConversionReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Secret{}, builder.WithPredicates(predicateutil.ByLabel(kubelb.LabelManagedBy, kubelb.LabelControllerName))). + Complete(r) +} diff --git a/internal/controllers/ccm/sync_secret_controller.go b/internal/controllers/ccm/sync_secret_controller.go new file mode 100644 index 0000000..d56c883 --- /dev/null +++ b/internal/controllers/ccm/sync_secret_controller.go @@ -0,0 +1,170 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ccm + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + "k8c.io/kubelb/internal/kubelb" + + "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + SyncSecretControllerName = "sync-secret-controller" +) + +// SyncSecretReconciler reconciles an Ingress Object +type SyncSecretReconciler struct { + ctrlclient.Client + + LBClient ctrlclient.Client + ClusterName string + + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +// +kubebuilder:rbac:groups=kubelb.k8c.io,resources=syncsecrets,verbs=get;list;watch;create;update;patch;delete + +func (r *SyncSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("name", req.NamespacedName) + + log.Info("Reconciling SyncSecret") + + resource := &kubelbv1alpha1.SyncSecret{} + if err := r.Get(ctx, req.NamespacedName, resource); err != nil { + if kerrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + // Resource is marked for deletion + if resource.DeletionTimestamp != nil { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + return r.cleanup(ctx, resource) + } + // Finalizer doesn't exist so clean up is already done + return reconcile.Result{}, nil + } + + // Add finalizer if it doesn't exist + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the SyncSecret") + return ctrl.Result{Requeue: true}, nil + } + + if err := r.Update(ctx, resource); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) + } + } + + err := r.reconcile(ctx, log, resource) + if err != nil { + log.Error(err, "reconciling failed") + } + + return reconcile.Result{}, err +} + +func (r *SyncSecretReconciler) reconcile(ctx context.Context, _ logr.Logger, object *kubelbv1alpha1.SyncSecret) error { + if object.Labels == nil { + object.Labels = make(map[string]string) + } + object.Labels[kubelb.LabelOriginNamespace] = object.Namespace + object.Labels[kubelb.LabelOriginName] = object.Name + + object.Namespace = r.ClusterName + object.Finalizers = []string{} + object.Name = string(object.UID) + object.SetUID("") // Reset UID to generate a new UID for the object + object.SetResourceVersion("") + + return CreateOrUpdateSyncSecret(ctx, r.LBClient, object) +} + +func CreateOrUpdateSyncSecret(ctx context.Context, client ctrlclient.Client, obj *kubelbv1alpha1.SyncSecret) error { + key := ctrlclient.ObjectKey{Namespace: obj.Namespace, Name: obj.Name} + existingObj := &kubelbv1alpha1.SyncSecret{} + if err := client.Get(ctx, key, existingObj); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to get SyncSecret: %w", err) + } + err := client.Create(ctx, obj) + if err != nil { + return fmt.Errorf("failed to create SyncSecret: %w", err) + } + return nil + } + + // Update the object if it is different from the existing one. + if equality.Semantic.DeepEqual(existingObj.Data, obj.Data) && + equality.Semantic.DeepEqual(existingObj.StringData, obj.StringData) && + equality.Semantic.DeepEqual(existingObj.Type, obj.Type) && + equality.Semantic.DeepEqual(existingObj.Labels, obj.Labels) && + equality.Semantic.DeepEqual(existingObj.Annotations, obj.Annotations) { + return nil + } + + // Required to update the object. + obj.ResourceVersion = existingObj.ResourceVersion + obj.UID = existingObj.UID + + if err := client.Update(ctx, obj); err != nil { + return fmt.Errorf("failed to update SyncSecret: %w", err) + } + return nil +} + +func (r *SyncSecretReconciler) cleanup(ctx context.Context, object *kubelbv1alpha1.SyncSecret) (ctrl.Result, error) { + resource := &kubelbv1alpha1.SyncSecret{ + ObjectMeta: metav1.ObjectMeta{Name: string(object.UID), Namespace: r.ClusterName}, + } + + err := r.LBClient.Delete(ctx, resource) + if err != nil && !kerrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete object %s from LB cluster: %w", object.Name, err) + } + + controllerutil.RemoveFinalizer(object, CleanupFinalizer) + if err := r.Update(ctx, object); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) + } + + return reconcile.Result{}, nil +} + +func (r *SyncSecretReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&kubelbv1alpha1.SyncSecret{}). + Complete(r) +} diff --git a/internal/controllers/kubelb/route_controller.go b/internal/controllers/kubelb/route_controller.go index e70eea6..64a902a 100644 --- a/internal/controllers/kubelb/route_controller.go +++ b/internal/controllers/kubelb/route_controller.go @@ -412,10 +412,13 @@ func (r *RouteReconciler) shouldReconcile(ctx context.Context, route *kubelbv1al // There is no source defined. return false, false, nil } - //nolint:gosimple - var resource client.Object - resource = &route.Spec.Source.Kubernetes.Route - switch resource := resource.(type) { + + resource, err := unstructured.ConvertUnstructuredToObject(&route.Spec.Source.Kubernetes.Route) + if err != nil { + return false, false, fmt.Errorf("failed to convert route to object: %w", err) + } + + switch v := resource.(type) { case *v1.Ingress: // Ensure that Ingress is enabled if config.Spec.Ingress.Disable { @@ -432,7 +435,7 @@ func (r *RouteReconciler) shouldReconcile(ctx context.Context, route *kubelbv1al return false, true, nil } - if resource.Name != "kubelb" { + if v.Name != "kubelb" { return false, false, nil } @@ -449,7 +452,7 @@ func (r *RouteReconciler) shouldReconcile(ctx context.Context, route *kubelbv1al } default: - log.Error(fmt.Errorf("Resource %v is not supported", resource.GetObjectKind().GroupVersionKind().GroupKind().String()), "cannot proceed") + log.Error(fmt.Errorf("Resource %v is not supported", v.GetObjectKind().GroupVersionKind().GroupKind().String()), "cannot proceed") return false, false, nil } return true, false, nil diff --git a/internal/controllers/kubelb/sync_secret_controller.go b/internal/controllers/kubelb/sync_secret_controller.go new file mode 100644 index 0000000..f53e998 --- /dev/null +++ b/internal/controllers/kubelb/sync_secret_controller.go @@ -0,0 +1,186 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelb + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + SyncSecretControllerName = "sync-secret-controller" +) + +// SyncSecretReconciler reconciles an Ingress Object +type SyncSecretReconciler struct { + ctrlclient.Client + Namespace string + EnvoyProxyTopology EnvoyProxyTopology + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +// +kubebuilder:rbac:groups=kubelb.k8c.io,resources=syncsecrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete + +func (r *SyncSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("name", req.NamespacedName) + + log.Info("Reconciling SyncSecret") + + resource := &kubelbv1alpha1.SyncSecret{} + if err := r.Get(ctx, req.NamespacedName, resource); err != nil { + if kerrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + // Resource is marked for deletion + if resource.DeletionTimestamp != nil { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + return r.cleanup(ctx, resource) + } + // Finalizer doesn't exist so clean up is already done + return reconcile.Result{}, nil + } + + // Add finalizer if it doesn't exist + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the SyncSecret") + return ctrl.Result{Requeue: true}, nil + } + + if err := r.Update(ctx, resource); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) + } + } + + err := r.reconcile(ctx, log, resource) + if err != nil { + log.Error(err, "reconciling failed") + } + + return reconcile.Result{}, err +} + +func (r *SyncSecretReconciler) reconcile(ctx context.Context, _ logr.Logger, object *kubelbv1alpha1.SyncSecret) error { + secret := &corev1.Secret{} + // Copy the SyncSecret to a Secret + secret.Data = object.Data + secret.StringData = object.StringData + secret.Type = object.Type + secret.Labels = object.Labels + secret.Annotations = object.Annotations + secret.Namespace = object.Namespace + if r.EnvoyProxyTopology.IsGlobalTopology() { + secret.Namespace = r.Namespace + } + + // Name needs to be randomized so using the UID of the SyncSecret. + secret.Name = string(object.UID) + + ownerReference := metav1.OwnerReference{ + APIVersion: object.APIVersion, + Kind: object.Kind, + Name: object.Name, + UID: object.UID, + } + + // Set owner reference for the resource. + secret.SetOwnerReferences([]metav1.OwnerReference{ownerReference}) + + return CreateOrUpdateSecret(ctx, r.Client, secret) +} + +func CreateOrUpdateSecret(ctx context.Context, client ctrlclient.Client, obj *corev1.Secret) error { + key := ctrlclient.ObjectKey{Namespace: obj.Namespace, Name: obj.Name} + existingObj := &corev1.Secret{} + if err := client.Get(ctx, key, existingObj); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to get Secret: %w", err) + } + err := client.Create(ctx, obj) + if err != nil { + return fmt.Errorf("failed to create Secret: %w", err) + } + return nil + } + + // Update the object if it is different from the existing one. + if equality.Semantic.DeepEqual(existingObj.Data, obj.Data) && + equality.Semantic.DeepEqual(existingObj.StringData, obj.StringData) && + equality.Semantic.DeepEqual(existingObj.Type, obj.Type) && + equality.Semantic.DeepEqual(existingObj.Labels, obj.Labels) && + equality.Semantic.DeepEqual(existingObj.Annotations, obj.Annotations) { + return nil + } + + // Required to update the object. + obj.ResourceVersion = existingObj.ResourceVersion + obj.UID = existingObj.UID + + if err := client.Update(ctx, obj); err != nil { + return fmt.Errorf("failed to update Secret: %w", err) + } + return nil +} + +func (r *SyncSecretReconciler) cleanup(ctx context.Context, object *kubelbv1alpha1.SyncSecret) (ctrl.Result, error) { + resource := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: string(object.UID), Namespace: object.Namespace}, + } + + if r.EnvoyProxyTopology.IsGlobalTopology() { + resource.Namespace = r.Namespace + } + + err := r.Delete(ctx, resource) + if err != nil && !kerrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete secret %s from LB cluster: %w", resource.Name, err) + } + + controllerutil.RemoveFinalizer(object, CleanupFinalizer) + if err := r.Update(ctx, object); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) + } + + return reconcile.Result{}, nil +} + +func (r *SyncSecretReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&kubelbv1alpha1.SyncSecret{}). + Complete(r) +} diff --git a/internal/controllers/kubelb/tenant_migration_controller.go b/internal/controllers/kubelb/tenant_migration_controller.go index a7a84bc..0b7610c 100644 --- a/internal/controllers/kubelb/tenant_migration_controller.go +++ b/internal/controllers/kubelb/tenant_migration_controller.go @@ -43,7 +43,7 @@ const ( TenantMigrationControllerName = "tenant-migration-controller" ) -// TenantMigrationReconciler reconciles an Ingress Object +// TenantMigrationReconciler is responsible for migrating namespace to a 1:1 tenant mapping. type TenantMigrationReconciler struct { ctrlclient.Client diff --git a/internal/controllers/utils.go b/internal/controllers/utils.go index 5a312be..223a94d 100644 --- a/internal/controllers/utils.go +++ b/internal/controllers/utils.go @@ -29,38 +29,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ) -var _ predicate.Predicate = &MatchingAnnotationPredicate{} - -type MatchingAnnotationPredicate struct { - AnnotationName string - AnnotationValue string -} - -// Create returns true if the Create event should be processed -func (r *MatchingAnnotationPredicate) Create(e event.CreateEvent) bool { - return r.Match(e.Object.GetAnnotations()) -} - -// Delete returns true if the Delete event should be processed -func (r *MatchingAnnotationPredicate) Delete(e event.DeleteEvent) bool { - return r.Match(e.Object.GetAnnotations()) -} - -// Update returns true if the Update event should be processed -func (r *MatchingAnnotationPredicate) Update(e event.UpdateEvent) bool { - return r.Match(e.ObjectNew.GetAnnotations()) -} - -// Generic returns true if the Generic event should be processed -func (r *MatchingAnnotationPredicate) Generic(e event.GenericEvent) bool { - return r.Match(e.Object.GetAnnotations()) -} - -func (r *MatchingAnnotationPredicate) Match(annotations map[string]string) bool { - val, ok := annotations[r.AnnotationName] - return !(ok && val == "") && val == r.AnnotationValue -} - // Helper functions to check and remove string from a slice of strings. func ContainsString(slice []string, s string) bool { for _, item := range slice { diff --git a/internal/resources/gatewayapi/gateway/gateway.go b/internal/resources/gatewayapi/gateway/gateway.go index d0075be..4c3cfd7 100644 --- a/internal/resources/gatewayapi/gateway/gateway.go +++ b/internal/resources/gatewayapi/gateway/gateway.go @@ -24,6 +24,7 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" + util "k8c.io/kubelb/internal/util/kubernetes" "k8s.io/apimachinery/pkg/api/equality" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -72,6 +73,19 @@ func CreateOrUpdateGateway(ctx context.Context, log logr.Logger, client ctrlclie // Process annotations. object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + + // Process secrets. + for i, listener := range object.Spec.Listeners { + if listener.TLS != nil { + for _, reference := range listener.TLS.CertificateRefs { + secretName := util.GetSecretNameIfExists(ctx, client, string(reference.Name), object.Namespace) + if secretName != "" { + object.Spec.Listeners[i].TLS.CertificateRefs[0].Name = gwapiv1.ObjectName(secretName) + } + } + } + } + object.Namespace = namespace object.SetUID("") // Reset UID to generate a new UID for the Gateway object diff --git a/internal/resources/ingress/ingress.go b/internal/resources/ingress/ingress.go index 80011a5..28a4628 100644 --- a/internal/resources/ingress/ingress.go +++ b/internal/resources/ingress/ingress.go @@ -24,6 +24,7 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" + util "k8c.io/kubelb/internal/util/kubernetes" networkingv1 "k8s.io/api/networking/v1" v1 "k8s.io/api/networking/v1" @@ -71,6 +72,16 @@ func CreateOrUpdateIngress(ctx context.Context, log logr.Logger, client ctrlclie // Process annotations. object.Annotations = kubelb.PropagateAnnotations(object.Annotations, annotations) + // Process secrets. + if object.Spec.TLS != nil { + for i := range object.Spec.TLS { + secretName := util.GetSecretNameIfExists(ctx, client, object.Spec.TLS[i].SecretName, object.Namespace) + if secretName != "" { + object.Spec.TLS[i].SecretName = secretName + } + } + } + // Update name and other fields before creating/updating the object. object.Name = kubelb.GenerateName(globalTopology, string(object.UID), object.Name, object.Namespace) object.Namespace = namespace diff --git a/internal/resources/service/service.go b/internal/resources/service/service.go index 737ffb7..e7a33fe 100644 --- a/internal/resources/service/service.go +++ b/internal/resources/service/service.go @@ -161,6 +161,10 @@ func CreateOrUpdateService(ctx context.Context, client ctrlclient.Client, obj *c return nil } + // Required to update the object. + obj.ResourceVersion = existingObj.ResourceVersion + obj.UID = existingObj.UID + if err := client.Update(ctx, obj); err != nil { return fmt.Errorf("failed to update Service: %w", err) } diff --git a/internal/util/kubernetes/secret.go b/internal/util/kubernetes/secret.go new file mode 100644 index 0000000..9408688 --- /dev/null +++ b/internal/util/kubernetes/secret.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + + "k8c.io/kubelb/internal/kubelb" + + corev1 "k8s.io/api/core/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetSecretNameIfExists(ctx context.Context, client ctrlclient.Client, name, namespace string) string { + secrets := corev1.SecretList{} + err := client.List(ctx, &secrets, ctrlclient.MatchingLabels{kubelb.LabelOriginName: name, kubelb.LabelOriginNamespace: namespace}) + if err != nil { + return "" + } + if len(secrets.Items) == 0 { + return "" + } + return secrets.Items[0].Name +} diff --git a/internal/util/predicate/predicate.go b/internal/util/predicate/predicate.go new file mode 100644 index 0000000..b02daf3 --- /dev/null +++ b/internal/util/predicate/predicate.go @@ -0,0 +1,136 @@ +/* +Copyright 2024 The KubeLB Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package predicate + +import ( + "k8s.io/apimachinery/pkg/util/sets" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// Factory returns a predicate func that applies the given filter function +// on CREATE, UPDATE and DELETE events. For UPDATE events, the filter is applied +// to both the old and new object and OR's the result. +func Factory(filter func(o ctrlruntimeclient.Object) bool) predicate.Funcs { + return TypedFactory(filter) +} + +func TypedFactory[T ctrlruntimeclient.Object](filter func(o T) bool) predicate.TypedFuncs[T] { + if filter == nil { + return predicate.TypedFuncs[T]{} + } + + return predicate.TypedFuncs[T]{ + CreateFunc: func(e event.TypedCreateEvent[T]) bool { + return filter(e.Object) + }, + UpdateFunc: func(e event.TypedUpdateEvent[T]) bool { + return filter(e.ObjectOld) || filter(e.ObjectNew) + }, + DeleteFunc: func(e event.TypedDeleteEvent[T]) bool { + return filter(e.Object) + }, + } +} + +// MultiFactory returns a predicate func that applies the given filter functions +// to the respective events for CREATE, UPDATE and DELETE. For UPDATE events, the +// filter is applied to both the old and new object and OR's the result. +func MultiFactory(createFilter func(o ctrlruntimeclient.Object) bool, updateFilter func(o ctrlruntimeclient.Object) bool, deleteFilter func(o ctrlruntimeclient.Object) bool) predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return createFilter(e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return updateFilter(e.ObjectOld) || updateFilter(e.ObjectNew) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return deleteFilter(e.Object) + }, + } +} + +// ByNamespace returns a predicate func that only includes objects in the given namespace. +func ByNamespace(namespace string) predicate.Funcs { + return Factory(func(o ctrlruntimeclient.Object) bool { + return o.GetNamespace() == namespace + }) +} + +// ByName returns a predicate func that only includes objects in the given names. +func ByName(names ...string) predicate.Funcs { + return TypedByName[ctrlruntimeclient.Object](names...) +} + +func TypedByName[T ctrlruntimeclient.Object](names ...string) predicate.TypedFuncs[T] { + namesSet := sets.New(names...) + return TypedFactory(func(o T) bool { + return namesSet.Has(o.GetName()) + }) +} + +// ByLabel returns a predicate func that only includes objects with the given label. +func ByLabel(key, value string) predicate.Funcs { + return Factory(func(o ctrlruntimeclient.Object) bool { + labels := o.GetLabels() + if labels != nil { + if existingValue, ok := labels[key]; ok { + if existingValue == value { + return true + } + } + } + return false + }) +} + +// ByLabel returns a predicate func that only includes objects that have a specific label key (value is ignored). +func ByLabelExists(key string) predicate.Funcs { + return Factory(func(o ctrlruntimeclient.Object) bool { + labels := o.GetLabels() + if labels != nil { + if _, ok := labels[key]; ok { + return true + } + } + return false + }) +} + +// ByAnnotation returns a predicate func that only includes objects with the given annotation. +func ByAnnotation(key, value string, checkValue bool) predicate.Funcs { + return Factory(func(o ctrlruntimeclient.Object) bool { + annotations := o.GetAnnotations() + if annotations != nil { + if existingValue, ok := annotations[key]; ok { + if !checkValue { + return true + } + if existingValue == value { + return true + } + } + } + return false + }) +} + +// TrueFilter is a helper filter implementation that always returns true, e.g. for use with MultiFactory. +func TrueFilter(_ ctrlruntimeclient.Object) bool { + return true +}