Migration from Namespaces to Tenants (#36)

* Migration from namespaces to tenants Signed-off-by: Waleed Malik <[email protected]> * Make migration-controller optional Signed-off-by: Waleed Malik <[email protected]> * Log instead of erroring out if tenant is not found Signed-off-by: Waleed Malik <[email protected]> * Tenant migration Signed-off-by: Waleed Malik <[email protected]> --------- Signed-off-by: Waleed Malik <[email protected]>
kubermatic · Aug 2, 2024 · d38bce6 · d38bce6
1 parent 7bbc480
commit d38bce6
Show file tree

Hide file tree

Showing 8 changed files with 359 additions and 50 deletions.
diff --git a/.prow/verify.yaml b/.prow/verify.yaml
@@ -133,6 +133,7 @@ presubmits:
               cpu: 4
 
   - name: pull-kubelb-e2e-tests
+    optional: true
     always_run: true
     decorate: true
     clone_uri: "ssh://[email protected]/kubermatic/kubelb.git"

diff --git a/api/kubelb.k8c.io/v1alpha1/tenant_types.go b/api/kubelb.k8c.io/v1alpha1/tenant_types.go
@@ -25,17 +25,26 @@ import (
 
 // TenantSpec defines the desired state of Tenant
 type TenantSpec struct {
-	// TODO: To be structured and implemented
-	// LoadBalancer *LoadBalancerSettings `json:"loadBalancer,omitempty"`
-	// Ingress      *IngressSettings      `json:"ingress,omitempty"`
-	// GatewayAPI   *GatewayAPISettings   `json:"gatewayAPI,omitempty"`
+	LoadBalancer LoadBalancerSettings `json:"loadBalancer,omitempty"`
+	// Ingress      IngressSettings      `json:"ingress,omitempty"`
+	// GatewayAPI   GatewayAPISettings   `json:"gatewayAPI,omitempty"`
 }
 
-// type LoadBalancerSettings struct {
-// 	// Class is the class of the load balancer to use.
-// 	// +optional
-// 	Class *string `json:"class,omitempty"`
-// }
+type LoadBalancerSettings struct {
+	// Class is the class of the load balancer to use.
+	// +optional
+	Class *string `json:"class,omitempty"`
+
+	// PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value.
+	// This will have a higher precedence than the annotations specified at the Config level.
+	// +optional
+	PropagatedAnnotations *map[string]string `json:"propagatedAnnotations,omitempty"`
+
+	// PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored.
+	// This will have a higher precedence than the value specified at the Config level.
+	// +optional
+	PropagateAllAnnotations *bool `json:"propagateAllAnnotations,omitempty"`
+}
 
 // type IngressSettings struct {
 // 	// Class is the class of the ingress to use.

diff --git a/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go b/api/kubelb.k8c.io/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_tenants.yaml
@@ -38,6 +38,25 @@ spec:
             type: object
           spec:
             description: TenantSpec defines the desired state of Tenant
+            properties:
+              loadBalancer:
+                properties:
+                  class:
+                    description: Class is the class of the load balancer to use.
+                    type: string
+                  propagateAllAnnotations:
+                    description: |-
+                      PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored.
+                      This will have a higher precedence than the value specified at the Config level.
+                    type: boolean
+                  propagatedAnnotations:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value.
+                      This will have a higher precedence than the annotations specified at the Config level.
+                    type: object
+                type: object
             type: object
           status:
             description: TenantStatus defines the observed state of Tenant

diff --git a/cmd/kubelb/main.go b/cmd/kubelb/main.go
@@ -39,14 +39,15 @@ import (
 )
 
 type options struct {
-	metricsAddr          string
-	envoyCPMetricsAddr   string
-	envoyListenAddress   string
-	enableLeaderElection bool
-	probeAddr            string
-	kubeconfig           string
-	enableDebugMode      bool
-	namespace            string
+	metricsAddr                     string
+	envoyCPMetricsAddr              string
+	envoyListenAddress              string
+	enableLeaderElection            bool
+	probeAddr                       string
+	kubeconfig                      string
+	enableDebugMode                 bool
+	namespace                       string
+	enableTenantMigrationController bool
 }
 
 var (
@@ -71,6 +72,8 @@ func main() {
 	flag.BoolVar(&opt.enableDebugMode, "debug", false, "Enables debug mode")
 	flag.StringVar(&opt.namespace, "namespace", "", "The namespace where the controller will run.")
 
+	flag.BoolVar(&opt.enableTenantMigrationController, "enable-tenant-migration", true, "Enables a controller that performs automated migration from namespaces to tenants")
+
 	if flag.Lookup("kubeconfig") == nil {
 		flag.StringVar(&opt.kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	}
@@ -202,6 +205,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	if opt.enableTenantMigrationController {
+		if err = (&kubelb.TenantMigrationReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   mgr.GetScheme(),
+			Log:      ctrl.Log.WithName("controllers").WithName(kubelb.TenantMigrationControllerName),
+			Recorder: mgr.GetEventRecorderFor(kubelb.TenantMigrationControllerName),
+		}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", kubelb.TenantMigrationControllerName)
+			os.Exit(1)
+		}
+	}
+
 	go func() {
 		setupLog.Info("starting kubelb envoy manager")
 

diff --git a/config/crd/bases/kubelb.k8c.io_tenants.yaml b/config/crd/bases/kubelb.k8c.io_tenants.yaml
@@ -38,6 +38,25 @@ spec:
             type: object
           spec:
             description: TenantSpec defines the desired state of Tenant
+            properties:
+              loadBalancer:
+                properties:
+                  class:
+                    description: Class is the class of the load balancer to use.
+                    type: string
+                  propagateAllAnnotations:
+                    description: |-
+                      PropagateAllAnnotations defines whether all annotations will be propagated to the LoadBalancer service. If set to true, PropagatedAnnotations will be ignored.
+                      This will have a higher precedence than the value specified at the Config level.
+                    type: boolean
+                  propagatedAnnotations:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      PropagatedAnnotations defines the list of annotations(key-value pairs) that will be propagated to the LoadBalancer service. Keep the `value` field empty in the key-value pair to allow any value.
+                      This will have a higher precedence than the annotations specified at the Config level.
+                    type: object
+                type: object
             type: object
           status:
             description: TenantStatus defines the observed state of Tenant

diff --git a/internal/controllers/kubelb/loadbalancer_controller.go b/internal/controllers/kubelb/loadbalancer_controller.go
@@ -20,7 +20,6 @@ import (
 	"context"
 	"fmt"
 	"reflect"
-	"strings"
 
 	kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1"
 	"k8c.io/kubelb/internal/config"
@@ -180,9 +179,10 @@ func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalan
 
 	log.V(2).Info("verify service")
 
-	ns := &corev1.Namespace{}
-	if err := r.Get(ctx, types.NamespacedName{Name: loadBalancer.Namespace}, ns); err != nil {
-		return err
+	tenant, err := getTenantForNamespace(ctx, r.Client, namespace)
+	if err != nil {
+		// This should never happen as the namespace should always have a tenant owner. We simply log this and continue.
+		log.V(5).Info("Tenant not found for namespace", "namespace", namespace)
 	}
 
 	labels := map[string]string{
@@ -205,10 +205,10 @@ func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalan
 			Name:        svcName,
 			Namespace:   namespace,
 			Labels:      labels,
-			Annotations: propagateAnnotations(ns.Annotations, loadBalancer.Annotations),
+			Annotations: propagateAnnotations(tenant, loadBalancer.Annotations),
 		},
 	}
-	err := r.Get(ctx, types.NamespacedName{
+	err = r.Get(ctx, types.NamespacedName{
 		Name:      svcName,
 		Namespace: namespace,
 	}, service)
@@ -254,7 +254,7 @@ func (r *LoadBalancerReconciler) reconcileService(ctx context.Context, loadBalan
 			ports = append(ports, allocatedPort)
 		}
 
-		for k, v := range propagateAnnotations(ns.Annotations, loadBalancer.Annotations) {
+		for k, v := range propagateAnnotations(tenant, loadBalancer.Annotations) {
 			service.Annotations[k] = v
 		}
 		service.Spec.Ports = ports
@@ -441,36 +441,29 @@ func filterServicesPredicate() predicate.TypedPredicate[client.Object] {
 	}
 }
 
-func propagateAnnotations(permitted map[string]string, loadbalancer map[string]string) map[string]string {
-	if config.GetConfig().Spec.PropagateAllAnnotations {
-		return loadbalancer
+func propagateAnnotations(tenant *kubelbv1alpha1.Tenant, loadbalancer map[string]string) map[string]string {
+	permitted := make(map[string]string)
+	if tenant != nil {
+		if tenant.Spec.LoadBalancer.PropagateAllAnnotations != nil && *tenant.Spec.LoadBalancer.PropagateAllAnnotations {
+			return loadbalancer
+		}
+		if tenant.Spec.LoadBalancer.PropagatedAnnotations != nil {
+			permitted = *tenant.Spec.LoadBalancer.PropagatedAnnotations
+		}
 	}
 
-	a := make(map[string]string)
-	permittedMap := make(map[string][]string)
-	for k, v := range permitted {
-		if strings.HasPrefix(k, kubelbv1alpha1.PropagateAnnotation) {
-			filter := strings.SplitN(k, "=", 2)
-			if len(filter) <= 1 {
-				permittedMap[v] = []string{}
-			} else {
-				// optional value filter provided
-				filterValues := strings.Split(filter[1], ",")
-				for i, v := range filterValues {
-					filterValues[i] = strings.TrimSpace(v)
-				}
-				permittedMap[filter[0]] = filterValues
-			}
+	if permitted == nil {
+		if config.GetConfig().Spec.PropagateAllAnnotations {
+			return loadbalancer
 		}
+		permitted = config.GetConfig().Spec.PropagatedAnnotations
 	}
 
-	if config.GetConfig().Spec.PropagatedAnnotations != nil {
-		globallyPermitted := config.GetConfig().Spec.PropagatedAnnotations
-		for k, v := range globallyPermitted {
-			// Annotations specified at namespace level have a higher precedence.
-			if _, found := permittedMap[k]; !found {
-				permittedMap[k] = []string{v}
-			}
+	a := make(map[string]string)
+	permittedMap := make(map[string][]string)
+	for k, v := range permitted {
+		if _, found := permittedMap[k]; !found {
+			permittedMap[k] = []string{v}
 		}
 	}
 
@@ -524,3 +517,23 @@ func (r *LoadBalancerReconciler) enqueueLoadBalancersForConfig() handler.MapFunc
 		return result
 	}
 }
+
+func getTenantForNamespace(ctx context.Context, client ctrlruntimeclient.Client, namespace string) (*kubelbv1alpha1.Tenant, error) {
+	ns := &corev1.Namespace{}
+	err := client.Get(ctx, types.NamespacedName{Name: namespace}, ns)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, owner := range ns.GetOwnerReferences() {
+		if owner.Kind == "Tenant" {
+			tenant := &kubelbv1alpha1.Tenant{}
+			err := client.Get(ctx, types.NamespacedName{Name: owner.Name}, tenant)
+			if err != nil {
+				return nil, err
+			}
+			return tenant, nil
+		}
+	}
+	return nil, fmt.Errorf("no tenant found for namespace %s", namespace)
+}