Global envoy proxy (#50)

* Introduce global envoy proxy topology Signed-off-by: Waleed Malik <[email protected]> * Add port lookup service Signed-off-by: Waleed Malik <[email protected]> * Formatting fixes Signed-off-by: Waleed Malik <[email protected]> * Add RBAC for kubelb controller to manage configmaps Signed-off-by: Waleed Malik <[email protected]> * Map arbitrary ports with the targetted node ports Signed-off-by: Waleed Malik <[email protected]> * Remove redundant code Signed-off-by: Waleed Malik <[email protected]> * Take care of de-allocating ports when load balancer is removed Signed-off-by: Waleed Malik <[email protected]> * Fix typos Signed-off-by: Waleed Malik <[email protected]> * Remove circular dependency for EnvoyResourceIdentifierPattern Signed-off-by: Waleed Malik <[email protected]> * Fix cleanup for services Signed-off-by: Waleed Malik <[email protected]> * Fix port lookup for services Signed-off-by: Waleed Malik <[email protected]> * Rework port lookup for better state management Signed-off-by: Waleed Malik <[email protected]> * Requeue load-balancers when the service is updated Signed-off-by: Waleed Malik <[email protected]> * Only queue up LBs when status of service has changed Signed-off-by: Waleed Malik <[email protected]> * Refactored code Since we already have specific labels that are used by KubeLB when creating services we don't really need to add additional label for managed-by Signed-off-by: Waleed Malik <[email protected]> * Use downwards API for kubelb namespace Signed-off-by: Waleed Malik <[email protected]> --------- Signed-off-by: Waleed Malik <[email protected]>
kubermatic · Sep 25, 2023 · 569ebf9 · 569ebf9
1 parent 5cd08bd
commit 569ebf9
Show file tree

Hide file tree

Showing 12 changed files with 540 additions and 52 deletions.
diff --git a/cmd/ccm/main.go b/cmd/ccm/main.go
@@ -160,7 +160,7 @@ func main() {
 
 	if err = (&ccm.KubeLBServiceReconciler{
 		Client:          mgr.GetClient(),
-		KubeLBMananger:  kubeLBMgr,
+		KubeLBManager:   kubeLBMgr,
 		Log:             ctrl.Log.WithName("kubelb.service.reconciler"),
 		Scheme:          mgr.GetScheme(),
 		CloudController: enableCloudController,

diff --git a/cmd/kubelb/main.go b/cmd/kubelb/main.go
@@ -24,6 +24,7 @@ import (
 	kubelbk8ciov1alpha1 "k8c.io/kubelb/pkg/api/kubelb.k8c.io/v1alpha1"
 	"k8c.io/kubelb/pkg/controllers/kubelb"
 	"k8c.io/kubelb/pkg/envoy"
+	portlookup "k8c.io/kubelb/pkg/port-lookup"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -40,6 +41,7 @@ type options struct {
 	enableLeaderElection bool
 	probeAddr            string
 	enableDebugMode      bool
+	namespace            string
 
 	// Envoy configuration
 	envoyProxyTopology string
@@ -65,8 +67,19 @@ func main() {
 	flag.BoolVar(&opt.enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller kubelb. Enabling this will ensure there is only one active controller kubelb.")
 	flag.BoolVar(&opt.enableDebugMode, "debug", false, "Enables debug mode")
-	flag.StringVar(&opt.envoyProxyTopology, "envoy-proxy-topology", "shared", "The deployment topology for Envoy Proxy. Valid values are: shared, dedicated.")
+	flag.StringVar(&opt.envoyProxyTopology, "envoy-proxy-topology", "shared", "The deployment topology for Envoy Proxy. Valid values are: shared, dedicated, and global.")
 	flag.IntVar(&opt.envoyProxyReplicas, "envoy-proxy-replicas", 1, "Number of replicas for envoy proxy.")
+	flag.StringVar(&opt.namespace, "namespace", "", "The namespace where the controller will run.")
+
+	if len(opt.namespace) == 0 {
+		// Retrieve controller namespace
+		ns, _ := os.LookupEnv("NAMESPACE")
+		if len(ns) == 0 {
+			setupLog.Error(nil, "invalid value for --envoy-proxy-topology. Valid values are: shared, dedicated, and global")
+			os.Exit(1)
+		}
+		opt.namespace = ns
+	}
 
 	opts := zap.Options{
 		Development: true,
@@ -75,8 +88,9 @@ func main() {
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
-	if opt.envoyProxyTopology != "shared" && opt.envoyProxyTopology != "dedicated" {
-		setupLog.Error(nil, "invalid value for --envoy-proxy-topology. Valid values are: shared, dedicated")
+	envoyProxyTopology := kubelb.EnvoyProxyTopology(opt.envoyProxyTopology)
+	if envoyProxyTopology != kubelb.EnvoyProxyTopologyDedicated && envoyProxyTopology != kubelb.EnvoyProxyTopologyShared && envoyProxyTopology != kubelb.EnvoyProxyTopologyGlobal {
+		setupLog.Error(nil, "invalid value for --envoy-proxy-topology. Valid values are: shared, dedicated, and global")
 		os.Exit(1)
 	}
 
@@ -90,6 +104,7 @@ func main() {
 		LeaderElection:                opt.enableLeaderElection,
 		LeaderElectionID:              "19f32e7b.kubelb.k8c.io",
 		LeaderElectionReleaseOnCancel: true,
+		LeaderElectionNamespace:       opt.namespace,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start kubelb")
@@ -110,14 +125,26 @@ func main() {
 	// setup signal handler
 	ctx := ctrl.SetupSignalHandler()
 
+	// For Global topology, we need to ensure that the port lookup configmap exists. If it doesn't, we create it since it's managed by this controller.
+	var portAllocator *portlookup.PortAllocator
+	if envoyProxyTopology == kubelb.EnvoyProxyTopologyGlobal {
+		portAllocator = portlookup.NewPortAllocator(mgr.GetClient(), opt.namespace)
+		if err := portAllocator.LoadState(ctx, mgr.GetAPIReader()); err != nil {
+			setupLog.Error(err, ("unable to load port lookup state"))
+			os.Exit(1)
+		}
+	}
+
 	if err = (&kubelb.LoadBalancerReconciler{
 		Client:             mgr.GetClient(),
 		Cache:              mgr.GetCache(),
 		Scheme:             mgr.GetScheme(),
 		EnvoyCache:         envoyServer.Cache,
 		EnvoyBootstrap:     envoyServer.GenerateBootstrap(),
-		EnvoyProxyTopology: kubelb.EnvoyProxyTopology(opt.envoyProxyTopology),
+		EnvoyProxyTopology: envoyProxyTopology,
 		EnvoyProxyReplicas: opt.envoyProxyReplicas,
+		Namespace:          opt.namespace,
+		PortAllocator:      portAllocator,
 	}).SetupWithManager(mgr, ctx); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "LoadBalancer")
 		os.Exit(1)

diff --git a/config/kubelb/manager.yaml b/config/kubelb/manager.yaml
@@ -58,6 +58,11 @@ spec:
         - --envoy-proxy-topology=shared
         - --envoy-proxy-replicas=3
         image: controller:latest
+        env:
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
         securityContext:
           allowPrivilegeEscalation: false
           runAsUser: 65532

diff --git a/config/kubelb/rbac/role.yaml b/config/kubelb/rbac/role.yaml
@@ -5,6 +5,18 @@ metadata:
   creationTimestamp: null
   name: kubelb
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

diff --git a/pkg/controllers/ccm/service_controller.go b/pkg/controllers/ccm/service_controller.go
@@ -43,7 +43,7 @@ const LBFinalizerName = "kubelb.k8c.io/tcplb-finalizer"
 type KubeLBServiceReconciler struct {
 	ctrlclient.Client
 
-	KubeLBMananger  ctrl.Manager
+	KubeLBManager   ctrl.Manager
 	Log             logr.Logger
 	Scheme          *runtime.Scheme
 	ClusterName     string
@@ -113,7 +113,7 @@ func (r *KubeLBServiceReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	desiredTcpLB := kubelb.MapLoadBalancer(&service, clusterEndpoints, r.ClusterName)
 	log.V(6).Info("desired", "LoadBalancer", desiredTcpLB)
 
-	kubelbClient := r.KubeLBMananger.GetClient()
+	kubelbClient := r.KubeLBManager.GetClient()
 
 	var actualTcpLB kubelbk8ciov1alpha1.TCPLoadBalancer
 
@@ -167,7 +167,7 @@ func (r *KubeLBServiceReconciler) cleanupService(ctx context.Context, log logr.L
 		return ctrl.Result{}, nil
 	}
 
-	kubelbClient := r.KubeLBMananger.GetClient()
+	kubelbClient := r.KubeLBManager.GetClient()
 	desiredTcpLB := kubelb.MapLoadBalancer(service, clusterEndpoints, r.ClusterName)
 	log.V(1).Info("deleting TCPLoadBalancer", "name", desiredTcpLB)
 
@@ -242,7 +242,7 @@ func (r *KubeLBServiceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	}
 
 	kubeLBWatch := &source.Kind{Type: &kubelbk8ciov1alpha1.TCPLoadBalancer{}}
-	if err = kubeLBWatch.InjectCache(r.KubeLBMananger.GetCache()); err != nil {
+	if err = kubeLBWatch.InjectCache(r.KubeLBManager.GetCache()); err != nil {
 		return fmt.Errorf("failed to inject cache: %w", err)
 	}