restart envoy pods at boot

kubermatic · Dec 4, 2022 · c521da9 · c521da9
1 parent 28c8a42
commit c521da9
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 5 deletions.
diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -17,18 +17,25 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
+	"os"
+	"time"
+
 	"github.com/spf13/pflag"
-	kubelbk8ciov1alpha1 "k8c.io/kubelb/pkg/api/kubelb.k8c.io/v1alpha1"
-	"k8c.io/kubelb/pkg/controllers/manager"
-	"k8c.io/kubelb/pkg/envoy"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
-	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	kubelbk8ciov1alpha1 "k8c.io/kubelb/pkg/api/kubelb.k8c.io/v1alpha1"
+	"k8c.io/kubelb/pkg/controllers/manager"
+	"k8c.io/kubelb/pkg/envoy"
+	"k8c.io/kubelb/pkg/kubelb"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -120,6 +127,36 @@ func main() {
 
 	// +kubebuilder:scaffold:builder
 
+	//restart existing envoy pods to reset their snapshot version
+	go func() {
+		client := mgr.GetClient()
+
+		//wait for the controller to come up
+		time.Sleep(20 * time.Second)
+
+		var pods corev1.PodList
+		err = client.List(context.Background(), &pods, &ctrlclient.ListOptions{})
+		if err != nil {
+			setupLog.Error(err, "unable to list pods")
+			os.Exit(1)
+		}
+
+		for _, pod := range pods.Items {
+
+			value, ok := pod.ObjectMeta.Labels[kubelb.LabelAppKubernetesManagedBy]
+			if !ok || value != kubelb.LabelControllerName {
+				continue
+			}
+
+			setupLog.Info("resetting envoy", "pod", pod.Name)
+
+			err = client.Delete(context.Background(), &pod, &ctrlclient.DeleteOptions{})
+			if err != nil {
+				setupLog.Error(err, "failed to restart", "pod", pod.Name)
+			}
+		}
+	}()
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")

diff --git a/config/manager/rbac/role.yaml b/config/manager/rbac/role.yaml
@@ -19,6 +19,14 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - delete
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:

diff --git a/pkg/controllers/manager/tcploadbalancer_controller.go b/pkg/controllers/manager/tcploadbalancer_controller.go
@@ -55,6 +55,7 @@ type TCPLoadBalancerReconciler struct {
 // +kubebuilder:rbac:groups=kubelb.k8c.io,resources=tcploadbalancers/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups="",resources=pods,verbs=list;watch;delete
 
 func (r *TCPLoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
@@ -190,6 +191,8 @@ func (r *TCPLoadBalancerReconciler) reconcileDeployment(ctx context.Context, tcp
 			})
 		}
 
+		deployment.Labels = map[string]string{kubelb.LabelAppKubernetesName: tcpLoadBalancer.Name, kubelb.LabelAppKubernetesManagedBy: kubelb.LabelControllerName}
+
 		deployment.Spec.Replicas = &replicas
 		deployment.Spec.Selector = &v1.LabelSelector{
 			MatchLabels: map[string]string{kubelb.LabelAppKubernetesName: tcpLoadBalancer.Name},
@@ -198,7 +201,7 @@ func (r *TCPLoadBalancerReconciler) reconcileDeployment(ctx context.Context, tcp
 		deployment.Spec.Template.ObjectMeta = v1.ObjectMeta{
 			Name:      tcpLoadBalancer.Name,
 			Namespace: tcpLoadBalancer.Namespace,
-			Labels:    map[string]string{kubelb.LabelAppKubernetesName: tcpLoadBalancer.Name},
+			Labels:    map[string]string{kubelb.LabelAppKubernetesName: tcpLoadBalancer.Name, kubelb.LabelAppKubernetesManagedBy: kubelb.LabelControllerName},
 		}
 
 		var containers []corev1.Container

diff --git a/pkg/kubelb/utils.go b/pkg/kubelb/utils.go
@@ -24,6 +24,7 @@ import (
 
 const LabelOriginNamespace = "kubelb.k8c.io/origin-ns"
 const LabelOriginName = "kubelb.k8c.io/origin-name"
+const LabelControllerName = "kubelb"
 
 const LabelAppKubernetesName = "app.kubernetes.io/name"            //mysql
 const LabelAppKubernetesInstance = "app.kubernetes.io/instance"    //mysql-abcxzy"