Merge pull request #3 from ecooke-macu/ec-002

Fix most of issue #2

+semver:fix

charts/multicluster-ingress/templates/configmap.yaml

@@ -30,6 +30,8 @@ data:
   HeartbeatTimeout: "{{ .Values.config.heartbeatTimeout }}"
   HeartbeatSetInterval: "{{ .Values.config.heartbeatSetInterval }}"
   Serilog__MinimumLevel__Default: "{{ .Values.config.logLevel }}"
+
+  PerioidRefreshInterval: {{ .Values.config.periodicRefreshInterval }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -55,26 +57,6 @@ data:
       "HeartbeatSetInterval": 0,
       "HeartbeatTimeout": 0,
       "NameserverNames": {},
-      "Peers": []
+      "Peers": [],
+      "PerioidRefreshInterval": 0
     }
-
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: mcingress
-type: Opaque
-data:
-  {{- if (lookup "v1" "Secret" .Release.Namespace "mcingress").data }}
-  ClusterSalt: {{ (lookup "v1" "Secret" .Release.Namespace "mcingress").data.ClusterSalt }}
-  {{ else }}
-  ClusterSalt: {{ randAscii 64 | b64enc | b64enc }}
-  {{ end }}
-
-  {{- range $index, $remote := .Values.config.apiKeys }}
-  Authentication__ApiKeys__{{ $index}}__Key: {{ $remote.Key | b64enc }}
-  {{- end }}
-
-  {{- range $index, $peer := .Values.config.peers }}
-  Peers__{{ $index }}__Key: {{ $peer.Key | b64enc }}
-  {{- end }}

charts/multicluster-ingress/templates/dns-service.yaml

@@ -8,6 +8,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
 spec:
+  {{- if ne .Values.dnsServer.service.loadBalancerIP "" }}
+  loadBalancerIP: {{ .Values.dnsServer.service.loadBalancerIP }}
+  {{- end }}
   type: {{ .Values.dnsServer.service.type }}
   ports:
   - port: 53

charts/multicluster-ingress/templates/secret.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.config.createSecret }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mcingress
+type: Opaque
+data:
+  {{- if (lookup "v1" "Secret" .Release.Namespace "mcingress").data }}
+  ClusterSalt: {{ (lookup "v1" "Secret" .Release.Namespace "mcingress").data.ClusterSalt }}
+  {{ else }}
+  ClusterSalt: {{ randAscii 64 | b64enc | b64enc }}
+  {{ end }}
+
+  {{- range $index, $remote := .Values.config.apiKeys }}
+  Authentication__ApiKeys__{{ $index}}__Key: {{ $remote.Key | b64enc }}
+  {{- end }}
+
+  {{- range $index, $peer := .Values.config.peers }}
+  Peers__{{ $index }}__Key: {{ $peer.Key | b64enc }}
+  {{- end }}
+{{- end }}

charts/multicluster-ingress/values.yaml

@@ -60,7 +60,10 @@ dnsServer:
     name: dns-server
     annotations: {}
   service:
+    enabled: true
     type: LoadBalancer
+    # Set the loadBalancerIP to the IP you want for your DNS server, leave empty to make it automatically set by the cluster.
+    loadBalancerIP: ''
   podSecurityContext: {}
     # fsGroup: 2000
   securityContext:
@@ -70,12 +73,12 @@ dnsServer:
     readOnlyRootFilesystem: true
   resources:
     limits:
-      # 300m is recommended minimum for startup, while running, 100m should be plenty
-      cpu: 300m
-      memory: 128Mi
+      # 500m is recommended minimum for startup, while running, 100m should be plenty
+      cpu: 500m
+      memory: 256Mi
     requests:
       cpu: 100m
-      memory: 64Mi
+      memory: 128Mi
   volumeMounts: []
   nodeSelector: {}
   affinity: {}
@@ -144,6 +147,17 @@ orchestrator:
     component: orchestrator
 
 config:
+  ################################################################################################
+  ### Secret configuration for the cluster
+  # If you want to create your secret yourself, for example, via a SealedSecret
+  # set this to false.
+  # The secret will need the following values
+  # * ClusterSalt - 64 bytes of random data not base64 encoded
+  # * Authentication__ApiKeys__##__Key - The hash of the key that remote peers use to talk to this instance
+  #                                      of the multicluster ingress controller
+  # * Peers__{{ $index }}__Key - The keys to talk to the remote peers
+  createSecret: true
+
   ################################################################################################
   ### Local cluster configuration
   # Name of the local cluster cache object
@@ -199,18 +213,20 @@ config:
   # it includes a base 64 encoded hashe of a key and a identifier used by the local cluster
   # that can communicate with this cluster
   apiKeys: []
-
-  # W26VCNAgvtUa5B38iNmFLPUa3hoUYY3L152qJPyJOeQ7UVu0G+XSDL1+lq6zNqnJ8HgVpgJoQulb9KRX690UGQ==
+  # If you are creating the secret yourself, don't add the `Key` field.
   # - Identifier: remote-1
   #   Key: W26VCNAgvtUa5B38iNmFLPUa3hoUYY3L152qJPyJOeQ7UVu0G+XSDL1+lq6zNqnJ8HgVpgJoQulb9KRX690UGQ==
 
-  # this is the list of peers that we will communicate with and send our updates to
+  # This is the list of peers that we will communicate with and send our updates to
   # it includes the api key, identifier and the url to send the updates to
+  # If you are creating the secret yourself, don't add the `Key` field.
   peers: []
   # - Identifier: remote-1
   #   Key: abcd
   #   Url: http://remote-1-api-server.domain.tld
 
+  # How frequently to refresh the cluster to maintain consistency if an event is not caught in the cluster
+  periodicRefreshInterval: 300
 rbac:
   enabled: true
 

src/Vecc.K8s.MultiCluster.Api/Program.cs

@@ -164,7 +164,9 @@
     {
         logger.LogInformation("Starting the operator leader watcher");
         var leaderStateChanged = app.Services.GetRequiredService<OperatorLeader>();
-        while (true)
+        var lifecycle = app.Lifetime;
+
+        while (!lifecycle.ApplicationStopping.IsCancellationRequested)
         {
             await Task.Yield();
             await Task.Delay(1000);
@@ -180,6 +182,44 @@
         });
     }));
 
+    if (options.PerioidRefreshInterval <= 0)
+    {
+        logger.LogInformation("Perioid refresh interval is {interval} which is <= 0, disabling periodic refresher.", options.PerioidRefreshInterval);
+    }
+    else
+    {
+        processTasks.Add(Task.Run(async () =>
+        {
+            logger.LogInformation("Starting the periodic refresher");
+            var lifecycle = app.Lifetime;
+            var leaderStatus = app.Services.GetRequiredService<LeaderStatus>();
+
+            while (!lifecycle.ApplicationStopping.IsCancellationRequested)
+            {
+                await Task.Yield();
+                await Task.Delay(options.PerioidRefreshInterval * 1000);
+                using var scope = logger.BeginScope(new { PeriodicRefreshId = Guid.NewGuid() });
+
+                if (leaderStatus.IsLeader)
+                {
+                    logger.LogInformation("Initiating periodic refresh");
+                    try
+                    {
+                        await hostnameSynchronizer.SynchronizeLocalClusterAsync();
+                    }
+                    catch (Exception ex)
+                    {
+                        logger.LogError(ex, "Error during periodic refresh");
+                    }
+                }
+                else
+                {
+                    logger.LogTrace("Not the leader, skipping periodic refresh");
+                }
+            };
+        }).ContinueWith(_ => logger.LogInformation("Periodic refresher stopped")));
+    }
+
     processTasks.Add(Task.Run(() =>
     {
         logger.LogInformation("Running API Server for health checks");
@@ -192,10 +232,9 @@
 {
     logger.LogInformation("Running the orchestrator");
 
-    var hostnameSynchronizer = app.Services.GetRequiredService<IHostnameSynchronizer>();
-
     processTasks.Add(Task.Run(() =>
     {
+        var hostnameSynchronizer = app.Services.GetRequiredService<IHostnameSynchronizer>();
         logger.LogInformation("Starting cluster heartbeat watcher");
         return hostnameSynchronizer.WatchClusterHeartbeatsAsync().ContinueWith(_ => logger.LogInformation("Cluster heartbeat watcher stopped"));
     }));
@@ -211,6 +250,46 @@
         }
     }).ContinueWith(_ => logger.LogInformation("Orchestrator leader watcher stopped")));
 
+
+    if (options.PerioidRefreshInterval <= 0)
+    {
+        logger.LogInformation("Perioid refresh interval is {interval} which is <= 0, disabling periodic refresher.", options.PerioidRefreshInterval);
+    }
+    else
+    {
+        processTasks.Add(Task.Run(async () =>
+        {
+            logger.LogInformation("Starting the periodic refresher");
+            var lifecycle = app.Lifetime;
+            var leaderStatus = app.Services.GetRequiredService<LeaderStatus>();
+            var cache = app.Services.GetRequiredService<ICache>();
+
+            while (!lifecycle.ApplicationStopping.IsCancellationRequested)
+            {
+                await Task.Yield();
+                await Task.Delay(options.PerioidRefreshInterval * 1000);
+                using var scope = logger.BeginScope(new { PeriodicRefreshId = Guid.NewGuid() });
+
+                if (leaderStatus.IsLeader)
+                {
+                    logger.LogInformation("Initiating periodic refresh");
+                    try
+                    {
+                        await cache.SynchronizeCachesAsync();
+                    }
+                    catch (Exception ex)
+                    {
+                        logger.LogError(ex, "Error during periodic refresh");
+                    }
+                }
+                else
+                {
+                    logger.LogTrace("Not the leader, skipping periodic refresh");
+                }
+            };
+        }).ContinueWith(_ => logger.LogInformation("Periodic refresher stopped")));
+    }
+
     processTasks.Add(Task.Run(() =>
     {
         logger.LogInformation("Running API Server for health checks");

src/Vecc.K8s.MultiCluster.Api/Properties/launchSettings.json

@@ -1,16 +1,5 @@
 {
   "profiles": {
-    "http": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      },
-      "dotnetRunMessages": true,
-      "applicationUrl": "http://localhost:5141",
-      "commandLineArgs": "--dns-server"
-    },
     "IIS Express": {
       "commandName": "IISExpress",
       "launchBrowser": true,
@@ -24,6 +13,26 @@
       "launchBrowser": true,
       "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/swagger",
       "publishAllPorts": true
+    },
+    "Orchestrator": {
+      "commandName": "Project",
+      "commandLineArgs": "--orchestrator",
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:5141"
+    },
+    "Dns-Server": {
+      "commandName": "Project",
+      "commandLineArgs": "--dns-server",
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:5141"
     }
   },
   "$schema": "https://json.schemastore.org/launchsettings.json",

src/Vecc.K8s.MultiCluster.Api/Services/Default/DefaultDnsHost.cs

@@ -15,10 +15,30 @@ public DefaultDnsHost(ILogger<DefaultDnsHost> logger, IDnsServer dnsServer, IHos
             _shutdownCancellationToken = lifetime.ApplicationStopping;
         }
 
-        public Task StartAsync()
+        public async Task StartAsync()
         {
-            _logger.LogInformation("Starting dns server");
-            return _dnsServer.ExecuteAsync(_shutdownCancellationToken);
+            while (!_shutdownCancellationToken.IsCancellationRequested)
+            {
+                _logger.LogInformation("Starting dns server");
+
+                try
+                {
+                    await _dnsServer.ExecuteAsync(_shutdownCancellationToken);
+
+                    if (!_shutdownCancellationToken.IsCancellationRequested)
+                    {
+                        _logger.LogError("Unexpected shutdown of the dns server, restarting.");
+                    }
+                }
+                catch (Exception ex)
+                {
+                    if (!_shutdownCancellationToken.IsCancellationRequested)
+                    {
+                        _logger.LogError(ex, "Crash in the dns server, restarting.");
+                    }
+                }
+            }
+            _logger.LogInformation("Shutdown of dns server complete");
         }
     }
 }

src/Vecc.K8s.MultiCluster.Api/Services/Default/DefaultHostnameSynchronizer.cs

@@ -61,8 +61,11 @@ public DefaultHostnameSynchronizer(
         public async Task SynchronizeLocalClusterAsync()
         {
             _synchronizeLocalClusterHolder.WaitOne();
+            using var scope = _logger.BeginScope(new { SyncId = Guid.NewGuid() });
             try
             {
+                _logger.LogInformation("Waiting a second for the cluster to settle.");
+                await Task.Delay(1000); //wait one second to allow resources to fully complete.
                 _logger.LogInformation("Synchronizing local cluster");
                 var ipAddresses = new Dictionary<string, List<HostIP>>();
                 var localClusterIdentifier = _multiClusterOptions.Value.ClusterIdentifier;