From fef1cbc960656ff538af30e6bcff304e38405573 Mon Sep 17 00:00:00 2001 From: Lionel Jouin Date: Tue, 7 Nov 2023 13:19:03 +0100 Subject: [PATCH] Gateway routes imported/exported/preferred metrics Collect metrics of routes imported, exported and preferred for each gateways. The metrics are extracted and parsed from show protocol all command in bird. --- cmd/frontend/internal/frontend/metrics.go | 260 +++++++++ .../internal/frontend/metrics_test.go | 499 ++++++++++++++++++ cmd/frontend/internal/frontend/service.go | 11 +- cmd/frontend/main.go | 15 +- docs/observability/dashboard.json | 219 +++++++- docs/observability/metrics.md | 11 +- pkg/metrics/const.go | 3 + 7 files changed, 1011 insertions(+), 7 deletions(-) create mode 100644 cmd/frontend/internal/frontend/metrics.go create mode 100644 cmd/frontend/internal/frontend/metrics_test.go diff --git a/cmd/frontend/internal/frontend/metrics.go b/cmd/frontend/internal/frontend/metrics.go new file mode 100644 index 00000000..eeb54b2a --- /dev/null +++ b/cmd/frontend/internal/frontend/metrics.go @@ -0,0 +1,260 @@ +/* +Copyright (c) 2023 Nordix Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package frontend + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "sync" + + nspAPI "github.com/nordix/meridio/api/nsp/v1" + "github.com/nordix/meridio/cmd/frontend/internal/bird" + meridioMetrics "github.com/nordix/meridio/pkg/metrics" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +type GatewayMetrics struct { + meter metric.Meter + gateways map[string]*nspAPI.Gateway + metricAttributes []metric.ObserveOption + RoutingService *bird.RoutingService + mu sync.Mutex +} + +func NewGatewayMetrics(metricAttributes []metric.ObserveOption) *GatewayMetrics { + meter := otel.GetMeterProvider().Meter(meridioMetrics.METER_NAME) + gm := &GatewayMetrics{ + meter: meter, + metricAttributes: metricAttributes, + } + + return gm +} + +func (gm *GatewayMetrics) Set(gateways []*nspAPI.Gateway) { + gatewaysMap := map[string]*nspAPI.Gateway{} + + for _, gateway := range gateways { + gatewaysMap[fmt.Sprintf("NBR-%s", gateway.Name)] = gateway + } + + gm.mu.Lock() + defer gm.mu.Unlock() + gm.gateways = gatewaysMap +} + +// Collect collects the metrics for the gateways. +func (gm *GatewayMetrics) Collect() error { + if gm.RoutingService == nil { + return errors.New("routing service not set for gateway metrics") + } + + lp, err := gm.RoutingService.LookupCli() + if err != nil { + return fmt.Errorf("frontend metrics, failed to RoutingService.LookupCli: %w", err) + } + + _, err = gm.meter.Int64ObservableCounter( + meridioMetrics.MERIDIO_ATTRACTOR_GATEWAY_ROUTES_IMPORTED, + metric.WithUnit("routes"), + metric.WithDescription("Counts number of routes imported for a gateway."), + metric.WithInt64Callback(func(ctx context.Context, observer metric.Int64Observer) error { + return gm.observe( + ctx, + observer, + func(birdStats *BirdStats) int64 { + return int64(birdStats.routesImported) + }, + lp, + ) + }), + ) + if err != nil { + return fmt.Errorf("frontend metrics, failed to Int64ObservableCounter (%s): %w", meridioMetrics.MERIDIO_ATTRACTOR_GATEWAY_ROUTES_IMPORTED, err) + } + + _, err = gm.meter.Int64ObservableCounter( + meridioMetrics.MERIDIO_ATTRACTOR_GATEWAY_ROUTES_EXPORTED, + metric.WithUnit("routes"), + metric.WithDescription("Counts number of routes exported for a gateway."), + metric.WithInt64Callback(func(ctx context.Context, observer metric.Int64Observer) error { + return gm.observe( + ctx, + observer, + func(birdStats *BirdStats) int64 { + return int64(birdStats.routesExported) + }, + lp, + ) + }), + ) + if err != nil { + return fmt.Errorf("frontend metrics, failed to Int64ObservableCounter (%s): %w", meridioMetrics.MERIDIO_ATTRACTOR_GATEWAY_ROUTES_EXPORTED, err) + } + + _, err = gm.meter.Int64ObservableCounter( + meridioMetrics.MERIDIO_ATTRACTOR_GATEWAY_ROUTES_PREFERRED, + metric.WithUnit("routes"), + metric.WithDescription("Counts number of routes preferred for a gateway."), + metric.WithInt64Callback(func(ctx context.Context, observer metric.Int64Observer) error { + return gm.observe( + ctx, + observer, + func(birdStats *BirdStats) int64 { + return int64(birdStats.routesPreferred) + }, + lp, + ) + }), + ) + if err != nil { + return fmt.Errorf("frontend metrics, failed to Int64ObservableCounter (%s): %w", meridioMetrics.MERIDIO_ATTRACTOR_GATEWAY_ROUTES_PREFERRED, err) + } + return nil +} + +func (gm *GatewayMetrics) observe(ctx context.Context, observer metric.Int64Observer, valueFunc func(*BirdStats) int64, lp string) error { + gm.mu.Lock() + defer gm.mu.Unlock() + + spaOutput, err := gm.RoutingService.ShowProtocolSessions(ctx, lp, "NBR-*") + if err != nil { + return fmt.Errorf("frontend metrics, failed to RoutingService.ShowProtocolSessions: %w", err) + } + + birdStats := ParseShowProtocolsAll(spaOutput) + + for _, birdStat := range birdStats { + gateway, exists := gm.gateways[birdStat.gatewayName] + if !exists { + continue + } + + metricAttributes := []metric.ObserveOption{ + metric.WithAttributes(attribute.String("Protocol", getProtocolName(gateway))), + metric.WithAttributes(attribute.String("Gateway", gateway.Name)), + metric.WithAttributes(attribute.String("IP", gateway.Address)), + } + metricAttributes = append(metricAttributes, gm.metricAttributes...) + + observer.Observe( + valueFunc(birdStat), + metricAttributes..., + ) + } + + return nil +} + +func getProtocolName(gateway *nspAPI.Gateway) string { + res := "unknown" + switch strings.ToLower(gateway.Protocol) { + case "bgp": + res = "BGP" + case "static": + res = "Static" + } + if gateway.Bfd { + res = fmt.Sprintf("%s+BFD", res) + } + return res +} + +type BirdStats struct { + gatewayName string + routesImported int + routesExported int + routesPreferred int +} + +func (bs *BirdStats) String() string { + return fmt.Sprintf("%s (%d, %d, %d)", bs.gatewayName, bs.routesImported, bs.routesExported, bs.routesPreferred) +} + +var regex = regexp.MustCompile(`(?m) Routes:.*`) + +func ParseShowProtocolsAll(output string) []*BirdStats { + res := []*BirdStats{} + + protocols := strings.Split(output, "\n\n") + + for _, protocol := range protocols { + protocolName, _, found := strings.Cut(protocol, " ") + + if !found { + continue + } + + newStat := &BirdStats{ + gatewayName: protocolName, + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + } + + routes := regex.FindAllString(protocol, -1) + + if len(routes) <= 0 { + res = append(res, newStat) + continue + } + + routesStats := strings.ReplaceAll(routes[0], "Routes:", "") + routesStats = strings.ReplaceAll(routesStats, "imported", "") + routesStats = strings.ReplaceAll(routesStats, "exported", "") + routesStats = strings.ReplaceAll(routesStats, "preferred", "") + routesStats = strings.ReplaceAll(routesStats, " ", "") + routesStatsSlice := strings.Split(routesStats, ",") + + if len(routesStatsSlice) != 3 { + res = append(res, newStat) + continue + } + + routesImported, err := strconv.Atoi(routesStatsSlice[0]) + if err != nil { + res = append(res, newStat) + continue + } + + routesExported, err := strconv.Atoi(routesStatsSlice[1]) + if err != nil { + res = append(res, newStat) + continue + } + + routesPreferred, err := strconv.Atoi(routesStatsSlice[2]) + if err != nil { + res = append(res, newStat) + continue + } + + newStat.routesImported = routesImported + newStat.routesExported = routesExported + newStat.routesPreferred = routesPreferred + + res = append(res, newStat) + } + + return res +} diff --git a/cmd/frontend/internal/frontend/metrics_test.go b/cmd/frontend/internal/frontend/metrics_test.go new file mode 100644 index 00000000..d777290e --- /dev/null +++ b/cmd/frontend/internal/frontend/metrics_test.go @@ -0,0 +1,499 @@ +/* +Copyright (c) 2023 Nordix Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package frontend + +import ( + "reflect" + "testing" +) + +func TestParseShowProtocolsAll(t *testing.T) { + type args struct { + output string + } + tests := []struct { + name string + args args + want []*BirdStats + }{ + { + name: "t1", + args: args{ + output: t1, + }, + want: []*BirdStats{ + { + gatewayName: "BIRD", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + { + gatewayName: "VIP4", + routesImported: 1, + routesExported: 0, + routesPreferred: 1, + }, + { + gatewayName: "VIP6", + routesImported: 1, + routesExported: 0, + routesPreferred: 1, + }, + { + gatewayName: "kernel1", + routesImported: 0, + routesExported: 1, + routesPreferred: 0, + }, + { + gatewayName: "kernel2", + routesImported: 0, + routesExported: 1, + routesPreferred: 0, + }, + { + gatewayName: "DROP4", + routesImported: 1, + routesExported: 0, + routesPreferred: 1, + }, + { + gatewayName: "DROP6", + routesImported: 1, + routesExported: 0, + routesPreferred: 1, + }, + { + gatewayName: "kernel3", + routesImported: 0, + routesExported: 1, + routesPreferred: 0, + }, + { + gatewayName: "kernel4", + routesImported: 0, + routesExported: 1, + routesPreferred: 0, + }, + { + gatewayName: "NBR-gateway-v4-a-1", + routesImported: 1, + routesExported: 1, + routesPreferred: 1, + }, + { + gatewayName: "NBR-gateway-v6-a-1", + routesImported: 1, + routesExported: 1, + routesPreferred: 1, + }, + { + gatewayName: "NBR-BFD", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + }, + }, + { + name: "t2", + args: args{ + output: t2, + }, + want: []*BirdStats{ + { + gatewayName: "BIRD", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + { + gatewayName: "kernel1", + routesImported: 0, + routesExported: 1, + routesPreferred: 0, + }, + { + gatewayName: "kernel2", + routesImported: 0, + routesExported: 1, + routesPreferred: 0, + }, + { + gatewayName: "DROP4", + routesImported: 1, + routesExported: 0, + routesPreferred: 1, + }, + { + gatewayName: "DROP6", + routesImported: 1, + routesExported: 0, + routesPreferred: 1, + }, + { + gatewayName: "kernel3", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + { + gatewayName: "kernel4", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + { + gatewayName: "NBR-gateway-v4-a-1", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + { + gatewayName: "NBR-gateway-v6-a-1", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + { + gatewayName: "NBR-BFD", + routesImported: 0, + routesExported: 0, + routesPreferred: 0, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ParseShowProtocolsAll(tt.args.output); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseShowProtocolsAll() = %v, want %v", got, tt.want) + } + }) + } +} + +var t1 = `BIRD 2.0.10 ready. +Name Proto Table State Since Info +device1 Device --- up 09:19:34.280 + +VIP4 Static master4 up 09:40:08.628 + Channel ipv4 + State: UP + Table: master4 + Preference: 110 + Input filter: ACCEPT + Output filter: REJECT + Routes: 1 imported, 0 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 0 0 0 --- 0 + Export withdraws: 0 --- --- --- 0 + +VIP6 Static master6 up 09:40:08.628 + Channel ipv6 + State: UP + Table: master6 + Preference: 110 + Input filter: ACCEPT + Output filter: REJECT + Routes: 1 imported, 0 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 0 0 0 --- 0 + Export withdraws: 0 --- --- --- 0 + +kernel1 Kernel drop4 up 09:19:43.404 + Channel ipv4 + State: UP + Table: drop4 + Preference: 10 + Input filter: REJECT + Output filter: ACCEPT + Routes: 0 imported, 1 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 1 0 0 --- 1 + Export withdraws: 0 --- --- --- 0 + +kernel2 Kernel drop6 up 09:19:43.404 + Channel ipv6 + State: UP + Table: drop6 + Preference: 10 + Input filter: REJECT + Output filter: ACCEPT + Routes: 0 imported, 1 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 1 0 0 --- 1 + Export withdraws: 0 --- --- --- 0 + +DROP4 Static drop4 up 09:19:43.403 + Channel ipv4 + State: UP + Table: drop4 + Preference: 0 + Input filter: ACCEPT + Output filter: REJECT + Routes: 1 imported, 0 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 0 0 0 --- 0 + Export withdraws: 0 --- --- --- 0 + +DROP6 Static drop6 up 09:19:43.403 + Channel ipv6 + State: UP + Table: drop6 + Preference: 0 + Input filter: ACCEPT + Output filter: REJECT + Routes: 1 imported, 0 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 0 0 0 --- 0 + Export withdraws: 0 --- --- --- 0 + +kernel3 Kernel master4 up 09:19:43.403 + Channel ipv4 + State: UP + Table: master4 + Preference: 10 + Input filter: REJECT + Output filter: cluster_breakout + Routes: 0 imported, 1 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 6 0 3 --- 3 + Export withdraws: 4 --- --- --- 2 + +kernel4 Kernel master6 up 09:19:43.403 + Channel ipv6 + State: UP + Table: master6 + Preference: 10 + Input filter: REJECT + Output filter: cluster_breakout + Routes: 0 imported, 1 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 6 0 3 --- 3 + Export withdraws: 4 --- --- --- 2 + +NBR-gateway-v4-a-1 BGP --- up 09:40:08.169 Established + BGP state: Established + Neighbor address: 169.254.100.150%ext-vlan0 + Neighbor port: 10179 + Neighbor AS: 4248829953 + Local AS: 8103 + Neighbor ID: 169.254.100.150 + Local capabilities + Multiprotocol + AF announced: ipv4 + Route refresh + 4-octet AS numbers + Enhanced refresh + Neighbor capabilities + Multiprotocol + AF announced: ipv4 ipv6 + Route refresh + 4-octet AS numbers + Enhanced refresh + Session: external AS4 + Source address: 169.254.100.1 + Hold timer: 1.724/3 + Keepalive timer: 0.746/1 + Channel ipv4 + State: UP + Table: master4 + Preference: 100 + Input filter: cluster_breakout + Output filter: cluster_access + Routes: 1 imported, 1 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 2 1 0 --- 1 + Export withdraws: 0 --- --- --- 0 + BGP Next hop: 169.254.100.1 + +NBR-gateway-v6-a-1 BGP --- up 09:40:08.002 Established + BGP state: Established + Neighbor address: 100:100::150%ext-vlan0 + Neighbor port: 10179 + Neighbor AS: 4248829953 + Local AS: 8103 + Neighbor ID: 169.254.100.150 + Local capabilities + Multiprotocol + AF announced: ipv6 + Route refresh + 4-octet AS numbers + Enhanced refresh + Neighbor capabilities + Multiprotocol + AF announced: ipv4 ipv6 + Route refresh + 4-octet AS numbers + Enhanced refresh + Session: external AS4 + Source address: 100:100::1 + Hold timer: 2.506/3 + Keepalive timer: 0.233/1 + Channel ipv6 + State: UP + Table: master6 + Preference: 100 + Input filter: cluster_breakout + Output filter: cluster_access + Routes: 1 imported, 1 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 2 1 0 --- 1 + Export withdraws: 0 --- --- --- 0 + BGP Next hop: 100:100::1 fe80::fe:34ff:fe07:4bc6 + +NBR-BFD BFD --- up 09:19:34.280` + +var t2 = `BIRD 2.0.10 ready. +Name Proto Table State Since Info +device1 Device --- up 09:19:34.280 + +kernel1 Kernel drop4 up 09:19:43.404 + Channel ipv4 + State: UP + Table: drop4 + Preference: 10 + Input filter: REJECT + Output filter: ACCEPT + Routes: 0 imported, 1 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 1 0 0 --- 1 + Export withdraws: 0 --- --- --- 0 + +kernel2 Kernel drop6 up 09:19:43.404 + Channel ipv6 + State: UP + Table: drop6 + Preference: 10 + Input filter: REJECT + Output filter: ACCEPT + Routes: 0 imported, 1 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 1 0 0 --- 1 + Export withdraws: 0 --- --- --- 0 + +DROP4 Static drop4 up 09:19:43.403 + Channel ipv4 + State: UP + Table: drop4 + Preference: 0 + Input filter: ACCEPT + Output filter: REJECT + Routes: 1 imported, 0 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 0 0 0 --- 0 + Export withdraws: 0 --- --- --- 0 + +DROP6 Static drop6 up 09:19:43.403 + Channel ipv6 + State: UP + Table: drop6 + Preference: 0 + Input filter: ACCEPT + Output filter: REJECT + Routes: 1 imported, 0 exported, 1 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 1 0 0 0 1 + Import withdraws: 0 0 --- 0 0 + Export updates: 0 0 0 --- 0 + Export withdraws: 0 --- --- --- 0 + +kernel3 Kernel master4 up 09:19:43.403 + Channel ipv4 + State: UP + Table: master4 + Preference: 10 + Input filter: REJECT + Output filter: cluster_breakout + Routes: 0 imported, 0 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 6 0 3 --- 3 + Export withdraws: 6 --- --- --- 3 + +kernel4 Kernel master6 up 09:19:43.403 + Channel ipv6 + State: UP + Table: master6 + Preference: 10 + Input filter: REJECT + Output filter: cluster_breakout + Routes: 0 imported, 0 exported, 0 preferred + Route change stats: received rejected filtered ignored accepted + Import updates: 0 0 0 0 0 + Import withdraws: 0 0 --- 0 0 + Export updates: 6 0 3 --- 3 + Export withdraws: 6 --- --- --- 3 + +NBR-gateway-v4-a-1 BGP --- start 10:39:56.122 Active Socket: Connection closed + BGP state: Active + Neighbor address: 169.254.100.150%ext-vlan0 + Neighbor AS: 4248829953 + Local AS: 8103 + Connect delay: 3.509/5 + Last error: Socket: Connection closed + Channel ipv4 + State: DOWN + Table: master4 + Preference: 100 + Input filter: cluster_breakout + Output filter: cluster_access + +NBR-gateway-v6-a-1 BGP --- start 10:39:56.122 Active Socket: Connection closed + BGP state: Active + Neighbor address: 100:100::150%ext-vlan0 + Neighbor AS: 4248829953 + Local AS: 8103 + Connect delay: 3.505/5 + Last error: Socket: Connection closed + Channel ipv6 + State: DOWN + Table: master6 + Preference: 100 + Input filter: cluster_breakout + Output filter: cluster_access + +NBR-BFD BFD --- up 09:19:34.280` diff --git a/cmd/frontend/internal/frontend/service.go b/cmd/frontend/internal/frontend/service.go index 6e18b9d5..b6f10162 100644 --- a/cmd/frontend/internal/frontend/service.go +++ b/cmd/frontend/internal/frontend/service.go @@ -45,7 +45,7 @@ import ( const rulePriorityVIP int = 100 // FrontEndService - -func NewFrontEndService(ctx context.Context, c *feConfig.Config) *FrontEndService { +func NewFrontEndService(ctx context.Context, c *feConfig.Config, gatewayMetrics *GatewayMetrics) *FrontEndService { logger := log.FromContextOrGlobal(ctx).WithValues("class", "FrontEndService") targetRegistryClient := nspAPI.NewTargetRegistryClient(c.NSPConn) @@ -87,6 +87,7 @@ func NewFrontEndService(ctx context.Context, c *feConfig.Config) *FrontEndServic authCh: authCh, logger: logger, config: c, + gatewayMetrics: gatewayMetrics, } if len(frontEndService.vrrps) > 0 { @@ -94,6 +95,8 @@ func NewFrontEndService(ctx context.Context, c *feConfig.Config) *FrontEndServic frontEndService.dropIfNoPeer = false } + gatewayMetrics.RoutingService = frontEndService.routingService + logger.Info("Created", "object", frontEndService) return frontEndService } @@ -133,6 +136,11 @@ type FrontEndService struct { authCh chan struct{} // used by secretDatabase to signal updates to FE Service logger logr.Logger config *feConfig.Config + gatewayMetrics *GatewayMetrics +} + +func (fes *FrontEndService) GetRoutingService() *bird.RoutingService { + return fes.routingService } // CleanUp - @@ -232,6 +240,7 @@ func (fes *FrontEndService) SetNewConfig(ctx context.Context, c interface{}) err logger.V(2).Info("Gateway configuration changed") fes.checkAuthentication(ctx) } + fes.gatewayMetrics.Set(c.Gateways) configChange = configChange || gwConfigChange } default: diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go index 832fc5dc..cc24bf0b 100644 --- a/cmd/frontend/main.go +++ b/cmd/frontend/main.go @@ -131,12 +131,18 @@ func main() { logger.Error(err, "NSP connection state monitor") } + gatewayMetrics := frontend.NewGatewayMetrics([]metric.ObserveOption{ + metric.WithAttributes(attribute.String("Hostname", hostname)), + metric.WithAttributes(attribute.String("Trench", config.TrenchName)), + metric.WithAttributes(attribute.String("Attractor", config.AttractorName)), + }) + // create and start frontend service c := &feConfig.Config{ Config: config, NSPConn: conn, } - fe := frontend.NewFrontEndService(ctx, c) + fe := frontend.NewFrontEndService(ctx, c, gatewayMetrics) defer fe.CleanUp() health.SetServingStatus(ctx, health.TargetRegistryCliSvc, true) // NewFrontEndService() creates Target Registry Client @@ -187,6 +193,13 @@ func main() { return } + err = gatewayMetrics.Collect() + if err != nil { + logger.Error(err, "Unable to start gateway metrics collector") + cancel() + return + } + metricsServer := metrics.Server{ IP: "", Port: config.MetricsPort, diff --git a/docs/observability/dashboard.json b/docs/observability/dashboard.json index 08875f47..0f34aa5d 100644 --- a/docs/observability/dashboard.json +++ b/docs/observability/dashboard.json @@ -1225,6 +1225,223 @@ "title": "Interface Metrics", "transformations": [], "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "meridio_attracted_gateway_routes_exported_total", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Exported", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "meridio_attracted_gateway_routes_imported_total", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Imported", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "meridio_attracted_gateway_routes_preferred_total", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Preferred", + "useBackend": false + } + ], + "title": "Gateways", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "container": true, + "endpoint": true, + "instance": true, + "job": true, + "namespace": true, + "otel_scope_name": true, + "pod": true + }, + "indexByName": { + "Attractor": 1, + "Gateway": 2, + "Hostname": 3, + "IP": 4, + "Protocol": 5, + "Time": 7, + "Trench": 0, + "Value": 6, + "__name__": 8, + "container": 9, + "endpoint": 10, + "instance": 11, + "job": 12, + "namespace": 13, + "otel_scope_name": 14, + "pod": 15 + }, + "renameByName": {} + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Attractor": { + "aggregations": [], + "operation": "groupby" + }, + "Gateway": { + "aggregations": [], + "operation": "groupby" + }, + "Hostname": { + "aggregations": [], + "operation": "groupby" + }, + "IP": { + "aggregations": [], + "operation": "groupby" + }, + "Protocol": { + "aggregations": [], + "operation": "groupby" + }, + "Trench": { + "aggregations": [], + "operation": "groupby" + }, + "Value #Exported": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #Imported": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #Preferred": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #Exported (lastNotNull)": "Exported", + "Value #Imported (lastNotNull)": "Imported", + "Value #Preferred (lastNotNull)": "Preferred" + } + } + } + ], + "type": "table" } ], "refresh": "5s", @@ -1254,6 +1471,6 @@ "timezone": "", "title": "Meridio", "uid": "f0339d9f-4744-441c-972b-f8b294fb7ff8", - "version": 3, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/docs/observability/metrics.md b/docs/observability/metrics.md index 9e1d8b43..0c2c9c31 100644 --- a/docs/observability/metrics.md +++ b/docs/observability/metrics.md @@ -54,14 +54,17 @@ Reports the latency with a target. * Conduit * IP -### meridio.attractor.gateway.status (Planned) +### meridio.attracted.gateway.routes.`METRIC_TYPE` -Gateway status in the attractor instance. +`METRIC_TYPE`: imported, exported, preferred -* Type: Gauge (Health Metric) +Number of `METRIC_TYPE` routes for a gateway in the attractor instance. + +* Type: Counter * Attributes: * Pod Name * Trench - * Conduit * Attactor * Gateway + * IP + * Protocol diff --git a/pkg/metrics/const.go b/pkg/metrics/const.go index 2d4fadf3..7bebe0c1 100644 --- a/pkg/metrics/const.go +++ b/pkg/metrics/const.go @@ -28,6 +28,9 @@ const ( MERIDIO_INTERFACE_TX_ERRORS = "meridio.interface.tx_errors" MERIDIO_INTERFACE_RX_DROPPED = "meridio.interface.rx_dropped" MERIDIO_INTERFACE_TX_DROPPED = "meridio.interface.tx_dropped" + MERIDIO_ATTRACTOR_GATEWAY_ROUTES_IMPORTED = "meridio.attracted.gateway.routes.imported" + MERIDIO_ATTRACTOR_GATEWAY_ROUTES_EXPORTED = "meridio.attracted.gateway.routes.exported" + MERIDIO_ATTRACTOR_GATEWAY_ROUTES_PREFERRED = "meridio.attracted.gateway.routes.preferred" METER_NAME = "Meridio" )