diff --git a/examples/custom_strategy.py b/examples/custom_strategy.py index df6c22a3..c3623ef6 100644 --- a/examples/custom_strategy.py +++ b/examples/custom_strategy.py @@ -3,7 +3,7 @@ import pydantic as pd import robusta_krr -from robusta_krr.api.models import MetricsPodData, K8sObjectData, ResourceRecommendation, ResourceType, RunResult +from robusta_krr.api.models import K8sObjectData, MetricsPodData, ResourceRecommendation, ResourceType, RunResult from robusta_krr.api.strategies import BaseStrategy, StrategySettings from robusta_krr.core.integrations.prometheus.metrics import MaxCPULoader, MaxMemoryLoader diff --git a/pyproject.toml b/pyproject.toml index 1f3e075b..b6ae903f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [tool.poetry] name = "robusta-krr" -version = "1.4.1-dev" +version = "1.5.0-dev" description = "Robusta's Resource Recommendation engine for Kubernetes" authors = ["Pavel Zhukov <33721692+LeaveMyYard@users.noreply.github.com>"] license = "MIT" readme = "README.md" -packages = [{include = "robusta_krr"}] +packages = [{ include = "robusta_krr" }] [tool.black] line-length = 120 @@ -24,7 +24,7 @@ krr = "robusta_krr.main:run" [tool.poetry.dependencies] python = ">=3.9,<3.12" -typer = {extras = ["all"], version = "^0.7.0"} +typer = { extras = ["all"], version = "^0.7.0" } pydantic = "1.10.7" kubernetes = "^26.1.0" prometheus-api-client = "^0.5.3" @@ -52,4 +52,4 @@ build-backend = "poetry.core.masonry.api" [project] -name = "robusta_krr" \ No newline at end of file +name = "robusta_krr" diff --git a/robusta_krr/__init__.py b/robusta_krr/__init__.py index 24a4deb3..d063250e 100644 --- a/robusta_krr/__init__.py +++ b/robusta_krr/__init__.py @@ -1,4 +1,4 @@ from .main import run -__version__ = "1.4.1-dev" +__version__ = "1.5.0-dev" __all__ = ["run", "__version__"] diff --git a/robusta_krr/api/models.py b/robusta_krr/api/models.py index c6eb8f5f..3a453ce9 100644 --- a/robusta_krr/api/models.py +++ b/robusta_krr/api/models.py @@ -1,9 +1,4 @@ -from robusta_krr.core.abstract.strategies import ( - PodsTimeData, - MetricsPodData, - ResourceRecommendation, - RunResult, -) +from robusta_krr.core.abstract.strategies import MetricsPodData, PodsTimeData, ResourceRecommendation, RunResult from robusta_krr.core.models.allocations import RecommendationValue, ResourceAllocations, ResourceType from robusta_krr.core.models.objects import K8sObjectData, PodData from robusta_krr.core.models.result import ResourceScan, Result diff --git a/robusta_krr/core/abstract/metrics.py b/robusta_krr/core/abstract/metrics.py index 1dbeda71..3b6f19c5 100644 --- a/robusta_krr/core/abstract/metrics.py +++ b/robusta_krr/core/abstract/metrics.py @@ -1,5 +1,5 @@ -from abc import ABC, abstractmethod import datetime +from abc import ABC, abstractmethod from robusta_krr.core.abstract.strategies import PodsTimeData from robusta_krr.core.models.objects import K8sObjectData diff --git a/robusta_krr/core/abstract/strategies.py b/robusta_krr/core/abstract/strategies.py index d91f8148..9e6b2692 100644 --- a/robusta_krr/core/abstract/strategies.py +++ b/robusta_krr/core/abstract/strategies.py @@ -3,7 +3,7 @@ import abc import datetime from textwrap import dedent -from typing import Annotated, Generic, Literal, Optional, TypeVar, get_args, TYPE_CHECKING, Sequence +from typing import TYPE_CHECKING, Annotated, Generic, Literal, Optional, Sequence, TypeVar, get_args import numpy as np import pydantic as pd @@ -12,8 +12,8 @@ from robusta_krr.core.models.result import K8sObjectData, ResourceType if TYPE_CHECKING: - from robusta_krr.core.integrations.prometheus.metrics import PrometheusMetric from robusta_krr.core.abstract.metrics import BaseMetric # noqa: F401 + from robusta_krr.core.integrations.prometheus.metrics import PrometheusMetric SelfRR = TypeVar("SelfRR", bound="ResourceRecommendation") diff --git a/robusta_krr/core/integrations/kubernetes.py b/robusta_krr/core/integrations/kubernetes.py index 8ee8afab..915f79b8 100644 --- a/robusta_krr/core/integrations/kubernetes.py +++ b/robusta_krr/core/integrations/kubernetes.py @@ -1,19 +1,19 @@ import asyncio from concurrent.futures import ThreadPoolExecutor -from typing import AsyncGenerator, Optional, Union, Callable, AsyncIterator -import aiostream +from typing import AsyncGenerator, AsyncIterator, Callable, Optional, Union +import aiostream from kubernetes import client, config # type: ignore from kubernetes.client import ApiException from kubernetes.client.models import ( V1Container, V1DaemonSet, V1Deployment, + V1HorizontalPodAutoscalerList, V1Job, V1LabelSelector, V1Pod, V1StatefulSet, - V1HorizontalPodAutoscalerList, V2HorizontalPodAutoscaler, V2HorizontalPodAutoscalerList, ) @@ -22,7 +22,6 @@ from robusta_krr.core.models.result import ResourceAllocations from robusta_krr.utils.configurable import Configurable - from .rollout import RolloutAppsV1Api AnyKubernetesAPIObject = Union[V1Deployment, V1DaemonSet, V1StatefulSet, V1Pod, V1Job] diff --git a/robusta_krr/core/integrations/prometheus/loader.py b/robusta_krr/core/integrations/prometheus/loader.py index 70d839ad..1829ef1a 100644 --- a/robusta_krr/core/integrations/prometheus/loader.py +++ b/robusta_krr/core/integrations/prometheus/loader.py @@ -11,7 +11,7 @@ from robusta_krr.core.models.objects import K8sObjectData from robusta_krr.utils.configurable import Configurable -from .metrics_service.prometheus_metrics_service import PrometheusMetricsService, PrometheusNotFound +from .metrics_service.prometheus_metrics_service import PrometheusMetricsService from .metrics_service.thanos_metrics_service import ThanosMetricsService from .metrics_service.victoria_metrics_service import VictoriaMetricsService diff --git a/robusta_krr/core/integrations/prometheus/metrics/base.py b/robusta_krr/core/integrations/prometheus/metrics/base.py index b6be241b..177d9083 100644 --- a/robusta_krr/core/integrations/prometheus/metrics/base.py +++ b/robusta_krr/core/integrations/prometheus/metrics/base.py @@ -2,12 +2,12 @@ import abc import asyncio +import copy import datetime import enum -import copy from concurrent.futures import ThreadPoolExecutor -from typing import Any, TYPE_CHECKING, Optional from functools import reduce +from typing import TYPE_CHECKING, Any, Optional import numpy as np import pydantic as pd diff --git a/robusta_krr/core/integrations/prometheus/metrics/cpu.py b/robusta_krr/core/integrations/prometheus/metrics/cpu.py index 31cb38fa..302e09dc 100644 --- a/robusta_krr/core/integrations/prometheus/metrics/cpu.py +++ b/robusta_krr/core/integrations/prometheus/metrics/cpu.py @@ -1,7 +1,6 @@ from robusta_krr.core.models.objects import K8sObjectData -from robusta_krr.core.abstract.strategies import PodsTimeData -from .base import QueryMetric, QueryRangeMetric, FilterJobsMixin, BatchedRequestMixin +from .base import BatchedRequestMixin, FilterJobsMixin, QueryMetric, QueryRangeMetric class CPULoader(QueryRangeMetric, FilterJobsMixin, BatchedRequestMixin): diff --git a/robusta_krr/core/integrations/prometheus/metrics/memory.py b/robusta_krr/core/integrations/prometheus/metrics/memory.py index c5685a93..ad84abe8 100644 --- a/robusta_krr/core/integrations/prometheus/metrics/memory.py +++ b/robusta_krr/core/integrations/prometheus/metrics/memory.py @@ -1,8 +1,6 @@ -from robusta_krr.core.abstract.strategies import PodsTimeData - from robusta_krr.core.models.objects import K8sObjectData -from .base import QueryMetric, QueryRangeMetric, FilterJobsMixin, BatchedRequestMixin +from .base import BatchedRequestMixin, FilterJobsMixin, QueryMetric, QueryRangeMetric class MemoryLoader(QueryRangeMetric, FilterJobsMixin, BatchedRequestMixin): diff --git a/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py b/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py index 57bd3c41..728dc520 100644 --- a/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py +++ b/robusta_krr/core/integrations/prometheus/metrics_service/prometheus_metrics_service.py @@ -1,13 +1,11 @@ import asyncio import datetime -import time from concurrent.futures import ThreadPoolExecutor from typing import List, Optional from kubernetes.client import ApiClient from prometheus_api_client import PrometheusApiClientException from prometrix import PrometheusNotFound, get_custom_prometheus_connect -from requests.exceptions import ConnectionError, HTTPError from robusta_krr.core.abstract.strategies import PodsTimeData from robusta_krr.core.models.config import Config @@ -83,9 +81,7 @@ def __init__( headers |= {"Authorization": self.auth_header} elif not self.config.inside_cluster and self.api_client is not None: self.api_client.update_params_for_auth(headers, {}, ["BearerToken"]) - self.prom_config = generate_prometheus_config( - config, url=self.url, headers=headers, metrics_service=self - ) + self.prom_config = generate_prometheus_config(config, url=self.url, headers=headers, metrics_service=self) self.prometheus = get_custom_prometheus_connect(self.prom_config) def check_connection(self): @@ -95,7 +91,7 @@ def check_connection(self): PrometheusNotFound: If the connection to Prometheus cannot be established. """ self.prometheus.check_prometheus_connection() - + async def query(self, query: str) -> dict: loop = asyncio.get_running_loop() return await loop.run_in_executor(self.executor, lambda: self.prometheus.custom_query(query=query)) @@ -217,7 +213,9 @@ async def load_pods(self, object: K8sObjectData, period: datetime.timedelta) -> current_pods_set = {pod["metric"]["pod"] for pod in current_pods} del current_pods - object.pods += set([ - PodData(name=pod["metric"]["pod"], deleted=pod["metric"]["pod"] not in current_pods_set) - for pod in related_pods - ]) + object.pods += set( + [ + PodData(name=pod["metric"]["pod"], deleted=pod["metric"]["pod"] not in current_pods_set) + for pod in related_pods + ] + ) diff --git a/robusta_krr/core/integrations/prometheus/prometheus_utils.py b/robusta_krr/core/integrations/prometheus/prometheus_utils.py index d4c6fd40..450c60c9 100644 --- a/robusta_krr/core/integrations/prometheus/prometheus_utils.py +++ b/robusta_krr/core/integrations/prometheus/prometheus_utils.py @@ -1,8 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import boto3 from prometrix import AWSPrometheusConfig, CoralogixPrometheusConfig, PrometheusConfig, VictoriaMetricsPrometheusConfig from robusta_krr.core.models.config import Config +if TYPE_CHECKING: + from robusta_krr.core.integrations.prometheus.metrics_service.prometheus_metrics_service import ( + PrometheusMetricsService, + ) + class ClusterNotSpecifiedException(Exception): """ @@ -13,10 +22,10 @@ class ClusterNotSpecifiedException(Exception): def generate_prometheus_config( - config: Config, url: str, headers: dict[str, str], metrics_service: "PrometheusMetricsService" + config: Config, url: str, headers: dict[str, str], metrics_service: PrometheusMetricsService ) -> PrometheusConfig: from .metrics_service.victoria_metrics_service import VictoriaMetricsService - + baseconfig = { "url": url, "disable_ssl": not config.prometheus_ssl_enabled, diff --git a/robusta_krr/core/integrations/rollout.py b/robusta_krr/core/integrations/rollout.py index 677e00d9..389c65b2 100644 --- a/robusta_krr/core/integrations/rollout.py +++ b/robusta_krr/core/integrations/rollout.py @@ -1,6 +1,5 @@ -from kubernetes import client import six - +from kubernetes import client from kubernetes.client.exceptions import ApiTypeError, ApiValueError diff --git a/robusta_krr/core/models/config.py b/robusta_krr/core/models/config.py index 8aef1a72..a3b7be08 100644 --- a/robusta_krr/core/models/config.py +++ b/robusta_krr/core/models/config.py @@ -7,7 +7,6 @@ from robusta_krr.core.abstract import formatters from robusta_krr.core.abstract.strategies import AnyStrategy, BaseStrategy - from robusta_krr.core.models.objects import KindLiteral diff --git a/robusta_krr/core/models/objects.py b/robusta_krr/core/models/objects.py index 63049c41..23f73459 100644 --- a/robusta_krr/core/models/objects.py +++ b/robusta_krr/core/models/objects.py @@ -1,10 +1,9 @@ -from typing import Optional, Literal +from typing import Literal, Optional import pydantic as pd from robusta_krr.core.models.allocations import ResourceAllocations - KindLiteral = Literal["deployment", "daemonset", "statefulset", "job", "rollout"] diff --git a/robusta_krr/core/runner.py b/robusta_krr/core/runner.py index f419c7cb..659e16f8 100644 --- a/robusta_krr/core/runner.py +++ b/robusta_krr/core/runner.py @@ -1,27 +1,20 @@ import asyncio import math +import os +import sys +import warnings from concurrent.futures import ThreadPoolExecutor from typing import Optional, Union -import sys, os + +from prometrix import PrometheusNotFound from slack_sdk import WebClient -import warnings from robusta_krr.core.abstract.strategies import ResourceRecommendation, RunResult from robusta_krr.core.integrations.kubernetes import KubernetesLoader -from robusta_krr.core.integrations.prometheus import ( - PrometheusMetricsLoader, - ClusterNotSpecifiedException, -) -from prometrix import PrometheusNotFound +from robusta_krr.core.integrations.prometheus import ClusterNotSpecifiedException, PrometheusMetricsLoader from robusta_krr.core.models.config import Config from robusta_krr.core.models.objects import K8sObjectData -from robusta_krr.core.models.result import ( - ResourceAllocations, - ResourceScan, - ResourceType, - Result, - StrategyData, -) +from robusta_krr.core.models.result import ResourceAllocations, ResourceScan, ResourceType, Result, StrategyData from robusta_krr.utils.configurable import Configurable from robusta_krr.utils.logo import ASCII_LOGO from robusta_krr.utils.progress_bar import ProgressBar @@ -76,22 +69,21 @@ def _process_result(self, result: Result) -> None: file_name = self.config.file_output elif self.config.slack_output: file_name = self.config.slack_output - with open(file_name, 'w') as target_file: + with open(file_name, "w") as target_file: sys.stdout = target_file self.print_result(formatted, rich=getattr(Formatter, "__rich_console__", False)) sys.stdout = sys.stdout - if (self.config.slack_output): + if self.config.slack_output: client = WebClient(os.environ["SLACK_BOT_TOKEN"]) warnings.filterwarnings("ignore", category=UserWarning) client.files_upload( - channels=f'#{self.config.slack_output}', + channels=f"#{self.config.slack_output}", title="KRR Report", - file=f'./{file_name}', - initial_comment=f'Kubernetes Resource Report for {(" ".join(self.config.namespaces))}' + file=f"./{file_name}", + initial_comment=f'Kubernetes Resource Report for {(" ".join(self.config.namespaces))}', ) os.remove(file_name) - def __get_resource_minimal(self, resource: ResourceType) -> float: if resource == ResourceType.CPU: return 1 / 1000 * self.config.cpu_min_value diff --git a/robusta_krr/main.py b/robusta_krr/main.py index 2522fafe..b9a2d3e2 100644 --- a/robusta_krr/main.py +++ b/robusta_krr/main.py @@ -5,11 +5,11 @@ from datetime import datetime from typing import List, Literal, Optional, Union from uuid import UUID -from pydantic import ValidationError # noqa: F401 -from rich import print # noqa: F401 import typer import urllib3 +from pydantic import ValidationError # noqa: F401 +from rich import print # noqa: F401 from robusta_krr import formatters as concrete_formatters # noqa: F401 from robusta_krr.core.abstract import formatters @@ -185,7 +185,7 @@ def {func_name}( {strategy_settings}, ) -> None: '''Run KRR using the `{func_name}` strategy''' - + try: config = Config( kubeconfig=kubeconfig, diff --git a/robusta_krr/strategies/simple.py b/robusta_krr/strategies/simple.py index fa59fac2..f4adee3d 100644 --- a/robusta_krr/strategies/simple.py +++ b/robusta_krr/strategies/simple.py @@ -3,15 +3,15 @@ from robusta_krr.core.abstract.strategies import ( BaseStrategy, - PodsTimeData, - MetricsPodData, K8sObjectData, + MetricsPodData, + PodsTimeData, ResourceRecommendation, ResourceType, RunResult, StrategySettings, ) -from robusta_krr.core.integrations.prometheus.metrics import PercentileCPULoader, MaxMemoryLoader, PrometheusMetric +from robusta_krr.core.integrations.prometheus.metrics import MaxMemoryLoader, PercentileCPULoader, PrometheusMetric class SimpleStrategySettings(StrategySettings): diff --git a/robusta_krr/utils/configurable.py b/robusta_krr/utils/configurable.py index 4fbd804e..183fc295 100644 --- a/robusta_krr/utils/configurable.py +++ b/robusta_krr/utils/configurable.py @@ -1,5 +1,4 @@ import abc -from inspect import getframeinfo, stack from typing import Literal, Union from rich.console import Console @@ -62,7 +61,6 @@ def debug(self, message: str = "") -> None: """ if self.debug_active: - caller = getframeinfo(stack()[1][0]) self.console.print( self.__add_prefix( message, diff --git a/tests/conftest.py b/tests/conftest.py index 7b570a7f..81800c39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import random from datetime import datetime, timedelta -from unittest.mock import AsyncMock, patch, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import numpy as np import pytest