Merge pull request #1935 from bunkerity/dev

Merge branch "dev" into branch "staging"

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## v1.6.0-rc3 - ????/??/??
 
+- [AUTOCONF] Add the possibility to add/override settings via ConfigMap in Kubernetes using the `bunkerweb.io/CONFIG_TYPE=settings` annotation
 - [UI] Add support page for easy logs and configuration sharing while anonymizing sensitive data
 - [LINUX] Support Fedora 41
 

docs/advanced.md

@@ -867,7 +867,7 @@ Some integrations provide more convenient ways to apply configurations, such as
     metadata:
       name: cfg-bunkerweb-all-server-http
       annotations:
-    	bunkerweb.io/CONFIG_TYPE: "server-http"
+    	  bunkerweb.io/CONFIG_TYPE: "server-http"
     data:
       myconf: |
     	location /hello {
@@ -878,6 +878,22 @@ Some integrations provide more convenient ways to apply configurations, such as
     	}
     ```
 
+    !!! tip "Custom Extra Config"
+        Since the `1.6.0-rc3` version, you can add/override settings using the `bunkerweb.io/CONFIG_TYPE=settings` annotation. Here is an example :
+
+        ```yaml
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: cfg-bunkerweb-extra-settings
+          annotations:
+            bunkerweb.io/CONFIG_TYPE: "settings"
+        data:
+          USE_ANTIBOT: "captcha" # multisite setting that will be applied to all services that do not override it
+          USE_REDIS: "yes" # global setting that will be applied globally
+          ...
+        ```
+
 === "Linux"
 
     When using the [Linux integration](integrations.md#linux), custom configurations must be written to the /etc/bunkerweb/configs folder.

src/autoconf/Config.py

@@ -30,6 +30,7 @@ def __init__(self, ctrl_type: Union[Literal["docker"], Literal["swarm"], Literal
         )
         self.__configs = {config_type: {} for config_type in self._supported_config_types}
         self.__config = {}
+        self.__extra_config = {}
 
         self._db = Database(self.__logger)
 
@@ -44,6 +45,12 @@ def _update_settings(self):
 
     def __get_full_env(self) -> dict:
         config = {"SERVER_NAME": "", "MULTISITE": "yes"}
+        for service in self.__services:
+            server_name = service["SERVER_NAME"].split(" ")[0]
+            if not server_name:
+                continue
+            config["SERVER_NAME"] += f" {server_name}"
+
         for service in self.__services:
             server_name = service["SERVER_NAME"].split(" ")[0]
             if not server_name:
@@ -53,7 +60,12 @@ def __get_full_env(self) -> dict:
                     continue
 
                 is_global = False
-                success, err = self._db.is_valid_setting(variable, value=value, multisite=True)
+                success, err = self._db.is_valid_setting(
+                    variable,
+                    value=value,
+                    multisite=True,
+                    extra_services=config["SERVER_NAME"].split(" "),
+                )
                 if not success:
                     if self._type == "kubernetes":
                         success, err = self._db.is_valid_setting(variable, value=value)
@@ -65,15 +77,24 @@ def __get_full_env(self) -> dict:
                         continue
 
                 if is_global or variable.startswith(f"{server_name}_"):
+                    if variable == "SERVER_NAME":
+                        self.__logger.warning("Global variable SERVER_NAME can't be set via annotations, ignoring it")
+                        continue
                     config[variable] = value
                     continue
                 config[f"{server_name}_{variable}"] = value
-            config["SERVER_NAME"] += f" {server_name}"
         config["SERVER_NAME"] = config["SERVER_NAME"].strip()
         return config
 
-    def update_needed(self, instances: List[Dict[str, Any]], services: List[Dict[str, str]], configs: Optional[Dict[str, Dict[str, bytes]]] = None) -> bool:
+    def update_needed(
+        self,
+        instances: List[Dict[str, Any]],
+        services: List[Dict[str, str]],
+        configs: Optional[Dict[str, Dict[str, bytes]]] = None,
+        extra_config: Optional[Dict[str, str]] = None,
+    ) -> bool:
         configs = configs or {}
+        extra_config = extra_config or {}
 
         # Use sets for comparing lists of dictionaries
         if set(map(str, self.__instances)) != set(map(str, instances)):
@@ -88,6 +109,10 @@ def update_needed(self, instances: List[Dict[str, Any]], services: List[Dict[str
             self.__logger.debug(f"Configs changed: {self.__configs} -> {configs}")
             return True
 
+        if set(map(str, self.__extra_config.items())) != set(map(str, extra_config.items())):
+            self.__logger.debug(f"Extra config changed: {self.__extra_config} -> {extra_config}")
+            return True
+
         return False
 
     def have_to_wait(self) -> bool:
@@ -133,6 +158,7 @@ def apply(
         services: List[Dict[str, str]],
         configs: Optional[Dict[str, Dict[str, bytes]]] = None,
         first: bool = False,
+        extra_config: Optional[Dict[str, str]] = None,
     ) -> bool:
         success = True
 
@@ -154,9 +180,9 @@ def apply(
         if configs != self.__configs or first:
             self.__configs = configs
             changes.append("custom_configs")
-        if "instances" in changes or "services" in changes:
+        if "instances" in changes or "services" in changes or extra_config != self.__extra_config:
             old_env = self.__config.copy()
-            new_env = self.__get_full_env()
+            new_env = self.__get_full_env() | extra_config
             if old_env != new_env or first:
                 self.__config = new_env
                 changes.append("config")

src/autoconf/Controller.py

@@ -16,6 +16,7 @@ def __init__(self, *args, **kwargs):
         self._instances = []
         self._services = []
         self._configs = {config_type: {} for config_type in self._supported_config_types}
+        self._extra_config = {}
         self._logger = setup_logger(f"{self._type}-controller", getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")))
         self._namespaces = None
         namespaces = getenv("NAMESPACES")

src/autoconf/IngressController.py

@@ -4,7 +4,7 @@
 from os import getenv
 from time import sleep
 from traceback import format_exc
-from typing import List
+from typing import List, Tuple
 from kubernetes import client, config, watch
 from kubernetes.client import Configuration
 from kubernetes.client.exceptions import ApiException
@@ -122,6 +122,7 @@ def _to_services(self, controller_service) -> List[dict]:
 
         namespace = controller_service.metadata.namespace
         services = []
+        server_names = set()
 
         # parse rules
         for rule in controller_service.spec.rules:
@@ -131,6 +132,7 @@ def _to_services(self, controller_service) -> List[dict]:
 
             service = {}
             service["SERVER_NAME"] = rule.host
+            server_names.add(rule.host)
             if not rule.http:
                 services.append(service)
                 continue
@@ -182,10 +184,21 @@ def _to_services(self, controller_service) -> List[dict]:
         # parse annotations
         if controller_service.metadata.annotations:
             for service in services:
+                server_name = service["SERVER_NAME"].strip().split(" ")[0]
+
                 for annotation, value in controller_service.metadata.annotations.items():
                     if not annotation.startswith("bunkerweb.io/"):
                         continue
-                    service[annotation.replace("bunkerweb.io/", "", 1)] = value
+                    setting = annotation.replace("bunkerweb.io/", "", 1)
+                    success, _ = self._db.is_valid_setting(setting, value=value, multisite=True)
+                    if success and not setting.startswith(f"{server_name}_"):
+                        if any(setting.startswith(f"{s}_") for s in server_names):
+                            continue
+                        if setting == "SERVER_NAME":
+                            self._logger.warning("Variable SERVER_NAME can't be set globally via annotations, ignoring it")
+                            continue
+                        setting = f"{server_name}_{setting}"
+                    service[setting] = value
 
                 # Handle stream services
                 for server_name in service["SERVER_NAME"].strip().split(" "):
@@ -242,8 +255,9 @@ def _to_services(self, controller_service) -> List[dict]:
 
         return services
 
-    def get_configs(self) -> dict:
+    def get_configs(self) -> Tuple[dict, dict]:
         configs = {config_type: {} for config_type in self._supported_config_types}
+        config = {}
         for configmap in self.__corev1.list_config_map_for_all_namespaces(watch=False).items:
             if (
                 not configmap.metadata.annotations
@@ -253,25 +267,34 @@ def get_configs(self) -> dict:
                 continue
 
             config_type = configmap.metadata.annotations["bunkerweb.io/CONFIG_TYPE"]
-            if config_type not in self._supported_config_types:
+            if config_type not in set(self._supported_config_types) | {"settings"}:
                 self._logger.warning(f"Ignoring unsupported CONFIG_TYPE {config_type} for ConfigMap {configmap.metadata.name}")
                 continue
             elif not configmap.data:
                 self._logger.warning(f"Ignoring blank ConfigMap {configmap.metadata.name}")
                 continue
 
-            config_site = ""
-            if "bunkerweb.io/CONFIG_SITE" in configmap.metadata.annotations:
-                if not self._is_service_present(configmap.metadata.annotations["bunkerweb.io/CONFIG_SITE"]):
-                    self._logger.warning(
-                        f"Ignoring config {configmap.metadata.name} because {configmap.metadata.annotations['bunkerweb.io/CONFIG_SITE']} doesn't exist"
-                    )
-                    continue
-                config_site = f"{configmap.metadata.annotations['bunkerweb.io/CONFIG_SITE']}/"
+            if config_type == "settings":
+                for config_name, config_data in configmap.data.items():
+                    if not self._db.is_valid_setting(config_name, value=config_data, accept_prefixed=False):
+                        self._logger.warning(
+                            f"Ignoring invalid setting {config_name} for ConfigMap {configmap.metadata.name} (the setting must exist and should not be prefixed)"
+                        )
+                        continue
+                    config[config_name] = config_data
+            else:
+                config_site = ""
+                if "bunkerweb.io/CONFIG_SITE" in configmap.metadata.annotations:
+                    if not self._is_service_present(configmap.metadata.annotations["bunkerweb.io/CONFIG_SITE"]):
+                        self._logger.warning(
+                            f"Ignoring config {configmap.metadata.name} because {configmap.metadata.annotations['bunkerweb.io/CONFIG_SITE']} doesn't exist"
+                        )
+                        continue
+                    config_site = f"{configmap.metadata.annotations['bunkerweb.io/CONFIG_SITE']}/"
 
-            for config_name, config_data in configmap.data.items():
-                configs[config_type][f"{config_site}{config_name}"] = config_data
-        return configs
+                for config_name, config_data in configmap.data.items():
+                    configs[config_type][f"{config_site}{config_name}"] = config_data
+        return config, configs
 
     def __process_event(self, event):
         obj = event["object"]
@@ -362,9 +385,9 @@ def __watch(self, watch_type):
                         self._update_settings()
                         self._instances = self.get_instances()
                         self._services = self.get_services()
-                        self._configs = self.get_configs()
+                        self._extra_config, self._configs = self.get_configs()
 
-                        if not to_apply and not self.update_needed(self._instances, self._services, configs=self._configs):
+                        if not to_apply and not self.update_needed(self._instances, self._services, self._configs, self._extra_config):
                             if locked:
                                 self.__internal_lock.release()
                                 locked = False
@@ -414,7 +437,7 @@ def __watch(self, watch_type):
                     sleep(10)
 
     def apply_config(self) -> bool:
-        return self.apply(self._instances, self._services, configs=self._configs, first=not self._loaded)
+        return self.apply(self._instances, self._services, configs=self._configs, first=not self._loaded, extra_config=self._extra_config)
 
     def process_events(self):
         self._set_autoconf_load_db()

src/common/core/bunkernet/bunkernet.lua

@@ -15,6 +15,7 @@ local HTTP_INTERNAL_SERVER_ERROR = ngx.HTTP_INTERNAL_SERVER_ERROR
 local HTTP_OK = ngx.HTTP_OK
 local timer_at = ngx.timer.at
 local get_phase = ngx.get_phase
+local worker = ngx.worker
 local get_version = utils.get_version
 local get_integration = utils.get_integration
 local get_deny_status = utils.get_deny_status
@@ -44,8 +45,6 @@ function bunkernet:initialize(ctx)
 			self.bunkernet_id = id
 			self.version = get_version(self.ctx)
 			self.integration = get_integration(self.ctx)
-		else
-			self.logger:log(ERR, "can't get BunkerNet ID from datastore : " .. err)
 		end
 	end
 end
@@ -253,6 +252,21 @@ function bunkernet:log_stream()
 end
 
 function bunkernet:timer()
+
+	-- Only execute on worker 0
+	if worker.id() ~= 0 then
+		return self:ret(true, "skipped")
+	end
+
+	-- Check if BunkerNet is activated
+	local is_needed, err = has_variable("USE_BUNKERNET", "yes")
+	if is_needed == nil then
+		return self:ret(false, "can't check USE_BUNKERNET variable : " .. err)
+	end
+	if not is_needed then
+		return self:ret(true, "no service uses BunkerNet, skipping init")
+	end
+
 	local ret = true
 	local ret_err = "success"
 
@@ -264,7 +278,7 @@ function bunkernet:timer()
 
 	-- Loop on reports
 	local reports = {}
-	for i = 0, len do
+	for i = 1, len do
 		-- Pop the report and decode it
 		local report, report_err = self.datastore:lpop("plugin_bunkernet_reports")
 		if not report then

src/common/db/Database.py

@@ -341,7 +341,14 @@ def _db_session(self) -> Any:
                     session.remove()
 
     def is_valid_setting(
-        self, setting: str, *, value: Optional[str] = None, multisite: bool = False, session: Optional[scoped_session] = None
+        self,
+        setting: str,
+        *,
+        value: Optional[str] = None,
+        multisite: bool = False,
+        session: Optional[scoped_session] = None,
+        accept_prefixed: bool = True,
+        extra_services: Optional[List[str]] = None,
     ) -> Tuple[bool, str]:
         """Check if the setting exists in the database, if it's valid and if the value is valid"""
 
@@ -354,15 +361,21 @@ def check_setting(session: scoped_session, setting: str, value: Optional[str], m
 
                 db_setting = session.query(Settings).filter_by(id=setting).first()
 
-                if not db_setting:
-                    for service in session.query(Services).with_entities(Services.id):
-                        if setting.startswith(f"{service.id}_"):
-                            db_setting = session.query(Settings).filter_by(id=setting.replace(f"{service.id}_", "")).first()
-                            multisite = True
+                if accept_prefixed and not db_setting:
+                    for service in extra_services or []:
+                        if setting.startswith(f"{service}_"):
+                            db_setting = session.query(Settings).filter_by(id=setting.replace(f"{service}_", "")).first()
                             break
 
                     if not db_setting:
-                        return False, "missing"
+                        for service in session.query(Services).with_entities(Services.id):
+                            if setting.startswith(f"{service.id}_"):
+                                db_setting = session.query(Settings).filter_by(id=setting.replace(f"{service.id}_", "")).first()
+                                multisite = True
+                                break
+
+                if not db_setting:
+                    return False, "missing"
 
                 if multisite and db_setting.context != "multisite":
                     return False, "not multisite"

src/linux/Dockerfile-rhel

@@ -43,11 +43,11 @@ WORKDIR /usr/share/bunkerweb
 RUN export MAKEFLAGS="-j$(nproc)" && \
     mkdir -p deps/python && \
     easy_install-3.9 pip && \
-    pip install --no-cache-dir --require-hashes --break-system-packages --ignore-installed -r /tmp/requirements-deps.txt && \
-    pip install --no-cache-dir --require-hashes --break-system-packages --target deps/python $(for file in $(ls /tmp/req/requirements*.txt) ; do echo "-r ${file}" ; done | xargs) && \
+    pip install --no-cache-dir --require-hashes -r /tmp/requirements-deps.txt && \
+    pip install --no-cache-dir --require-hashes --force-reinstall --target deps/python $(for file in $(ls /tmp/req/requirements*.txt) ; do echo "-r ${file}" ; done | xargs) && \
     python3 -m venv /tmp/venv && \
     . /tmp/venv/bin/activate && \
-    pip install --no-cache-dir --require-hashes --break-system-packages -r /tmp/req/requirements-ui.txt && \
+    pip install --no-cache-dir --require-hashes --force-reinstall -r /tmp/req/requirements-ui.txt && \
     rm -rf deps/python/zope* && \
     mv /tmp/venv/lib/python*/site-packages/zope* deps/python/
 

src/linux/Dockerfile-rhel9

@@ -44,11 +44,11 @@ WORKDIR /usr/share/bunkerweb
 RUN export MAKEFLAGS="-j$(nproc)" && \
     mkdir -p deps/python && \
     python3 -m ensurepip --upgrade && \
-    python3 -m pip install --no-cache-dir --require-hashes --break-system-packages --ignore-installed -r /tmp/requirements-deps.txt && \
-    python3 -m pip install --no-cache-dir --require-hashes --break-system-packages --target deps/python $(for file in $(ls /tmp/req/requirements*.txt) ; do echo "-r ${file}" ; done | xargs) && \
+    python3 -m pip install --no-cache-dir --require-hashes -r /tmp/requirements-deps.txt && \
+    python3 -m pip install --no-cache-dir --require-hashes --force-reinstall --target deps/python $(for file in $(ls /tmp/req/requirements*.txt) ; do echo "-r ${file}" ; done | xargs) && \
     python3 -m venv /tmp/venv && \
     . /tmp/venv/bin/activate && \
-    python3 -m pip install --no-cache-dir --require-hashes --break-system-packages -r /tmp/req/requirements-ui.txt && \
+    python3 -m pip install --no-cache-dir --require-hashes --force-reinstall -r /tmp/req/requirements-ui.txt && \
     rm -rf deps/python/zope* && \
     mv /tmp/venv/lib/python*/site-packages/zope* deps/python/