decouple redis from netbox (cisagov#580)

.dockerignore

@@ -39,8 +39,8 @@ scripts
 !scripts/malcolm_utils.py
 zeek-logs
 suricata-logs
-netbox/netbox/media
-netbox/netbox/postgres
-netbox/netbox/redis
+redis
+netbox/media
+netbox/postgres
 netbox/custom-plugins
 zeek/custom

.github/workflows/redis-build-and-push-ghcr.yml

@@ -6,7 +6,7 @@ on:
       - main
       - development
     paths:
-      - 'netbox/health-scripts/container_health_redis.sh'
+      - 'redis-scripts/container_health.sh'
       - 'Dockerfiles/redis.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/capa-build.sh'

.gitignore

@@ -23,6 +23,7 @@
 .tmp
 .configured
 config.*/
+/redis
 
 # development
 .envrc

Dockerfiles/redis.Dockerfile

@@ -24,7 +24,7 @@ ENV TERM xterm
 COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
 ADD --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/
-ADD --chmod=755 netbox/health-scripts/container_health_redis.sh /usr/local/bin/container_health.sh
+ADD --chmod=755 redis-scripts/container_health.sh /usr/local/bin/container_health.sh
 
 RUN apk update --no-cache && \
     apk upgrade --no-cache && \
@@ -37,7 +37,7 @@ ENTRYPOINT ["/sbin/tini", \
             "--", \
             "/usr/local/bin/docker-uid-gid-setup.sh", \
             "/usr/local/bin/service_check_passthrough.sh", \
-            "-s", "netbox-redis"]
+            "-s", "redis"]
 
 # to be populated at build-time:
 ARG BUILD_DATE

config/netbox-common.env.example

@@ -20,5 +20,4 @@ NETBOX_CACHE_TTL=30
 # Whether to disable Malcolm's NetBox instance ('true') or not ('false')
 NETBOX_DISABLED=true
 NETBOX_POSTGRES_DISABLED=true
-NETBOX_REDIS_DISABLED=true
 NETBOX_CRON=true

config/netbox.env.example

@@ -44,11 +44,11 @@ MEDIA_ROOT=/opt/netbox/netbox/media
 METRICS_ENABLED=false
 NAPALM_TIMEOUT=10
 REDIS_CACHE_DATABASE=1
-REDIS_CACHE_HOST=netbox-redis-cache
+REDIS_CACHE_HOST=redis-cache
 REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY=false
 REDIS_CACHE_SSL=false
 REDIS_DATABASE=0
-REDIS_HOST=netbox-redis
+REDIS_HOST=redis
 REDIS_INSECURE_SKIP_TLS_VERIFY=false
 REDIS_SSL=false
 RELEASE_CHECK_URL=

docker-compose-dev.yml

@@ -1146,8 +1146,8 @@ services:
       VIRTUAL_HOST: 'netbox.malcolm.local'
     depends_on:
     - netbox-postgres
-    - netbox-redis
-    - netbox-redis-cache
+    - redis
+    - redis-cache
     volumes:
     - type: bind
       bind:
@@ -1228,7 +1228,7 @@ services:
       timeout: 15s
       retries: 3
       start_period: 45s
-  netbox-redis:
+  redis:
     image: ghcr.io/idaholab/malcolm/redis:25.02.0
     profiles: ["malcolm"]
     userns_mode: keep-id
@@ -1244,16 +1244,16 @@ services:
     restart: "no"
     stdin_open: false
     tty: true
-    hostname: netbox-redis
+    hostname: redis
     networks:
     - default
     env_file:
     - ./config/process.env
     - ./config/ssl.env
     - ./config/netbox-common.env
-    - ./config/netbox-redis.env
+    - ./config/redis.env
     environment:
-      VIRTUAL_HOST: 'netbox-redis.malcolm.local'
+      VIRTUAL_HOST: 'redis.malcolm.local'
     command:
     - sh
     - -c
@@ -1268,15 +1268,15 @@ services:
     - type: bind
       bind:
         create_host_path: false
-      source: ./netbox/redis
+      source: ./redis
       target: /data
     healthcheck:
       test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 60s
       timeout: 15s
       retries: 3
       start_period: 45s
-  netbox-redis-cache:
+  redis-cache:
     image: ghcr.io/idaholab/malcolm/redis:25.02.0
     profiles: ["malcolm"]
     userns_mode: keep-id
@@ -1292,16 +1292,16 @@ services:
     restart: "no"
     stdin_open: false
     tty: true
-    hostname: netbox-redis-cache
+    hostname: redis-cache
     networks:
     - default
     env_file:
     - ./config/process.env
     - ./config/ssl.env
     - ./config/netbox-common.env
-    - ./config/netbox-redis-cache.env
+    - ./config/redis-cache.env
     environment:
-      VIRTUAL_HOST: 'netbox-redis-cache.malcolm.local'
+      VIRTUAL_HOST: 'redis-cache.malcolm.local'
     command:
     - sh
     - -c

docker-compose.yml

@@ -1020,8 +1020,8 @@ services:
       VIRTUAL_HOST: 'netbox.malcolm.local'
     depends_on:
     - netbox-postgres
-    - netbox-redis
-    - netbox-redis-cache
+    - redis
+    - redis-cache
     volumes:
     - type: bind
       bind:
@@ -1099,7 +1099,7 @@ services:
       timeout: 15s
       retries: 3
       start_period: 45s
-  netbox-redis:
+  redis:
     image: ghcr.io/idaholab/malcolm/redis:25.02.0
     profiles: ["malcolm"]
     userns_mode: keep-id
@@ -1112,16 +1112,16 @@ services:
     restart: "no"
     stdin_open: false
     tty: true
-    hostname: netbox-redis
+    hostname: redis
     networks:
     - default
     env_file:
     - ./config/process.env
     - ./config/ssl.env
     - ./config/netbox-common.env
-    - ./config/netbox-redis.env
+    - ./config/redis.env
     environment:
-      VIRTUAL_HOST: 'netbox-redis.malcolm.local'
+      VIRTUAL_HOST: 'redis.malcolm.local'
     command:
     - sh
     - -c
@@ -1136,15 +1136,15 @@ services:
     - type: bind
       bind:
         create_host_path: false
-      source: ./netbox/redis
+      source: ./redis
       target: /data
     healthcheck:
       test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 60s
       timeout: 15s
       retries: 3
       start_period: 45s
-  netbox-redis-cache:
+  redis-cache:
     image: ghcr.io/idaholab/malcolm/redis:25.02.0
     profiles: ["malcolm"]
     userns_mode: keep-id
@@ -1157,16 +1157,16 @@ services:
     restart: "no"
     stdin_open: false
     tty: true
-    hostname: netbox-redis-cache
+    hostname: redis-cache
     networks:
     - default
     env_file:
     - ./config/process.env
     - ./config/ssl.env
     - ./config/netbox-common.env
-    - ./config/netbox-redis-cache.env
+    - ./config/redis-cache.env
     environment:
-      VIRTUAL_HOST: 'netbox-redis-cache.malcolm.local'
+      VIRTUAL_HOST: 'redis-cache.malcolm.local'
     command:
     - sh
     - -c

docs/contributing-local-modifications.md

@@ -494,7 +494,7 @@ services:
           create_host_path: false
         source: ./netbox/postgres
         target: /var/lib/postgresql/data
-  netbox-redis:
+  redis:
     volumes:
       - type: bind
         bind:
@@ -505,9 +505,9 @@ services:
       - type: bind
         bind:
           create_host_path: false
-        source: ./netbox/redis
+        source: ./redis
         target: /data
-  netbox-redis-cache:
+  redis-cache:
     volumes:
       - type: bind
         bind:

docs/development.md

@@ -21,7 +21,7 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{
 * `htadmin` - configuration for the `htadmin` user account management container
 * `logstash` - code and configuration for the `logstash` container that parses Zeek logs and forwards them to the `opensearch` container
 * `malcolm-iso` - code and configuration for building an [installer ISO](malcolm-iso.md#ISO) for a minimal Debian-based Linux installation for running Malcolm
-* `netbox` - code and configuration for the `netbox`, `netbox-postgres`, `netbox-redis`, and `netbox-redis-cache` containers which provide asset management capabilities
+* `netbox` - code and configuration for the `netbox` and `netbox-postgres`containers which provide asset management capabilities
 * `nginx` - configuration for the `nginx` reverse-proxy container
 * `opensearch` - an initially empty directory where the OpenSearch database instance will reside
 * `opensearch-backup` - an initially empty directory for storing OpenSearch [index snapshots](index-management.md#IndexManagement) 

docs/kubernetes.md

@@ -283,15 +283,15 @@ logstash-deployment-6fbc9fdcd5-hwx8s           | Running | 10.42.2.22 | ReplicaS
 netbox-deployment-cdcff4977-hbbw5              | Running | 10.42.2.23 | ReplicaSet | agent1      | 807.64m   | 702.86Mi     | netbox-container:0             | netbox:{{ site.malcolm.version }}            |
 suricata-offline-deployment-6ccdb89478-z5696   | Running | 10.42.2.19 | ReplicaSet | agent1      | 0.22m     | 34.88Mi      | suricata-offline-container:0   | suricata:{{ site.malcolm.version }}          |
 dashboards-deployment-69b5465db-vz88g          | Running | 10.42.1.14 | ReplicaSet | agent2      | 0.94m     | 100.12Mi     | dashboards-container:0         | dashboards:{{ site.malcolm.version }}        |
-netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2      | 3.57m     | 7.36Mi       | netbox-redis-cache-container:0 | redis:{{ site.malcolm.version }}             |
+redis-cache-deployment-5f77d47b8b-z7t2z        | Running | 10.42.1.15 | ReplicaSet | agent2      | 3.57m     | 7.36Mi       | redis-cache-container:0        | redis:{{ site.malcolm.version }}             |
 suricata-live-deployment-6494c77759-9rlnt      | Running | 10.42.1.16 | ReplicaSet | agent2      | 0.02m     | 9.69Mi       | suricata-live-container:0      | suricata:{{ site.malcolm.version }}          |
 freq-deployment-cfd84fd97-dnngf                | Running | 10.42.1.17 | ReplicaSet | agent2      | 0.2m      | 26.36Mi      | freq-container:0               | freq:{{ site.malcolm.version }}              |
 arkime-deployment-56999cdd66-s98pp             | Running | 10.42.1.18 | ReplicaSet | agent2      | 4.15m     | 113.07Mi     | arkime-container:0             | arkime:{{ site.malcolm.version }}            |
 pcap-monitor-deployment-594ff674c4-fsm7m       | Running | 10.42.1.19 | ReplicaSet | agent2      | 1.24m     | 48.44Mi      | pcap-monitor-container:0       | pcap-monitor:{{ site.malcolm.version }}      |
 pcap-capture-deployment-7c8bf6957-jzpzn        | Running | 10.42.1.20 | ReplicaSet | agent2      | 0.02m     | 9.64Mi       | pcap-capture-container:0       | pcap-capture:{{ site.malcolm.version }}      |
 netbox-postgres-deployment-5879b8dffc-kkt56    | Running | 10.42.1.21 | ReplicaSet | agent2      | 70.91m    | 33.02Mi      | netbox-postgres-container:0    | postgresql:{{ site.malcolm.version }}        |
 htadmin-deployment-6fc46888b9-sq6ln            | Running | 10.42.1.23 | ReplicaSet | agent2      | 0.14m     | 30.53Mi      | htadmin-container:0            | htadmin:{{ site.malcolm.version }}           |
-netbox-redis-deployment-5bcd8f6c96-j5xpf       | Running | 10.42.1.24 | ReplicaSet | agent2      | 1.46m     | 7.34Mi       | netbox-redis-container:0       | redis:{{ site.malcolm.version }}             |
+redis-deployment-5bcd8f6c96-j5xpf              | Running | 10.42.1.24 | ReplicaSet | agent2      | 1.46m     | 7.34Mi       | redis-container:0              | redis:{{ site.malcolm.version }}             |
 nginx-proxy-deployment-69fcc4968d-f68tq        | Running | 10.42.1.22 | ReplicaSet | agent2      | 0.31m     | 22.63Mi      | nginx-proxy-container:0        | nginx-proxy:{{ site.malcolm.version }}       |
 opensearch-deployment-75498799f6-4zmwd         | Running | 10.42.1.25 | ReplicaSet | agent2      | 89.8m     | 11.03Gi      | opensearch-container:0         | opensearch:{{ site.malcolm.version }}        |
 ```
@@ -556,8 +556,8 @@ agent1    | agent1   | 192.168.56.11 | agent1      | k3s           | 6000m     |
 agent2    | agent2   | 192.168.56.12 | agent2      | k3s           | 6000m     | 552.71m   | 9.21%       | 19.55Gi      | 13.27Gi      | 61.28Gi       | 12           |
 
 Pod Name                                       | State   | Pod IP     | Pod Kind   | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts        | Container Image              |
-netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6  | ReplicaSet | agent2      | 1.89m     | 7.24Mi       | netbox-redis-cache-container:0 | redis:{{ site.malcolm.version }}             |
-netbox-redis-deployment-5bcd8f6c96-bkzmh       | Running | 10.42.2.5  | ReplicaSet | agent2      | 1.62m     | 7.52Mi       | netbox-redis-container:0       | redis:{{ site.malcolm.version }}             |
+redis-cache-deployment-5f77d47b8b-jr9nt        | Running | 10.42.2.6  | ReplicaSet | agent2      | 1.89m     | 7.24Mi       | redis-cache-container:0        | redis:{{ site.malcolm.version }}             |
+redis-deployment-5bcd8f6c96-bkzmh              | Running | 10.42.2.5  | ReplicaSet | agent2      | 1.62m     | 7.52Mi       | redis-container:0              | redis:{{ site.malcolm.version }}             |
 dashboards-helper-deployment-69dc54f6b6-ks7ps  | Running | 10.42.2.4  | ReplicaSet | agent2      | 12.95m    | 40.75Mi      | dashboards-helper-container:0  | dashboards-helper:{{ site.malcolm.version }} |
 freq-deployment-cfd84fd97-5bwp6                | Running | 10.42.2.8  | ReplicaSet | agent2      | 0.11m     | 26.33Mi      | freq-container:0               | freq:{{ site.malcolm.version }}              |
 pcap-capture-deployment-7c8bf6957-hkvkn        | Running | 10.42.2.12 | ReplicaSet | agent2      | 0.02m     | 9.21Mi       | pcap-capture-container:0       | pcap-capture:{{ site.malcolm.version }}      |

docs/malcolm-config.md

@@ -59,7 +59,7 @@ Although the configuration script automates many of the following configuration
     - `FREQ_SEVERITY_THRESHOLD` - when [severity scoring](severity.md#Severity) is enabled, this variable indicates the entropy threshold for assigning severity to events with entropy scores calculated by [`freq`](https://github.com/MarkBaggett/freq); a lower value will only assign severity scores to fewer domain names with higher entropy (e.g., `2.0` for `NQZHTFHRMYMTVBQJE.COM`), while a higher value will assign severity scores to more domain names with lower entropy (e.g., `7.5` for `naturallanguagedomain.example.org`) (default `2.0`)
     - `SENSITIVE_COUNTRY_CODES` - when [severity scoring](severity.md#Severity) is enabled, this variable defines a comma-separated list of sensitive countries (using [ISO 3166-1 alpha-2 codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Current_codes)) (default `'AM,AZ,BY,CN,CU,DZ,GE,HK,IL,IN,IQ,IR,KG,KP,KZ,LY,MD,MO, PK,RU,SD,SS,SY,TJ,TM,TW,UA,UZ'`, taken from the U.S. Department of Energy Sensitive Country List)
     - `TOTAL_MEGABYTES_SEVERITY_THRESHOLD` - when [severity scoring](severity.md#Severity) is enabled, this variable indicates the size threshold (in megabytes) for assigning severity to large connections or file transfers (default `1000`)
-* **`netbox-common.env`**, `netbox.env`, `netbox-secret.env`, `netbox-postgres.env`, `netbox-redis-cache.env` and `netbox-redis.env` - settings related to [NetBox](https://netbox.dev/) and [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis)
+* **`netbox-common.env`**, `netbox.env`, `netbox-secret.env`, and `netbox-postgres.env` - settings related to [NetBox](https://netbox.dev/) and [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis)
     - `NETBOX_DISABLED` - if set to `true`, Malcolm will **not** start and manage a [NetBox](asset-interaction-analysis.md#AssetInteractionAnalysis) instance (default `true`)
     - `NETBOX_ENRICHMENT` - if set to `true`, Logstash will [enrich network traffic metadata](asset-interaction-analysis.md#NetBoxEnrichment) via NetBox API calls
     - `NETBOX_DEFAULT_SITE` - specifies the default NetBox [site name](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/) for use when [enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment) if a specific site is not otherwise specified for the source of the data (default `Malcolm`)
@@ -99,6 +99,7 @@ Although the configuration script automates many of the following configuration
 * **`process.env`** - settings for how the processes running inside Malcolm containers are executed
     - `PUID` and `PGID` - Docker runs all its containers as the privileged `root` user by default. For better security, Malcolm immediately drops to non-privileged user accounts for executing internal processes wherever possible. The `PUID` (**p**rocess **u**ser **ID**) and `PGID` (**p**rocess **g**roup **ID**) environment variables allow Malcolm to map internal non-privileged user accounts to a corresponding [user account](https://en.wikipedia.org/wiki/User_identifier) on the host. Note a few (including the `logstash` and `netbox` containers) may take a few extra minutes during startup if `PUID` and `PGID` are set to values other than the default `1000`. This is expected and should not affect operation after the initial startup.
     - `MALCOLM_PROFILE` - Specifies the [profile](https://docs.docker.com/compose/profiles/) which determines the Malcolm containers to run (`malcolm` to run all containers, `hedgehog` to run only [capture-related containers](https://github.com/idaholab/Malcolm/issues/254))
+* **`redis.env`** and **`redis-cache.env`** - Settings related to the Redis in-memory database
 * **`ssl.env`** - TLS-related settings used by many containers
 * **`suricata.env`**, **`suricata-live.env`** and **`suricata-offline.env`** - settings for [Suricata](https://suricata.io/)
     - `SURICATA_AUTO_ANALYZE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will automatically be analyzed by Suricata, and the resulting logs will also be imported (default `false`)

docs/quickstart.md

@@ -46,7 +46,7 @@ Pulling htadmin           ... done
 Pulling logstash          ... done
 Pulling netbox            ... done
 Pulling netbox-postgresql ... done
-Pulling netbox-redis      ... done
+Pulling redis             ... done
 Pulling nginx-proxy       ... done
 Pulling opensearch        ... done
 Pulling pcap-capture      ... done

docs/ubuntu-install-example.md

@@ -252,7 +252,7 @@ Pulling htadmin           ... done
 Pulling logstash          ... done
 Pulling netbox            ... done
 Pulling netbox-postgresql ... done
-Pulling netbox-redis      ... done
+Pulling redis             ... done
 Pulling nginx-proxy       ... done
 Pulling opensearch        ... done
 Pulling pcap-capture      ... done
@@ -302,8 +302,8 @@ malcolm-htadmin-1              "/usr/local/bin/dock…"   htadmin              r
 malcolm-logstash-1             "/usr/local/bin/dock…"   logstash             running (starting)   …
 malcolm-netbox-1               "/usr/bin/tini -- /u…"   netbox               running (starting)   …
 malcolm-netbox-postgres-1      "/usr/bin/docker-uid…"   netbox-postgres      running (starting)   …
-malcolm-netbox-redis-1         "/sbin/tini -- /usr/…"   netbox-redis         running (starting)   …
-malcolm-netbox-redis-cache-1   "/sbin/tini -- /usr/…"   netbox-redis-cache   running (starting)   …
+malcolm-redis-1                "/sbin/tini -- /usr/…"   redis                running (starting)   …
+malcolm-redis-cache-1          "/sbin/tini -- /usr/…"   redis-cache          running (starting)   …
 malcolm-nginx-proxy-1          "/usr/local/bin/dock…"   nginx-proxy          running (starting)   …
 malcolm-opensearch-1           "/usr/local/bin/dock…"   opensearch           running (starting)   …
 malcolm-pcap-capture-1         "/usr/local/bin/dock…"   pcap-capture         running              …