From 8676f37b17ce6e17b1e489d51959a5bffd12ba1c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 11 Feb 2025 12:07:56 -0700 Subject: [PATCH] decouple redis from netbox (cisagov/Malcolm#580) and reorganized some of the other netbox password stuff --- config/netbox-postgres.env.example | 12 +++++++-- config/netbox-secret.env.example | 4 --- config/netbox.env.example | 4 --- config/redis-cache.env.example | 2 -- config/redis.env.example | 2 ++ docker-compose-dev.yml | 4 ++- docker-compose.yml | 4 ++- docs/malcolm-config.md | 2 +- kubernetes/16-redis-cache.yml | 2 +- kubernetes/18-netbox.yml | 4 +++ scripts/control.py | 43 +++++++++--------------------- scripts/malcolm_common.py | 1 - 12 files changed, 36 insertions(+), 48 deletions(-) delete mode 100644 config/redis-cache.env.example diff --git a/config/netbox-postgres.env.example b/config/netbox-postgres.env.example index 6814396b9..8e62f7a0d 100644 --- a/config/netbox-postgres.env.example +++ b/config/netbox-postgres.env.example @@ -1,4 +1,12 @@ +DB_HOST=netbox-postgres + POSTGRES_DB=netbox -POSTGRES_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXX +DB_NAME=netbox + POSTGRES_USER=netbox -K8S_SECRET=True \ No newline at end of file +DB_USER=netbox + +POSTGRES_PASSWORD=xxxxxxxxxxxxxxxx +DB_PASSWORD=xxxxxxxxxxxxxxxx + +K8S_SECRET=True diff --git a/config/netbox-secret.env.example b/config/netbox-secret.env.example index d629b84df..f523212e0 100644 --- a/config/netbox-secret.env.example +++ b/config/netbox-secret.env.example @@ -1,11 +1,7 @@ -DB_PASSWORD=xxxxxxxxxxxxxxxx -DB_USER=netbox EMAIL_PASSWORD= EMAIL_USERNAME=netbox NAPALM_PASSWORD= NAPALM_USERNAME= -REDIS_CACHE_PASSWORD=xxxxxxxxxxxxxxxx -REDIS_PASSWORD=xxxxxxxxxxxxxxxx SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx SUPERUSER_API_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx SUPERUSER_NAME=admin diff --git a/config/netbox.env.example b/config/netbox.env.example index 31ba6fec5..c15e047d5 100644 --- a/config/netbox.env.example +++ b/config/netbox.env.example @@ -26,8 +26,6 @@ REMOTE_AUTH_STAFF_USERS= REMOTE_AUTH_SUPERUSER_GROUPS=administrator REMOTE_AUTH_SUPERUSERS= -DB_HOST=netbox-postgres -DB_NAME=netbox EMAIL_FROM=netbox@bar.com EMAIL_PORT=25 EMAIL_SERVER=localhost @@ -44,11 +42,9 @@ MEDIA_ROOT=/opt/netbox/netbox/media METRICS_ENABLED=false NAPALM_TIMEOUT=10 REDIS_CACHE_DATABASE=1 -REDIS_CACHE_HOST=redis-cache REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY=false REDIS_CACHE_SSL=false REDIS_DATABASE=0 -REDIS_HOST=redis REDIS_INSECURE_SKIP_TLS_VERIFY=false REDIS_SSL=false RELEASE_CHECK_URL= diff --git a/config/redis-cache.env.example b/config/redis-cache.env.example deleted file mode 100644 index b5e80205d..000000000 --- a/config/redis-cache.env.example +++ /dev/null @@ -1,2 +0,0 @@ -REDIS_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXX -K8S_SECRET=True \ No newline at end of file diff --git a/config/redis.env.example b/config/redis.env.example index b5e80205d..c60b0751f 100644 --- a/config/redis.env.example +++ b/config/redis.env.example @@ -1,2 +1,4 @@ +REDIS_HOST=redis +REDIS_CACHE_HOST=redis-cache REDIS_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXX K8S_SECRET=True \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 86b65f865..b8bd7bb58 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1142,6 +1142,8 @@ services: - ./config/netbox-common.env - ./config/netbox.env - ./config/netbox-secret.env + - ./config/netbox-postgres.env + - ./config/redis.env environment: VIRTUAL_HOST: 'netbox.malcolm.local' depends_on: @@ -1299,7 +1301,7 @@ services: - ./config/process.env - ./config/ssl.env - ./config/netbox-common.env - - ./config/redis-cache.env + - ./config/redis.env environment: VIRTUAL_HOST: 'redis-cache.malcolm.local' command: diff --git a/docker-compose.yml b/docker-compose.yml index 58eb4d0da..353313c3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1016,6 +1016,8 @@ services: - ./config/netbox-common.env - ./config/netbox.env - ./config/netbox-secret.env + - ./config/netbox-postgres.env + - ./config/redis.env environment: VIRTUAL_HOST: 'netbox.malcolm.local' depends_on: @@ -1164,7 +1166,7 @@ services: - ./config/process.env - ./config/ssl.env - ./config/netbox-common.env - - ./config/redis-cache.env + - ./config/redis.env environment: VIRTUAL_HOST: 'redis-cache.malcolm.local' command: diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 30bfc0823..1d581b394 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -99,7 +99,7 @@ Although the configuration script automates many of the following configuration * **`process.env`** - settings for how the processes running inside Malcolm containers are executed - `PUID` and `PGID` - Docker runs all its containers as the privileged `root` user by default. For better security, Malcolm immediately drops to non-privileged user accounts for executing internal processes wherever possible. The `PUID` (**p**rocess **u**ser **ID**) and `PGID` (**p**rocess **g**roup **ID**) environment variables allow Malcolm to map internal non-privileged user accounts to a corresponding [user account](https://en.wikipedia.org/wiki/User_identifier) on the host. Note a few (including the `logstash` and `netbox` containers) may take a few extra minutes during startup if `PUID` and `PGID` are set to values other than the default `1000`. This is expected and should not affect operation after the initial startup. - `MALCOLM_PROFILE` - Specifies the [profile](https://docs.docker.com/compose/profiles/) which determines the Malcolm containers to run (`malcolm` to run all containers, `hedgehog` to run only [capture-related containers](https://github.com/idaholab/Malcolm/issues/254)) -* **`redis.env`** and **`redis-cache.env`** - Settings related to the Redis in-memory database +* **`redis.env`** - Settings related to the Redis in-memory database * **`ssl.env`** - TLS-related settings used by many containers * **`suricata.env`**, **`suricata-live.env`** and **`suricata-offline.env`** - settings for [Suricata](https://suricata.io/) - `SURICATA_AUTO_ANALYZE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will automatically be analyzed by Suricata, and the resulting logs will also be imported (default `false`) diff --git a/kubernetes/16-redis-cache.yml b/kubernetes/16-redis-cache.yml index 32b1c6a0a..047505c97 100644 --- a/kubernetes/16-redis-cache.yml +++ b/kubernetes/16-redis-cache.yml @@ -55,7 +55,7 @@ spec: - configMapRef: name: netbox-common-env - secretRef: - name: redis-cache-env + name: redis-env env: - name: VIRTUAL_HOST value: "redis-cache.malcolm.local" diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index fd4cbc1bb..d3e843849 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -61,6 +61,10 @@ spec: name: netbox-env - secretRef: name: netbox-secret-env + - secretRef: + name: redis-env + - secretRef: + name: netbox-postgres-env env: - name: VIRTUAL_HOST value: "netbox.malcolm.local" diff --git a/scripts/control.py b/scripts/control.py index cbf6d1df8..3a0004d4b 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1881,13 +1881,12 @@ def authSetup(): preExistingPasswordFound = False preExistingPasswords = { - 'netbox-postgres.env': ('POSTGRES_PASSWORD',), - 'redis-cache.env': ('REDIS_PASSWORD',), + 'netbox-postgres.env': ( + 'POSTGRES_PASSWORD', + 'DB_PASSWORD', + ), 'redis.env': ('REDIS_PASSWORD',), 'netbox-secret.env': ( - 'DB_PASSWORD', - 'REDIS_CACHE_PASSWORD', - 'REDIS_PASSWORD', 'SECRET_KEY', 'SUPERUSER_PASSWORD', 'SUPERUSER_API_TOKEN', @@ -1918,24 +1917,24 @@ def authSetup(): netboxKeyAlphabet = string.ascii_letters + string.digits + '%@<=>?~^_-' netboxPostGresPassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) redisPassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) - redisCachePassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) netboxSuPassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) netboxSuToken = ''.join(secrets.choice(netboxPwAlphabet) for i in range(40)) netboxSecretKey = ''.join(secrets.choice(netboxKeyAlphabet) for i in range(50)) with open('netbox-postgres.env', 'w') as f: + f.write('DB_HOST=netbox-postgres\n') f.write('POSTGRES_DB=netbox\n') - f.write(f'POSTGRES_PASSWORD={netboxPostGresPassword}\n') + f.write('DB_NAME=netbox\n') f.write('POSTGRES_USER=netbox\n') + f.write('DB_USER=netbox\n') + f.write(f'POSTGRES_PASSWORD={netboxPostGresPassword}\n') + f.write(f'DB_PASSWORD={netboxPostGresPassword}\n') f.write('K8S_SECRET=True\n') os.chmod('netbox-postgres.env', stat.S_IRUSR | stat.S_IWUSR) - with open('redis-cache.env', 'w') as f: - f.write(f'REDIS_PASSWORD={redisCachePassword}\n') - f.write('K8S_SECRET=True\n') - os.chmod('redis-cache.env', stat.S_IRUSR | stat.S_IWUSR) - with open('redis.env', 'w') as f: + f.write(f'REDIS_HOST=redis\n') + f.write(f'REDIS_CACHE_HOST=redis-cache\n') f.write(f'REDIS_PASSWORD={redisPassword}\n') f.write('K8S_SECRET=True\n') os.chmod('redis.env', stat.S_IRUSR | stat.S_IWUSR) @@ -1949,25 +1948,7 @@ def authSetup(): for line in envFile: line = line.rstrip("\n") - if line.startswith('DB_PASSWORD'): - line = re.sub( - r'(DB_PASSWORD\s*=\s*)(.*?)$', - fr"\g<1>{netboxPostGresPassword}", - line, - ) - elif line.startswith('REDIS_CACHE_PASSWORD'): - line = re.sub( - r'(REDIS_CACHE_PASSWORD\s*=\s*)(.*?)$', - fr"\g<1>{redisCachePassword}", - line, - ) - elif line.startswith('REDIS_PASSWORD'): - line = re.sub( - r'(REDIS_PASSWORD\s*=\s*)(.*?)$', - fr"\g<1>{redisPassword}", - line, - ) - elif line.startswith('SECRET_KEY'): + if line.startswith('SECRET_KEY'): line = re.sub( r'(SECRET_KEY\s*=\s*)(.*?)$', fr"\g<1>{netboxSecretKey}", diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 54a7b470c..006513af0 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -793,7 +793,6 @@ def MalcolmAuthFilesExist(configDir=None): and os.path.isfile(os.path.join(MalcolmPath, os.path.join('htadmin', 'config.ini'))) and os.path.isfile(os.path.join(configDirToCheck, 'netbox-secret.env')) and os.path.isfile(os.path.join(configDirToCheck, 'netbox-postgres.env')) - and os.path.isfile(os.path.join(configDirToCheck, 'redis-cache.env')) and os.path.isfile(os.path.join(configDirToCheck, 'redis.env')) and os.path.isfile(os.path.join(configDirToCheck, 'auth.env')) and os.path.isfile(os.path.join(MalcolmPath, '.opensearch.primary.curlrc'))