From eeb54018f2047c54836ad98f37fc75391817f83a Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 13 May 2022 15:52:43 -0600 Subject: [PATCH 01/12] bump development for v6.0.1 --- README.md | 70 +++++++++++++++++------------------ docker-compose-standalone.yml | 34 ++++++++--------- docker-compose.yml | 34 ++++++++--------- docs/web/download.md | 4 +- sensor-iso/README.md | 2 +- 5 files changed, 72 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index ee376e9ca..8f417dc5c 100644 --- a/README.md +++ b/README.md @@ -184,23 +184,23 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 6.0.0 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 6.0.0 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 6.0.0 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 6.0.0 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/filebeat-oss 6.0.0 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/file-monitor 6.0.0 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 6.0.0 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/freq 6.0.0 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 6.0.0 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 6.0.0 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 6.0.0 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/nginx-proxy 6.0.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 6.0.0 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 6.0.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 6.0.0 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/suricata 6.0.0 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 6.0.0 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 6.0.1 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 6.0.1 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 6.0.1 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 6.0.1 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/filebeat-oss 6.0.1 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/file-monitor 6.0.1 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 6.0.1 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/freq 6.0.1 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 6.0.1 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 6.0.1 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 6.0.1 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/nginx-proxy 6.0.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 6.0.1 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 6.0.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 6.0.1 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/suricata 6.0.1 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 6.0.1 xxxxxxxxxxxx 3 days ago 1GB ``` #### Import from pre-packaged tarballs @@ -3427,7 +3427,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-6.0.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-6.0.1.iso" … ``` @@ -3834,23 +3834,23 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/api 6.0.0 xxxxxxxxxxxx 3 days ago 158MB -malcolmnetsec/arkime 6.0.0 xxxxxxxxxxxx 3 days ago 816MB -malcolmnetsec/dashboards 6.0.0 xxxxxxxxxxxx 3 days ago 1.02GB -malcolmnetsec/dashboards-helper 6.0.0 xxxxxxxxxxxx 3 days ago 184MB -malcolmnetsec/filebeat-oss 6.0.0 xxxxxxxxxxxx 3 days ago 624MB -malcolmnetsec/file-monitor 6.0.0 xxxxxxxxxxxx 3 days ago 588MB -malcolmnetsec/file-upload 6.0.0 xxxxxxxxxxxx 3 days ago 259MB -malcolmnetsec/freq 6.0.0 xxxxxxxxxxxx 3 days ago 132MB -malcolmnetsec/htadmin 6.0.0 xxxxxxxxxxxx 3 days ago 242MB -malcolmnetsec/logstash-oss 6.0.0 xxxxxxxxxxxx 3 days ago 1.35GB -malcolmnetsec/name-map-ui 6.0.0 xxxxxxxxxxxx 3 days ago 143MB -malcolmnetsec/nginx-proxy 6.0.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/opensearch 6.0.0 xxxxxxxxxxxx 3 days ago 1.17GB -malcolmnetsec/pcap-capture 6.0.0 xxxxxxxxxxxx 3 days ago 121MB -malcolmnetsec/pcap-monitor 6.0.0 xxxxxxxxxxxx 3 days ago 213MB -malcolmnetsec/suricata 6.0.0 xxxxxxxxxxxx 3 days ago 278MB -malcolmnetsec/zeek 6.0.0 xxxxxxxxxxxx 3 days ago 1GB +malcolmnetsec/api 6.0.1 xxxxxxxxxxxx 3 days ago 158MB +malcolmnetsec/arkime 6.0.1 xxxxxxxxxxxx 3 days ago 816MB +malcolmnetsec/dashboards 6.0.1 xxxxxxxxxxxx 3 days ago 1.02GB +malcolmnetsec/dashboards-helper 6.0.1 xxxxxxxxxxxx 3 days ago 184MB +malcolmnetsec/filebeat-oss 6.0.1 xxxxxxxxxxxx 3 days ago 624MB +malcolmnetsec/file-monitor 6.0.1 xxxxxxxxxxxx 3 days ago 588MB +malcolmnetsec/file-upload 6.0.1 xxxxxxxxxxxx 3 days ago 259MB +malcolmnetsec/freq 6.0.1 xxxxxxxxxxxx 3 days ago 132MB +malcolmnetsec/htadmin 6.0.1 xxxxxxxxxxxx 3 days ago 242MB +malcolmnetsec/logstash-oss 6.0.1 xxxxxxxxxxxx 3 days ago 1.35GB +malcolmnetsec/name-map-ui 6.0.1 xxxxxxxxxxxx 3 days ago 143MB +malcolmnetsec/nginx-proxy 6.0.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/opensearch 6.0.1 xxxxxxxxxxxx 3 days ago 1.17GB +malcolmnetsec/pcap-capture 6.0.1 xxxxxxxxxxxx 3 days ago 121MB +malcolmnetsec/pcap-monitor 6.0.1 xxxxxxxxxxxx 3 days ago 213MB +malcolmnetsec/suricata 6.0.1 xxxxxxxxxxxx 3 days ago 278MB +malcolmnetsec/zeek 6.0.1 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 9b6965a7e..218e2dcd2 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -163,7 +163,7 @@ x-pcap-capture-variables: &pcap-capture-variables services: opensearch: - image: malcolmnetsec/opensearch:6.0.0 + image: malcolmnetsec/opensearch:6.0.1 restart: "no" stdin_open: false tty: true @@ -203,7 +203,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: malcolmnetsec/dashboards-helper:6.0.0 + image: malcolmnetsec/dashboards-helper:6.0.1 restart: "no" stdin_open: false tty: true @@ -233,7 +233,7 @@ services: retries: 3 start_period: 30s dashboards: - image: malcolmnetsec/dashboards:6.0.0 + image: malcolmnetsec/dashboards:6.0.1 restart: "no" stdin_open: false tty: true @@ -256,7 +256,7 @@ services: retries: 3 start_period: 210s logstash: - image: malcolmnetsec/logstash-oss:6.0.0 + image: malcolmnetsec/logstash-oss:6.0.1 restart: "no" stdin_open: false tty: true @@ -301,7 +301,7 @@ services: retries: 3 start_period: 600s filebeat: - image: malcolmnetsec/filebeat-oss:6.0.0 + image: malcolmnetsec/filebeat-oss:6.0.1 restart: "no" stdin_open: false tty: true @@ -333,7 +333,7 @@ services: retries: 3 start_period: 60s arkime: - image: malcolmnetsec/arkime:6.0.0 + image: malcolmnetsec/arkime:6.0.1 restart: "no" stdin_open: false tty: true @@ -373,7 +373,7 @@ services: retries: 3 start_period: 210s zeek: - image: malcolmnetsec/zeek:6.0.0 + image: malcolmnetsec/zeek:6.0.1 restart: "no" stdin_open: false tty: true @@ -404,7 +404,7 @@ services: retries: 3 start_period: 60s suricata: - image: malcolmnetsec/suricata:6.0.0 + image: malcolmnetsec/suricata:6.0.1 restart: "no" stdin_open: false tty: true @@ -429,7 +429,7 @@ services: retries: 3 start_period: 120s file-monitor: - image: malcolmnetsec/file-monitor:6.0.0 + image: malcolmnetsec/file-monitor:6.0.1 restart: "no" stdin_open: false tty: true @@ -454,7 +454,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: malcolmnetsec/pcap-capture:6.0.0 + image: malcolmnetsec/pcap-capture:6.0.1 restart: "no" stdin_open: false tty: true @@ -474,7 +474,7 @@ services: volumes: - ./pcap/upload:/pcap pcap-monitor: - image: malcolmnetsec/pcap-monitor:6.0.0 + image: malcolmnetsec/pcap-monitor:6.0.1 restart: "no" stdin_open: false tty: true @@ -499,7 +499,7 @@ services: retries: 3 start_period: 90s upload: - image: malcolmnetsec/file-upload:6.0.0 + image: malcolmnetsec/file-upload:6.0.1 restart: "no" stdin_open: false tty: true @@ -527,7 +527,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:6.0.0 + image: malcolmnetsec/htadmin:6.0.1 restart: "no" stdin_open: false tty: true @@ -551,7 +551,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:6.0.0 + image: malcolmnetsec/freq:6.0.1 restart: "no" stdin_open: false tty: true @@ -571,7 +571,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:6.0.0 + image: malcolmnetsec/name-map-ui:6.0.1 restart: "no" stdin_open: false tty: true @@ -594,7 +594,7 @@ services: retries: 3 start_period: 60s api: - image: malcolmnetsec/api:6.0.0 + image: malcolmnetsec/api:6.0.1 command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -614,7 +614,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: malcolmnetsec/nginx-proxy:6.0.0 + image: malcolmnetsec/nginx-proxy:6.0.1 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index 8e7e56ccc..abd123bb8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -166,7 +166,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: malcolmnetsec/opensearch:6.0.0 + image: malcolmnetsec/opensearch:6.0.1 restart: "no" stdin_open: false tty: true @@ -209,7 +209,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: malcolmnetsec/dashboards-helper:6.0.0 + image: malcolmnetsec/dashboards-helper:6.0.1 restart: "no" stdin_open: false tty: true @@ -242,7 +242,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: malcolmnetsec/dashboards:6.0.0 + image: malcolmnetsec/dashboards:6.0.1 restart: "no" stdin_open: false tty: true @@ -268,7 +268,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:6.0.0 + image: malcolmnetsec/logstash-oss:6.0.1 restart: "no" stdin_open: false tty: true @@ -319,7 +319,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:6.0.0 + image: malcolmnetsec/filebeat-oss:6.0.1 restart: "no" stdin_open: false tty: true @@ -354,7 +354,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: malcolmnetsec/arkime:6.0.0 + image: malcolmnetsec/arkime:6.0.1 restart: "no" stdin_open: false tty: true @@ -400,7 +400,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:6.0.0 + image: malcolmnetsec/zeek:6.0.1 restart: "no" stdin_open: false tty: true @@ -435,7 +435,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: malcolmnetsec/suricata:6.0.0 + image: malcolmnetsec/suricata:6.0.1 restart: "no" stdin_open: false tty: true @@ -463,7 +463,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:6.0.0 + image: malcolmnetsec/file-monitor:6.0.1 restart: "no" stdin_open: false tty: true @@ -491,7 +491,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:6.0.0 + image: malcolmnetsec/pcap-capture:6.0.1 restart: "no" stdin_open: false tty: true @@ -514,7 +514,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: malcolmnetsec/pcap-monitor:6.0.0 + image: malcolmnetsec/pcap-monitor:6.0.1 restart: "no" stdin_open: false tty: true @@ -542,7 +542,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:6.0.0 + image: malcolmnetsec/file-upload:6.0.1 restart: "no" stdin_open: false tty: true @@ -570,7 +570,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:6.0.0 + image: malcolmnetsec/htadmin:6.0.1 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -597,7 +597,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:6.0.0 + image: malcolmnetsec/freq:6.0.1 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -620,7 +620,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:6.0.0 + image: malcolmnetsec/name-map-ui:6.0.1 build: context: . dockerfile: Dockerfiles/name-map-ui.Dockerfile @@ -646,7 +646,7 @@ services: retries: 3 start_period: 60s api: - image: malcolmnetsec/api:6.0.0 + image: malcolmnetsec/api:6.0.1 build: context: . dockerfile: Dockerfiles/api.Dockerfile @@ -672,7 +672,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:6.0.0 + image: malcolmnetsec/nginx-proxy:6.0.1 restart: "no" stdin_open: false tty: true diff --git a/docs/web/download.md b/docs/web/download.md index 28d0ede21..acc7d44dd 100644 --- a/docs/web/download.md +++ b/docs/web/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-6.0.0.iso](/iso/malcolm-6.0.0.iso) (4.0GiB) | [`b76907ede156e59813d332f7fe4d38d93c782a6d8265d884b5c4d39c337c5958`](/iso/malcolm-6.0.0.iso.sha256.txt) | +| [malcolm-6.0.1.iso](/iso/malcolm-6.0.1.iso) (4.0GiB) | [`xxxxxxxx`](/iso/malcolm-6.0.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-6.0.0.iso](/iso/hedgehog-6.0.0.iso) (2.3GiB) | [`a93151c37502fad1a672b3ffc228a89ec84cf5be549fa98fe0d86ad91667d7f9`](/iso/hedgehog-6.0.0.iso.sha256.txt) | +| [hedgehog-6.0.1.iso](/iso/hedgehog-6.0.1.iso) (2.3GiB) | [`xxxxxxxx`](/iso/hedgehog-6.0.1.iso.sha256.txt) | ## Warning diff --git a/sensor-iso/README.md b/sensor-iso/README.md index 76644c655..72c9ec1f6 100644 --- a/sensor-iso/README.md +++ b/sensor-iso/README.md @@ -429,7 +429,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-6.0.0.iso" +Finished, created "/sensor-build/hedgehog-6.0.1.iso" … ``` From cde946fd2f3bf58514e4f83c497f426b537f087e Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 16 May 2022 08:50:03 -0600 Subject: [PATCH 02/12] added some local environment files to .gitignore and .dockerignore --- .dockerignore | 2 ++ .gitignore | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.dockerignore b/.dockerignore index 428da8804..f2f88aa93 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,8 @@ **/.ldap_config_defaults **/htpasswd **/malcolm_*images.tar.gz +.envrc +.direnv .trigger_workflow_build .tmp docker-compose*yml diff --git a/.gitignore b/.gitignore index fd3573453..721b6a4c1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ /htadmin/metadata # development +.envrc +.direnv .vagrant malcolm_*images.tar.gz *.iso From 9a7b6ca00a910d4bd60e006fbdb1f10763cc96d9 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 17 May 2022 10:25:42 -0600 Subject: [PATCH 03/12] Fix download of docker compose from github --- scripts/install.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 1530ee688..cd2d6aa01 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -29,7 +29,7 @@ from malcolm_common import * ################################################################################################### -DOCKER_COMPOSE_INSTALL_VERSION = "1.29.2" +DOCKER_COMPOSE_INSTALL_VERSION = "2.5.0" DEB_GPG_KEY_FINGERPRINT = '0EBFCD88' # used to verify GPG key for Docker Debian repository @@ -1372,8 +1372,11 @@ def install_docker_compose(self): result = False dockerComposeCmd = 'docker-compose' - if not Which(dockerComposeCmd, debug=self.debug) and os.path.isfile('/usr/local/bin/docker-compose'): - dockerComposeCmd = '/usr/local/bin/docker-compose' + if not Which(dockerComposeCmd, debug=self.debug): + if os.path.isfile('/usr/libexec/docker/cli-plugins/docker-compose'): + dockerComposeCmd = '/usr/libexec/docker/cli-plugins/docker-compose' + elif os.path.isfile('/usr/local/bin/docker-compose'): + dockerComposeCmd = '/usr/local/bin/docker-compose' # first see if docker-compose is already installed and runnable (try non-root and root) err, out = self.run_process([dockerComposeCmd, 'version'], privileged=False) @@ -1391,14 +1394,14 @@ def install_docker_compose(self): unames = [] err, out = self.run_process((['uname', '-s'])) if (err == 0) and (len(out) > 0): - unames.append(out[0]) + unames.append(out[0].lower()) err, out = self.run_process((['uname', '-m'])) if (err == 0) and (len(out) > 0): - unames.append(out[0]) + unames.append(out[0].lower()) if len(unames) == 2: # download docker-compose from github and save it to a temporary file tempFileName = os.path.join(self.tempDirName, dockerComposeCmd) - dockerComposeUrl = f"https://github.com/docker/compose/releases/download/{DOCKER_COMPOSE_INSTALL_VERSION}/docker-compose-{unames[0]}-{unames[1]}" + dockerComposeUrl = f"https://github.com/docker/compose/releases/download/v{DOCKER_COMPOSE_INSTALL_VERSION}/docker-compose-{unames[0]}-{unames[1]}" if DownloadToFile(dockerComposeUrl, tempFileName, debug=self.debug): os.chmod(tempFileName, 493) # 493 = 0o755, mark as executable # put docker-compose into /usr/local/bin From f5b54b08194a5d546b6ad3fa176af7ccff536fba Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 17 May 2022 10:28:41 -0600 Subject: [PATCH 04/12] Fix download of docker compose from github --- malcolm-iso/config/hooks/normal/0910-agg-build.hook.chroot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/malcolm-iso/config/hooks/normal/0910-agg-build.hook.chroot b/malcolm-iso/config/hooks/normal/0910-agg-build.hook.chroot index e23802f37..6d7f7c7f4 100755 --- a/malcolm-iso/config/hooks/normal/0910-agg-build.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0910-agg-build.hook.chroot @@ -1,7 +1,7 @@ #!/bin/bash -DOCKER_COMPOSE_VER="1.29.2" -DOCKER_COMPOSE_URL="https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VER/docker-compose-$(uname -s)-$(uname -m)" +DOCKER_COMPOSE_VER="2.5.0" +DOCKER_COMPOSE_URL="https://github.com/docker/compose/releases/download/v$DOCKER_COMPOSE_VER/docker-compose-$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m)" # install docker-compose curl -o /usr/local/bin/docker-compose -sSL "$DOCKER_COMPOSE_URL" && chmod 755 /usr/local/bin/docker-compose From b68312351abe4a8b86f76e77b410f2b49cc995ca Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 17 May 2022 12:14:42 -0600 Subject: [PATCH 05/12] A few readme tweaks --- README.md | 61 +++++++++++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 8f417dc5c..2bfaf8322 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ You can help steer Malcolm's development by sharing your ideas and feedback. Ple * [STIG compliance exceptions](#STIGExceptions) * [CIS benchmark compliance exceptions](#CISExceptions) * [Known issues](#Issues) -* [Installation example using Ubuntu 20.04 LTS](#InstallationExample) +* [Installation example using Ubuntu 22.04 LTS](#InstallationExample) * [Upgrading Malcolm](#UpgradePlan) * [Modifying or Contributing to Malcolm](#Contributing) * [Forks](#Forks) @@ -140,7 +140,7 @@ See [**Building from source**](#Build) to read how you can use GitHub [workflow ### Getting Malcolm -For a `TL;DR` example of downloading, configuring, and running Malcolm on a Linux platform, see [Installation example using Ubuntu 20.04 LTS](#InstallationExample). +For a `TL;DR` example of downloading, configuring, and running Malcolm on a Linux platform, see [Installation example using Ubuntu 22.04 LTS](#InstallationExample). The scripts to control Malcolm require Python 3. The [`install.py`](#ConfigAndTuning) script requires the [requests](https://docs.python-requests.org/en/latest/) module for Python 3, and will make use of the [pythondialog](https://pythondialog.sourceforge.io/) module for user interaction (on Linux) if it is available. @@ -3614,7 +3614,7 @@ After Malcolm ingests your data (or, more specifically, after it has ingested a ![Refreshing the OpenSearch Dashboards cached index pattern](./docs/images/screenshots/dashboards_refresh_index.png) -## Installation example using Ubuntu 20.04 LTS +## Installation example using Ubuntu 22.04 LTS Here's a step-by-step example of getting [Malcolm from GitHub](https://github.com/idaholab/Malcolm/tree/main), configuring your system and your Malcolm instance, and running it on a system running Ubuntu Linux. Your mileage may vary depending on your individual system configuration, but this should be a good starting point. @@ -3645,28 +3645,27 @@ user@host:~$ cd Malcolm/ Next, run the `install.py` script to configure your system. Replace `user` in this example with your local account username, and follow the prompts. Most questions have an acceptable default you can accept by pressing the `Enter` key. Depending on whether you are installing Malcolm from the release tarball or inside of a git working copy, the questions below will be slightly different, but for the most part are the same. ``` -user@host:~/Downloads$ sudo ./install.py -Installing required packages: ['apache2-utils', 'make', 'openssl'] +user@host:~/Malcolm$ sudo ./scripts/install.py +Installing required packages: ['apache2-utils', 'make', 'openssl', 'python3-dialog'] -"docker info" failed, attempt to install Docker? (Y/n): y +"docker info" failed, attempt to install Docker? (Y/n): y Attempt to install Docker using official repositories? (Y/n): y Installing required packages: ['apt-transport-https', 'ca-certificates', 'curl', 'gnupg-agent', 'software-properties-common'] Installing docker packages: ['docker-ce', 'docker-ce-cli', 'containerd.io'] Installation of docker packages apparently succeeded -Add a non-root user to the "docker" group? (y/n): y +Add a non-root user to the "docker" group?: y Enter user account: user -Add another non-root user to the "docker" group? (y/n): n +Add another non-root user to the "docker" group?: n "docker-compose version" failed, attempt to install docker-compose? (Y/n): y Install docker-compose directly from docker github? (Y/n): y Download and installation of docker-compose apparently succeeded - fs.file-max increases allowed maximum for file handles fs.file-max= appears to be missing from /etc/sysctl.conf, append it? (Y/n): y @@ -3679,32 +3678,31 @@ fs.inotify.max_queued_events= appears to be missing from /etc/sysctl.conf, appen fs.inotify.max_user_instances increases allowed maximum monitor file watchers fs.inotify.max_user_instances= appears to be missing from /etc/sysctl.conf, append it? (Y/n): y - vm.max_map_count increases allowed maximum for memory segments vm.max_map_count= appears to be missing from /etc/sysctl.conf, append it? (Y/n): y - net.core.somaxconn increases allowed maximum for socket connections net.core.somaxconn= appears to be missing from /etc/sysctl.conf, append it? (Y/n): y - vm.swappiness adjusts the preference of the system to swap vs. drop runtime memory pages vm.swappiness= appears to be missing from /etc/sysctl.conf, append it? (Y/n): y - vm.dirty_background_ratio defines the percentage of system memory fillable with "dirty" pages before flushing vm.dirty_background_ratio= appears to be missing from /etc/sysctl.conf, append it? (Y/n): y - vm.dirty_ratio defines the maximum percentage of dirty system memory before committing everything vm.dirty_ratio= appears to be missing from /etc/sysctl.conf, append it? (Y/n): y - /etc/security/limits.d/limits.conf increases the allowed maximums for file handles and memlocked segments /etc/security/limits.d/limits.conf does not exist, create it? (Y/n): y ``` -At this point, **if you are installing from the a release tarball** you will be asked if you would like to extract the contents of the tarball and to specify the installation directory: +If you are configuring Malcolm from within a git working copy, `install.py` will now exit. Run `install.py` again like you did at the beginning of the example, only remove the `sudo` and add `--configure` to run `install.py` in "configuration only" mode. +``` +user@host:~/Malcolm$ ./scripts/install.py --configure +``` + +Alternately, if you are configuring Malcolm from the release tarball you will be asked if you would like to extract the contents of the tarball and to specify the installation directory and `install.py` will continue: ``` Extract Malcolm runtime files from /home/user/Downloads/malcolm_20190611_095410_ce2d8de.tar.gz (Y/n): y @@ -3712,11 +3710,6 @@ Enter installation path for Malcolm [/home/user/Downloads/malcolm]: /home/user/M Malcolm runtime files extracted to /home/user/Malcolm ``` -Alternatively, **if you are configuring Malcolm from within a git working copy**, `install.py` will now exit. Run `install.py` again like you did at the beginning of the example, only remove the `sudo` and add `--configure` to run `install.py` in "configuration only" mode. -``` -user@host:~/Malcolm$ ./scripts/install.py --configure -``` - Now that any necessary system configuration changes have been made, the local Malcolm instance will be configured: ``` Malcolm processes will run as UID 1000 and GID 1000. Is this OK? (Y/n): y @@ -3742,9 +3735,11 @@ Authenticate against Lightweight Directory Access Protocol (LDAP) server? (y/N): Configure OpenSearch index state management? (y/N): n +Automatically analyze all PCAP files with Suricata? (Y/n): y + Automatically analyze all PCAP files with Zeek? (Y/n): y -Perform reverse DNS lookup locally for source and destination IP addresses in Zeek logs? (y/N): n +Perform reverse DNS lookup locally for source and destination IP addresses in logs? (y/N): n Perform hardware vendor OUI lookups for MAC addresses? (Y/n): y @@ -3778,7 +3773,7 @@ Lookup extracted file hashes with VirusTotal? (y/N): n Download updated scanner signatures periodically? (Y/n): y -Should Malcolm capture network traffic to PCAP files? (y/N): y +Should Malcolm capture network traffic to PCAP files? (y/N): y Specify capture interface(s) (comma-separated): eth0 @@ -3787,28 +3782,28 @@ Capture packets using netsniff-ng? (Y/n): y Capture packets using tcpdump? (y/N): n Malcolm has been installed to /home/user/Malcolm. See README.md for more information. -Scripts for starting and stopping Malcolm and changing authentication-related settings can be found -in /home/user/Malcolm/scripts. +Scripts for starting and stopping Malcolm and changing authentication-related settings can be found in /home/user/Malcolm/scripts. ``` At this point you should **reboot your computer** so that the new system settings can be applied. After rebooting, log back in and return to the directory to which Malcolm was installed (or to which the git working copy was cloned). Now we need to [set up authentication](#AuthSetup) and generate some unique self-signed TLS certificates. You can replace `analyst` in this example with whatever username you wish to use to log in to the Malcolm web interface. ``` -user@host:~/Malcolm$ ./scripts/auth_setup -Store administrator username/password for local Malcolm access? (Y/n): +user@host:~/Malcolm$ ./scripts/auth_setup + +Store administrator username/password for local Malcolm access? (Y/n): y Administrator username: analyst -analyst password: -analyst password (again): +analyst password: +analyst password (again): -(Re)generate self-signed certificates for HTTPS access (Y/n): +(Re)generate self-signed certificates for HTTPS access (Y/n): y -(Re)generate self-signed certificates for a remote log forwarder (Y/n): +(Re)generate self-signed certificates for a remote log forwarder (Y/n): y -Store username/password for forwarding Logstash events to a secondary, external OpenSearch instance (y/N): +Store username/password for forwarding Logstash events to a secondary, external OpenSearch instance (y/N): n -Store username/password for email alert sender account (y/N): +Store username/password for email alert sender account (see https://opensearch.org/docs/latest/monitoring-plugins/alerting/monitors/#authenticate-sender-account) (y/N): n ``` For now, rather than [build Malcolm from scratch](#Build), we'll pull images from [Docker Hub](https://hub.docker.com/u/malcolmnetsec): From cdac0e82e41a0255c9c64e6ac8366a1e20395157 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 17 May 2022 12:16:29 -0600 Subject: [PATCH 06/12] A few readme tweaks --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2bfaf8322..d6b9cf81c 100644 --- a/README.md +++ b/README.md @@ -3869,7 +3869,7 @@ Attaching to malcolm_nginx-proxy_1, malcolm_dashboards_1, malcolm_filebeat_1, ma It will take several minutes for all of Malcolm's components to start up. Logstash will take the longest, probably 3 to 5 minutes. You'll know Logstash is fully ready when you see Logstash spit out a bunch of starting up messages, ending with this: ``` … -logstash_1 | [2019-06-11T15:45:42,009][INFO ][logstash.agent ] Pipelines running {:count=>4, :running_pipelines=>[:"malcolm-output", :"malcolm-input", :"malcolm-zeek", :"malcolm-enrichment"], :non_running_pipelines=>[]} +logstash_1 | [2019-06-11T15:45:42,009][INFO ][logstash.agent ] Pipelines running {:count=>5, :running_pipelines=>[:"malcolm-output", :"malcolm-input", :"malcolm-suricata", :"malcolm-zeek", :"malcolm-enrichment"], :non_running_pipelines=>[]} logstash_1 | [2019-06-11T15:45:42,599][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} … ``` From 85131135122be4bfda0a87051abf4257a057cfa5 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 23 May 2022 07:37:28 -0600 Subject: [PATCH 07/12] added corelight zeek plugins for VMware Workspace ONE Access and Identity Manager RCE vulnerability (CVE-2022-22954) and DCE/RPC remote code execution vulnerability (CVE-2022-26809) --- Dockerfiles/zeek.Dockerfile | 4 ++-- README.md | 8 +++++--- shared/bin/zeek_install_plugins.sh | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index f664e6a46..12c54b673 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -165,8 +165,8 @@ ADD shared/bin/zeek_intel_setup.sh /usr/local/bin/entrypoint.sh # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22 ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY__OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|ANALYZER_SPICY_GENISYS_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::S7comm|Zeek::TDS)" -ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 20 -ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" +ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 22 +ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" RUN mkdir -p /tmp/logs && \ cd /tmp/logs && \ diff --git a/README.md b/README.md index d6b9cf81c..2179424dd 100644 --- a/README.md +++ b/README.md @@ -266,19 +266,21 @@ Malcolm leverages the following excellent open source tools, among others. * Andrew Klaus's [zeek-httpattacks](https://github.com/precurse/zeek-httpattacks) plugin for detecting noncompliant HTTP requests * ICS protocol analyzers for Zeek published by [DHS CISA](https://github.com/cisagov/ICSNPP) and [Idaho National Lab](https://github.com/idaholab/ICSNPP) * Corelight's ["bad neighbor" (CVE-2020-16898)](https://github.com/corelight/CVE-2020-16898) plugin - * Corelight's ["OMIGOD" (CVE-2021-38647)](https://github.com/corelight/CVE-2021-38647) plugin * Corelight's ["Log4Shell" (CVE-2021-44228)](https://github.com/corelight/cve-2021-44228) plugin - * Corelight's [Microsoft Excel privilege escalation detection (CVE-2021-42292)](https://github.com/corelight/CVE-2021-42292) plugin + * Corelight's ["OMIGOD" (CVE-2021-38647)](https://github.com/corelight/CVE-2021-38647) plugin * Corelight's [Apache HTTP server 2.4.49-2.4.50 path traversal/RCE vulnerability (CVE-2021-41773)](https://github.com/corelight/CVE-2021-41773) plugin * Corelight's [bro-xor-exe](https://github.com/corelight/bro-xor-exe-plugin) plugin * Corelight's [callstranger-detector](https://github.com/corelight/callstranger-detector) plugin * Corelight's [community ID](https://github.com/corelight/zeek-community-id) flow hashing plugin + * Corelight's [DCE/RPC remote code execution vulnerability (CVE-2022-26809)](https://github.com/corelight/cve-2022-26809) plugin + * Corelight's [HTTP More Filenames](https://github.com/corelight/http-more-files-names) plugin * Corelight's [HTTP protocol stack vulnerability (CVE-2021-31166)](https://github.com/corelight/CVE-2021-31166) plugin * Corelight's [pingback](https://github.com/corelight/pingback) plugin * Corelight's [ripple20](https://github.com/corelight/ripple20) plugin * Corelight's [SIGred](https://github.com/corelight/SIGred) plugin + * Corelight's [VMware Workspace ONE Access and Identity Manager RCE vulnerability (CVE-2022-22954)](https://github.com/corelight/cve-2022-22954) plugin * Corelight's [Zerologon](https://github.com/corelight/zerologon) plugin - * Corelight's [HTTP More Filenames](https://github.com/corelight/http-more-files-names) plugin +* Corelight's [Microsoft Excel privilege escalation detection (CVE-2021-42292)](https://github.com/corelight/CVE-2021-42292) plugin * J-Gras' [Zeek::AF_Packet](https://github.com/J-Gras/zeek-af_packet-plugin) plugin * Johanna Amann's [CVE-2020-0601](https://github.com/0xxon/cve-2020-0601) ECC certificate validation plugin and [CVE-2020-13777](https://github.com/0xxon/cve-2020-13777) GnuTLS unencrypted session ticket detection plugin * Lexi Brent's [EternalSafety](https://github.com/0xl3x1/zeek-EternalSafety) plugin diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index 5faac654c..bf9d85f0e 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -92,6 +92,8 @@ ZKG_GITHUB_URLS=( "https://github.com/corelight/CVE-2021-41773" "https://github.com/corelight/CVE-2021-42292" "https://github.com/corelight/cve-2021-44228" + "https://github.com/corelight/cve-2022-22954" + "https://github.com/corelight/cve-2022-26809" "https://github.com/corelight/http-more-files-names" "https://github.com/corelight/pingback" "https://github.com/corelight/ripple20" From e77ec1a5658e0bf258b02a298df1aec623dbe46c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 23 May 2022 17:20:41 -0600 Subject: [PATCH 08/12] bump opensearch and dashboards to v1.3.2 --- Dockerfiles/dashboards.Dockerfile | 6 +++--- Dockerfiles/opensearch.Dockerfile | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index 4ed21d401..d9b6deb53 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -14,10 +14,10 @@ ENV PGROUP "dashboarder" ENV TERM xterm -ARG OPENSEARCH_VERSION="1.3.1" +ARG OPENSEARCH_VERSION="1.3.2" ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION -ARG OPENSEARCH_DASHBOARDS_VERSION="1.3.1" +ARG OPENSEARCH_DASHBOARDS_VERSION="1.3.2" ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION # base system dependencies for checking out and building plugins @@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \ # runtime ################################################################## -FROM opensearchproject/opensearch-dashboards:1.3.1 +FROM opensearchproject/opensearch-dashboards:1.3.2 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index e48d279e9..f44d84cf7 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:1.3.1 +FROM opensearchproject/opensearch:1.3.2 # Copyright (c) 2022 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" @@ -40,15 +40,18 @@ RUN yum install -y openssl util-linux procps && \ echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0\ncompatibility.override_main_response_version: true\nbootstrap.memory_lock: true' > /usr/share/opensearch/config/opensearch.yml && \ sed -i "s/#[[:space:]]*\([0-9]*-[0-9]*:-XX:-\(UseConcMarkSweepGC\|UseCMSInitiatingOccupancyOnly\)\)/\1/" /usr/share/opensearch/config/jvm.options && \ sed -i "s/^[0-9][0-9]*\(-:-XX:\(+UseG1GC\|G1ReservePercent\|InitiatingHeapOccupancyPercent\)\)/$($JAVA_HOME/bin/java -version 2>&1 | grep version | awk '{print $3}' | tr -d '\"' | cut -d. -f1)\1/" /usr/share/opensearch/config/jvm.options && \ - chown -R $PUSER:$PGROUP /usr/share/opensearch/config/opensearch.yml && \ - sed -i "s/^\([[:space:]]*\)\(performance-analyzer-agent-cli\)/\1# \2/" /usr/share/opensearch/opensearch-docker-entrypoint.sh && \ - sed -i '/[^#].*OPENSEARCH_HOME\/bin\/opensearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/share/opensearch/opensearch-docker-entrypoint.sh + mkdir -p /usr/share/opensearch/ca-trust && \ + chown -R $PUSER:$PGROUP /usr/share/opensearch/config/opensearch.yml /usr/share/opensearch/ca-trust && \ + sed -i "s/^\([[:space:]]*\)\([^#].*performance-analyzer-agent-cli\)/\1# \2/" /usr/share/opensearch/opensearch-docker-entrypoint.sh && \ + sed -i '/^[[:space:]]*[^#].*runOpensearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/share/opensearch/opensearch-docker-entrypoint.sh # just used for initial keystore creation ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ ADD shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/ +VOLUME ["/usr/share/opensearch/ca-trust"] + ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"] CMD ["/usr/share/opensearch/opensearch-docker-entrypoint.sh"] From a6ddc26eea350690ecff5a7627c2030eeee3d679 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 24 May 2022 14:37:52 -0600 Subject: [PATCH 09/12] bump alpine to v3.16 for docker image base (https://alpinelinux.org/posts/Alpine-3.16.0-released.html) --- Dockerfiles/dashboards-helper.Dockerfile | 2 +- Dockerfiles/name-map-ui.Dockerfile | 12 ++++++------ Dockerfiles/nginx.Dockerfile | 2 +- README.md | 6 +++--- name-map-ui/config/supervisord.conf | 2 +- shared/bin/docker-uid-gid-setup.sh | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index 667d917dd..805af2701 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15 +FROM alpine:3.16 # Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/name-map-ui.Dockerfile b/Dockerfiles/name-map-ui.Dockerfile index 421778e7f..536367205 100644 --- a/Dockerfiles/name-map-ui.Dockerfile +++ b/Dockerfiles/name-map-ui.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15 +FROM alpine:3.16 # Copyright (c) 2022 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" @@ -26,13 +26,13 @@ ENV LISTJS_VERSION v1.5.0 RUN apk update --no-cache && \ apk upgrade --no-cache && \ - apk --no-cache add bash php7 php7-fpm php7-mysqli php7-json php7-openssl php7-curl php7-fileinfo \ - php7-zlib php7-xml php7-phar php7-intl php7-dom php7-xmlreader php7-ctype php7-session \ - php7-mbstring php7-gd nginx supervisor curl inotify-tools file psmisc shadow + apk --no-cache add bash php8 php8-fpm php8-mysqli php8-json php8-openssl php8-curl php8-fileinfo \ + php8-zlib php8-xml php8-phar php8-intl php8-dom php8-xmlreader php8-ctype php8-session \ + php8-mbstring php8-gd nginx supervisor curl inotify-tools file psmisc shadow COPY name-map-ui/config/nginx.conf /etc/nginx/nginx.conf -COPY name-map-ui/config/fpm-pool.conf /etc/php7/php-fpm.d/www.conf -COPY name-map-ui/config/php.ini /etc/php7/conf.d/custom.ini +COPY name-map-ui/config/fpm-pool.conf /etc/php8/php-fpm.d/www.conf +COPY name-map-ui/config/php.ini /etc/php8/conf.d/custom.ini COPY name-map-ui/config/supervisord.conf /etc/supervisord.conf COPY name-map-ui/config/supervisor_logstash_ctl.conf /etc/supervisor/logstash/supervisord.conf COPY name-map-ui/scripts/*.sh /usr/local/bin/ diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile index 6ac145a78..f2d964b55 100644 --- a/Dockerfiles/nginx.Dockerfile +++ b/Dockerfiles/nginx.Dockerfile @@ -7,7 +7,7 @@ # jwilder/nginx-proxy - https://github.com/jwilder/nginx-proxy/blob/master/Dockerfile.alpine #################################################################################### -FROM alpine:3.15 +FROM alpine:3.16 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/README.md b/README.md index 2179424dd..a0b0a3134 100644 --- a/README.md +++ b/README.md @@ -410,7 +410,7 @@ Then, go take a walk or something since it will be a while. When you're done, yo * `malcolmnetsec/api` (based on `python:3-slim`) * `malcolmnetsec/arkime` (based on `debian:11-slim`) -* `malcolmnetsec/dashboards-helper` (based on `alpine:3.15`) +* `malcolmnetsec/dashboards-helper` (based on `alpine:3.16`) * `malcolmnetsec/dashboards` (based on `opensearchproject/opensearch-dashboards`) * `malcolmnetsec/file-monitor` (based on `debian:11-slim`) * `malcolmnetsec/file-upload` (based on `debian:11-slim`) @@ -418,8 +418,8 @@ Then, go take a walk or something since it will be a while. When you're done, yo * `malcolmnetsec/freq` (based on `debian:11-slim`) * `malcolmnetsec/htadmin` (based on `debian:11-slim`) * `malcolmnetsec/logstash-oss` (based on `opensearchproject/logstash-oss-with-opensearch-output-plugin`) -* `malcolmnetsec/name-map-ui` (based on `alpine:3.15`) -* `malcolmnetsec/nginx-proxy` (based on `alpine:3.15`) +* `malcolmnetsec/name-map-ui` (based on `alpine:3.16`) +* `malcolmnetsec/nginx-proxy` (based on `alpine:3.16`) * `malcolmnetsec/opensearch` (based on `opensearchproject/opensearch`) * `malcolmnetsec/pcap-capture` (based on `debian:11-slim`) * `malcolmnetsec/pcap-monitor` (based on `debian:11-slim`) diff --git a/name-map-ui/config/supervisord.conf b/name-map-ui/config/supervisord.conf index 646fdec97..e2a1735fe 100644 --- a/name-map-ui/config/supervisord.conf +++ b/name-map-ui/config/supervisord.conf @@ -17,7 +17,7 @@ supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface serverurl=unix:///tmp/supervisor-main.sock [program:php-fpm] -command=/usr/sbin/php-fpm7 -F +command=/usr/sbin/php-fpm8 -F stopasgroup=true killasgroup=true stdout_logfile=/dev/fd/1 diff --git a/shared/bin/docker-uid-gid-setup.sh b/shared/bin/docker-uid-gid-setup.sh index e2e64474d..908d22061 100755 --- a/shared/bin/docker-uid-gid-setup.sh +++ b/shared/bin/docker-uid-gid-setup.sh @@ -36,7 +36,7 @@ else fi # execute the entrypoint command specified -su --shell /bin/bash --preserve-environment ${EXEC_USER} << EOF +su -s /bin/bash -p ${EXEC_USER} << EOF export USER="${EXEC_USER}" export HOME="${USER_HOME}" whoami From a0debe34830a13e9094b1c5bead566cec5faeabe Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 24 May 2022 15:00:52 -0600 Subject: [PATCH 10/12] added requests to dashboards-helper --- Dockerfiles/dashboards-helper.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index 805af2701..6ef03995f 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -73,7 +73,7 @@ RUN apk update --no-cache && \ apk upgrade --no-cache && \ apk --no-cache add bash python3 py3-pip curl procps psmisc npm shadow jq && \ npm install -g http-server && \ - pip3 install supervisor humanfriendly && \ + pip3 install supervisor humanfriendly requests && \ curl -fsSLO "$SUPERCRONIC_URL" && \ echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \ chmod +x "$SUPERCRONIC" && \ From c1c3babde6ff79a33ea9e86e4cf2980d91cabddd Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 24 May 2022 21:25:49 -0600 Subject: [PATCH 11/12] replace deprecated JAVA_HOME variables --- Dockerfiles/logstash.Dockerfile | 4 ++-- Dockerfiles/opensearch.Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index f34a657fd..c7beb572a 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -41,7 +41,7 @@ RUN cd /opt && \ mkdir -p ./logstash-filter-fingerprint && \ curl -sSL "$FINGERPRINT_URL" | tar xzvf - -C ./logstash-filter-fingerprint --strip-components 1 && \ sed -i "s/\('logstash-mixin-ecs_compatibility_support'\),.*/\1/" ./logstash-filter-fingerprint/logstash-filter-fingerprint.gemspec && \ - /bin/bash -lc "export JAVA_HOME=$(realpath $(dirname $(find /usr/lib/jvm -name javac -type f))/../) && cd /opt/logstash-filter-fingerprint && ( bundle install || bundle install ) && gem build logstash-filter-fingerprint.gemspec && bundle info logstash-filter-fingerprint" + /bin/bash -lc "export LS_JAVA_HOME=$(realpath $(dirname $(find /usr/lib/jvm -name javac -type f))/../) && cd /opt/logstash-filter-fingerprint && ( bundle install || bundle install ) && gem build logstash-filter-fingerprint.gemspec && bundle info logstash-filter-fingerprint" FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.3 @@ -75,7 +75,7 @@ ENV LOGSTASH_PARSE_PIPELINE_ADDRESSES $LOGSTASH_PARSE_PIPELINE_ADDRESSES ENV LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_INTERNAL $LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_INTERNAL ENV LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL $LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL ENV LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES $LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES -ENV JAVA_HOME=/usr/share/logstash/jdk +ENV LS_JAVA_HOME=/usr/share/logstash/jdk USER root diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index f44d84cf7..384418ee1 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -26,7 +26,7 @@ ENV MALCOLM_API_URL $MALCOLM_API_URL ARG DISABLE_INSTALL_DEMO_CONFIG=true ENV DISABLE_INSTALL_DEMO_CONFIG $DISABLE_INSTALL_DEMO_CONFIG -ENV JAVA_HOME=/usr/share/opensearch/jdk +ENV OPENSEARCH_JAVA_HOME=/usr/share/opensearch/jdk USER root @@ -39,7 +39,7 @@ RUN yum install -y openssl util-linux procps && \ /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer --purge && \ echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0\ncompatibility.override_main_response_version: true\nbootstrap.memory_lock: true' > /usr/share/opensearch/config/opensearch.yml && \ sed -i "s/#[[:space:]]*\([0-9]*-[0-9]*:-XX:-\(UseConcMarkSweepGC\|UseCMSInitiatingOccupancyOnly\)\)/\1/" /usr/share/opensearch/config/jvm.options && \ - sed -i "s/^[0-9][0-9]*\(-:-XX:\(+UseG1GC\|G1ReservePercent\|InitiatingHeapOccupancyPercent\)\)/$($JAVA_HOME/bin/java -version 2>&1 | grep version | awk '{print $3}' | tr -d '\"' | cut -d. -f1)\1/" /usr/share/opensearch/config/jvm.options && \ + sed -i "s/^[0-9][0-9]*\(-:-XX:\(+UseG1GC\|G1ReservePercent\|InitiatingHeapOccupancyPercent\)\)/$($OPENSEARCH_JAVA_HOME/bin/java -version 2>&1 | grep version | awk '{print $3}' | tr -d '\"' | cut -d. -f1)\1/" /usr/share/opensearch/config/jvm.options && \ mkdir -p /usr/share/opensearch/ca-trust && \ chown -R $PUSER:$PGROUP /usr/share/opensearch/config/opensearch.yml /usr/share/opensearch/ca-trust && \ sed -i "s/^\([[:space:]]*\)\([^#].*performance-analyzer-agent-cli\)/\1# \2/" /usr/share/opensearch/opensearch-docker-entrypoint.sh && \ From 259abfae055efe60cf17b0a07f151fc4ef485af3 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 25 May 2022 07:55:25 -0600 Subject: [PATCH 12/12] update references for a CVE --- logstash/maps/notice_authors.yaml | 9 +++++---- logstash/maps/notice_license.yaml | 7 ++++--- logstash/maps/notice_reference.yaml | 9 +++++---- logstash/pipelines/zeek/11_zeek_logs.conf | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/logstash/maps/notice_authors.yaml b/logstash/maps/notice_authors.yaml index f9d074ec3..35f0d0f54 100644 --- a/logstash/maps/notice_authors.yaml +++ b/logstash/maps/notice_authors.yaml @@ -1,7 +1,8 @@ -"EternalSafety": "Lexi Brent" "ATTACK": "MITRE" -"HTTPATTACKS": "Andrew Klaus" "Corelight": "Corelight" -"SNIFFPASS": "Andrew Klaus" "CVE_2020_0601": "Johanna Amann" -"CVE_2020_13777": "Johanna Amann" \ No newline at end of file +"CVE_2020_13777": "Johanna Amann" +"EternalSafety": "Lexi Brent" +"HTTPATTACKS": "Andrew Klaus" +"SNIFFPASS": "Andrew Klaus" +"VMWareRCE2022": "Corelight" \ No newline at end of file diff --git a/logstash/maps/notice_license.yaml b/logstash/maps/notice_license.yaml index 5f7b6f66c..5d3c0ed18 100644 --- a/logstash/maps/notice_license.yaml +++ b/logstash/maps/notice_license.yaml @@ -1,7 +1,8 @@ -"EternalSafety": "BSD-3-Clause License" "ATTACK": " BSD-3-Clause License" -"HTTPATTACKS": "BSD-2-Clause License" -"SNIFFPASS": "BSD-3-Clause License" "Corelight": "https://github.com/corelight" "CVE_2020_0601": "https://raw.githubusercontent.com/0xxon/cve-2020-0601/master/COPYING" "CVE_2020_13777": "https://raw.githubusercontent.com/0xxon/cve-2020-13777/master/COPYING" +"EternalSafety": "BSD-3-Clause License" +"HTTPATTACKS": "BSD-2-Clause License" +"SNIFFPASS": "BSD-3-Clause License" +"VMWareRCE2022": "https://github.com/corelight" diff --git a/logstash/maps/notice_reference.yaml b/logstash/maps/notice_reference.yaml index 2e4f215e1..9b9b39341 100644 --- a/logstash/maps/notice_reference.yaml +++ b/logstash/maps/notice_reference.yaml @@ -1,7 +1,8 @@ -"EternalSafety": "https://github.com/0xl3x1/zeek-EternalSafety" "ATTACK": "https://github.com/mitre-attack/bzar" -"HTTPATTACKS": "https://github.com/precurse/zeek-httpattacks" "Corelight": "https://github.com/corelight" -"SNIFFPASS": "https://github.com/cybera/zeek-sniffpass" "CVE_2020_0601": "https://github.com/0xxon/cve-2020-0601" -"CVE_2020_13777": "https://github.com/0xxon/cve-2020-13777" \ No newline at end of file +"CVE_2020_13777": "https://github.com/0xxon/cve-2020-13777" +"EternalSafety": "https://github.com/0xl3x1/zeek-EternalSafety" +"HTTPATTACKS": "https://github.com/precurse/zeek-httpattacks" +"SNIFFPASS": "https://github.com/cybera/zeek-sniffpass" +"VMWareRCE2022": "https://github.com/corelight" \ No newline at end of file diff --git a/logstash/pipelines/zeek/11_zeek_logs.conf b/logstash/pipelines/zeek/11_zeek_logs.conf index fbcc4c2e4..38cba1c8d 100644 --- a/logstash/pipelines/zeek/11_zeek_logs.conf +++ b/logstash/pipelines/zeek/11_zeek_logs.conf @@ -4877,6 +4877,7 @@ filter { if (([zeek][notice][category] =~ /^CVE/) or ([zeek][notice][category] == "EternalSafety") or ([zeek][notice][category] == "Ripple20") or + ([zeek][notice][category] == "VMWareRCE2022") or ([zeek][notice][category] == "Zerologon")) { # ECS - zeek.notice.category -> vulnerability.* (https://www.elastic.co/guide/en/ecs/current/ecs-vulnerability.html) mutate { id => "mutate_merge_ecs_vulnerability_category_notice_category"