Merge pull request #8683 from cfpb/cfgov-helmed

Add CFGOV Helm Charts for Local Deployment to K8s

.dockerignore

@@ -17,10 +17,12 @@
 !dump-data.sh
 !Dockerfile
 !frontend.sh
+!index.sh
 !initial-data.sh
 !jest.config.js
 !package.json
 !refresh-data.sh
+!test.sql.gz
 !yarn.lock
 
 # ...but DO ignore these others

.github/workflows/helm.yml

@@ -0,0 +1,21 @@
+name: Helm testing
+on:
+  pull_request:
+    paths:
+      - helm/**
+
+jobs:
+  helm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Install Helm dependencies
+        run: helm dependency update helm
+
+      - name: Lint Helm chart
+        run: helm lint --strict -f helm/values.local.yaml helm

.gitignore

@@ -145,6 +145,10 @@ cfgov/regulations3k/jinja2/regulations3k/regulations3k-service-worker.js.map
 cfgov/regulations3k/jinja2/regulations3k/workbox-*.js
 cfgov/regulations3k/jinja2/regulations3k/workbox-*.js.map
 
+# Helm Charts #
+helm/apache/
+helm/charts/
+
 # Apache #
 ##########
 cfgov/apache/logs

.prettierignore

@@ -6,6 +6,7 @@ cfgov/retirement_api/data/
 cfgov/static_built/
 **/fixtures/
 collectstatic/
+helm/
 
 # Ignore all HTML files:
 *.html

Dockerfile

@@ -150,6 +150,10 @@ ENV ALLOWED_HOSTS '["*"]'
 # Copy the application code over
 COPY cfgov ./cfgov/
 COPY static.in ./static.in/
+COPY refresh-data.sh .
+COPY initial-data.sh .
+COPY index.sh .
+COPY test.sql.gz .
 
 # Copy our static build over from node-builder
 COPY --from=node-builder ${APP_HOME} ${APP_HOME}
@@ -170,4 +174,4 @@ RUN chown -R ${USERNAME}:${USERNAME} ${APP_HOME}
 USER $USERNAME
 
 # Run Gunicorn
-CMD gunicorn cfgov.wsgi:application -b :8000
+CMD gunicorn --reload cfgov.wsgi:application -b :8000

cfgov/apache/dockerfile

@@ -0,0 +1,24 @@
+FROM httpd:2.4-alpine
+
+# Copy the Apache configuration file
+COPY ./conf /usr/local/apache2/conf
+COPY ./conf.d /usr/local/apache2/conf.d
+COPY ./conf.modules.d /usr/local/apache2/conf.modules.d
+
+# Add Apache ENV variables
+ENV APACHE_PORT="80"
+ENV APACHE_USER="www-data"
+ENV APACHE_GROUP="www-data"
+ENV APACHE_SERVER_ROOT="/usr/local/apache2/"
+ENV APACHE_UPLOADS_F_ALIAS="/src/consumerfinance.gov/cfgov/f/"
+ENV STATIC_PATH="/tmp"
+ENV ERROR_LOG="/proc/self/fd/2"
+ENV ACCESS_LOG="/proc/self/fd/1"
+ENV LIMIT_REQUEST_BODY="0"
+ENV APACHE_STATUS_ALLOW_FROM="127.0.0.1"
+ENV APACHE_PROCESS_COUNT="4"
+ENV CFGOV_APPLICATION_HOST="localhost"
+
+EXPOSE 80
+
+CMD ["httpd-foreground"]

cfgov/cfgov/settings/base.py

@@ -368,6 +368,7 @@
                 os.getenv("ES_USER", "admin"),
                 os.getenv("ES_PASS", "admin"),
             ),
+            "use_ssl": True, 
             "verify_certs": False,
         }
     }

cfgov/cfgov/settings/local.py

@@ -102,3 +102,6 @@
 # If DEPLOY_ENVIRONMENT hasn't been set by the environment in base.py,
 # default it to local.
 DEPLOY_ENVIRONMENT = DEPLOY_ENVIRONMENT or "local"
+
+# Disable use_ssl for functional tests
+OPENSEARCH_DSL["default"]["use_ssl"] = False

cfgov/cfgov/settings/test.py

@@ -66,6 +66,10 @@
 OPENSEARCH_DSL_AUTO_REFRESH = False
 OPENSEARCH_DSL_AUTOSYNC = False
 
+# Disable `use_ssl` for unit tests
+OPENSEARCH_DSL["default"]["use_ssl"] = False
+
+
 SKIP_DJANGO_MIGRATIONS = os.getenv("SKIP_DJANGO_MIGRATIONS", False)
 if SKIP_DJANGO_MIGRATIONS:
     for _db in DATABASES.values():

docs/running-k8s.md

@@ -0,0 +1,113 @@
+# Running in Kubernetes
+
+This document provides a comprehensive guide to running in Kubernetes using Helm.
+
+## Table of Contents
+
+1. [Introduction](#introduction)
+2. [Requirements](#requirements)
+3. [Helm Chart](#helm-chart)
+   - [chart.yaml](#chartyaml)
+   - [values.yaml](#valuesyaml)
+   - [templates/NOTES.txt](#templatesnotestxt)
+4. [Deploying the Helm Chart](#deploying-the-helm-chart)
+5. [Uninstalling the Helm Deployment](#uninstalling-the-helm-deployment)
+
+## Introduction
+
+This guide explains the setup and deployment process for the CFPB Helm Chart, which includes PostgreSQL, OpenSearch, and the CFGOV application. The Helm chart is configured to support local deployments.
+
+## Requirements
+
+1. **Local Kubernetes Cluster**:
+
+    - Ensure you have a local Kubernetes cluster running. We only support [Docker Desktop](https://www.docker.com/products/docker-desktop) and [colima](https://github.com/abiosoft/colima).
+
+2. **kubectl**:
+
+    - Install `kubectl`, the Kubernetes command-line tool, by following the [official installation guide](https://kubernetes.io/docs/tasks/tools/install-kubectl/).
+
+3. **Helm**:
+
+    - Install Helm, the package manager for Kubernetes, by following the [official installation guide](https://helm.sh/docs/intro/install/).
+
+## Helm Chart
+
+### chart.yaml
+
+In our `chart.yaml` we bring two Helm chart dependencies:
+
+- Opensearch
+- Postgresql
+
+### values.yaml
+
+The `values.yaml` file contains our local configuration for the deployment of CFGOV.
+
+#### Init Containers
+
+We have two init containers:
+
+1. Busybox:
+
+    - Starts up once we have our Postgresql and Opensearch Pods running.
+    - Serves as a pre-cursor to our cfgov-migrations container.
+
+2. cfgov-migrations
+
+    - Runs the django migrations and the `refresh-data.sh` script and populates it with test data `test.sql.gz`.
+    - Indexes our database to Opensearch.
+
+#### Containers
+
+1. cfgov:
+
+    - Uses the cfgov image, found in the repo's root `dockerfile`
+    - Serves up our Django application on Port 8000 of the cfgov pod
+
+2. cfgov-apache:
+
+    - Built from `cfgov-apache` image built from the `apache/dockerfile`
+    - Serves as a webserver to proxy to our cfgov container
+
+### notes.txt
+
+Our [notes.txt](https://helm.sh/docs/chart_template_guide/notes_files/) is split to work for local deployments and deployments to production.
+
+```txt
+{{- if .Values.localDeployment }}
+{{ .Files.Get "notes/NOTES-local.txt" }}
+{{- else }}
+{{ .Files.Get "notes/NOTES-production.txt" }}
+{{- end }}
+```
+
+## Deploying the CFGOV Helm Chart
+
+Deploying the CFGOV Helm chart is simple with the `helm-init.sh` script.
+
+To deploy the CFGOV Helm Chart you must:
+
+1. make sure you have a the cfgov and cfgov-apache image built
+2. have a local K8s cluster running (docker desktop, colima)
+
+If these criteria are not met, the `helm-init.sh` script will not run.
+
+## Viewing the Helm Chart
+
+You can either user a K8s IDE ([lens](https://k8slens.dev/), [k9s](https://k9scli.io)) or manually portfward to view the application running.
+
+Review the output of your deployment for more information on manually portforwarding.
+
+| Pod        | Container    | Port             |
+| ---------- | ------------ | ---------------- |
+| cfgov      | cfgov        | 8000             |
+| cfgov      | cfogv-apache | 80               |
+| postgresql | postgresql   | 5432             |
+| opensearch | opensearch   | 9200 (http)      |
+|            |              | 9300 (transport) |
+|            |              | 9600 (metrics)   |
+
+## Uninstall the CFGOV Helm Chart
+
+Running `helm-uninstall.sh` will uninstall the helm deploymennt of the CFGOV Helm Chart, as well as remove Persistent Volume Claims for the Open Search Pod and the Postgresql Pod.

helm-init.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Fail if any command fails.
+set -e
+
+# Define color codes for pretty output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+OUTPUT_FILE=$(mktemp)
+
+# Check if helm is installed
+if ! command -v helm &> /dev/null; then
+    echo -e "${RED}Helm could not be found. Please install it before running this script.${NC}"
+    exit 1
+fi
+
+# Check if kubectl is installed
+if ! command -v kubectl &> /dev/null; then
+    echo -e "${RED}Kubectl could not be found. Please install it before running this script.${NC}"
+    exit 1
+fi
+
+if ! docker images -q --filter=reference='cfgov' | grep -q .; then
+    echo -e "${RED}Docker image 'cfgov' could not be found."
+    docker build . -t cfgov
+fi
+
+if ! docker images -q --filter=reference='cfgov-apache' | grep -q .; then
+    echo -e "${RED}Docker image 'cfgov-apache' could not be found."
+    docker build ./cfgov/apache/. -t cfgov-apache
+fi
+
+# Get the current Kubernetes cluster context
+CLUSTER=$(kubectl config current-context)
+
+# Check if a Kubernetes cluster is configured
+if [ -z "$CLUSTER" ]; then
+    echo -e "${RED}No Kubernetes cluster is currently configured. Please configure a cluster before running this script.${NC}"
+    exit 1
+fi
+
+echo -e "${GREEN}Current Kubernetes cluster: $CLUSTER${NC}"
+
+# Check if the current cluster is a local cluster (docker-desktop or colima)
+if [[ "$CLUSTER" != "docker-desktop" &&  "$CLUSTER" != "colima" ]]; then
+    echo -e "${YELLOW}"
+    echo -e "This script is intended to be run on a local Kubernetes cluster (i.e. docker-desktop or colima)."
+    echo -e "Your current cluster is ${RED}$CLUSTER${YELLOW}."
+    echo -e "Please switch to docker-desktop or colima before running this script."
+    echo -e "${NC}"
+    echo -e "${RED}Exiting helm-init${NC}"
+    exit 1
+fi
+
+# Define the Helm directory
+HELM_DIR="./helm"
+
+
+# Check if the ./cfgov/apache directory exists
+if [ -d "./cfgov/apache" ]; then
+    echo -e "${GREEN}Moving ./cfgov/apache to $HELM_DIR${NC}"
+    cp -r ./cfgov/apache $HELM_DIR
+else
+    echo -e "${RED}Directory ./cfgov/apache does not exist.${NC}"
+    exit 1
+fi
+
+# Check if the ./helm/charts directory exists
+if [ ! -d "./helm/charts" ]; then
+    echo -e "${GREEN}Building Helm charts...${NC}"
+    helm dependency update $HELM_DIR
+    helm dependency build $HELM_DIR
+else
+    # Update Helm dependencies if SKIP_DEP_UPDATE is not set
+    if [ -z "$SKIP_DEP_UPDATE" ]; then
+        echo -e "${GREEN}Updating Helm dependencies...${NC}"
+        helm dependency update $HELM_DIR
+    fi
+fi
+
+# Upgrade or install the Helm release
+echo -e "${GREEN}Upgrading/Installing Helm release...${NC}"
+helm upgrade --install cfgov $HELM_DIR -f $HELM_DIR/values.local.yaml >> $OUTPUT_FILE 2>&1
+
+# Remove the copied apache directory
+echo -e "${GREEN}Cleaning up...${NC}"
+rm -r $HELM_DIR/apache
+
+echo -e "${GREEN}Helm initialization script completed successfully.${NC}"
+
+cat $OUTPUT_FILE
+rm $OUTPUT_FILE

helm-uninstall.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Fail if any command fails.
+set -e
+
+# Define color codes for pretty output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
+# Check if helm is installed
+if ! command -v helm &> /dev/null; then
+    echo -e "${RED}Helm could not be found. Please install it before running this script.${NC}"
+    exit 1
+fi
+
+# Define the release name and namespace
+RELEASE_NAME="cfgov"
+NAMESPACE="default"
+
+# Uninstall the Helm release
+echo -e "${GREEN}Uninstalling Helm release ${RELEASE_NAME}...${NC}"
+helm uninstall $RELEASE_NAME --namespace $NAMESPACE
+
+# Uninstall the PVCs for the PostgreSQL and OpenSearch pods
+kubectl delete pvc data-cfgov-postgresql-0 --namespace $NAMESPACE
+kubectl delete pvc opensearch-cluster-master-opensearch-cluster-master-0 --namespace $NAMESPACE
+
+# Check if the uninstall was successful
+if [ $? -eq 0 ]; then
+    echo -e "${GREEN}Helm release ${RELEASE_NAME} uninstalled successfully.${NC}"
+else
+    echo -e "${RED}Failed to uninstall Helm release ${RELEASE_NAME}.${NC}"
+    exit 1
+fi
+
+# Optionally, delete the namespace if it was created specifically for this release
+# echo -e "${GREEN}Deleting namespace ${NAMESPACE}...${NC}"
+# kubectl delete namespace $NAMESPACE
+
+echo -e "${GREEN}Helm uninstallation script completed successfully.${NC}"

helm/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/