Clean up

.dock

@@ -16,17 +16,6 @@ env_var CI "{CI:-0}"
 env_var APP_UID $(user_id)
 env_var APP_GID $(group_id)
 
-# Configuration for uploading artifacts to Nexus
-env_var NEXUS_GROUP $project
-optional_env_var NEXUS_USER
-optional_env_var NEXUS_PASSWORD
-
-# Allow user to deploy an Docker image tag that has already been pushed
-optional_env_var DEPLOY_TAG
-
-# Used by Jenkins jobs to unlock Arcanus chest during deployment
-optional_env_var ARCANUS_PASSWORD
-
 # Expose Docker Hub credentials if they exist
 if [ -d "$HOME/.docker" ]; then
   volume "$HOME/.docker:/home/app/.docker"

.dockerignore

@@ -1,6 +1,4 @@
 # Exclude files matching these patterns from the Docker build context by
 # default.
-.arcanus/unprotected.key
 .git
 dock
-jenkins

.overcommit.yml

@@ -1,19 +1,10 @@
-# Don't need this for a private repo
-verify_signatures: false
+verify_signatures: true
 
 CommitMsg:
-  GerritChangeId:
-    enabled: true
-
   HardTabs:
     enabled: true
 
 PreCommit:
-  AuthorEmail:
-    enabled: true
-    description: 'Checking author email for Brigade domain'
-***REMOVED***
-
   HardTabs:
     enabled: true
     exclude:

Dockerfile

@@ -1,7 +1,4 @@
-# Defines the set of steps needed to create an image suitable for deploying the
-# Verifier to Marathon.
-
-FROM brigade/centos:7.4.1708-latest
+FROM centos:7.4.1708
 
 # Copy all source code into the container
 # See the .dockerignore file for a list of files that are excluded

README.md

@@ -4,61 +4,35 @@ This is the Elasticsearch-based system which resolves buckets of fuzzy contact
 info to one or more voter registration records. The most interesting
 entrypoints is the `match_many` routines in `matching.py`.
 
-For further documentation and rationale see:
-***REMOVED***
-
 By nature, the algorithm filters rather little and ranks rather much.
 
 In `indexing.INDEX_SETTINGS`, the `number_of_replicas` is presently 2.
 
 ## Development
+TODO: The dock integration broke when forked and opensourced, and needs to be fixed.
 
 To get started with development for the verifier, you need to install
 Docker and our `dock` utility:
 
-***REMOVED***
-
 You can then get started by running `dock` in the root of the repository:
 
 ```bash
-dock
+$ dock
 ```
 
 You'll see additional instructions after running `dock`.
 
 ## Deployment
-
-The verifier is deployed to staging after every commit merged:
-
-***REMOVED***
-
-You can manually deploy by running:
-
-```
-jenkins/deploy [staging | production]
-```
-
-To deploy an image tag that has already been pushed, specify
-the `DEPLOY_TAG` environment variable:
-
-```
-DEPLOY_TAG=2016-12-27_23-18-47_7970879 jenkins/deploy production
-```
-
-This avoids building and pushing the image (which can take time).
-See ***REMOVED*** for a list of
-available tags.
+TODO
 
 ## Testing
 
 You can run tests in a container with:
 
 ```bash
-jenkins/test
+$ make test
 ```
 
-This is the same script that Jenkins would run.
-
 ## JSON Schema
 
 To generate a new version of the schema.json file:

bin/helpers.bash

@@ -0,0 +1,83 @@
+set -euo pipefail
+
+source /***REMOVED***
+source /***REMOVED***
+
+# Need to include Python version since some packages may link against libraries
+# specific to that version
+env-artifact-name() {
+  echo "env-$(python --version 2>&1 | tr ' ' '-')"
+}
+
+install-python-packages() {
+  local artifact_name="$(env-artifact-name)"
+  local artifact_path="$(git rev-parse --show-toplevel)/env"
+  local cache_key="$(fingerprint requirements.txt)"
+  local cache_path="$HOME/.cache/$artifact_name/${cache_key}"
+
+  if [ ! -d "$artifact_path" ]; then
+    section "Installing Python packages in virtual environment..."
+    with-artifact setup-virtualenv "$cache_key" "$cache_path" "$artifact_name" "$artifact_path"
+  fi
+}
+
+activate-virtualenv() {
+  # We need to define a PS1 prompt so that this script works
+  PS1='[\h \W]\$ ' source env/bin/activate
+}
+
+setup-virtualenv() {
+  (virtualenv env && \
+    activate-virtualenv && \
+    pip install -r requirements.txt --no-cache-dir && \
+    virtualenv --relocatable env)
+}
+
+wait-for-elasticsearch() {
+  echo "Waiting for Elasticsearch..."
+  dockerize -timeout 30s -wait http://${ELASTICSEARCH_HOST:-127.0.0.1}:9200 \
+    echo "Elasticsearch ready!" && return
+
+  error "Could not connect to Elasticsearch!"
+  docker-compose logs --tail=20 elasticsearch
+  return 1
+}
+
+# We need to vary this name based on the Ruby version since native extensions
+# are compiled against the Ruby libraries present, meaning some gems are not
+# transferrable between Ruby versions.
+gems-artifact-name() {
+  echo "gems-$(cat .ruby-version)"
+}
+
+install-gems() {
+  local artifact_name="$(gems-artifact-name)"
+  local artifact_path=/usr/local/bundle
+  local cache_key="$(fingerprint Gemfile Gemfile.lock)"
+  local cache_path="$HOME/.cache/$artifact_name/${cache_key}"
+
+  section "Installing gems..."
+  with-artifact download-gems "$cache_key" "$cache_path" "$artifact_name" "$artifact_path"
+}
+
+download-gems() {
+  local artifact_name="$(gems-artifact-name)"
+  local artifact_path=/usr/local/bundle
+
+  local bundler_version=$(awk '/DEPENDENCIES/,/ bundler /' Gemfile.lock | tail -n1 | grep -oE '[0-9\.]+')
+
+  # Attempt to download latest gem bundle with the hope that it won't be too
+  # different from the one for current Gemfile. This significantly speeds up
+  # test runs which make changes to the Gemfile, since we don't generate the
+  # bundle from scratch.
+  if download-latest-artifact "$artifact_name" "$artifact_path"; then
+    echo "Downloaded latest '$artifact_name' bundle as base."
+    echo "Installing new gems on top of base and then pruning unused gems..."
+  else
+    warn "WARN: Unable to download latest '$artifact_name' artifact. Installing from scratch..."
+  fi
+
+  echo "Installing gem bundle..."
+  chronic gem install bundler --conservative --version $bundler_version
+  chronic bundle install --jobs=$(nproc) --retry=3 --clean
+}

dock/bootstrap_Dockerfile

@@ -1,7 +1,7 @@
 # Defines the set of steps needed to create an image suitable for deploying the
 # Verifier to Marathon.
 
-FROM brigade/ruby:2.3.1-latest
+FROM ruby:2.3.8-latest
 
 # Used to maintain isolated Python environment in the repo
 RUN pip install virtualenv

dock/datastore_setup

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-source /src/script/helpers.bash
+source /src/bin/helpers.bash
 
 ELASTICSEARCH_HOST=docker-host wait-for-elasticsearch

dock/development_Dockerfile

@@ -1,7 +1,7 @@
 # Defines the set up steps necessary to create an image to use Dock with this
 # repository, allowing you to easily develop or test the Verifier.
 
-FROM brigade/dock:latest
+FROM dock:latest
 
 # Used to maintain isolated Python environment in the repo
 RUN pip install virtualenv

dock/initialize

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-source /src/script/helpers.bash
+source /src/bin/helpers.bash
 
 section "Activating virtual environment..."
 cd /src

dock/run

@@ -1,7 +1,3 @@
 #!/bin/bash
 
-if [[ -n "${ARCANUS_PASSWORD}" || -f "/src/.arcanus/unprotected.key" ]]; then
-  eval "$(arcanus export --shell)"
-fi
-
 cd /src && env/bin/gunicorn web:app -b 0.0.0.0:3000 -w 8 --log-file=-

dock/start-docker

@@ -14,7 +14,6 @@ groupadd --non-unique -g $APP_GID docker
 # Start daemon and wait for it to be ready
 dockerd \
   --storage-driver=overlay \
-***REMOVED***
   --icc=false \
   >/dev/null 2>&1 &
 for i in {1..5}; do

dock/start-everything

@@ -4,7 +4,7 @@
 # It starts up core backend services and displays instructions for getting
 # started.
 
-source $(dirname $0)/../script/helpers.bash
+source $(dirname $0)/../bin/helpers.bash
 
 section "Downloading latest version of all images..."
 docker-compose pull

docker-compose.yml

@@ -13,11 +13,10 @@ services:
     environment:
       APP_UID:
       APP_GID:
-      ARCANUS_PASSWORD: ${ARCANUS_PASSWORD_VERIFIER}
       ELASTICSEARCH_HOSTS: http://docker-host:9200/
       FLASK_ENV: 'development'
     ports:
-      - '11012:3000'
+      - '3000:3000'
     volumes:
       - '.:/src'
     working_dir: '/src'
@@ -39,16 +38,15 @@ services:
 
   elasticsearch:
     extends:
-      file: $BASE_COMPOSE
       service: elasticsearch_dev_base
-    image: 'brigade/elasticsearch:1.7-latest'
+    image: 'elasticsearch:1.7-latest'
     command: ['elasticsearch', '--action.write_consistency=one']
 
   elasticsearch_test:
     extends:
       file: $BASE_COMPOSE
       service: elasticsearch_test_base
-    image: 'brigade/elasticsearch:1.7-latest'
+    image: 'elasticsearch:1.7-latest'
     command: ['elasticsearch', '--action.write_consistency=one']
 
 volumes: