diff --git a/.dock b/.dock index 3931864..5d64321 100644 --- a/.dock +++ b/.dock @@ -1,8 +1,9 @@ -# vi: ft=sh - # Configures the development/test environment for the Verifier. -dockerfile Dockerfile +project=verifier + +build_context dock +dockerfile dock/Development-Dockerfile privileged true # Need extended privileges to run Docker daemon in container pull_latest true @@ -16,18 +17,32 @@ ci() { env_var APP_UID $(user_id) env_var APP_GID $(group_id) +# Configuration for uploading artifacts to Nexus +env_var NEXUS_GROUP $project +optional_env_var NEXUS_USER +optional_env_var NEXUS_PASSWORD + # Tests expect a Docker daemon to be available, so we allocate a volume so the # daemon running in the container uses a separate layer cache from the host. -volume "$(repo_path)/script/dock-start-docker-daemon:/entrypoint.d/start-docker-daemon:ro" +volume "$(repo_path)/dock/start-docker:/entrypoint.d/start-docker:ro" volume "$(container_name)_docker:/var/lib/docker:rw" -# Credentials used to push/pull images to/from the Brigade org on Docker Hub -volume "$HOME/.docker:/home/app/.docker" +# Shared amongst all Verifier-related builds to coordinate artifact uploading. +volume "${project}_tmp:/tmp" +volume "${project}_cache:/home/app/.cache" + +# Expose Docker Hub credentials if they exist +if [ -d "$HOME/.docker" ]; then + volume "$HOME/.docker:/home/app/.docker" +fi if ! ci; then publish 3000:3000 # Gunicorn - publish 9200:9200; publish 9300:9300 # Elasticsearch + publish 9200:9200 # Elasticsearch HTTP + publish 9300:9300 # Elasticsearch Transport fi # If no command is specified, start a running instance in a container -default_command script/dock-start-everything +default_command dock/start-everything + +# vi: ft=sh diff --git a/.dockerignore b/.dockerignore index 8d7f738..2fa47d2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,6 @@ # This speeds up the start time of any `dock` command. # # Whitelist files/patterns for inclusion by prepending "!" to them. -* +.git +dock +jenkins diff --git a/.overcommit.yml b/.overcommit.yml index b807251..f888757 100644 --- a/.overcommit.yml +++ b/.overcommit.yml @@ -1,5 +1,8 @@ # Ensure we use the version of Overcommit (and other gems) specified in the Gemfile -gemfile: Gemfile +gemfile: .overcommit_gems.rb + +# Don't need this for a private repo +verify_signatures: false CommitMsg: GerritChangeId: @@ -9,10 +12,6 @@ CommitMsg: enabled: true PreCommit: - ALL: - exclude: - - 'vendor/**/*' - AuthorEmail: enabled: true description: 'Checking author email for Brigade domain' @@ -20,18 +19,14 @@ PreCommit: HardTabs: enabled: true - description: 'Checking for hard tabs' exclude: - 'Makefile' JsonSyntax: enabled: true - MergeConflicts: - enabled: true - TrailingWhitespace: enabled: true -# Don't need this for a private repo -verify_signatures: false + YamlSyntax: + enabled: true diff --git a/.overcommit_gems.rb b/.overcommit_gems.rb new file mode 100644 index 0000000..126cd7c --- /dev/null +++ b/.overcommit_gems.rb @@ -0,0 +1,11 @@ +# Specifies the gems available during an Overcommit hook run. +# +# After updating this file, run: +# +# bundle install --gemfile=.overcommit_gems.rb +# +# ...to update the associated lock file. + +source 'https://rubygems.org' do + gem 'overcommit', '0.37.0' +end diff --git a/.overcommit_gems.rb.lock b/.overcommit_gems.rb.lock new file mode 100644 index 0000000..2b54c0b --- /dev/null +++ b/.overcommit_gems.rb.lock @@ -0,0 +1,19 @@ +GEM + remote: https://rubygems.org/ + specs: + childprocess (0.5.9) + ffi (~> 1.0, >= 1.0.11) + ffi (1.9.14) + iniparse (1.4.2) + overcommit (0.37.0) + childprocess (~> 0.5.8) + iniparse (~> 1.4) + +PLATFORMS + ruby + +DEPENDENCIES + overcommit (= 0.37.0)! + +BUNDLED WITH + 1.12.5 diff --git a/Dockerfile b/Dockerfile index f8e0c0c..255a51d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,8 @@ -# Defines the set up steps necessary to create an image to use Dock with this -# repository, allowing you to easily develop or test the Verifier. +# Defines the set of steps needed to create an image suitable for deploying the +# Verifier to Marathon. -FROM brigade/ruby:2.3.1-latest +FROM brigade/centos:7.2.1511-latest -# Install Docker-related software. We can't just mount the host executables in -# the container since the host may be a Mac and thus using a different build -***REMOVED*** - | tar -xzf - -C /usr/local/bin --strip-components=1 \ -***REMOVED*** - | tar -xzf - -C /usr/local/bin \ - && chmod +x /usr/local/bin/docker-compose - -RUN yum install -y \ - # Needed to install virtualenv via easy_install - python-setuptools \ - # Used by Chronos job to do monthly imports - pv \ - - # Used to maintain isolated Python environment in the repo - && easy_install virtualenv \ - && yum clean all +# Copy all source code into the container +# See the .dockerignore file for a list of files that are excluded +ADD . /src diff --git a/Gemfile b/Gemfile deleted file mode 100644 index eda8b5b..0000000 --- a/Gemfile +++ /dev/null @@ -1,7 +0,0 @@ -source 'https://rubygems.org' - -# For deploying the Verifier -***REMOVED*** - -# For running git hooks -gem 'overcommit' diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 23295b8..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,55 +0,0 @@ -GEM - remote: https://rubygems.org/ -***REMOVED*** - specs: - activesupport (4.2.6) - i18n (~> 0.7) - json (~> 1.7, >= 1.7.7) - minitest (~> 5.1) - thread_safe (~> 0.3, >= 0.3.4) - tzinfo (~> 1.1) - bli (0.1.12) - activesupport (~> 4) - chamber (~> 2.8) - colorize (~> 0) - docker-api (~> 1.22) - multipart-post (~> 2) - net-ssh (~> 3) - poseidon (~> 0) - thor (~> 0) - chamber (2.8.0) - hashie (~> 3.3) - thor (~> 0.19.1) - childprocess (0.5.9) - ffi (~> 1.0, >= 1.0.11) - colorize (0.7.7) - docker-api (1.26.2) - excon (>= 0.38.0) - json - excon (0.47.0) - ffi (1.9.10) - hashie (3.4.3) - i18n (0.7.0) - iniparse (1.4.2) - json (1.8.3) - minitest (5.8.4) - multipart-post (2.0.0) - net-ssh (3.0.2) - overcommit (0.33.0) - childprocess (~> 0.5.8) - iniparse (~> 1.4) - poseidon (0.0.5) - thor (0.19.1) - thread_safe (0.3.5) - tzinfo (1.2.2) - thread_safe (~> 0.1) - -PLATFORMS - ruby - -DEPENDENCIES - bli! - overcommit - -BUNDLED WITH - 1.11.2 diff --git a/Makefile b/Makefile index 87640ab..b00616b 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,3 @@ schema.json: test: python -m unittest discover --pattern='test*.py' - -bli: - gem install docker-api --version=1.28.0 --no-document -***REMOVED*** - -jenkins_ci: bli - DEBUG=1 bli test - -jenkins_build: bli - DEBUG=1 bli build diff --git a/bin/build b/bin/build deleted file mode 100755 index 971b9f3..0000000 --- a/bin/build +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/local/bin/dock bash - -# Builds and deploys a verifier Docker image and source code tarball, pushing -# to the Docker Hub and uploading to Nexus, respectively. -# -# Once both artifacts are uploaded, you can deploy with the bin/deploy script. - -make jenkins_build diff --git a/bin/sync-targetsmart-s3-to-hdfs b/bin/sync-targetsmart-s3-to-hdfs deleted file mode 100755 index dc4f103..0000000 --- a/bin/sync-targetsmart-s3-to-hdfs +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Usage: -***REMOVED*** -# -***REMOVED*** -# -set -euo pipefail - -latest=${1:-"20150908analysis"} -***REMOVED*** -if [ -z "${AWS_ACCESS_KEY_ID}" ]; then - echo "Make sure AWS_ACCESS_KEY_ID environment variable is set!" -***REMOVED*** - exit 1 -fi -if [ -z "${AWS_SECRET_ACCESS_KEY}" ]; then - echo "Make sure AWS_SECRET_ACCESS_KEY environment variable is set!" -***REMOVED*** - exit 1 -fi - -# Due to https://issues.apache.org/jira/browse/HADOOP-11621, we cannot simply -# run: -***REMOVED*** -# Rather, we must use a different client library to find all filenames and pass -# them as separate arguments into `hadoop distcp`. -***REMOVED*** - -# Note: To keep the filenames predictable across different versions of the -***REMOVED*** -***REMOVED*** -***REMOVED*** -# data, we will have to manually remove the old version's file from the -# top level. Note that this copies the latest data into two distinct directories -# because hdfs does not support symlinks. -ssh -t $gateway sudo -u verifier "/bin/bash -exc \" -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** diff --git a/deploy.yml b/deploy.yml deleted file mode 100644 index 86f81eb..0000000 --- a/deploy.yml +++ /dev/null @@ -1,91 +0,0 @@ -# <% $deploy_variables = {} unless defined?($deploy_variables) %> -default: &default - build: - cache: - - /app/env - commands: - - virtualenv env - - source env/bin/activate - - pip install -r requirements.txt --no-cache-dir - - virtualenv --relocatable env - - deploy: - nexus: { username: 'brigade-deployment', password: '8HBF5mMcfgkBSUp', group: 'verifier-deploy', tar_uid: 4060, tar_gid: 4060 } - docker: { docker_tag_repo: 'brigade/verifier-deploy' } - - run: - web: env/bin/gunicorn web:app -b 0.0.0.0:3000 --log-file=- - test: source env/bin/activate && make test - - dependencies: - - elasticsearch - - env: &default_env - APP_DIRECTORY: '/mnt/mesos/sandbox' - APP_UID: 4060 # pinned in cuisine/data_bags/users/verifier.json - APP_GID: 4060 # pinned in cuisine/data_bags/users/verifier.json - FLASK_ENV: 'production' - STATSD_HOST: 'docker-host' - SENTRY_DSN: 'https://c975cacd7f9148bd94ba4738fc039fb2:50e1b5fd696f444abf83c0984f6effe6@app.getsentry.com/55635' - - marathon: - verifier: &default_marathon_verifier - args: ['./env/bin/gunicorn', 'web:app', '-b', '0.0.0.0:3000', '--log-file=-'] - mem: 256 - cpus: 0.2 - instances: 2 - healthChecks: - - protocol: HTTP - intervalSeconds: 10 - timeoutSeconds: 10 - path: '/health' - maxConsecutiveFailures: 0 # don't auto-kill tasks that fail health check - container: - type: 'DOCKER' - docker: - image: '<%= $deploy_variables[:docker_image] %>' - network: 'BRIDGE' - portMappings: - - containerPort: 3000 - hostPort: 0 - protocol: 'tcp' - uris: - - '<%= $deploy_variables[:nexus_url] %>' - -test: - <<: *default - -############################################################################### -# EDGE -############################################################################### -edge: - <<: *default - - env: - <<: *default_env -***REMOVED*** - -############################################################################### -# STAGING -############################################################################### -staging: - <<: *default - - env: - <<: *default_env -***REMOVED*** - -############################################################################### -# PRODUCTION -############################################################################### -production: - <<: *default - - env: - <<: *default_env -***REMOVED*** - - marathon: - verifier: - <<: *default_marathon_verifier - instances: 4 diff --git a/dock/.dockerignore b/dock/.dockerignore new file mode 100644 index 0000000..8d7f738 --- /dev/null +++ b/dock/.dockerignore @@ -0,0 +1,5 @@ +# Exclude all files from the Docker build context by default. +# This speeds up the start time of any `dock` command. +# +# Whitelist files/patterns for inclusion by prepending "!" to them. +* diff --git a/dock/Development-Dockerfile b/dock/Development-Dockerfile new file mode 100644 index 0000000..9b5f8f4 --- /dev/null +++ b/dock/Development-Dockerfile @@ -0,0 +1,13 @@ +# Defines the set up steps necessary to create an image to use Dock with this +# repository, allowing you to easily develop or test the Verifier. + +FROM brigade/centos:7.2.1511-latest + +***REMOVED*** + | tar -xzf - -C /usr/local/bin --strip-components=1 \ +***REMOVED*** + | tar -xzf - -C /usr/local/bin \ + && chmod +x /usr/local/bin/docker-compose + +# Used to maintain isolated Python environment in the repo +RUN pip install virtualenv diff --git a/dock/start-docker b/dock/start-docker new file mode 100755 index 0000000..a3ca3ac --- /dev/null +++ b/dock/start-docker @@ -0,0 +1,31 @@ +#!/bin/bash + +# Helper script that waits for a Docker daemon to start inside the container +# before continuing + +# Don't do anything if daemon already running +docker info >/dev/null 2>&1 && exit + +# Magic: if you create a group named "docker" the Docker daemon will +# automatically set the group of the daemon's socket to "docker". Thus we create +# a group with the same GID as the app's group. +groupadd --non-unique -g $APP_GID docker + +# Start daemon and wait for it to be ready +dockerd \ + --storage-driver=overlay \ +***REMOVED*** + --icc=false \ + >/dev/null 2>&1 & +for i in {1..5}; do + if docker info >/dev/null 2>&1; then + break + fi + echo "Waiting for Docker daemon to start..." + sleep 1 +done + +if ! docker info >/dev/null 2>&1; then + echo "Unable to start Docker daemon!" + exit 1 +fi diff --git a/dock/start-everything b/dock/start-everything new file mode 100755 index 0000000..8ef13aa --- /dev/null +++ b/dock/start-everything @@ -0,0 +1,47 @@ +#!/usr/local/bin/dock bash + +# Executed when you run `dock` in the root of this repo without any arguments. +# It starts up core backend services and displays instructions for getting +# started. + +source $(dirname $0)/../script/helpers.bash + +install-python-packages + +section "Downloading latest version of all images..." +docker-compose pull + +# Ensure services are shutdown cleanly when exiting the shell +exit_handler='trap "docker-compose stop; exit" EXIT INT QUIT TERM' +eval "$exit_handler" + +section "Starting up backend services..." +docker-compose up --build -d elasticsearch gunicorn + +wait-for-elasticsearch +activate-virtualenv + +cat <` in order +# to shut down this container that we still gracefully shutdown services. +exec bash --rcfile <(echo $exit_handler) -i diff --git a/docker-compose.yml b/docker-compose.yml index dd61ad9..c0f2e46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,7 @@ services: gunicorn: depends_on: - elasticsearch - build: - context: . - dockerfile: Dockerfile + image: brigade/centos:7.2.1511-latest environment: APP_UID: APP_GID: