From a92e3132d66c2aa478a1a39ce71265d4d1103211 Mon Sep 17 00:00:00 2001 From: Jeremy Satterfield Date: Tue, 25 Aug 2020 15:57:30 +0000 Subject: [PATCH] update to more flexible workflow scripts --- .env | 15 ++++ docker/init/db/empty | 0 .../utils/adminer/plugins/fill-login-form.php | 78 ++++++++++++++++ scripts/README.md | 90 ++++++++++++++----- scripts/attach | 13 ++- scripts/check | 56 +++++++++--- scripts/cleanup | 30 +++---- scripts/console | 45 +++++++--- scripts/debug | 32 ++++--- scripts/install | 57 +++++++++--- scripts/manage | 38 ++++++++ scripts/migrate | 15 +++- scripts/restart | 5 +- scripts/restore_db | 41 +++++---- scripts/run_util | 3 +- scripts/setup | 12 ++- scripts/setup_docker | 5 +- scripts/start | 63 +++++++++++-- scripts/stop | 49 +++++++++- scripts/teardown | 5 +- scripts/test | 23 +++-- scripts/tools | 45 +++++++++- scripts/update | 25 +++--- scripts/utils/dbadmin | 65 ++++++++++++++ scripts/utils/{pgdump => dbdump} | 17 ++-- scripts/utils/flower | 56 ++++++++++++ scripts/utils/pgadmin | 59 ------------ 27 files changed, 736 insertions(+), 206 deletions(-) create mode 100644 docker/init/db/empty create mode 100644 docker/utils/adminer/plugins/fill-login-form.php create mode 100755 scripts/manage create mode 100755 scripts/utils/dbadmin rename scripts/utils/{pgdump => dbdump} (79%) create mode 100755 scripts/utils/flower delete mode 100755 scripts/utils/pgadmin diff --git a/.env b/.env index 60a5429..0d59c4f 100644 --- a/.env +++ b/.env @@ -1,3 +1,18 @@ +# How are you using Docker for development? +# full - Docker for running app and all dependencies (recommmended) +# deps - Docker for running dependencies, but running app natively +# none - Running app and all dependencies natively +DEV_DOCKER_WORKFLOW=full +#DEV_DOCKER_WORKFLOW=deps +#DEV_DOCKER_WORKFLOW=none + +## ENVIRONMENT CONFIG +# choose a debugger tool, pdb, ipdb, bpdb, etc. +#PYTHONBREAKPOINT=ipdb.set_trace +# put ipython configs somewhere persistent, so you have history between Docker reboots +# Not needed for deps or none workflows. +#IPYTHONDIR=/usr/src/app/.ipython + ## DOCKER COMPOSE SETTINGS #COMPOSE_APP_PATH=. diff --git a/docker/init/db/empty b/docker/init/db/empty new file mode 100644 index 0000000..e69de29 diff --git a/docker/utils/adminer/plugins/fill-login-form.php b/docker/utils/adminer/plugins/fill-login-form.php new file mode 100644 index 0000000..9dd1522 --- /dev/null +++ b/docker/utils/adminer/plugins/fill-login-form.php @@ -0,0 +1,78 @@ +system = $system; + $this->server = $server; + $this->name = $name; + $this->pass = $pass; + $this->database = $database; + } + + public function loginForm(){ + if(empty($_GET[DRIVER]) && empty($_GET["username"]) && empty($_GET["db"]) ){?> + > + document.addEventListener("DOMContentLoaded", function(event) { + var dr = qs("option[value='system; ?>']"); + if(dr){ dr.selected = true; } + + server)){ ?> + var s = qs("input[name='auth[server]']"); + if(s && s.value.trim() == ""){ s.value = "server ?>"; } + name)){ ?> + var l = qs("input[name='auth[username]']"); + if(l && l.value.trim() == ""){ l.value = "name ?>"; } + pass)){ ?> + var p = qs("input[name='auth[password]']"); + if(p && p.value.trim() == ""){ p.value = "pass ?>"; } + database)){ ?> + var d = qs("input[name='auth[db]']"); + if(d && d.value.trim() == ""){ d.value = "database ?>"; } + + }); + + diff --git a/scripts/README.md b/scripts/README.md index 010a97b..3654518 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,5 +1,5 @@ Developer Workflow Scripts -=========================================== +========================== The scripts in this directory are intended to improve developer workflow when working with the Docker toolchain. All scripts contain a help message outlining the purpose and usage of each @@ -8,6 +8,15 @@ quite complex, and these scripts are intended to get you kickstarted, but you li use the `docker` and `docker-compose` commands directly at some point in the near future. The [Docker docs](https://docs.docker.com/reference/) are your friend. +You can configure which Docker services are managed by these scripts by changing the +DEV_DOCKER_WORKFLOW variable in your .env file. Depending on this configuration some scripts will +act differently. + +* full - application and dependency services are all run in Docker +* deps - only dependencies are run in Docker, devs are responsible for setting up application +environment +* none - Docker is not used at all, devs are responsible for setting up all dependencies. + setup ----- @@ -16,6 +25,8 @@ This is the script intended to prepare your system by installing the Docker tool the images and restoring a fresh database for the project. Due to the installation process for Mac, you should run `setup_docker` first. +Supported Workflows: `full`, partial for `deps` + setup_docker ------------ @@ -23,6 +34,8 @@ This script will install Docker and docker-compose. On Linux systems this is cal `setup` script and does not need to be run separately. For Mac, this script should be run before running `setup`. +Supported Workflows: `full`, `deps` + start ----- @@ -36,49 +49,61 @@ If you provide the `-d` flag to detach from the services and run them in the bac be returned to your shell. You can use the `attach` script to connect to all or specific services at any time. You will need to use the `stop` script to stop services that are detached. +Supported Workflows: `full`, partial for `deps` + stop ---- This script will stop all or specfic services, whether they are detached or attached in another terminal. +Supported Workflows: `full`, partial for `deps` + update ------ This script is intended to be run when pulling down a large changeset, i.e. pulling a new branch or -latest master. It will rebuild your images and containers, installing new requirements, and run +latest main branch. It will rebuild your images and containers, install new requirements, and run migrations. +Supported Workflows: `full`, `deps`, `none` + install ------- This is intended to abstract away a large number of complexities when working with the project. You -should specify whether you installing a Python or System (apt) package and this script will -take the appropriate steps to make sure the package is installed and added to the correct -requirements file (pyproject.toml or package.json). You can provide the `--dev` and `--no-save` -flags to modify this functionality as needed. +should specify whether you installing a Python or System (apt) package and this script will take +the appropriate steps to make sure the package is installed and added to the correct requirements +file. You can provide the `--dev` and `--no-save` flags to modify this functionality as needed. + +Supported Workflows: `full`, partial for `deps` and `none` migrate ------- -Use this script to abstract some complexities in managing migrations within Docker. When run -directly, the `run` command is assumed and migrations will be run, but you can also use the `make` -and `show` commands to create and list migrations. Most options for equivalent Django commands -should translate directly, when provided after the command. +Use this script to abstract some complexities in managing migrations. When run directly, the `run` +command is assumed and migrations will be run, but you can also use the `make` and `show` commands +to create and list migrations. Most options for equivalent Django commands should translate +directly, when provided after the command. + +Supported Workflows: `full`, `deps`, `none` test ---- -This will run the appropriate unittesting suite. While it currently only supports Python/Django -tests, it is intended that in the future javascript tests will be decoupled from the build process -allowing this to manage those as well. +This will run the unittesting suite. Any additional args passed to this command are passed directly +to the test runner. + +Supported Workflows: `full`, `deps`, `none` console ------- This will allow you to connect to the services via the appropriate interactive console. Currently supported are `bash` to use bash on the primary web service, `sudo` to use bash as sudo, `python` -to use Django's shell_plus command, and `mysql` to get a MySQL console. +to use Django's shell_plus command, and `sql` to get a SQL console. + +Supported Workflows: `full`, partial for `deps` and `none` attach ------ @@ -86,6 +111,8 @@ attach This can be used to attach to a specific service's output console. Whether it's running in detached mode or you simply want to see that services output by itself in a separate terminal. +Supported Workflows: `full`, partial for `deps` + restore_db ---------- @@ -95,6 +122,16 @@ files located in `docker/init/db`. The restore files **must** exist for this scr All migrations in your current checkout of the codebase will be run at the end of the restore unless using the `--no-migrate` flag. +Supported Workflows: `full`, `deps` + +manage +------ + +This is simply a convenience command which calls Django's `manage.py` command in the appropriate +environment based on selected dev workflow and passes all arg to the command. + +Supported Workflows: `full`, `deps`, `none` + cleanup ------- @@ -109,6 +146,8 @@ cache. However, this takes up a large amount of space on your system and it's re occasionally run `cleanup --docker` to also remove any Docker artifacts that are no longer used, freeing up space on your system +Supported Workflows: `full`, `deps`, partial for `none` + teardown -------- @@ -116,6 +155,8 @@ This will utterly and completely destroy your Docker environment, including all database volumes. There should never be any side affects outside of that environment, which can be rebuilt using `setup`, but you will be waiting a while for it to rebuild. +Supported Workflows: `full`, `deps` + tools ----- @@ -129,14 +170,23 @@ This script will allow you to run certain, short lived, utilities that are not s running or developing the application, but can sometimes come in handy. See the help text of this command to see all available utilities. -### pgadmin +### flower + +This starts a web-based tool for monitoring Celery. It's recommended that you set `FLOWER_PORT` in +your .env file to expose the service on a non-random port. + +Supported Workflows: `full`, `deps` -pgAdmin4 is a web-based tool for monitoring and managing Postgres databases. You will need to -configure a server connection the first time you run it (and anytime it's volume gets deleted). Use -the network alias and default Postgres username and password found in the docker-compose file for -the db service. +### dbadmin -### pgdump +Adminer is a web-based tool for monitoring and managing databases. On pageload the default +credentials shoul dbe pre-populated. + +Supported Workflows: `full`, `deps` + +### dbdump This will dump the contents of the database to a file. By default it only dumps the `base_app` database to a gzipped file. + +Supported Workflows: `full`, `deps`, `none` diff --git a/scripts/attach b/scripts/attach index eb78650..8d2265a 100755 --- a/scripts/attach +++ b/scripts/attach @@ -1,16 +1,17 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" case $key in -h|--help|?) - echo "Attach to a running service." + echo "Watch output of a running service." echo cecho "Usage: $(basename $0) " default bold echo @@ -28,5 +29,11 @@ while [[ $# -gt 0 ]]; do shift done set -- "${POSITIONAL[@]}" +target=$1 -docker-compose logs -f $1 +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW $target) ]; then + docker-compose logs -f $target +else + cecho "Cannot attach to '$target' while DEV_DOCKER_WORKFLOW set as '$DEV_DOCKER_WORKFLOW'. Please review your .env file." red bold + exit 1 +fi diff --git a/scripts/check b/scripts/check index 9269858..bcedb71 100755 --- a/scripts/check +++ b/scripts/check @@ -1,18 +1,18 @@ #!/bin/bash -set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env -fix= +fail_fast= POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" case $key in - -f|--fix) - fix=1 + -f|--fail-fast) + fail_fast=1 ;; -h|--help|?) echo "Run system and linting checks against repository." @@ -20,6 +20,7 @@ while [[ $# -gt 0 ]]; do cecho "Usage: $(basename $0) " default bold echo echo "Options:" + echo " -f --fail-fast Stop running checks on the first failure." echo " -h --help This help message" exit 0 ;; @@ -32,12 +33,45 @@ done set -- "${POSITIONAL[@]}" args=$@ +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW app) ]; then + runner="docker-compose run --rm --no-deps -e TESTING=True -e DJANGO_SETTINGS_MODULE=base_app.settings app" +else + export TESTING=True + export DJANGO_SETTINGS_MODULE=base_app.settings + runner="" +fi + + +exit_status=0 + +run_it(){ + title=$1 + cmd=$2 + + echo + cecho "Running $title..." blue bold + $runner $cmd + status=$? -echo -cecho "Running Django System Checks..." blue bold -docker-compose exec app ./manage.py check --settings=base_app.settings.test + echo + if [ "$status" -ne 0 ]; then + cecho "$title completed with failures: status $status" red bold + if [ "$fail_fast" ]; then + exit $status + else + if [ "$exit_status" -eq 0 ]; then + exit_status=$status + fi + fi + else + cecho "$title complete" green normal + fi +} -echo -cecho "Running Linting..." blue bold -docker-compose exec app flake8 +run_it "Django System Checks" "./manage.py check" +run_it "Code Style Checks" "flake8" +run_it "Linter" "prospector --tool pylint --messages-only --die-on-tool-error" +run_it "Type Checking" "mypy ." +run_it "Security Scan" "bandit --ini setup.cfg -r ." +exit $exit_status diff --git a/scripts/cleanup b/scripts/cleanup index 8bea977..fbb7407 100755 --- a/scripts/cleanup +++ b/scripts/cleanup @@ -1,20 +1,17 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env no_input= -docker= POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" case $key in - -d|--docker) - docker=1 - ;; -y|--no-input) no_input=1 ;; @@ -23,9 +20,6 @@ while [[ $# -gt 0 ]]; do echo cecho "Usage: $(basename $0) " default bold echo - echo "Services:" - echo "$(docker-compose config --services)" - echo echo "Options:" echo " -y --no-input Assume yes for all user input" echo " -h --help This help message" @@ -50,20 +44,20 @@ if [ ! "$no_input" ]; then fi fi -echo -cecho "Stopping running services..." default bold -$script_base/stop - -echo -cecho "Deleting python cache files..." default bold -find . \( -type f -name "*.pyc" -o -type d -name __pycache__ \) -delete +if [ "$(which docker)" ] && [ ! "$DEV_DOCKER_WORKFLOW" = "none" ]; then + echo + cecho "Stopping running services..." default bold + $script_base/stop -q -if [ "$(which docker)" ]; then echo cecho "Deleting stopped Docker images, cache and dangling images..." default bold docker system prune -f echo - cecho "Dropping static file and Redis volumes..." default bold - docker volume rm -f base_app_python-packages + cecho "Dropping cache data volumes..." default bold + docker volume rm -f base_app_cache-data fi + +echo +cecho "Deleting python cache files..." default bold +find . \( -type f -name "*.pyc" -o -type d -name __pycache__ \) -delete diff --git a/scripts/console b/scripts/console index a849bf7..737bc99 100755 --- a/scripts/console +++ b/scripts/console @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env target=app @@ -21,11 +22,11 @@ while [[ $# -gt 0 ]]; do cecho "Usage: $(basename $0) []" default bold echo echo "Console Types:" - echo " bash - Bash (Default)" - echo " sudo - Bash as sudo" - echo " py | python - Django Shell Plus w/ IPython" - echo " sql | psql - PostgreSQL as root" - echo " redis - Redis" + echo " bash - Bash (Default)" + echo " sudo - Bash as sudo" + echo " py | python - Django Shell Plus w/ IPython" + echo " sql | psql - PostgreSQL as root" + echo " cache | redis - Redis" echo echo "Options:" echo " -h --help This help message" @@ -46,21 +47,31 @@ else shell="bash" fi +docker_flags= +docker_required= + case $shell in bash) - cmd="$target bash" + cmd=bash + docker_required=1 ;; sudo) - cmd="-u root $target bash" + cmd=bash + docker_flags+=" -u root" + docker_required=1 ;; py|python) - cmd="app ./manage.py shell_plus" + cmd="./manage.py shell_plus" + target=app ;; sql|psql) - cmd="-u root db psql -U postgres" + cmd="psql -U postgres" + target=db + docker_flags+=" -u root" ;; - redis) - cmd="redis redis-cli" + cache|redis) + cmd=redis-cli + target=cache ;; *) cecho "You must provide a valid shell type. Please see '$(basename $0) --help' for more information." red bold @@ -68,5 +79,11 @@ case $shell in ;; esac - -docker-compose exec -e COLUMNS="`tput cols`" -e LINES="`tput lines`" $cmd +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW $target) ]; then + docker-compose exec -e COLUMNS="`tput cols`" -e LINES="`tput lines`" $docker_flags $target $cmd +elif [ "$docker_required" ]; then + cecho "Cannot run console '$shell' on target '$target' while DEV_DOCKER_WORKFLOW set as '$DEV_DOCKER_WORKFLOW'. Please review your .env file." red bold + exit 1 +else + $cmd +fi diff --git a/scripts/debug b/scripts/debug index 28c2bba..1747fd7 100755 --- a/scripts/debug +++ b/scripts/debug @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env POSITIONAL=() while [[ $# -gt 0 ]]; do @@ -12,13 +13,16 @@ while [[ $# -gt 0 ]]; do -h|--help|?) echo "Run a specific service in interactive debug mode." echo - cecho "Usage: $(basename $0) SERVICE" default bold + cecho "Usage: $(basename $0) []" default bold + echo + echo "Services:" + echo "$(docker-compose config --services)" echo echo "Options:" echo " -h --help This help message" echo echo "Run 'docker-compose up --help' to see more options" - exit 0 + exit 0 ;; *) POSITIONAL+=("$key") @@ -28,13 +32,19 @@ while [[ $# -gt 0 ]]; do done set -- "${POSITIONAL[@]}" -service=$1 -if [[ ! $service ]]; then - cecho "You must provide the name of the service you would like to debug." red bold - exit 1; +if [ "$1" ]; then + service=$1 +else + service=app fi -cecho "Restarting '$service' in interactive debug mode. Kill the process to return to normal." yellow bold -docker-compose stop $service -docker-compose run --name ${service}_debug --rm --service-ports $service \ - && docker-compose up -d $service +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW $service) ]; then + cecho "Restarting '$service' in interactive debug mode. Kill the process to return to normal." yellow bold + docker-compose stop $service + docker-compose run --name base_app_debug-$service --rm --service-ports $service \ + && docker-compose up -d $service +else + cecho "Since the '$service' service is not managed by Docker, we cannot restart in debug mode." red bold + cecho "Please start debug mode for this servie manually or review your .env file." red bold + exit 1 +fi diff --git a/scripts/install b/scripts/install index 87a1092..ce27ab6 100755 --- a/scripts/install +++ b/scripts/install @@ -1,12 +1,14 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env no_save= no_input= +no_pkg= dev= install_type= @@ -23,6 +25,9 @@ while [[ $# -gt 0 ]]; do --dev) dev=1 ;; + --) + no_pkg=1 + ;; -h|--help|?) echo "Install a package." echo @@ -30,11 +35,11 @@ while [[ $# -gt 0 ]]; do echo echo "Package Types:" echo " py | python" - echo " apt | linux" + echo " apt | linux | system" echo echo "Options:" - echo " --dev Install package for development only (JS and Python only)" - echo " --no-save Do not save packages, install only (JS and Python only)" + echo " --dev Install package for development only (Python only)" + echo " --no-save Do not save packages, install only (Python only)" echo " -y --no-input Assume yes for all user input" echo " -h --help This help message" exit 0 @@ -51,23 +56,55 @@ install_type=$1 shift args=$@ -if [ ! "$args" ]; then +if [ ! "$args" ] && [ ! "$no_pkg" ]; then cecho "You must provide at least one package name to install." red bold exit 2 fi +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW app) ]; then + runner="docker-compose run --rm --no-deps -u root app" + in_docker=1 +else + runner="sudo" + in_docker= +fi + case $install_type in py|python) - docker-compose exec -u root app python -m pip install $args + if [ "$no_save" ]; then + $runner pip install $args + else + flags= + if [ "$dev" ]; then + flags+=" --dev" + fi + if [ "$no_input" ]; then + flags+=" --no-interaction" + fi + if [ "$no_pkg" ]; then + $runner pip install $flags $args + else + $runner pip install $flags $args + fi + fi ;; - apt|linux) + apt|linux|system) + if [ ! "$in_docker" ] && [ ! " ('Debian' 'Ubuntu')[@] " =~ "$(detect_platform)" ]; then + cecho "This script cannot handle installs for your native system, $PLATFORM." red bold + cecho "Please install the package manually or review your .env file." red bold + exit 3 + fi + + flags= if [ ! "$no_input" ]; then cecho "Linux packages are not saved upon install. If these packages are needed " yellow bold - cecho "permenantly, please make sure to update '$(resolve_path "$script_base/..")/Dockerfile' accordingly." yellow bold + cecho "permenantly, please make sure to update '$(resolve_path "$script_base/../docker")/Dockerfile' accordingly." yellow bold read -n1 -r -p "Press any key to continue..." + else + flags+=" -y" fi - docker-compose exec -u root app apt-get update - docker-compose exec -u root app apt-get install -y $args + $runner apt-get update + $runner apt-get install $flags $args ;; *) cecho "You must provide a valid package type to install. Use '$(basename $0) --help' for most information." yellow bold diff --git a/scripts/manage b/scripts/manage new file mode 100755 index 0000000..70a262b --- /dev/null +++ b/scripts/manage @@ -0,0 +1,38 @@ +#!/bin/bash +set -e +script_base=$(dirname "$0") +source $script_base/tools +cd $script_base/.. + +load_env + +POSITIONAL=() +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -h|--help|?) + echo "Execute Django's manage.py script in the configured environment." + echo + cecho "Usage: $(basename $0) [args...]" default bold + echo + echo "Options:" + echo " -h --help This help message" + echo + echo "All other args are passed directly to manage.py" + exit 0 + ;; + *) + POSITIONAL+=("$key") + ;; + esac + shift +done +set -- "${POSITIONAL[@]}" + +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW app) ]; then + runner="docker-compose run --rm app" +else + runner= +fi +$runner ./manage.py $@ + diff --git a/scripts/migrate b/scripts/migrate index 0a28bce..9514163 100755 --- a/scripts/migrate +++ b/scripts/migrate @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env cmd= case $1 in @@ -71,13 +72,19 @@ while [[ $# -gt 0 ]]; do done set -- "${POSITIONAL[@]}" +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW app) ]; then + runner="docker-compose run --rm -u $UID app" +else + runner= +fi + case $cmd in run) opts= if [ "$fake" ]; then opts+="--fake " fi - docker-compose exec app ./manage.py migrate --noinput $opts $@ + $runner ./manage.py migrate --noinput $opts $@ ;; make) opts= @@ -90,9 +97,9 @@ case $cmd in if [ "$name" ]; then opts+="--name $name " fi - docker-compose exec -u $UID app ./manage.py makemigrations $opts $@ + $runner ./manage.py makemigrations $opts $@ ;; show) - docker-compose exec app ./manage.py showmigrations $@ + $runner ./manage.py showmigrations $@ ;; esac diff --git a/scripts/restart b/scripts/restart index 0ef1765..75c72e1 100755 --- a/scripts/restart +++ b/scripts/restart @@ -1,16 +1,17 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" case $key in -h|--help|?) - echo "Stop all or specfic running Docker services." + echo "Restart all or specfic running Docker services." echo cecho "Usage: $(basename $0) [SERVICE...]" default bold exit 0 diff --git a/scripts/restore_db b/scripts/restore_db index 320d25a..0a03507 100755 --- a/scripts/restore_db +++ b/scripts/restore_db @@ -1,13 +1,13 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env no_input= no_migrate= -skip_check= POSITIONAL=() while [[ $# -gt 0 ]]; do @@ -19,9 +19,6 @@ while [[ $# -gt 0 ]]; do --no-migrate) no_migrate=1 ;; - --skip-check) - skip_check=1 - ;; -h|--help|?) echo "Drop existing database service and restore from database backups" echo @@ -30,7 +27,6 @@ while [[ $# -gt 0 ]]; do echo "Options:" echo " --no-migrate Skip running migrations" echo " -y --no-input Assume yes on all user input" - echo " --skip-check Don't check or fail if dump file missing" echo " -h --help This help message" exit 0 ;; @@ -42,23 +38,38 @@ while [[ $# -gt 0 ]]; do done set -- "${POSITIONAL[@]}" +if [ ! $(can_run_docker $DEV_DOCKER_WORKFLOW db) ]; then + cecho "This script currently only supports db restores when the db instance is managed by Docker." red bold + cecho "Please manually restore the db or review your .env file." red bold + exit 2 +fi + shopt -s nullglob dump_files=( docker/init/db/*.sql.gz docker/init/db/*.sql ) -if [ ! "$skip_check" ]; then - if (( ! ${#dump_files[@]} )); then - cecho "No database dumps were found to restore from. Please retrieve a database dump file" red bold - cecho "and place is in $(resolve_path "$script_base/../docker/init/db")." red bold +if (( ! ${#dump_files[@]} )); then + cecho "No database dumps were found to restore from. Please retrieve a database dump file" red bold + cecho "and place it in $(resolve_path "$script_base/../docker/init/db")." red bold + if [ "$no_input"]; then exit 1 + else + cecho "Otherwise the database will be created empty." red bold + read -n1 -r -p "Press 'Y' to continue or any other key to cancel..." key1 + echo + + if [ ! "$key1" ] || [[ "yY" != *"$key1"* ]]; then + cecho "Aborting." red + exit 0 + fi fi fi if [ ! "$no_input" ]; then cecho "Warning this will delete all data in your existing database service and replace" yellow bold cecho "it with data from any database dumps located in '$(resolve_path "$script_base/../docker/init/db")'." yellow bold - read -n1 -r -p "Press 'Y' to continue or any other key to cancel..." key + read -n1 -r -p "Press 'Y' to continue or any other key to cancel..." key2 + echo - if [ ! "$key" ] || [[ "yY" != *"$key"* ]]; then - echo + if [ ! "$key2" ] || [[ "yY" != *"$key2"* ]]; then cecho "Aborting." red exit 0 fi @@ -67,8 +78,8 @@ fi echo cecho "Deleting old data..." default bold $script_base/stop -docker-compose rm -svf db app -docker volume rm -f app_db-volume +docker-compose rm -svf db +docker volume rm -f base_app_db-data echo cecho "Restoring data..." default bold diff --git a/scripts/run_util b/scripts/run_util index 0adae37..60139b2 100755 --- a/scripts/run_util +++ b/scripts/run_util @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env util=$1 shift diff --git a/scripts/setup b/scripts/setup index aad13b6..412960a 100755 --- a/scripts/setup +++ b/scripts/setup @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env skip_install= skip_db= @@ -37,6 +38,13 @@ while [[ $# -gt 0 ]]; do done set -- "${POSITIONAL[@]}" +if [ "$DEV_DOCKER_WORKFLOW" == "none" ]; then + cecho "This setup script currently only supports setup when using Docker either to run" red bold + cecho "the entire app or just dependencies. You will need to manually setup your environment." red bold + cecho "If you intend to use Docker, please review your .env file." red bold + exit 1 +fi + if [ ! "$skip_install" ]; then case $(detect_platform) in OSX|OSX_BREW) @@ -64,5 +72,5 @@ docker-compose build if [ ! "$skip_db" ]; then echo cecho "Populating database..." blue bold - $script_base/restore_db -y --skip-check + $script_base/restore_db -y fi diff --git a/scripts/setup_docker b/scripts/setup_docker index 8266d4d..38b8ca7 100755 --- a/scripts/setup_docker +++ b/scripts/setup_docker @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env force= use_existing= @@ -64,7 +65,7 @@ fi PLATFORM="$(detect_platform)" case "$PLATFORM" in - "Debian") + "Debian" | "Ubuntu") if [[ "$PATH" != *"$HOME/.local/bin"* ]]; then cecho "It is recommended that you install docker-compose at the user level. This requires adding" yellow bold cecho "$HOME/.local/bin to your PATH in your .profile, .zprofile or equivalent. " yellow bold diff --git a/scripts/start b/scripts/start index 3934e7d..2b77beb 100755 --- a/scripts/start +++ b/scripts/start @@ -1,11 +1,13 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env detach= +quiet= POSITIONAL=() while [[ $# -gt 0 ]]; do @@ -23,6 +25,9 @@ while [[ $# -gt 0 ]]; do echo "Run 'docker-compose up --help' to see more options" exit 0 ;; + -q|--quiet) + quiet=1 + ;; -d|--detach) detach=1 ;; @@ -35,16 +40,62 @@ done set -- "${POSITIONAL[@]}" args=$@ +allowed=$(allowed_services $DEV_DOCKER_WORKFLOW) + +run_docker= +docker_args= +case $DEV_DOCKER_WORKFLOW in + full) + run_docker=1 + docker_args=$args + ;; + deps) + if [ "$args" ]; then + services= + for svc in $allowed; do + if [[ " $args[@] " =~ $svc ]]; then + services+=" $svc" + fi + done + echo $services + if [ "$services" ]; then + run_docker=1 + docker_args=$services + fi + else + if [ "$allowed" ]; then + run_docker=1 + docker_args=$allowed + fi + fi + ;; + none) + ;; +esac + teardown() { $script_base/stop $args } trap "teardown" INT -docker-compose up -d $args -if [ "$detach" ]; then - cecho "Starting in detached mode. Use '$script_base/attach ' to view output." blue bold -else - docker-compose logs -f $args || teardown +if [ "$run_docker" ]; then + + #if [ ! "$(docker network ls | grep app_global)" ]; then + #cecho "Shared network not found, creating..." yellow bold + #docker network create --driver=bridge app_global + #fi + + docker-compose up -d $docker_args + + if [ "$detach" ]; then + cecho "Starting in detached mode. Use '$script_base/attach ' to view output." blue bold + else + docker-compose logs -f --tail=100 $docker_args || teardown + fi +elif [ ! "$quiet" ]; then + cecho "There are no services that can be managed using this script. Please launch the" red bold + cecho "service manually or review your .env file." red bold + exit 1 fi diff --git a/scripts/stop b/scripts/stop index c316081..3a15a11 100755 --- a/scripts/stop +++ b/scripts/stop @@ -1,9 +1,12 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env + +quiet= POSITIONAL=() while [[ $# -gt 0 ]]; do @@ -15,6 +18,9 @@ while [[ $# -gt 0 ]]; do cecho "Usage: $(basename $0) [SERVICE...]" default bold exit 0 ;; + -q|--quiet) + quiet=1 + ;; *) POSITIONAL+=("$key") ;; @@ -23,4 +29,43 @@ while [[ $# -gt 0 ]]; do done set -- "${POSITIONAL[@]}" -docker-compose stop $@ +args=$@ +allowed=$(allowed_services $DEV_DOCKER_WORKFLOW) + +run_docker= +docker_args= +case $DEV_DOCKER_WORKFLOW in + full) + run_docker=1 + docker_args=$args + ;; + deps) + if [ "$args" ]; then + services= + for svc in $allowed; do + if [[ " $args[@] " =~ $svc ]]; then + services+=" $svc" + fi + done + if [ "$services" ]; then + run_docker=1 + docker_args=$services + fi + else + if [ "$allowed" ]; then + run_docker=1 + docker_args=$allowed + fi + fi + ;; + none) + ;; +esac + +if [ "$run_docker" ]; then + docker-compose stop $docker_args +elif [ ! "$quiet" ]; then + cecho "There are no services that can be managed using this script. Please launch the" red bold + cecho "service manually or review your .env file." red bold + exit 1 +fi diff --git a/scripts/teardown b/scripts/teardown index 9ede944..5bf2088 100755 --- a/scripts/teardown +++ b/scripts/teardown @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env force= no_input= @@ -19,7 +20,7 @@ while [[ $# -gt 0 ]]; do no_input=1 ;; -h|--help|?) - echo "Shutdowns all instances of this repository and deletes all related containers," + echo "Shuts down all instances of this repository and deletes all related containers," echo "images, volumes and networks." echo cecho "Usage: $(basename $0) " default bold diff --git a/scripts/test b/scripts/test index 46c43fb..e3cd439 100755 --- a/scripts/test +++ b/scripts/test @@ -1,18 +1,25 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env -watch= +flags="--nomigrations --capture=no" POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" case $key in -w|--watch) - watch=1 + flags+=" --looponfail" + ;; + -n|--new-db) + flags+=" --create-db" + ;; + -c|--with-coverage) + flags+=" --cov" ;; -h|--help) echo "Run unit tests." @@ -21,7 +28,10 @@ while [[ $# -gt 0 ]]; do echo echo "Options:" echo " -w --watch Rerun tests when changes are made" + echo " -n --new-db Re-create database." echo " -h --help This help message" + echo + echo "Note: any additional options provided will be passed to pytest." exit 0 ;; *) @@ -32,8 +42,9 @@ while [[ $# -gt 0 ]]; do done set -- "${POSITIONAL[@]}" -if [ "$watch" ]; then - docker-compose exec app pytest --looponfail --nomigrations $@ +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW app) ]; then + runner="docker-compose run --rm app" else - docker-compose exec app pytest --nomigrations $@ + runner= fi +$runner pytest $flags $@ diff --git a/scripts/tools b/scripts/tools index da81dda..a6a76cb 100755 --- a/scripts/tools +++ b/scripts/tools @@ -33,9 +33,10 @@ cecho(){ # detect_platform # detects Debian, OS X and OS X + brew detect_platform() { - if [ "$(which apt-get)" ]; then - echo "Debian" - elif [[ "$(uname -s)" = "Darwin" ]]; then + platform=$(uname -s) + if [[ $platform = "Linux" ]]; then + echo $(lsb_release -is) + elif [[ $platform = "Darwin" ]]; then if [ "$(which brew)" ]; then echo "OSX_BREW" else @@ -76,3 +77,41 @@ wait_until_false(){ sleep 1 done } + +load_env(){ + if [ ! "$NO_ENV_FILE" ]; then + if [ ! -e '.env' ]; then + cecho "Could not find an environment settings file (.env). Would you like to start one now?" yellow bold + read -n1 -r -p "Press 'Y' to generate file and continue or any other key to cancel..." key + echo + if [ ! "$key" ] || [[ "yY" != *"$key"* ]]; then + echo + cecho "An environment settings file (.env) is required. Please make a copy from .env-dist and try again." red + exit 1 + fi + + cp .env-dist .env + fi + + source .env + fi +} + +allowed_services(){ + workflow=$1 + if [ "$workflow" = "full" ]; then + echo "app" "task_worker" "db" "cache" + elif [ "$workflow" = "deps" ]; then + echo "db" "cache" + fi +} + + +can_run_docker(){ + workflow=$1 + target=$2 + + if [[ " $(allowed_services $DEV_DOCKER_WORKFLOW)[@] " =~ "$target" ]]; then + echo 1 + fi +} diff --git a/scripts/update b/scripts/update index 38007a4..7327dbe 100755 --- a/scripts/update +++ b/scripts/update @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/tools cd $script_base/.. -source $script_base/tools +load_env no_input= @@ -17,10 +18,7 @@ while [[ $# -gt 0 ]]; do -h|--help|?) echo "Reset environment after a large repository update." echo - cecho "Usage: $(basename $0) ]" default bold - echo - echo "Services:" - echo "$(docker-compose config --services)" + cecho "Usage: $(basename $0) " default bold echo echo "Options:" echo " -y --no-input Assume yes for all user input" @@ -48,12 +46,19 @@ fi $script_base/cleanup --no-input -echo -cecho "Rebuilding environment..." default bold -docker-compose build +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW app) ]; then + echo + cecho "Rebuilding environment..." default bold + docker-compose build + +else + echo + cecho "Installing Python requirements..." default bold + pip install -r requirements.txt +fi echo cecho "Running database migrations..." default bold -$script_base/start -d app db +$script_base/start -q -d app db $script_base/migrate -$script_base/stop +$script_base/stop -q diff --git a/scripts/utils/dbadmin b/scripts/utils/dbadmin new file mode 100755 index 0000000..c21aadf --- /dev/null +++ b/scripts/utils/dbadmin @@ -0,0 +1,65 @@ +#!/bin/bash +set -e +script_base=$(dirname "$0") +cd $script_base/../.. + +source $script_base/../tools + +detach= +port=8080 + +POSITIONAL=() +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -d|--detach) + detach=-d + ;; + -p|--port) + port=$2 + shift + ;; + -h|--help|?) + if [ "$2" != "quiet" ]; then + echo "A web-based administration tool for databases." + echo + cecho "Usage: run_util $(basename $0) " default bold + echo + else + echo "dbadmin" + echo "-------" + echo "Adminer - A web-based administration tool for databases." + fi + echo "Options:" + echo " -d --detach Run utility in the background" + echo " -p --port Specify port Adminer will be exposed on" + exit 0 + ;; + *) + POSITIONAL+=("$key") + ;; + esac + shift +done +set -- "${POSITIONAL[@]}" + +if [ "$DEV_DOCKER_WORKFLOW" == "none" ]; then + cecho "This utility requires docker to run. Please review your .env file." red bold + exit 1 +fi + +plugin_path=$(resolve_path $script_base/../../docker/utils/adminer/plugins/) +cecho "Starting DB Adminer..." default bold +cecho "http://localhost:$port" green bold +docker run --rm \ + --name dbadmin \ + -p $port:8080 \ + #--network base_app_default \ + --mount type=bind,source=$plugin_path/fill-login-form.php,target=/var/www/html/plugins-enabled/fill-login-form.php \ + -e "ADMINER_SYSTEM=pgsql" \ + -e "ADMINER_SERVER=db" \ + -e "ADMINER_USER=postgres" \ + -e "ADMINER_PASSWORD=postgres" \ + -e "ADMINER_DB=base_app" \ + $detach \ + adminer diff --git a/scripts/utils/pgdump b/scripts/utils/dbdump similarity index 79% rename from scripts/utils/pgdump rename to scripts/utils/dbdump index 6e2ce2f..96005bb 100755 --- a/scripts/utils/pgdump +++ b/scripts/utils/dbdump @@ -1,9 +1,10 @@ #!/bin/bash set -e script_base=$(dirname "$0") +source $script_base/../tools cd $script_base/../.. -source $script_base/../tools +load_env database=base_app output= @@ -54,6 +55,12 @@ while [[ $# -gt 0 ]]; do done set -- "${POSITIONAL[@]}" +if [ $(can_run_docker $DEV_DOCKER_WORKFLOW db) ]; then + runner="docker-compose exec -u postgres db" +else + runner= +fi + if [ ! "$output" ]; then if [ "$dump_all" ]; then database=full_database @@ -67,14 +74,14 @@ fi if [ "$nozip" ]; then if [ "$dump_all" ]; then - docker-compose exec -u postgres db sh -c "pg_dumpall" > $output + $runner pg_dumpall > $output else - docker-compose exec -u postgres db sh -c "pg_dump $database" > $output + $runner pg_dump -U postgres -t public.* -O -c -x --if-exists -T spatial_ref_sys $database > $output fi else if [ "$dump_all" ]; then - docker-compose exec -u postgres db sh -c "pg_dumpall" | gzip > $output + $runner pg_dumpall | gzip > $output else - docker-compose exec -u postgres db sh -c "pg_dump $database" | gzip > $output + $runner pg_dump -U postgres -t public.* -O -c -x --if-exists -T spatial_ref_sys $database | gzip > $output fi fi diff --git a/scripts/utils/flower b/scripts/utils/flower new file mode 100755 index 0000000..f92029d --- /dev/null +++ b/scripts/utils/flower @@ -0,0 +1,56 @@ +#!/bin/bash +set -e +script_base=$(dirname "$0") +source $script_base/../tools +cd $script_base/../.. + +load_env + +port=${FLOWER_PORT:-5555} +detach= + +POSITIONAL=() +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -d|--detach) + detach=-d + ;; + -p|--port) + port=$2 + shift + ;; + -h|--help|?) + if [ "$2" != "quiet" ]; then + echo "A web-based tool for monitoring and administration Celery" + echo + cecho "Usage: run_util $(basename $0) " default bold + echo + else + echo "flower" + echo "------" + echo "flower - A web-based tool for monitoring and administration Celery" + fi + echo "Options:" + echo " -p --port Port to bind" + exit 0 + ;; + *) + POSITIONAL+=("$key") + ;; + esac + shift +done +set -- "${POSITIONAL[@]}" + +if [ "$DEV_DOCKER_WORKFLOW" == "none" ]; then + cecho "This utility requires docker to run. Please review your .env file." red bold + exit 1 +fi + +docker-compose run --rm \ + --name flower \ + -p $port:5555 \ + $detach \ + task_worker \ + sh -c "flower -A base_app.celery" diff --git a/scripts/utils/pgadmin b/scripts/utils/pgadmin deleted file mode 100755 index d5bde3e..0000000 --- a/scripts/utils/pgadmin +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -set -e -script_base=$(dirname "$0") -cd $script_base/../.. - -source $script_base/../tools - -detach= -port=2345 - -POSITIONAL=() -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -d|--detach) - detach=-d - ;; - -p|--port) - port=$2 - shift - ;; - -h|--help|?) - if [ "$2" != "quiet" ]; then - echo "A web-based administration tool for PostgreSQL." - echo - cecho "Usage: run_util $(basename $0) " default bold - echo - else - echo "pgadmin" - echo "-------" - echo "pgAdmin 4 - A web-based administration tool for PostgreSQL." - fi - echo "Options:" - echo " -d --detach Run utility in the background" - echo " -p --port Specify port pgAdmin will be exposed on" - exit 0 - ;; - *) - POSITIONAL+=("$key") - ;; - esac - shift -done -set -- "${POSITIONAL[@]}" - -cecho "Starting pgAdmin 4..." default bold -cecho "http://localhost:$port" green bold -cecho "Email: admin" green bold -cecho "Password: admin" green bold -cecho "You will need to add the connection on first run." yellow bold -cecho "Hostname (alias) and credentials for postgres instance can be found in docker-compose file" yellow bold -docker run --rm \ - --name base_app-pgadmin \ - -p $port:80 \ - -v base_app_pgadmin:/var/lib/pgadmin \ - -e "PGADMIN_DEFAULT_EMAIL=admin" \ - -e "PGADMIN_DEFAULT_PASSWORD=admin" \ - $detach \ - dpage/pgadmin4