diff --git a/bootstrap/build_image.sh b/bootstrap/build_image.sh index 3734bfdb426..06cf632a4e6 100755 --- a/bootstrap/build_image.sh +++ b/bootstrap/build_image.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple script to build the Docker images. # This is intended to be invoked as a step in Argo to build the docker image. @@ -15,8 +15,8 @@ BUILDER_IMG=gcr.io/kubeflow-images-public/bootstrapper-builder BUILDER_IMG_VERSION=$(head -1 ${CONTEXT_DIR}/glide.lock | cut -d ' ' -f 2) # Wait for the Docker daemon to be available. -until docker ps -do sleep 3 +until docker ps; do + sleep 3 done # pull builder image from GCR or build it from local if required one doesn't exist. diff --git a/bootstrap/debug.sh b/bootstrap/debug.sh index 9ee20dd839d..75efac9042b 100755 --- a/bootstrap/debug.sh +++ b/bootstrap/debug.sh @@ -6,8 +6,7 @@ # See the [developer_guide.md](./developer_guide.md) for additional details. # -cleanup() -{ +cleanup() { if [[ -n $portforwardcommand ]]; then echo killing $portforwardcommand pkill -f $portforwardcommand @@ -15,16 +14,14 @@ cleanup() } trap cleanup EXIT -portforward() -{ +portforward() { local pod=$1 namespace=$2 from_port=$3 to_port=$4 cmd cmd='kubectl port-forward $pod ${from_port}:${to_port} --namespace=$namespace 2>&1>/dev/null &' portforwardcommand="${cmd% 2>&1>/dev/null &}" eval $cmd } -waitforpod() -{ +waitforpod() { local cmd="kubectl get pods --no-headers -oname --selector=app=kubeflow-bootstrapper --field-selector=status.phase=Running --namespace=kubeflow-admin | sed 's/^pod.*\///'" found=$(eval "$cmd") while [[ -z $found ]]; do sleep 1 @@ -33,8 +30,7 @@ waitforpod() echo $found } -waitforever() -{ +waitforever() { which gsleep >/dev/null if [[ $? == 1 ]]; then while true; do diff --git a/bootstrap/start.sh b/bootstrap/start.sh index f639cb6522f..044961b40be 100644 --- a/bootstrap/start.sh +++ b/bootstrap/start.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Startup script. # The purpose of this script is to create a unix user @@ -14,23 +14,23 @@ set -x # If it's run by deployment job, don't switch users if [ -v "DEPLOY_JOB" ]; then - cd /kubeflow - # Remove fake kubeconfig and start k8s proxy - rm ~/.kube/config - kubectl proxy --port=8111 & + cd /kubeflow + # Remove fake kubeconfig and start k8s proxy + rm ~/.kube/config + kubectl proxy --port=8111 & - # Recreate env since we have proper k8s creds - # TODO (inc0): Replace this with bootstrapper when we confirm that it works inside cluster - ks env rm default - ks env add default --server=http://127.0.0.1:8111 - ks env set default --namespace ${NAMESPACE} + # Recreate env since we have proper k8s creds + # TODO (inc0): Replace this with bootstrapper when we confirm that it works inside cluster + ks env rm default + ks env add default --server=http://127.0.0.1:8111 + ks env set default --namespace ${NAMESPACE} - if [ ! -d /opt/kubeflow/app/kubeflow ]; then - ks generate kubeflow-core kubeflow-core - cp -R /kubeflow /opt/kubeflow/app - fi - cd /opt/kubeflow/app/kubeflow - tail -f /dev/null + if [ ! -d /opt/kubeflow/app/kubeflow ]; then + ks generate kubeflow-core kubeflow-core + cp -R /kubeflow /opt/kubeflow/app + fi + cd /opt/kubeflow/app/kubeflow + tail -f /dev/null fi groupadd -g ${GROUP_ID} ${GROUP} useradd -r -u ${USER_ID} -g ${GROUP} --shell=/bin/bash ${USER} diff --git a/bootstrap/test_delete.sh b/bootstrap/test_delete.sh index a70cadb5b39..efeddebf1b9 100755 --- a/bootstrap/test_delete.sh +++ b/bootstrap/test_delete.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Script to delete kubeflow-ci namespace; deployment and source repo in sandbox project. # This is intended to be invoked as a step in Argo. @@ -17,10 +17,10 @@ gcloud container clusters get-credentials ${CLUSTER} --zone ${ZONE} --project ku # delete test namespace kubectl delete namespace ${NAMESPACE} -for i in 1 2 3 4 5 -do gcloud -q deployment-manager deployments delete ${DEPLOYMENT} --project=kubeflow-ci-deploy && break || sleep 30 +for i in 1 2 3 4 5; do + gcloud -q deployment-manager deployments delete ${DEPLOYMENT} --project=kubeflow-ci-deploy && break || sleep 30 done -for i in 1 2 3 4 5 -do gcloud -q source repos delete kubeflow-ci-deploy-kubeflow-config --project=kubeflow-ci-deploy && break || sleep 30 +for i in 1 2 3 4 5; do + gcloud -q source repos delete kubeflow-ci-deploy-kubeflow-config --project=kubeflow-ci-deploy && break || sleep 30 done diff --git a/bootstrap/test_setup.sh b/bootstrap/test_setup.sh index b7748bffb1b..4793d3a32ce 100755 --- a/bootstrap/test_setup.sh +++ b/bootstrap/test_setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple script to setup deployment service. # This is intended to be invoked as a step in Argo. diff --git a/build/check_boilerplate.sh b/build/check_boilerplate.sh index acd242b5914..a03bca8b66d 100755 --- a/build/check_boilerplate.sh +++ b/build/check_boilerplate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2015 The Kubeflow Authors All rights reserved. # diff --git a/components/centraldashboard/build_image.sh b/components/centraldashboard/build_image.sh index f062711e3a5..4d2ba0e7b96 100755 --- a/components/centraldashboard/build_image.sh +++ b/components/centraldashboard/build_image.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple script to build the Docker images. # This is intended to be invoked as a step in Argo to build the docker image. @@ -13,8 +13,8 @@ CONTEXT_DIR=$(dirname "$DOCKERFILE") PROJECT="${GCP_PROJECT}" # Wait for the Docker daemon to be available. -until docker ps -do sleep 3 +until docker ps; do + sleep 3 done gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} @@ -25,4 +25,4 @@ echo "GCP Project: "$PROJECT echo "Building centraldashboard using gcloud build" gcloud builds submit --tag=${IMAGE}:${TAG} --project=${PROJECT} . -echo "Finished building image" \ No newline at end of file +echo "Finished building image" diff --git a/components/gcp-click-to-deploy/ks-app/create_gcp_deploy_sa.sh b/components/gcp-click-to-deploy/ks-app/create_gcp_deploy_sa.sh index 7eed8a5fb5d..1e4bc32feb6 100755 --- a/components/gcp-click-to-deploy/ks-app/create_gcp_deploy_sa.sh +++ b/components/gcp-click-to-deploy/ks-app/create_gcp_deploy_sa.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Creates a service account for running the GCP deploy app. # We need a service account to work with external DNS. @@ -39,15 +39,14 @@ gcloud --project=${PROJECT} --quiet iam service-accounts create ${NAME} declare -a roles=("dns.admin") # Add policy bindings -for ROLE in "${roles[@]}" -do - gcloud projects add-iam-policy-binding ${DNS_PROJECT} \ - --member serviceAccount:${EMAIL} \ - --role roles/${ROLE} +for ROLE in "${roles[@]}"; do + gcloud projects add-iam-policy-binding ${DNS_PROJECT} \ + --member serviceAccount:${EMAIL} \ + --role roles/${ROLE} done # Get a new service account key -SECRET_FILE=~/tmp/${EMAIL}.json +SECRET_FILE=~/tmp/${EMAIL}.json rm -f ~${SECRET_FILE} gcloud --project=${PROJECT} iam service-accounts keys create ${SECRET_FILE} --iam-account ${EMAIL} diff --git a/components/k8s-model-server/http-proxy/build_image.sh b/components/k8s-model-server/http-proxy/build_image.sh index 015f3ea1809..aec6529352e 100755 --- a/components/k8s-model-server/http-proxy/build_image.sh +++ b/components/k8s-model-server/http-proxy/build_image.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple script to build the Docker images. # This is intended to be invoked as a step in Argo to build the docker image. @@ -11,12 +11,12 @@ CONTEXT_DIR=$(dirname "$DOCKERFILE") IMAGE=$2 # Wait for the Docker daemon to be available. -until docker ps -do sleep 3 +until docker ps; do + sleep 3 done docker build --pull -t ${IMAGE} \ - -f ${DOCKERFILE} ${CONTEXT_DIR} + -f ${DOCKERFILE} ${CONTEXT_DIR} gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} -gcloud docker -- push ${IMAGE} \ No newline at end of file +gcloud docker -- push ${IMAGE} diff --git a/components/k8s-model-server/images/build_image.sh b/components/k8s-model-server/images/build_image.sh index 015f3ea1809..aec6529352e 100755 --- a/components/k8s-model-server/images/build_image.sh +++ b/components/k8s-model-server/images/build_image.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple script to build the Docker images. # This is intended to be invoked as a step in Argo to build the docker image. @@ -11,12 +11,12 @@ CONTEXT_DIR=$(dirname "$DOCKERFILE") IMAGE=$2 # Wait for the Docker daemon to be available. -until docker ps -do sleep 3 +until docker ps; do + sleep 3 done docker build --pull -t ${IMAGE} \ - -f ${DOCKERFILE} ${CONTEXT_DIR} + -f ${DOCKERFILE} ${CONTEXT_DIR} gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} -gcloud docker -- push ${IMAGE} \ No newline at end of file +gcloud docker -- push ${IMAGE} diff --git a/components/k8s-model-server/inception-client/run.sh b/components/k8s-model-server/inception-client/run.sh index 5b8d98c29cb..0994ccb4ac0 100644 --- a/components/k8s-model-server/inception-client/run.sh +++ b/components/k8s-model-server/inception-client/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2018 The Kubeflow Authors All rights reserved. # @@ -17,11 +17,11 @@ SERVER=$1 PORT=$2 -if [ -z $SERVER ] ; then +if [ -z $SERVER ]; then SERVER=$INCEPTION_SERVICE_HOST fi -if [ -z $PORT ] ; then +if [ -z $PORT ]; then PORT=$INCEPTION_SERVICE_PORT fi diff --git a/components/tensorflow-notebook-image/build_image.sh b/components/tensorflow-notebook-image/build_image.sh index a106e6d234f..4fba8121e1a 100755 --- a/components/tensorflow-notebook-image/build_image.sh +++ b/components/tensorflow-notebook-image/build_image.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple script to build the Docker images. # This is intended to be invoked as a step in Argo to build the docker image. @@ -16,41 +16,40 @@ TAG=$3 # # TODO(jlewi): We should take in the json config file and then parse that. CONFIG_FILE=$4 -BASE_IMAGE=$(jq -r .BASE_IMAGE ${CONFIG_FILE}) -TF_PACKAGE=$(jq -r .TF_PACKAGE ${CONFIG_FILE}) +BASE_IMAGE=$(jq -r .BASE_IMAGE ${CONFIG_FILE}) +TF_PACKAGE=$(jq -r .TF_PACKAGE ${CONFIG_FILE}) TF_PACKAGE_PY_27=$(jq -r .TF_PACKAGE_PY_27 ${CONFIG_FILE}) TF_SERVING_VERSION=$(jq -r .TF_SERVING_VERSION ${CONFIG_FILE}) TFMA_VERSION=$(jq -r .TFMA_VERSION ${CONFIG_FILE}) TFDV_VERSION=$(jq -r .TFDV_VERSION ${CONFIG_FILE}) - # JQ returns null for non defined values. if [ ${BASE_IMAGE} == "null" ]; then - BASE_IMAGE="" + BASE_IMAGE="" fi if [ ${TFMA_VERSION} == "null" ]; then - TFMA_VERSION="" + TFMA_VERSION="" fi if [ ${TFDV_VERSION} == "null" ]; then - TFDV_VERSION="" + TFDV_VERSION="" fi # Wait for the Docker daemon to be available. -until docker ps -do sleep 3 +until docker ps; do + sleep 3 done docker build --pull \ - --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ - --build-arg "TF_PACKAGE=${TF_PACKAGE}" \ - --build-arg "TF_PACKAGE_PY_27=${TF_PACKAGE_PY_27}" \ - --build-arg "TF_SERVING_VERSION=${TF_SERVING_VERSION}" \ - --build-arg "TFMA_VERSION=${TFMA_VERSION}" \ - --build-arg "TFDV_VERSION=${TFDV_VERSION}" \ - -t "${IMAGE}:${TAG}" \ - -f ${DOCKERFILE} ${CONTEXT_DIR} + --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ + --build-arg "TF_PACKAGE=${TF_PACKAGE}" \ + --build-arg "TF_PACKAGE_PY_27=${TF_PACKAGE_PY_27}" \ + --build-arg "TF_SERVING_VERSION=${TF_SERVING_VERSION}" \ + --build-arg "TFMA_VERSION=${TFMA_VERSION}" \ + --build-arg "TFDV_VERSION=${TFDV_VERSION}" \ + -t "${IMAGE}:${TAG}" \ + -f ${DOCKERFILE} ${CONTEXT_DIR} gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} gcloud docker -- push "${IMAGE}:${TAG}" diff --git a/components/tensorflow-notebook-image/install.sh b/components/tensorflow-notebook-image/install.sh index ab7a49f2a13..cb81f2c7f67 100755 --- a/components/tensorflow-notebook-image/install.sh +++ b/components/tensorflow-notebook-image/install.sh @@ -1,28 +1,28 @@ -#!/bin/bash +#!/usr/bin/env bash # # An install script to be called from the DOCKERFILE set -ex conda create -n py2 python=2 source activate py2 -pip install --upgrade pip +pip install --upgrade pip -# TFX packages only supports python 2 +# TFX packages only supports python 2 pip --no-cache-dir install \ - ipykernel \ - ${TF_PACKAGE_PY_27} \ - tensorflow-transform \ - tensorflow-serving-api===${TF_SERVING_VERSION} + ipykernel \ + ${TF_PACKAGE_PY_27} \ + tensorflow-transform \ + tensorflow-serving-api===${TF_SERVING_VERSION} python -m ipykernel install # TODO(jlewi): Does TFDV not have jupyter extensions? if [[ $TFDV_VERSION ]]; then - pip install --no-cache-dir tensorflow-data-validation==$TFDV_VERSION + pip install --no-cache-dir tensorflow-data-validation==$TFDV_VERSION fi # tensorflow-model-analysis is only supported for TF 1.6 and above -# TODO temporarily remove tensorflow-model-analysis because of gpu problem +# TODO temporarily remove tensorflow-model-analysis because of gpu problem # https://github.com/kubeflow/kubeflow/pull/1759 if [[ $TFMA_VERSION ]]; then # if you find any dependency problems in the future @@ -31,7 +31,7 @@ if [[ $TFMA_VERSION ]]; then pip install --no-cache-dir tensorflow-model-analysis==$TFMA_VERSION # We use --system because if we don't the config is written to the home directory # and the changes are lost when we mount a PV over them. - jupyter nbextension enable --py --system widgetsnbextension + jupyter nbextension enable --py --system widgetsnbextension fi # TODO a quick fix for tensorflow_serving_api when install gpu @@ -46,9 +46,9 @@ jupyter labextension install @jupyter-widgets/jupyterlab-manager # Install common packages from requirements.txt for both python2 and python pip --no-cache-dir install -r /tmp/requirements.txt -source activate py2 -pip --no-cache-dir install -r /tmp/requirements.txt - +source activate py2 +pip --no-cache-dir install -r /tmp/requirements.txt + # Do chown in this layer for significant size savings -chown -R ${NB_USER}:users $HOME -chown -R ${NB_USER}:users $CONDA_DIR \ No newline at end of file +chown -R ${NB_USER}:users $HOME +chown -R ${NB_USER}:users $CONDA_DIR diff --git a/components/tensorflow-notebook-image/pvc-check.sh b/components/tensorflow-notebook-image/pvc-check.sh index 4f07fad0ca0..de56782e183 100755 --- a/components/tensorflow-notebook-image/pvc-check.sh +++ b/components/tensorflow-notebook-image/pvc-check.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2016 The Kubeflow Authors All rights reserved. # @@ -22,18 +22,18 @@ CONF_DIR=$HOME/.jupyter echo "checking if $HOME volume needs init..." if [ "$(ls --ignore=lost+found -A $HOME)" ]; then - # assume we are working with an existing volume via a PVC - echo "...$HOME already has content..." + # assume we are working with an existing volume via a PVC + echo "...$HOME already has content..." else - # clean volume, needs init - echo "...creating $WORK_DIR" - mkdir $WORK_DIR - mkdir $CONF_DIR + # clean volume, needs init + echo "...creating $WORK_DIR" + mkdir $WORK_DIR + mkdir $CONF_DIR - echo "...load initial content into $HOME..." - cp $SRC_CONF $CONF_DIR + echo "...load initial content into $HOME..." + cp $SRC_CONF $CONF_DIR - chown -R $NB_USER:users $(ls -A $HOME) + chown -R $NB_USER:users $(ls -A $HOME) fi echo "...done" diff --git a/components/tensorflow-notebook-image/start-notebook.sh b/components/tensorflow-notebook-image/start-notebook.sh index 8b051ea0c77..9c9467f9607 100755 --- a/components/tensorflow-notebook-image/start-notebook.sh +++ b/components/tensorflow-notebook-image/start-notebook.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2017 The Kubeflow Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/components/tensorflow-notebook-image/start-singleuser.sh b/components/tensorflow-notebook-image/start-singleuser.sh index e32c468d884..878081decb4 100755 --- a/components/tensorflow-notebook-image/start-singleuser.sh +++ b/components/tensorflow-notebook-image/start-singleuser.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2017 The Kubeflow Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,7 +46,7 @@ if [ ! -z "$JPY_HUB_API_URL" ]; then NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" fi -# check to see if a PV has been mounted +# check to see if a PV has been mounted . /usr/local/bin/pvc-check.sh # We delay enabling the Jupyter extension until runtime because diff --git a/components/tensorflow-notebook-image/start.sh b/components/tensorflow-notebook-image/start.sh index 24d41ab6584..7096fcc3d24 100755 --- a/components/tensorflow-notebook-image/start.sh +++ b/components/tensorflow-notebook-image/start.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2017 The Kubeflow Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,48 +16,48 @@ set -e # Handle special flags if we're root -if [ $(id -u) == 0 ] ; then - # Handle username change. Since this is cheap, do this unconditionally - usermod -d /home/$NB_USER -l $NB_USER jovyan +if [ $(id -u) == 0 ]; then + # Handle username change. Since this is cheap, do this unconditionally + usermod -d /home/$NB_USER -l $NB_USER jovyan - # Change UID of NB_USER to NB_UID if it does not match - if [ "$NB_UID" != $(id -u $NB_USER) ] ; then - echo "Set user UID to: $NB_UID" - usermod -u $NB_UID $NB_USER - for d in "$CONDA_DIR" "$JULIA_PKGDIR"; do - if [[ ! -z "$d" && -d "$d" ]]; then - echo "Set ownership to uid $NB_UID: $d" - chown -R $NB_UID "$d" - fi - done - fi + # Change UID of NB_USER to NB_UID if it does not match + if [ "$NB_UID" != $(id -u $NB_USER) ]; then + echo "Set user UID to: $NB_UID" + usermod -u $NB_UID $NB_USER + for d in "$CONDA_DIR" "$JULIA_PKGDIR"; do + if [[ ! -z "$d" && -d "$d" ]]; then + echo "Set ownership to uid $NB_UID: $d" + chown -R $NB_UID "$d" + fi + done + fi - # Change GID of NB_USER to NB_GID if NB_GID is passed as a parameter - if [ "$NB_GID" ] ; then - echo "Change GID to $NB_GID" - groupmod -g $NB_GID -o $(id -g -n $NB_USER) - fi + # Change GID of NB_USER to NB_GID if NB_GID is passed as a parameter + if [ "$NB_GID" ]; then + echo "Change GID to $NB_GID" + groupmod -g $NB_GID -o $(id -g -n $NB_USER) + fi - # Enable sudo if requested - if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then - echo "Granting $NB_USER sudo access" - echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook - fi + # Enable sudo if requested + if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then + echo "Granting $NB_USER sudo access" + echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook + fi - # Exec the command as NB_USER - echo "Execute the command as $NB_USER" - exec su $NB_USER -c "env PATH=$PATH $*" + # Exec the command as NB_USER + echo "Execute the command as $NB_USER" + exec su $NB_USER -c "env PATH=$PATH $*" else if [[ ! -z "$NB_UID" && "$NB_UID" != "$(id -u)" ]]; then - echo 'Container must be run as root to set $NB_UID' + echo 'Container must be run as root to set $NB_UID' fi if [[ ! -z "$NB_GID" && "$NB_GID" != "$(id -g)" ]]; then - echo 'Container must be run as root to set $NB_GID' + echo 'Container must be run as root to set $NB_GID' fi if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then - echo 'Container must be run as root to grant sudo permissions' + echo 'Container must be run as root to grant sudo permissions' fi - # Exec the command - echo "Execute the command" - exec $* + # Exec the command + echo "Execute the command" + exec $* fi diff --git a/hack/cherry_pick_pull.sh b/hack/cherry_pick_pull.sh index 733745c72ce..2e58d5f6ccc 100755 --- a/hack/cherry_pick_pull.sh +++ b/hack/cherry_pick_pull.sh @@ -79,10 +79,15 @@ fi declare -r BRANCH="$1" shift 1 -declare -r PULLS=( "$@" ) +declare -r PULLS=("$@") -function join { local IFS="$1"; shift; echo "$*"; } -declare -r PULLDASH=$(join - "${PULLS[@]/#/#}") # Generates something like "#12345-#56789" +join() { + local IFS="$1" + shift + echo "$*" +} + +declare -r PULLDASH=$(join - "${PULLS[@]/#/#}") # Generates something like "#12345-#56789" declare -r PULLSUBJ=$(join " " "${PULLS[@]/#/#}") # Generates something like "#12345 #56789" echo "+++ Updating remotes..." @@ -102,7 +107,8 @@ echo "+++ Creating local branch ${NEWBRANCHUNIQ}" cleanbranch="" prtext="" gitamcleanup=false -function return_to_kansas { + +return_to_kansas() { if [[ "${gitamcleanup}" == "true" ]]; then echo echo "+++ Aborting in-progress git am." @@ -122,10 +128,12 @@ function return_to_kansas { fi fi } + trap return_to_kansas EXIT SUBJECTS=() -function make-a-pr() { + +make-a-pr() { local rel="$(basename "${BRANCH}")" echo echo "+++ Creating a pull request on GitHub at ${GITHUB_USER}:${NEWBRANCH}" @@ -142,7 +150,7 @@ Cherry pick of ${PULLSUBJ} on ${rel}. ${numandtitle} EOF -hub pull-request -F "${prtext}" -h "${GITHUB_USER}:${NEWBRANCH}" -b "${MAIN_REPO_ORG}:${rel}" + hub pull-request -F "${prtext}" -h "${GITHUB_USER}:${NEWBRANCH}" -b "${MAIN_REPO_ORG}:${rel}" } git checkout -b "${NEWBRANCHUNIQ}" "${BRANCH}" @@ -159,8 +167,8 @@ for pull in "${PULLS[@]}"; do echo git am -3 "/tmp/${pull}.patch" || { conflicts=false - while unmerged=$(git status --porcelain | grep ^U) && [[ -n ${unmerged} ]] \ - || [[ -e "${REBASEMAGIC}" ]]; do + while unmerged=$(git status --porcelain | grep ^U) && [[ -n ${unmerged} ]] || + [[ -e "${REBASEMAGIC}" ]]; do conflicts=true # <-- We should have detected conflicts once echo echo "+++ Conflicts detected:" diff --git a/hack/retag_katib_images.sh b/hack/retag_katib_images.sh index 25a613d7170..3e08aed1394 100755 --- a/hack/retag_katib_images.sh +++ b/hack/retag_katib_images.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # One off script to retag Katib images for 0.2 set -ex @@ -9,31 +9,31 @@ docker pull katib/suggestion-grid:master docker pull katib/vizier-core:master docker tag mitdbg/modeldb-backend:latest \ - gcr.io/kubeflow-images-public/modeldb-backend:v0.2.0 + gcr.io/kubeflow-images-public/modeldb-backend:v0.2.0 docker tag katib/katib-frontend:master \ - gcr.io/kubeflow-images-public/katib-frontend:v0.2.0 + gcr.io/kubeflow-images-public/katib-frontend:v0.2.0 docker tag katib/suggestion-random:master \ - gcr.io/kubeflow-images-public/katib-suggestion-random:v0.2.0 + gcr.io/kubeflow-images-public/katib-suggestion-random:v0.2.0 docker tag katib/suggestion-grid:master \ - gcr.io/kubeflow-images-public/katib-suggestion-grid:v0.2.0 + gcr.io/kubeflow-images-public/katib-suggestion-grid:v0.2.0 docker tag katib/vizier-core:master \ - gcr.io/kubeflow-images-public/katib-vizier-core:v0.2.0 + gcr.io/kubeflow-images-public/katib-vizier-core:v0.2.0 gcloud docker -- push \ - gcr.io/kubeflow-images-public/modeldb-backend:v0.2.0 + gcr.io/kubeflow-images-public/modeldb-backend:v0.2.0 gcloud docker -- push \ - gcr.io/kubeflow-images-public/katib-frontend:v0.2.0 + gcr.io/kubeflow-images-public/katib-frontend:v0.2.0 gcloud docker -- push \ - gcr.io/kubeflow-images-public/katib-suggestion-random:v0.2.0 + gcr.io/kubeflow-images-public/katib-suggestion-random:v0.2.0 gcloud docker -- push \ - gcr.io/kubeflow-images-public/katib-suggestion-grid:v0.2.0 + gcr.io/kubeflow-images-public/katib-suggestion-grid:v0.2.0 gcloud docker -- push \ - gcr.io/kubeflow-images-public/katib-vizier-core:v0.2.0 \ No newline at end of file + gcr.io/kubeflow-images-public/katib-vizier-core:v0.2.0 diff --git a/kubeflow/core/configure_envoy_for_iap.sh b/kubeflow/core/configure_envoy_for_iap.sh index ee66074d46a..1b452733aa9 100644 --- a/kubeflow/core/configure_envoy_for_iap.sh +++ b/kubeflow/core/configure_envoy_for_iap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A script to modify envoy config to perform JWT validation # given the information for the service. @@ -9,29 +9,39 @@ PROJECT=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id) if [ -z ${PROJECT} ]; then -echo Error unable to fetch PROJECT from compute metadata -exit 1 + echo Error unable to fetch PROJECT from compute metadata + exit 1 fi PROJECT_NUM=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/numeric-project-id) if [ -z ${PROJECT_NUM} ]; then -echo Error unable to fetch PROJECT_NUM from compute metadata -exit 1 + echo Error unable to fetch PROJECT_NUM from compute metadata + exit 1 fi +checkIAP() { + # created by init container. + . /var/shared/healthz.env + + # If node port or backend id change, so does the JWT audience. + CURR_NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') + CURR_BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${CURR_NODE_PORT}- --format='value(id)') + [ "$BACKEND_ID" == "$CURR_BACKEND_ID" ] +} + # Activate the service account -for i in $(seq 1 10) -do gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} && break || sleep 10 +for i in $(seq 1 10); do + gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} && break || sleep 10 done # Print out the config for debugging gcloud config list NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') -while [[ -z ${BACKEND_ID} ]]; -do BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${NODE_PORT}- --format='value(id)'); -echo "Waiting for backend id PROJECT=${PROJECT} NAMESPACE=${NAMESPACE} SERVICE=${SERVICE} filter=name~k8s-be-${NODE_PORT}-..."; -sleep 2; +while [[ -z ${BACKEND_ID} ]]; do + BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${NODE_PORT}- --format='value(id)') + echo "Waiting for backend id PROJECT=${PROJECT} NAMESPACE=${NAMESPACE} SERVICE=${SERVICE} filter=name~k8s-be-${NODE_PORT}-..." + sleep 2 done echo BACKEND_ID=${BACKEND_ID} @@ -45,27 +55,17 @@ echo "JWT_AUDIENCE=${JWT_AUDIENCE}" > /var/shared/healthz.env echo "NODE_PORT=${NODE_PORT}" >> /var/shared/healthz.env echo "BACKEND_ID=${BACKEND_ID}" >> /var/shared/healthz.env -kubectl get configmap -n ${NAMESPACE} envoy-config -o jsonpath='{.data.envoy-config\.json}' | \ -sed -e "s|{{JWT_AUDIENCE}}|${JWT_AUDIENCE}|g" > /var/shared/envoy-config.json +kubectl get configmap -n ${NAMESPACE} envoy-config -o jsonpath='{.data.envoy-config\.json}' | + sed -e "s|{{JWT_AUDIENCE}}|${JWT_AUDIENCE}|g" >/var/shared/envoy-config.json echo "Restarting envoy" curl -s ${ENVOY_ADMIN}/quitquitquit -function checkIAP() { -# created by init container. -. /var/shared/healthz.env - -# If node port or backend id change, so does the JWT audience. -CURR_NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') -CURR_BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${CURR_NODE_PORT}- --format='value(id)') -[ "$BACKEND_ID" == "$CURR_BACKEND_ID" ] -} - # Verify IAP every 10 seconds. while true; do -if ! checkIAP; then - echo "$(date) WARN: IAP check failed, restarting container." - exit 1 -fi -sleep 10 + if ! checkIAP; then + echo "$(date) WARN: IAP check failed, restarting container." + exit 1 + fi + sleep 10 done diff --git a/kubeflow/core/ingress_bootstrap.sh b/kubeflow/core/ingress_bootstrap.sh index d0978a611c8..46a983596de 100644 --- a/kubeflow/core/ingress_bootstrap.sh +++ b/kubeflow/core/ingress_bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -x set -e @@ -11,7 +11,10 @@ set -e # After the certificate is obtained, patch the ingress with the tls spec to enable SSL on the GCLB. # Wait for certificate. -(until kubectl -n ${NAMESPACE} get secret ${TLS_SECRET_NAME} 2>/dev/null; do echo "Waiting for certificate..." ; sleep 2; done) +until kubectl -n ${NAMESPACE} get secret ${TLS_SECRET_NAME} 2>/dev/null; do + echo "Waiting for certificate..." + sleep 2 +done kubectl -n ${NAMESPACE} patch ingress ${INGRESS_NAME} --type='json' -p '[{"op": "add", "path": "/spec/tls", "value": [{"secretName": "'${TLS_SECRET_NAME}'", "hosts":["'${TLS_HOST_NAME}'"]}]}]' diff --git a/kubeflow/core/setup_backend.sh b/kubeflow/core/setup_backend.sh index 22781803a23..732c9bcde63 100644 --- a/kubeflow/core/setup_backend.sh +++ b/kubeflow/core/setup_backend.sh @@ -1,11 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple shell script to configure the backend timeouts and health checks by using gcloud. [ -z ${NAMESPACE} ] && echo Error NAMESPACE must be set && exit 1 [ -z ${SERVICE} ] && echo Error SERVICE must be set && exit 1 # Stagger init of replicas when acquiring lock -sleep $(( $RANDOM % 5 + 1 )) +sleep $(($RANDOM % 5 + 1)) # We acquire a lock because we want to ensure there is a single process # trying to modify the backend at a time. @@ -14,37 +14,39 @@ LOCK=$(jq -r ".metadata.annotations.backendlock" service.json) NOW=$(date -u +'%s') if [[ -z "${LOCK}" || "${LOCK}" == "null" ]]; then -LOCK_T=$NOW + LOCK_T=$NOW else -LOCK_T=$(echo "${LOCK}" | cut -d' ' -f2) + LOCK_T=$(echo "${LOCK}" | cut -d' ' -f2) fi -LOCK_AGE=$(( $NOW - $LOCK_T )) + +LOCK_AGE=$(($NOW - $LOCK_T)) LOCK_TTL=120 + if [[ -z "${LOCK}" || "${LOCK}" == "null" || "${LOCK_AGE}" -gt "${LOCK_TTL}" ]]; then -jq -r ".metadata.annotations.backendlock=\"$(hostname -s) ${NOW}\"" service.json > service_lock.json -kubectl apply -f service_lock.json 2>/dev/null -if [[ $? -eq 0 ]]; then - echo "Acquired lock on service annotation to configure backend." + jq -r ".metadata.annotations.backendlock=\"$(hostname -s) ${NOW}\"" service.json > service_lock.json + kubectl apply -f service_lock.json 2>/dev/null + if [[ $? -eq 0 ]]; then + echo "Acquired lock on service annotation to configure backend." + else + echo "WARN: Failed to acquire lock on service annotation." + exit 1 + fi else - echo "WARN: Failed to acquire lock on service annotation." + echo "WARN: Lock on service annotation already acquired by: $LOCK, age: $LOCK_AGE, TTL: $LOCK_TTL" + sleep 20 exit 1 fi -else -echo "WARN: Lock on service annotation already acquired by: $LOCK, age: $LOCK_AGE, TTL: $LOCK_TTL" -sleep 20 -exit 1 -fi PROJECT=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id) if [ -z ${PROJECT} ]; then -echo Error unable to fetch PROJECT from compute metadata -exit 1 + echo Error unable to fetch PROJECT from compute metadata + exit 1 fi PROJECT_NUM=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/numeric-project-id) if [ -z ${PROJECT_NUM} ]; then -echo Error unable to fetch PROJECT_NUM from compute metadata -exit 1 + echo Error unable to fetch PROJECT_NUM from compute metadata + exit 1 fi # Activate the service account @@ -53,20 +55,20 @@ gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS gcloud config list NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') -while [[ -z ${BACKEND_ID} ]]; -do BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${NODE_PORT}- --format='value(id)'); -echo "Waiting for backend id PROJECT=${PROJECT} NAMESPACE=${NAMESPACE} SERVICE=${SERVICE} filter=name~k8s-be-${NODE_PORT}- ..."; -sleep 2; +while [[ -z ${BACKEND_ID} ]]; do + BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${NODE_PORT}- --format='value(id)') + echo "Waiting for backend id PROJECT=${PROJECT} NAMESPACE=${NAMESPACE} SERVICE=${SERVICE} filter=name~k8s-be-${NODE_PORT}- ..." + sleep 2 done echo BACKEND_ID=${BACKEND_ID} NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') BACKEND_SERVICE=$(gcloud --project=${PROJECT} compute backend-services list --filter=name~k8s-be-${NODE_PORT}- --uri) -while [[ -z ${HEALTH_CHECK_URI} ]]; -do HEALTH_CHECK_URI=$(gcloud compute --project=${PROJECT} health-checks list --filter=name~k8s-be-${NODE_PORT}- --uri); -echo "Waiting for the healthcheck resource PROJECT=${PROJECT} NODEPORT=${NODE_PORT} SERVICE=${SERVICE}..."; -sleep 2; +while [[ -z ${HEALTH_CHECK_URI} ]]; do + HEALTH_CHECK_URI=$(gcloud compute --project=${PROJECT} health-checks list --filter=name~k8s-be-${NODE_PORT}- --uri) + echo "Waiting for the healthcheck resource PROJECT=${PROJECT} NODEPORT=${NODE_PORT} SERVICE=${SERVICE}..." + sleep 2 done # Since we create the envoy-ingress ingress object before creating the envoy @@ -90,29 +92,29 @@ echo "BACKEND_ID=${BACKEND_ID}" >> /var/shared/healthz.env # TODO(https://github.com/kubeflow/kubeflow/issues/942): We should publish the modified envoy # config as a config map and use that in the envoy sidecars. -kubectl get configmap -n ${NAMESPACE} envoy-config -o jsonpath='{.data.envoy-config\.json}' | \ -sed -e "s|{{JWT_AUDIENCE}}|${JWT_AUDIENCE}|g" > /var/shared/envoy-config.json +kubectl get configmap -n ${NAMESPACE} envoy-config -o jsonpath='{.data.envoy-config\.json}' | + sed -e "s|{{JWT_AUDIENCE}}|${JWT_AUDIENCE}|g" > /var/shared/envoy-config.json echo "Clearing lock on service annotation" kubectl patch svc "${SERVICE}" -p "{\"metadata\": { \"annotations\": {\"backendlock\": \"\" }}}" -function checkBackend() { -# created by init container. -. /var/shared/healthz.env +checkBackend() { + # created by init container. + . /var/shared/healthz.env -# If node port or backend id change, so does the JWT audience. -CURR_NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') -read -ra toks <<< "$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${CURR_NODE_PORT}- --format='value(id,timeoutSec)')" -CURR_BACKEND_ID="${toks[0]}" -CURR_BACKEND_TIMEOUT="${toks[1]}" -[[ "$BACKEND_ID" == "$CURR_BACKEND_ID" && "${CURR_BACKEND_TIMEOUT}" -eq 3600 ]] + # If node port or backend id change, so does the JWT audience. + CURR_NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') + read -ra toks <<<"$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${CURR_NODE_PORT}- --format='value(id,timeoutSec)')" + CURR_BACKEND_ID="${toks[0]}" + CURR_BACKEND_TIMEOUT="${toks[1]}" + [[ "$BACKEND_ID" == "$CURR_BACKEND_ID" && "${CURR_BACKEND_TIMEOUT}" -eq 3600 ]] } # Verify configuration every 10 seconds. while true; do -if ! checkBackend; then - echo "$(date) WARN: Backend check failed, restarting container." - exit 1 -fi -sleep 10 + if ! checkBackend; then + echo "$(date) WARN: Backend check failed, restarting container." + exit 1 + fi + sleep 10 done diff --git a/kubeflow/openmpi/assets/init.sh b/kubeflow/openmpi/assets/init.sh index d2b0bc8b8d2..40e23bcd71e 100644 --- a/kubeflow/openmpi/assets/init.sh +++ b/kubeflow/openmpi/assets/init.sh @@ -65,7 +65,7 @@ cp ${OPENMPI_DIR}/assets/ssh_config ${HOME}/.ssh/config # when this runs by non-root user. # we have to create ephemeral hostkeys -if run_by_non_root ; then +if run_by_non_root; then mkdir -p ${HOME}/.sshd cp ${OPENMPI_DIR}/assets/sshd_config ${HOME}/.sshd/sshd_config SSHD_CONFIG=${HOME}/.sshd/sshd_config diff --git a/releasing/add_katib_image_shas.sh b/releasing/add_katib_image_shas.sh index f499a4b9d81..f92fe706492 100755 --- a/releasing/add_katib_image_shas.sh +++ b/releasing/add_katib_image_shas.sh @@ -1,14 +1,14 @@ -#!/bin/bash +#!/usr/bin/env bash # # Fetch Katib image shas from GCR and them to image_tags.yaml # set -ex -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT_DIR="$( cd ${DIR}/.. && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd ${DIR}/.. && pwd)" IMAGES_FILE=${ROOT_DIR}/releasing/image_tags.yaml -# Assume the testing repo is checkout in git_kubeflow_testing because +# Assume the testing repo is checkout in git_kubeflow_testing because # we depend on the python code in that repo. export PYTHONPATH=${PYTHONPATH}:${ROOT_DIR}/../git_kubeflow-testing/py @@ -16,4 +16,4 @@ KATIB_TAG=v0.1.2-alpha-34-gb46378c # Fetch shas for Jupyter images python ${ROOT_DIR}/releasing/add_image_shas.py --pattern=.*katib/.*:${KATIB_TAG} \ - --images_file=${IMAGES_FILE} + --images_file=${IMAGES_FILE} diff --git a/releasing/run_apply_image_tags.sh b/releasing/run_apply_image_tags.sh index 18d7300e282..9353c56c311 100755 --- a/releasing/run_apply_image_tags.sh +++ b/releasing/run_apply_image_tags.sh @@ -1,10 +1,10 @@ -#!/bin/bash +#!/usr/bin/env bash # Wrapper script for running apply_image_tags.py set -ex PATTERN=$1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT_DIR="$( cd ${DIR}/.. && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd ${DIR}/.. && pwd)" # We need to add kubeflow/testing to the python path. # We assume its checked out as git_kubeflow-testing @@ -12,5 +12,5 @@ export PYTHONPATH=${PYTHONPATH}:${ROOT_DIR}/../git_kubeflow-testing/py # Update the TFJob operator image python ${ROOT_DIR}/releasing/apply_image_tags.py \ - --images_file=${ROOT_DIR}/releasing/image_tags.yaml \ - --pattern=${PATTERN} \ No newline at end of file + --images_file=${ROOT_DIR}/releasing/image_tags.yaml \ + --pattern=${PATTERN} diff --git a/releasing/update_components.sh b/releasing/update_components.sh index ee0214afd9b..22aeed0ac7f 100755 --- a/releasing/update_components.sh +++ b/releasing/update_components.sh @@ -2,7 +2,7 @@ # # This is a script to automate updating ksonnet components. Use this script # to update the image versions after tagging them with run_apply_image_tags.sh. -# +# # Usage: # update_components.sh component tag # @@ -14,10 +14,10 @@ set -ex COMPONENT=$1 TAG=$2 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT_DIR="$( cd ${DIR}/.. && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd ${DIR}/.. && pwd)" -# Assume the testing repo is checkout in git_kubeflow_testing because +# Assume the testing repo is checkout in git_kubeflow_testing because # we depend on the python code in that repo. export PYTHONPATH=${PYTHONPATH}:${ROOT_DIR}/../git_kubeflow-testing/py @@ -30,15 +30,15 @@ fi if [ "${COMPONENT}" == "tf-operator" ]; then echo "Updating TF operator..." python scripts/update_prototype.py \ - --file=${ROOT_DIR}/kubeflow/tf-training/prototypes/tf-job-operator.jsonnet \ - --values=tfJobImage=gcr.io/kubeflow-images-public/tf_operator:${TAG} + --file=${ROOT_DIR}/kubeflow/tf-training/prototypes/tf-job-operator.jsonnet \ + --values=tfJobImage=gcr.io/kubeflow-images-public/tf_operator:${TAG} echo "Done." - + elif [ "${COMPONENT}" == "pytorch-operator" ]; then echo "Updating PyTorch operator..." python scripts/update_prototype.py \ - --file=${ROOT_DIR}/kubeflow/pytorch-job/prototypes/pytorch-operator.jsonnet \ - --values=pytorchJobImage=gcr.io/kubeflow-images-public/pytorch_operator:${TAG} + --file=${ROOT_DIR}/kubeflow/pytorch-job/prototypes/pytorch-operator.jsonnet \ + --values=pytorchJobImage=gcr.io/kubeflow-images-public/pytorch_operator:${TAG} echo "Done." elif [ "${COMPONENT}" == "katib" ]; then @@ -53,21 +53,21 @@ elif [ "${COMPONENT}" == "katib" ]; then VALUES="${VALUES},studyJobControllerImage=gcr.io/kubeflow-images-public/katib/studyjob-controller:${TAG}" VALUES="${VALUES},metricsCollectorImage=gcr.io/kubeflow-images-public/katib/metrics-collector:${TAG}" python ${ROOT_DIR}/scripts/update_prototype.py \ - --file=${ROOT_DIR}/kubeflow/katib/prototypes/all.jsonnet \ - --values=${VALUES} + --file=${ROOT_DIR}/kubeflow/katib/prototypes/all.jsonnet \ + --values=${VALUES} echo "Done." elif [ "${COMPONENT}" == "centraldashboard" ]; then echo "Updating CentralDashboard..." python scripts/update_prototype.py \ - --file=${ROOT_DIR}/kubeflow/core/prototypes/centraldashboard.jsonnet \ - --values=image=gcr.io/kubeflow-images-public/centraldashboard:${TAG} + --file=${ROOT_DIR}/kubeflow/core/prototypes/centraldashboard.jsonnet \ + --values=image=gcr.io/kubeflow-images-public/centraldashboard:${TAG} echo "Done." elif [ "${COMPONENT}" == "jupyter-notebooks" ]; then echo "Updating Jupyter notebooks..." sed -i "s/tensorflow-\([0-9\.]*\)-notebook-\(.*\):v[0-9\.]*/tensorflow-\1-notebook-\2:${TAG}/" \ - kubeflow/core/ui/default/config.yaml + kubeflow/core/ui/default/config.yaml echo "Done." else diff --git a/releasing/update_image_tags.sh b/releasing/update_image_tags.sh index abd7b10f9fb..f93f6d074b5 100755 --- a/releasing/update_image_tags.sh +++ b/releasing/update_image_tags.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A script to automate updating the images to use in the ksonnet # components. @@ -10,12 +10,12 @@ # # If image_tags.yaml looks good invoke apply_tags.py set -ex -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT_DIR="$( cd ${DIR}/.. && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd ${DIR}/.. && pwd)" IMAGES_FILE=${ROOT_DIR}/releasing/image_tags.yaml -# Assume the testing repo is checkout in git_kubeflow_testing because +# Assume the testing repo is checkout in git_kubeflow_testing because # we depend on the python code in that repo. export PYTHONPATH=${PYTHONPATH}:${ROOT_DIR}/../git_kubeflow-testing/py @@ -26,8 +26,8 @@ RELEASE=v0.3.1 # Fetch shas for Jupyter images python ${ROOT_DIR}/releasing/add_image_shas.py --pattern=.*tensorflow.*1.*notebook.*:${JUPYTER_TAG} \ - --images_file=${IMAGES_FILE} + --images_file=${IMAGES_FILE} # Tag the Jupyter images we want with the desired relase tag. python ${ROOT_DIR}/releasing/add_image_tag.py --pattern=.*tensorflow.*1.*notebook.*:${JUPYTER_TAG} --tag=${RELEASE} \ - --images_file=${IMAGES_FILE} + --images_file=${IMAGES_FILE} diff --git a/scripts/autoformat_jsonnet.sh b/scripts/autoformat_jsonnet.sh index 1eceebe7c08..99a6fba89a3 100755 --- a/scripts/autoformat_jsonnet.sh +++ b/scripts/autoformat_jsonnet.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -ex # Copyright 2018 The Kubeflow Authors All rights reserved. # @@ -19,61 +19,59 @@ set -ex ALL_FILES=false -function usage() -{ - echo "autoformat_jsonnet.sh [--all]" - echo "" - echo "Autoformats .jsonnet and .libjsonnet files tracked by git." - echo "By default only files relative that are modified to origin/master are formatted" - echo "" - echo "Options:" - echo " --all : Formats all .jsonnet and .libjsonnet files." +usage() { + echo "autoformat_jsonnet.sh [--all]" + echo "" + echo "Autoformats .jsonnet and .libjsonnet files tracked by git." + echo "By default only files relative that are modified to origin/master are formatted" + echo "" + echo "Options:" + echo " --all : Formats all .jsonnet and .libjsonnet files." } while [ "$1" != "" ]; do - PARAM=`echo $1 | awk -F= '{print $1}'` - case $PARAM in - -h | --help) - usage - exit - ;; - --all) - ALL_FILES=true - ;; - *) - echo "ERROR: unknown parameter \"$PARAM\"" - usage - exit 1 - ;; - esac - shift + PARAM=$(echo $1 | awk -F= '{print $1}') + case $PARAM in + -h | --help) + usage + exit + ;; + --all) + ALL_FILES=true + ;; + *) + echo "ERROR: unknown parameter \"$PARAM\"" + usage + exit 1 + ;; + esac + shift done if $ALL_FILES; then - fmt_files=($(git ls-files -- '*.libsonnet' '*.jsonnet')) -else - # Checkout versions of the code that shouldn't be overwritten - raw=`git remote` - readarray -t remotes <<< "$raw" + fmt_files=($(git ls-files -- '*.libsonnet' '*.jsonnet')) +else + # Checkout versions of the code that shouldn't be overwritten + raw=$(git remote) + readarray -t remotes <<< "$raw" - repo_name='' - non_matching='' - for r in "${remotes[@]}" - do - url=`git remote get-url ${r}` + repo_name='' + non_matching='' + for r in "${remotes[@]}"; do + url=$(git remote get-url ${r}) # Period is in brackets because its a special character. if [[ ${url} =~ (git@github[.]com:kubeflow/.*|https://github[.]com/kubeflow/.*) ]]; then - repo_name=${r} + repo_name=${r} else - non_matching="${non_matching}${r} at ${url} did not match" + non_matching="${non_matching}${r} at ${url} did not match" fi - done - echo using ${repo_name} - if [ -z "$repo_name" ]; then - echo "Could not find remote repository pointing at git@github.com:kubeflow/.*.git in ${non_matching}" - exit 1 - fi - fmt_files=($(git diff --name-only ${repo_name}/master -- '*.libsonnet' '*.jsonnet')) + done + echo using ${repo_name} + if [ -z "$repo_name" ]; then + echo "Could not find remote repository pointing at git@github.com:kubeflow/.*.git in ${non_matching}" + exit 1 + fi + fmt_files=($(git diff --name-only ${repo_name}/master -- '*.libsonnet' '*.jsonnet')) fi # Need to execute from root because git will return full paths. @@ -86,13 +84,12 @@ cd ${ROOT} # Use // for comments # # TODO(jlewi): We should probably exclude vendor and k8s lib files. -for f in "${fmt_files[@]}" -do +for f in "${fmt_files[@]}"; do if [ ! -f $f ]; then echo "$f doesn't exist; it was probably deleted" continue fi - jsonnet fmt -i --string-style d --comment-style s --indent 2 $f + jsonnet fmt -i --string-style d --comment-style s --indent 2 $f echo "Autoformatted $f" done diff --git a/scripts/cleanup_kubeflow_ci.sh b/scripts/cleanup_kubeflow_ci.sh index 6da5c41c553..6a66c3563ec 100755 --- a/scripts/cleanup_kubeflow_ci.sh +++ b/scripts/cleanup_kubeflow_ci.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script cleans up ingress related compute engine resources like backend-services, # forwarding rules, health-checks, etc. from kubeflow-ci project which # do not belong to the kubeflow-testing cluster @@ -11,12 +11,12 @@ PROJECT="kubeflow-ci" # They use a common suffix for GCP resources KUBEFLOW_TESTING_ID=c3cb19bff97cde34 -function cleanup() { - gcloud compute ${1} list --project=${PROJECT} \ - | grep -v ${KUBEFLOW_TESTING_ID} \ - | awk '{print $1}' \ - | tail -n +2 \ - | xargs gcloud compute ${1} delete ${2} --quiet --project=${PROJECT} +cleanup() { + gcloud compute ${1} list --project=${PROJECT} | + grep -v ${KUBEFLOW_TESTING_ID} | + awk '{print $1}' | + tail -n +2 | + xargs gcloud compute ${1} delete ${2} --quiet --project=${PROJECT} } cleanup forwarding-rules --global diff --git a/scripts/download.sh b/scripts/download.sh index 40a645313c2..7bdf04d1ef8 100755 --- a/scripts/download.sh +++ b/scripts/download.sh @@ -1,13 +1,12 @@ -#!/bin/bash +#!/usr/bin/env bash # # Download the registry and scripts. # This is the first step in setting up Kubeflow. # Downloads it to the current directory set -ex - if [ ! -z "${KUBEFLOW_VERSION}" ]; then - KUBEFLOW_TAG=v${KUBEFLOW_VERSION} + KUBEFLOW_TAG=v${KUBEFLOW_VERSION} fi KUBEFLOW_TAG=${KUBEFLOW_TAG:-master} @@ -15,7 +14,7 @@ KUBEFLOW_TAG=${KUBEFLOW_TAG:-master} # Create a local copy of the Kubeflow source repo TMPDIR=$(mktemp -d /tmp/tmp.kubeflow-repo-XXXX) curl -L -o ${TMPDIR}/kubeflow.tar.gz https://github.com/kubeflow/kubeflow/archive/${KUBEFLOW_TAG}.tar.gz -tar -xzvf ${TMPDIR}/kubeflow.tar.gz -C ${TMPDIR} +tar -xzvf ${TMPDIR}/kubeflow.tar.gz -C ${TMPDIR} # GitHub seems to strip out the v in the file name. KUBEFLOW_SOURCE=$(find ${TMPDIR} -maxdepth 1 -type d -name "kubeflow*") diff --git a/scripts/gke/delete_deployment.sh b/scripts/gke/delete_deployment.sh index 397f4605ae2..b5592bb89f3 100755 --- a/scripts/gke/delete_deployment.sh +++ b/scripts/gke/delete_deployment.sh @@ -1,36 +1,35 @@ -#!/bin/bash +#!/usr/bin/env bash # Teardown the GCP deployment for Kubeflow. # We explicitly don't delete GCFS because we don't want to destroy # data. -# +# # Don't fail on error because some commands will fail if the resources were already deleted. -set -x +set -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" PROJECT=$1 DEPLOYMENT_NAME=$2 CONFIG_FILE=$3 gcloud deployment-manager --project=${PROJECT} deployments delete \ - ${DEPLOYMENT_NAME} \ - --quiet + ${DEPLOYMENT_NAME} \ + --quiet RESULT=$? if [ ${RESULT} -ne 0 ]; then - echo deleting the deployment did not work retry with abandon - gcloud deployment-manager --project=${PROJECT} deployments delete \ - ${DEPLOYMENT_NAME} \ - --quiet \ - --delete-policy=abandon - + echo deleting the deployment did not work retry with abandon + gcloud deployment-manager --project=${PROJECT} deployments delete \ + ${DEPLOYMENT_NAME} \ + --quiet \ + --delete-policy=abandon fi # Ensure resources are deleted. gcloud --project=${PROJECT} container clusters delete --zone=${ZONE} \ - ${DEPLOYMENT_NAME} --quiet + ${DEPLOYMENT_NAME} --quiet # Delete IAM bindings python "${DIR}/iam_patch.py" --action=remove \ @@ -43,7 +42,7 @@ declare -a accounts=("vm" "admin" "user") deleteSa() { local SA=$1 - O=`gcloud --project=${PROJECT} iam service-accounts describe ${SA} 2>&1` + O=$(gcloud --project=${PROJECT} iam service-accounts describe ${SA} 2>&1) local RESULT=$? if [ "${RESULT}" -ne 0 ]; then @@ -51,16 +50,15 @@ deleteSa() { return fi - return + return gcloud --project=${PROJECT} iam service-accounts delete \ - ${SA} \ - --quiet + ${SA} \ + --quiet } # now loop through the above array -for suffix in "${accounts[@]}"; -do - # Delete all role bindings. - SA=${DEPLOYMENT_NAME}-${suffix}@${PROJECT}.iam.gserviceaccount.com - deleteSa ${SA} +for suffix in "${accounts[@]}"; do + # Delete all role bindings. + SA=${DEPLOYMENT_NAME}-${suffix}@${PROJECT}.iam.gserviceaccount.com + deleteSa ${SA} done diff --git a/scripts/gke/teardown.sh b/scripts/gke/teardown.sh index 56767ab4175..ba6fbc21ee4 100755 --- a/scripts/gke/teardown.sh +++ b/scripts/gke/teardown.sh @@ -9,7 +9,7 @@ echo "WARNING teardown.sh is deprecated; use kfctl.sh" DEPLOYMENT_NAME=${DEPLOYMENT_NAME:-"kubeflow"} CONFIG_FILE=${CONFIG_FILE:-"cluster-kubeflow.yaml"} PROJECT=${PROJECT:-$(gcloud config get-value project 2>/dev/null)} -KUBEFLOW_DM_DIR=${KUBEFLOW_DM_DIR:-"`pwd`/${DEPLOYMENT_NAME}_deployment_manager_configs"} +KUBEFLOW_DM_DIR=${KUBEFLOW_DM_DIR:-"$(pwd)/${DEPLOYMENT_NAME}_deployment_manager_configs"} GCFS_INSTANCE=${GCFS_INSTANCE:-"${DEPLOYMENT_NAME}"} # GCP Zone ZONE=${ZONE:-$(gcloud config get-value compute/zone 2>/dev/null)} @@ -32,12 +32,12 @@ set -e # we need to obtain a fresh copy of the IAM policy. A stale # copy of IAM policy causes issues during deletion. gcloud deployment-manager deployments update \ - ${DEPLOYMENT_NAME} --config=${CONFIG_FILE} --project=${PROJECT} + ${DEPLOYMENT_NAME} --config=${CONFIG_FILE} --project=${PROJECT} gcloud deployment-manager deployments delete --quiet \ - ${DEPLOYMENT_NAME} --project=${PROJECT} + ${DEPLOYMENT_NAME} --project=${PROJECT} gcloud beta filestore instances delete ${GCFS_INSTANCE} \ - --project=${PROJECT} \ - --location=${ZONE} \ - --quiet + --project=${PROJECT} \ + --location=${ZONE} \ + --quiet diff --git a/scripts/gke/use_gcr_for_all_images.sh b/scripts/gke/use_gcr_for_all_images.sh index 8d083db376f..a73b49d06a2 100755 --- a/scripts/gke/use_gcr_for_all_images.sh +++ b/scripts/gke/use_gcr_for_all_images.sh @@ -1,11 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash # use_gcr_for_all_images.sh is a simple script which is intended to be run in a ksonnet # app directory. It sets the docker image params in all the components to use the images # from gcr.io registries instead of non-gcr.io registries. This is useful when deploying # private GKE clusters where one can only pull images from gcr.io # To push an image from DockerHub / Quay to gcr.io/kubeflow-images-public registry, use # the following bash function -# function sync_image() { +# sync_image() { # local source="${1}" # local target="gcr.io/kubeflow-images-public/${1}" # docker pull "${source}" @@ -18,22 +18,22 @@ set -x -if ks component list | awk '{print $1}' | grep -q "^argo$" ; then +if ks component list | awk '{print $1}' | grep -q "^argo$"; then ks param set argo workflowControllerImage gcr.io/kubeflow-images-public/argoproj/workflow-controller:v2.2.0 ks param set argo uiImage gcr.io/kubeflow-images-public/argoproj/argoui:v2.2.0 ks param set argo executorImage gcr.io/kubeflow-images-public/argoproj/argoexec:v2.2.0 fi -if ks component list | awk '{print $1}' | grep -q "^cert-manager$" ; then +if ks component list | awk '{print $1}' | grep -q "^cert-manager$"; then ks param set cert-manager certManagerImage gcr.io/kubeflow-images-public/quay.io/jetstack/cert-manager-controller:v0.2.4 ks param set cert-manager certManagerIngressShimImage gcr.io/kubeflow-images-public/quay.io/jetstack/cert-manager-ingress-shim:v0.2.4 fi -if ks component list | awk '{print $1}' | grep -q "^ambassador$" ; then +if ks component list | awk '{print $1}' | grep -q "^ambassador$"; then ks param set ambassador ambassadorImage gcr.io/kubeflow-images-public/quay.io/datawire/ambassador:0.37.0 fi -if ks component list | awk '{print $1}' | grep -q "^katib$" ; then +if ks component list | awk '{print $1}' | grep -q "^katib$"; then ks param set katib modeldbDatabaseImage gcr.io/kubeflow-images-public/mongo:3.4 ks param set katib vizierDbImage gcr.io/kubeflow-images-public/mysql:8.0.3 fi diff --git a/scripts/gke/util.sh b/scripts/gke/util.sh index 2bae56a6edc..0d05d1d44ad 100644 --- a/scripts/gke/util.sh +++ b/scripts/gke/util.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Define functions to customize the Kubeflow app for GCP. # @@ -13,7 +13,7 @@ gcpCreateSecretsDir() { # 1. We put the secrets in a directory with a .gitignore file # 2. We will delete the secrets immediately. if [ ! -f ${KUBEFLOW_SECRETS_DIR}/.gitignore ]; then - cat > ${KUBEFLOW_SECRETS_DIR}/.gitignore << EOF + cat > ${KUBEFLOW_SECRETS_DIR}/.gitignore <&1` + O=$(kubectl get secret --namespace=${K8S_NAMESPACE} ${SECRET} 2>&1) local RESULT=$? set -e @@ -99,10 +99,10 @@ downloadK8sManifests() { https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/stable/nvidia-driver-installer/cos/daemonset-preloaded.yaml curl -o ${KUBEFLOW_K8S_MANIFESTS_DIR}/rbac-setup.yaml \ - https://storage.googleapis.com/stackdriver-kubernetes/stable/rbac-setup.yaml + https://storage.googleapis.com/stackdriver-kubernetes/stable/rbac-setup.yaml curl -o ${KUBEFLOW_K8S_MANIFESTS_DIR}/agents.yaml \ - https://storage.googleapis.com/stackdriver-kubernetes/stable/agents.yaml + https://storage.googleapis.com/stackdriver-kubernetes/stable/agents.yaml } @@ -114,7 +114,7 @@ updateDeployment() { cd ${KUBEFLOW_DM_DIR} # Check if it already exists set +e - O=`gcloud deployment-manager --project=${PROJECT} deployments describe ${NAME} 2>&1` + O=$(gcloud deployment-manager --project=${PROJECT} deployments describe ${NAME} 2>&1) exists=$? set -e @@ -157,14 +157,14 @@ updateDM() { kubectl config set-context ${KUBEFLOW_K8S_CONTEXT} \ --namespace ${K8S_NAMESPACE} \ - --cluster $CURRENT_CLUSTER \ - --user $CURRENT_USER + --cluster $CURRENT_CLUSTER \ + --user $CURRENT_USER echo created context named: ${KUBEFLOW_K8S_CONTEXT} kubectl config use-context ${KUBEFLOW_K8S_CONTEXT} # Make yourself cluster admin set +e - O=`kubectl get clusterrolebinding default-admin 2>&1` + O=$(kubectl get clusterrolebinding default-admin 2>&1) RESULT=$? set -e @@ -175,7 +175,7 @@ updateDM() { fi set +e - O=`kubectl get namespace ${K8S_NAMESPACE} 2>&1` + O=$(kubectl get namespace ${K8S_NAMESPACE} 2>&1) RESULT=$? set -e @@ -206,7 +206,7 @@ createSecrets() { createGcpSecret ${USER_EMAIL} user-gcp-sa set +e - O=`kubectl get secret --namespace=${K8S_NAMESPACE} kubeflow-oauth 2>&1` + O=$(kubectl get secret --namespace=${K8S_NAMESPACE} kubeflow-oauth 2>&1) RESULT=$? set -e @@ -234,7 +234,7 @@ gcpKsApply() { cd "${KUBEFLOW_KS_DIR}" set +e - O=`ks env describe default 2>&1` + O=$(ks env describe default 2>&1) RESULT=$? set -e diff --git a/scripts/kfctl.sh b/scripts/kfctl.sh index 4fe91239863..06d56c3a988 100755 --- a/scripts/kfctl.sh +++ b/scripts/kfctl.sh @@ -154,7 +154,6 @@ ksApply() { set -x } - parseArgs() { # Parse all command line options while [[ $# -gt 0 ]]; do @@ -188,7 +187,7 @@ parseArgs() { # Check for gcp specific parameters to be set before proceeding if [ "${PLATFORM}" == "gcp" ]; then - # GCP Project + # GCP Project if [ -z "${PROJECT}" ]; then PROJECT=$(gcloud config get-value project 2>/dev/null) if [ -z "${PROJECT}" ]; then diff --git a/scripts/kfctl_generate_test.sh b/scripts/kfctl_generate_test.sh index 42f63f83403..7544f395f67 100755 --- a/scripts/kfctl_generate_test.sh +++ b/scripts/kfctl_generate_test.sh @@ -1,10 +1,10 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple and minimal test for kfctl.sh # TODO(jlewi): Should add this to our ci system set -ex -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TMPDIR=$(mktemp -d /tmp/tmp.kfctl-test-XXXX) cd ${TMPDIR} ${DIR}/kfctl.sh init kfapp --platform non diff --git a/scripts/run_gofmt.sh b/scripts/run_gofmt.sh index e3d28f5acd9..e680fa5b717 100755 --- a/scripts/run_gofmt.sh +++ b/scripts/run_gofmt.sh @@ -1,71 +1,67 @@ -#!/bin/bash +#!/usr/bin/env bash ALL_FILES=false -function usage() -{ - echo "run_gofmt [--all]" - echo "" - echo "Autoformats .go files tracked by git." - echo "By default only files relative that are modified to origin/master are formatted" - echo "" - echo "Options:" - echo " --all : Formats all .go files" +usage() { + echo "run_gofmt [--all]" + echo "" + echo "Autoformats .go files tracked by git." + echo "By default only files relative that are modified to origin/master are formatted" + echo "" + echo "Options:" + echo " --all : Formats all .go files" } # Checkout versions of the code that shouldn't be overwritten -raw=`git remote` -readarray -t remotes <<< "$raw" +raw=$(git remote) +readarray -t remotes <<<"$raw" repo_name='' -for r in "${remotes[@]}" -do - url=`git remote get-url ${r}` - # Period is in brackets because its a special character. - if [[ ${url} =~ git@github[.]com:kubeflow/.* ]]; then - repo_name=${r} - fi +for r in "${remotes[@]}"; do + url=$(git remote get-url ${r}) + # Period is in brackets because its a special character. + if [[ ${url} =~ git@github[.]com:kubeflow/.* ]]; then + repo_name=${r} + fi done echo using ${repo_name} if [ -z "$repo_name" ]; then - echo "Could not find remote repository pointing at git@github.com:kubeflow/kubeflow.git" - exit 1 + echo "Could not find remote repository pointing at git@github.com:kubeflow/kubeflow.git" + exit 1 fi - while [ "$1" != "" ]; do - PARAM=`echo $1 | awk -F= '{print $1}'` - case $PARAM in - -h | --help) - usage - exit - ;; - --all) - ALL_FILES=true - ;; - *) - echo "ERROR: unknown parameter \"$PARAM\"" - usage - exit 1 - ;; - esac - shift + PARAM=$(echo $1 | awk -F= '{print $1}') + case $PARAM in + -h | --help) + usage + exit + ;; + --all) + ALL_FILES=true + ;; + *) + echo "ERROR: unknown parameter \"$PARAM\"" + usage + exit 1 + ;; + esac + shift done if $ALL_FILES; then - fmt_files=($(git ls-files -- '*.go')) + fmt_files=($(git ls-files -- '*.go')) else - fmt_files=($(git diff --name-only ${repo_name}/master -- '*.go')) + fmt_files=($(git diff --name-only ${repo_name}/master -- '*.go')) fi # 2 spaces vertical indentation # Use double quotes for strings # Use // for comments -for f in "${fmt_files[@]}" -do +for f in "${fmt_files[@]}"; do if [[ "${f}" =~ "/vendor/" ]]; then - continue + continue fi gofmt -w "$f" goimports -w "$f" diff --git a/scripts/setup-minikube.sh b/scripts/setup-minikube.sh index 682855d1b24..89f6427aa11 100644 --- a/scripts/setup-minikube.sh +++ b/scripts/setup-minikube.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # A single commmand deployer script for Minikube with Kubeflow. # Linux or Mac OS X # @@ -11,15 +11,12 @@ NC='\033[0m' OSX="Darwin" LINUX="Linux" -PLATFORM=`uname` -if [[ $PLATFORM == $LINUX ]] -then - fpn=`uname -a` - if [[ $fpn =~ *(Ubuntu|Debian)* ]] - then +PLATFORM=$(uname) +if [[ $PLATFORM == $LINUX ]]; then + fpn=$(uname -a) + if [[ $fpn =~ *(Ubuntu|Debian)* ]]; then DIST_TYPE="Ubuntu" - elif [[ $fpn =~ *(CentOS|Fedora|Red Hat)* ]] - then + elif [[ $fpn =~ *(CentOS|Fedora|Red Hat)* ]]; then DIST_TYPE="CentOS" fi fi @@ -28,33 +25,29 @@ MINIKUBE_CMD="minikube start" MOUNT_LOCAL=false KUBEFLOW_VERSION=${KUBEFLOW_VERSION:-"master"} -GB1=$(( 1024 * 1024 * 1024 )) -MB1=$(( 1024 * 1024 )) -KB1=$(( 1024 )) +GB1=$((1024 * 1024 * 1024)) +MB1=$((1024 * 1024)) +KB1=$((1024)) -function main() { +main() { infer_minikube_settings - if cleanup_and_deploy_minikube - then - sleep 30 # Give minikube time to take its first breath + if cleanup_and_deploy_minikube; then + sleep 30 # Give minikube time to take its first breath deploy_kubeflow fi } -function install_ks_kubectl_minikube() { +install_ks_kubectl_minikube() { # Installing ksonnet if needed - KS_VERSION=`ks version | grep 'ksonnet version' | awk '{print $3}'` - if [[ $KS_VERSION != "0.11.0" ]] - then + KS_VERSION=$(ks version | grep 'ksonnet version' | awk '{print $3}') + if [[ $KS_VERSION != "0.11.0" ]]; then echo -e "${YELLOW}Installing ksonnet...${NC}" - if [[ ${HOST_PLATFORM} == $OSX ]] - then + if [[ ${HOST_PLATFORM} == $OSX ]]; then curl -OL https://github.com/ksonnet/ksonnet/releases/download/v0.11.0/ks_0.11.0_darwin_amd64.tar.gz tar zxf ks_0.11.0_darwin_amd64.tar.gz export PATH=$PATH:$(pwd)/ks_0.11.0_darwin_amd64 - elif [[ ${HOST_PLATFORM} == $LINUX ]] - then + elif [[ ${HOST_PLATFORM} == $LINUX ]]; then curl -OL https://github.com/ksonnet/ksonnet/releases/download/v0.11.0/ks_0.11.0_linux_amd64.tar.gz tar zxf ks_0.11.0_linux_amd64.tar.gz export PATH=$PATH:$(pwd)/ks_0.11.0_linux_amd64 @@ -63,24 +56,19 @@ function install_ks_kubectl_minikube() { fi # Installing kubectl if needed - if ! kubectl -h 2>&1 >/dev/null - then + if ! kubectl -h 2>&1 >/dev/null; then echo -e "${YELLOW}Installing kubectl...${NC}" - if [[ $PLATFORM == $OSX ]] - then + if [[ $PLATFORM == $OSX ]]; then brew install kubectl - elif [[ $PLATFORM == $LINUX ]] - then - if [[ $DIST_TYPE == "Ubuntu" ]] - then + elif [[ $PLATFORM == $LINUX ]]; then + if [[ $DIST_TYPE == "Ubuntu" ]]; then sudo apt-get update && sudo apt-get install -y apt-transport-https curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - - sudo touch /etc/apt/sources.list.d/kubernetes.list + sudo touch /etc/apt/sources.list.d/kubernetes.list echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list sudo apt-get update sudo apt-get install -y kubectl - elif [[ $DIST_TYPE == "CentOS" ]] - then + elif [[ $DIST_TYPE == "CentOS" ]]; then sudo cat < /etc/yum.repos.d/kubernetes.repo [kubernetes] name=Kubernetes @@ -97,94 +85,80 @@ EOF fi # Installing minikube if needed - if ! minikube -h 2>&1 >/dev/null - then + if ! minikube -h 2>&1 >/dev/null; then echo -e "${YELLOW}Installing minikube...$NC}" - if [[ $PLATFORM == $OSX ]] - then + if [[ $PLATFORM == $OSX ]]; then curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.28.0/minikube-darwin-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ - elif [[ $PLATFORM == $LINUX ]] - then + elif [[ $PLATFORM == $LINUX ]]; then curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.28.0/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ fi echo -e "${GREEN}[OK]${NC}" fi } -function cleanup_and_deploy_minikube() { +cleanup_and_deploy_minikube() { install_ks_kubectl_minikube # Stop and delete previous existence of Minikube VM. - mini_running=`minikube status | grep 'minikube:' | awk '{print $2}'` - if [[ $mini_running = "Running" ]] - then + mini_running=$(minikube status | grep 'minikube:' | awk '{print $2}') + if [[ $mini_running == "Running" ]]; then minikube stop fi minikube delete # start minikube with desired settings $MINIKUBE_CMD - if ! minikube status 2>&1 >/dev/null - then + if ! minikube status 2>&1 >/dev/null; then echo -e "${RED}Unable to start minikube. Please see errors above${NC}" return 1 fi - minikube_ip=`minikube ip` + minikube_ip=$(minikube ip) echo -e "Minikube dashboard at ${GREEN}http://$minikube_ip:30000/${NC}" } -function infer_minikube_settings() { +infer_minikube_settings() { # cpus local np=3 - if [[ $PLATFORM == $LINUX ]] - then - np=`nproc --all` - elif [[ $PLATFORM == $OSX ]] - then - np=`sysctl -n hw.ncpu` - fi - echo -n "Assign CPUs between 3..$(( np - 2 )) [$(( np / 2 ))]: " + if [[ $PLATFORM == $LINUX ]]; then + np=$(nproc --all) + elif [[ $PLATFORM == $OSX ]]; then + np=$(sysctl -n hw.ncpu) + fi + echo -n "Assign CPUs between 3..$((np - 2)) [$((np / 2))]: " read p - if [[ $p == "" ]] - then - p=$(( np / 2 )) + if [[ $p == "" ]]; then + p=$((np / 2)) fi MINIKUBE_CMD=$MINIKUBE_CMD" --cpus $p" # memory local mm=8 - if [[ $PLATFORM == $LINUX ]] - then - local mm_kb=`cat /proc/meminfo | grep 'MemTotal' | awk '{print $2}'` - mm=$(( mm_kb * KB1 / GB1 )) - elif [[ $PLATFORM == $OSX ]] - then - mm=$(( `sysctl -n hw.memsize` / GB1 )) - fi - if (( $mm < 16 )) - then + if [[ $PLATFORM == $LINUX ]]; then + local mm_kb=$(cat /proc/meminfo | grep 'MemTotal' | awk '{print $2}') + mm=$((mm_kb * KB1 / GB1)) + elif [[ $PLATFORM == $OSX ]]; then + mm=$(($(sysctl -n hw.memsize) / GB1)) + fi + if (($mm < 16)); then echo -e "${YELLOW}WARNING: Your system has low memory for an ML workload.${NC}" fi - echo -n "Assign memory (in GB) between 8..$(( mm - 5 )) [$(( mm / 2 ))]: " + echo -n "Assign memory (in GB) between 8..$((mm - 5)) [$((mm / 2))]: " read m - if [[ $m == "" ]] - then - m=$(( mm / 2 )) + if [[ $m == "" ]]; then + m=$((mm / 2)) fi - MINIKUBE_CMD=$MINIKUBE_CMD" --memory $(( m * KB1 ))" + MINIKUBE_CMD=$MINIKUBE_CMD" --memory $((m * KB1))" # disk local dd=40 - dd=`df -h . | awk '{print $4}' | tail -1 | sed 's/[A-Za-z]//g'` - if (( $dd < 40 )) - then + dd=$(df -h . | awk '{print $4}' | tail -1 | sed 's/[A-Za-z]//g') + if (($dd < 40)); then echo -e "${YELLOW}WARNING: Low available disk space.${NC}" fi - echo -n "Assign disk space (in GB) between 40..$(( dd / 2 )) [50]: " + echo -n "Assign disk space (in GB) between 40..$((dd / 2)) [50]: " read d - if [[ $d == "" ]] - then + if [[ $d == "" ]]; then d=50 fi MINIKUBE_CMD=$MINIKUBE_CMD" --disk-size ${d}g" @@ -192,30 +166,27 @@ function infer_minikube_settings() { # virtualizer echo -n "Choose a virtualizer/hypervisor installed on your system (virtualbox, vmwarefusion, kvm2, hyperkit): " read v - if [[ $v != "" ]] - then + if [[ $v != "" ]]; then MINIKUBE_CMD=$MINIKUBE_CMD" --vm-driver=$v" fi # Mount local dir echo -n "If you'd like to access a local directory in Jupyter, please enter the full path: " read l - if [[ $l != "" ]] - then + if [[ $l != "" ]]; then MINIKUBE_CMD=$MINIKUBE_CMD" --mount-string=$l:/mnt/local --mount" MOUNT_LOCAL=true fi } -function download_kubeflow_source() { +download_kubeflow_source() { curl -O https://raw.githubusercontent.com/kubeflow/kubeflow/${KUBEFLOW_VERSION}/scripts/download.sh /bin/bash ./download.sh } -function deploy_kubeflow() { +deploy_kubeflow() { download_kubeflow_source - if [ -e ./localapp ] - then + if [ -e ./localapp ]; then rm -rf ./localapp fi KUBEFLOW_REPO=$(pwd) MOUNT_LOCAL=${MOUNT_LOCAL} ./scripts/kfctl.sh init localapp --platform minikube diff --git a/scripts/update-changelog.sh b/scripts/update-changelog.sh index 4f79985e396..64327fd42f2 100755 --- a/scripts/update-changelog.sh +++ b/scripts/update-changelog.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2018 The Kubeflow Authors. # @@ -29,16 +29,15 @@ SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../.. cd ${SCRIPT_ROOT} -if [ "${GITHUB_TOKEN}" == "NO" ] -then - echo "Environment variable GITHUB_TOKEN is not set." - exit 1 +if [ "${GITHUB_TOKEN}" == "NO" ]; then + echo "Environment variable GITHUB_TOKEN is not set." + exit 1 fi github_changelog_generator -t ${GITHUB_TOKEN} -u kubeflow -p kubeflow \ - --exclude-labels community/discussion,cmmunity/question,duplicate,question,invalid,wontfix \ - --bug-labels kind/bug,problems/bug \ - --enhancement-labels improvement/optimization,kind/enhancement,improvement/enhancement,addition/feature,kind/feature \ - --enhancement-label "**Features and improvements:**" + --exclude-labels community/discussion,cmmunity/question,duplicate,question,invalid,wontfix \ + --bug-labels kind/bug,problems/bug \ + --enhancement-labels improvement/optimization,kind/enhancement,improvement/enhancement,addition/feature,kind/feature \ + --enhancement-label "**Features and improvements:**" cd - > /dev/null diff --git a/scripts/util-minikube.sh b/scripts/util-minikube.sh index b76ebf031ec..ec77e63546f 100755 --- a/scripts/util-minikube.sh +++ b/scripts/util-minikube.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Helper functions for minikube deployment of kubeflow # Linux or Mac OS X @@ -10,23 +10,20 @@ NC='\033[0m' MOUNT_LOCAL=${MOUNT_LOCAL:-"false"} -function is_kubeflow_ready() { +is_kubeflow_ready() { echo -en "${YELLOW}Getting kubeflow namespace ready...${NC}" local ns_ready=false - for i in {1..5} - do - kube_ns=`kubectl get namespaces | grep ${K8S_NAMESPACE} | wc -l` - if [ $kube_ns = 1 ] - then - ns_ready=true - echo -e "${GREEN}[OK]${NC}" - break + for i in {1..5}; do + kube_ns=$(kubectl get namespaces | grep ${K8S_NAMESPACE} | wc -l) + if [ $kube_ns = 1 ]; then + ns_ready=true + echo -e "${GREEN}[OK]${NC}" + break fi sleep 30 done - if [[ $ns_ready = false ]] - then + if [[ $ns_ready == false ]]; then return 1 fi @@ -34,29 +31,25 @@ function is_kubeflow_ready() { amb_up=0 tf_hub_up=0 echo -en "${YELLOW}Bringing kubeflow services up." - until (( "$amb_up" > 0 && "$tf_hub_up" > 0 )) - do + until (("$amb_up" > 0 && "$tf_hub_up" > 0)); do sleep 30 - amb_up=`kubectl -n ${K8S_NAMESPACE} get pods | grep Running | grep ambassador | wc -l` - tf_hub_up=`kubectl -n ${K8S_NAMESPACE} get pods | grep Running | grep jupyter | wc -l` + amb_up=$(kubectl -n ${K8S_NAMESPACE} get pods | grep Running | grep ambassador | wc -l) + tf_hub_up=$(kubectl -n ${K8S_NAMESPACE} get pods | grep Running | grep jupyter | wc -l) echo -n "." - if (( "$amb_up" > 0 && "$tf_hub_up" > 0 )) - then + if (("$amb_up" > 0 && "$tf_hub_up" > 0)); then svc_ready=true echo -e "${GREEN}[OK]${NC}" break fi done - if [ $svc_ready = false ] - then + if [ $svc_ready = false ]; then return 1 fi } -function create_local_fs_mount_spec() { - if $MOUNT_LOCAL - then +create_local_fs_mount_spec() { + if $MOUNT_LOCAL; then # Create a persistent volume cat < ${KUBEFLOW_KS_DIR}/pv.yaml kind: PersistentVolume @@ -84,8 +77,8 @@ spec: - minikube EOF - # Create a PVC attached to the volume - cat < ${KUBEFLOW_KS_DIR}/pv-claim.yaml + # Create a PVC attached to the volume + cat < ${KUBEFLOW_KS_DIR}/pv-claim.yaml kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -104,16 +97,15 @@ EOF # if user requested a local fs path to be mounted, make it accessible via # Jupyter Notebooks -function mount_local_fs() { - if $MOUNT_LOCAL - then +mount_local_fs() { + if $MOUNT_LOCAL; then kubectl create -f ${KUBEFLOW_KS_DIR}/pv.yaml kubectl create -n ${K8S_NAMESPACE} -f ${KUBEFLOW_KS_DIR}/pv-claim.yaml fi } # Setup various network tunnels -function setup_tunnels() { +setup_tunnels() { sleep 30 # Give services time to bind kubectl -n kubeflow port-forward svc/ambassador 8080:80 2>&1 >/dev/null & echo -e "Access Kubeflow dashboard at ${GREEN}http://localhost:8080/${NC}" diff --git a/scripts/util.sh b/scripts/util.sh index ae0c6a4079c..5c9163ce901 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -1,36 +1,35 @@ -#!/bin/bash +#!/usr/bin/env bash # Util functions to be used by scripts in this directory -function usage() { - echo "usage: kfctl " - echo "where command is one of" - echo "init - initialize something" - echo "apply -- apply some config" - echo "delete - delete some components" - echo - echo "what is one of" - echo "project - the GCP project" - echo "platform - platform resources (e.g. GCP, minikube); basically non K8s resources" - echo "k8s - kubernetes resources" - echo "help - print this message" +usage() { + echo "usage: kfctl " + echo "where command is one of" + echo "init - initialize something" + echo "apply -- apply some config" + echo "delete - delete some components" + echo + echo "what is one of" + echo "project - the GCP project" + echo "platform - platform resources (e.g. GCP, minikube); basically non K8s resources" + echo "k8s - kubernetes resources" + echo "help - print this message" } - -function check_install() { +check_install() { if ! which "${1}" &>/dev/null; then echo "You don't have ${1} installed. Please install ${1}." exit 1 fi } -function check_variable() { +check_variable() { if [[ -z "${1}" ]]; then echo "'${2}' environment variable is not set. Please set it using export ${2}=value." exit 1 fi } -function createKsApp() { +createKsApp() { # Create the ksonnet application. # All deployments should call this function to create a common ksonnet app. # They can then customize it as necessary. @@ -79,7 +78,7 @@ function createKsApp() { # cd ks_app # ks component rm spartakus # Generate a random 30 bit number - local usageId=$(((RANDOM<<15)|RANDOM)) + local usageId=$(((RANDOM << 15) | RANDOM)) ks generate spartakus spartakus --usageId=${usageId} --reportUsage=true echo "" echo "****************************************************************" @@ -99,7 +98,7 @@ function createKsApp() { ks generate application application } -function removeKsEnv() { +removeKsEnv() { pushd ${KUBEFLOW_KS_DIR} set +e O=$(ks env describe default 2>&1) @@ -114,10 +113,10 @@ function removeKsEnv() { popd } -function customizeKsAppWithDockerImage() { - # customize docker registry - if [[ ! -z "$KUBEFLOW_DOCKER_REGISTRY" ]]; then - find ${KUBEFLOW_KS_DIR} -name "*.libsonnet" -o -name "*.jsonnet" | xargs sed -i -e "s%gcr.io%$KUBEFLOW_DOCKER_REGISTRY%g" - find ${KUBEFLOW_KS_DIR} -name "*.libsonnet" -o -name "*.jsonnet" | xargs sed -i -e "s%quay.io%$KUBEFLOW_DOCKER_REGISTRY%g" - fi +customizeKsAppWithDockerImage() { + # customize docker registry + if [[ ! -z "$KUBEFLOW_DOCKER_REGISTRY" ]]; then + find ${KUBEFLOW_KS_DIR} -name "*.libsonnet" -o -name "*.jsonnet" | xargs sed -i -e "s%gcr.io%$KUBEFLOW_DOCKER_REGISTRY%g" + find ${KUBEFLOW_KS_DIR} -name "*.libsonnet" -o -name "*.jsonnet" | xargs sed -i -e "s%quay.io%$KUBEFLOW_DOCKER_REGISTRY%g" + fi } diff --git a/testing/install_minikube.sh b/testing/install_minikube.sh index 16f8d7ec318..068b185619d 100644 --- a/testing/install_minikube.sh +++ b/testing/install_minikube.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A helper script to run on a VM to install and start minikube. @@ -7,19 +7,25 @@ set -ex # Install Docker. sudo apt-get update -y sudo apt-get install -y \ - apt-transport-https \ - ca-certificates \ - curl \ - software-properties-common - -curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | sudo apt-key add - - + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + +curl -fsSL https://download.docker.com/linux/$( + . /etc/os-release + echo "$ID" +)/gpg | sudo apt-key add - + sudo add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \ + "deb [arch=amd64] https://download.docker.com/linux/$( + . /etc/os-release + echo "$ID" + ) \ $(lsb_release -cs) \ stable" - -sudo apt-get update -y + +sudo apt-get update -y sudo apt-get install docker-ce -y # Install kubectl @@ -28,8 +34,8 @@ chmod +x ./kubectl sudo mv kubectl /usr/local/bin/ # Install minikube -curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.25.0/minikube-linux-amd64 -chmod +x minikube +curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.25.0/minikube-linux-amd64 +chmod +x minikube sudo mv minikube /usr/local/bin/ # We need a large disk for Jupyter. diff --git a/testing/run.sh b/testing/run.sh index 4e403e30b01..ed6011bf6c4 100644 --- a/testing/run.sh +++ b/testing/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2018 The Kubeflow Authors All rights reserved. # @@ -22,19 +22,18 @@ set -ex # Trigger a workflow if [ -f /src/${REPO_OWNER}/${REPO_NAME}/prow_config.yaml ]; then python -m kubeflow.testing.run_e2e_workflow \ - --project=kubeflow-ci \ - --zone=us-east1-d \ - --cluster=kubeflow-testing \ - --bucket=kubernetes-jenkins \ - --config_file=/src/${REPO_OWNER}/${REPO_NAME}/prow_config.yaml \ - --repos_dir=/src + --project=kubeflow-ci \ + --zone=us-east1-d \ + --cluster=kubeflow-testing \ + --bucket=kubernetes-jenkins \ + --config_file=/src/${REPO_OWNER}/${REPO_NAME}/prow_config.yaml \ + --repos_dir=/src else python -m kubeflow.testing.run_e2e_workflow \ - --project=kubeflow-ci \ - --zone=us-east1-d \ - --cluster=kubeflow-testing \ - --bucket=kubernetes-jenkins \ - --component=workflows \ - --app_dir=/src/kubeflow/kubeflow/testing/workflows + --project=kubeflow-ci \ + --zone=us-east1-d \ + --cluster=kubeflow-testing \ + --bucket=kubernetes-jenkins \ + --component=workflows \ + --app_dir=/src/kubeflow/kubeflow/testing/workflows fi - diff --git a/testing/workflows/run.sh b/testing/workflows/run.sh index 4915923a8e9..3b92615a011 100755 --- a/testing/workflows/run.sh +++ b/testing/workflows/run.sh @@ -1,7 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple wrapper script to run a command in the e2e tests. -# This script performs common functions like +# This script performs common functions like # activating the service account. set -ex gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} @@ -11,18 +11,17 @@ echo Working Directory=$(pwd) # TODO(jlewi): We should add retries on error. # Retry up to 3 times -for i in `seq 1 3`; -do - set +e - "$@" - result=$? - set -e - if [[ ${result} -eq 0 ]]; then - echo command ran successfully - exit 0 - fi +for i in $(seq 1 3); do + set +e + "$@" + result=$? + set -e + if [[ ${result} -eq 0 ]]; then + echo command ran successfully + exit 0 + fi - echo Command failed: "$@" + echo Command failed: "$@" done echo "command didn't succeed" exit 1