Implement runner for e2e tests (#548)

* implement a runner for e2e tests

* move e2e tests to a Docker container

* integrate e2e tests into build pipelines

* add tests for multi-namespace support and logical backup jobs

* @FxKu implement the first e2e test for failovers

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+exclude=.git,__pycache__
+max-line-length=120

.gitignore

@@ -34,3 +34,59 @@ scm-source.json
 # diagrams
 *.aux
 *.log
+
+# Python
+# Adapted from https://github.com/github/gitignore/blob/master/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot

.travis.yml

@@ -15,8 +15,9 @@ before_install:
   - go get github.com/mattn/goveralls
 
 install:
-  - make deps
+  - make deps e2e-tools e2e-build
 
 script:
   - hack/verify-codegen.sh
   - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v
+  - make e2e-run

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean local test linux macos docker push scm-source.json
+.PHONY: clean local test linux macos docker push scm-source.json e2e-run e2e-tools e2e-build
 
 BINARY ?= postgres-operator
 BUILD_FLAGS ?= -v
@@ -34,7 +34,7 @@ ifdef CDP_PULL_REQUEST_NUMBER
 	CDP_TAG := -${CDP_BUILD_VERSION}
 endif
 
-
+KIND_PATH := $(GOPATH)/bin
 PATH := $(GOPATH)/bin:$(PATH)
 SHELL := env PATH=$(PATH) $(SHELL)
 
@@ -91,3 +91,16 @@ deps:
 test:
 	hack/verify-codegen.sh
 	@go test ./...
+
+e2e-build:
+	docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile .
+
+e2e-tools:
+	# install pinned version of 'kind' 
+	# leave the name as is to avoid overwriting official binary named `kind`
+	wget https://github.com/kubernetes-sigs/kind/releases/download/v0.3.0/kind-linux-amd64
+	chmod +x kind-linux-amd64
+	mv kind-linux-amd64 $(KIND_PATH)
+
+e2e-run: docker
+	e2e/run.sh

delivery.yaml

@@ -11,7 +11,7 @@ pipeline:
             apt-get update
         - desc: 'Install required build software'
           cmd: |
-            apt-get install -y make git apt-transport-https ca-certificates curl build-essential
+            apt-get install -y make git apt-transport-https ca-certificates curl build-essential python3 python3-pip
         - desc: 'Install go'
           cmd: |
             cd /tmp
@@ -41,6 +41,10 @@ pipeline:
             export PATH=$PATH:$HOME/go/bin
             cd $OPERATOR_TOP_DIR/postgres-operator
             go test ./...
+        - desc: 'Run e2e tests'
+          cmd: |
+            cd $OPERATOR_TOP_DIR/postgres-operator
+            make e2e-tools e2e-build e2e-run
         - desc: 'Push docker image'
           cmd: |
             export PATH=$PATH:$HOME/go/bin

docs/developer.md

@@ -322,6 +322,16 @@ Then you can for example check the Patroni logs:
 kubectl logs acid-minimal-cluster-0
 ```
 
+## End-to-end tests
+
+The operator provides reference e2e (end-to-end) tests to ensure various infra parts work smoothly together.
+Each e2e execution tests a Postgres operator image built from the current git branch. The test runner starts a [kind](https://kind.sigs.k8s.io/) (local k8s) cluster and Docker container with tests. The k8s API client from within the container connects to the `kind` cluster using the standard Docker `bridge` network.
+The tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The kind cluster is deleted if tests complete successfully.
+
+End-to-end tests are executed automatically during builds; to invoke them locally use `make e2e-run` from the project's top directory. Run `make e2e-tools e2e-build` to install `kind` and build the tests' image locally before the first run. 
+
+End-to-end tests are written in Python and use `flake8` for code quality. Please run flake8 [before submitting a PR](http://flake8.pycqa.org/en/latest/user/using-hooks.html).
+
 ## Introduce additional configuration parameters
 
 In the case you want to add functionality to the operator that shall be

e2e/Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:18.04
+LABEL maintainer="Team ACID @ Zalando <[email protected]>"
+
+WORKDIR /e2e
+
+COPY manifests ./manifests
+COPY e2e/requirements.txt e2e/tests ./
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \ 
+           python3 \
+           python3-setuptools \
+           python3-pip \
+           curl \
+    && pip3 install --no-cache-dir -r requirements.txt \
+    && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \
+    && chmod +x ./kubectl \
+    && mv ./kubectl /usr/local/bin/kubectl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]

e2e/kind-cluster-postgres-operator-e2e-tests.yaml

@@ -0,0 +1,6 @@
+kind: Cluster
+apiVersion: kind.sigs.k8s.io/v1alpha3
+nodes:
+- role: control-plane
+- role: worker
+- role: worker

e2e/requirements.txt

@@ -0,0 +1,3 @@
+kubernetes==9.0.0
+timeout_decorator==0.4.1
+pyyaml==5.1

e2e/run.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# enable unofficial bash strict mode
+set -o errexit
+set -o nounset
+set -o pipefail
+IFS=$'\n\t'
+
+readonly cluster_name="postgres-operator-e2e-tests"
+readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}"  | head -1)
+readonly e2e_test_image=${cluster_name}
+readonly kubeconfig_path="/tmp/kind-config-${cluster_name}"
+
+
+function start_kind(){
+
+  # avoid interference with previous test runs
+  if [[ $(kind-linux-amd64 get clusters | grep "^${cluster_name}*") != "" ]]
+  then
+    kind-linux-amd64 delete cluster --name ${cluster_name}
+  fi
+
+  kind-linux-amd64 create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml
+  kind-linux-amd64 load docker-image "${operator_image}" --name ${cluster_name}
+  KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})"
+  export KUBECONFIG
+}
+
+function set_kind_api_server_ip(){
+  # use the actual kubeconfig to connect to the 'kind' API server
+  # but update the IP address of the API server to the one from the Docker 'bridge' network
+  cp "${KUBECONFIG}" /tmp
+  readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase
+  readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" "${cluster_name}"-control-plane)
+  sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" "${kubeconfig_path}"
+}
+
+function run_tests(){
+  docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}"
+}
+
+function clean_up(){
+  unset KUBECONFIG 
+  kind-linux-amd64 delete cluster --name ${cluster_name}
+  rm -rf ${kubeconfig_path}
+}
+
+function main(){
+
+  trap "clean_up" QUIT TERM EXIT
+
+  start_kind
+  set_kind_api_server_ip
+  run_tests
+  exit 0
+}
+
+main "$@"