Add junit reports (jaegertracing#1810)

* Generate reports Signed-off-by: Israel Blancas <[email protected]> * Add reports Signed-off-by: Israel Blancas <[email protected]> * Fix issues after merging Signed-off-by: Israel Blancas <[email protected]> * Fix stop Kind cluster Signed-off-by: Israel Blancas <[email protected]> * Fix YQ call Signed-off-by: Israel Blancas <[email protected]> * Fix report generation Signed-off-by: Israel Blancas <[email protected]> * Not fail when the test fails Signed-off-by: Israel Blancas <[email protected]> * Fix path to KUTTL Signed-off-by: Israel Blancas <[email protected]> * Fix path to KUTTL installation Signed-off-by: Israel Blancas <[email protected]> * Fix path to KUTTL installation Signed-off-by: Israel Blancas <[email protected]> * Fix path to KUTTL installation Signed-off-by: Israel Blancas <[email protected]> * Fix path to Gomplate in template Signed-off-by: Israel Blancas <[email protected]> * Small fixes related to gomplate path Signed-off-by: Israel Blancas <[email protected]> * Set version for junitcli Signed-off-by: Israel Blancas <[email protected]>
indigostar-kr · Apr 5, 2022 · c3d421b · c3d421b
1 parent 399b087
commit c3d421b
Show file tree

Hide file tree

Showing 12 changed files with 94 additions and 42 deletions.
diff --git a/.gitignore b/.gitignore
@@ -91,4 +91,6 @@ kubeconfig
 bin
 ### Timestamp files to avoid rebuilding Docker images if not needed ###
 build-assert-job
-build-e2e-upgrade-image
+docker-e2e-upgrade-image
+### Reports for E2E tests
+reports
diff --git a/Makefile b/Makefile
@@ -166,6 +166,9 @@ cert-manager: cmctl
 	kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v${CERTMANAGER_VERSION}/cert-manager.yaml
 	cmctl check api --wait=5m
 
+undeploy-cert-manager:
+	kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v${CERTMANAGER_VERSION}/cert-manager.yaml
+
 cmctl:
 ifeq (, $(shell which cmctl))
 	@{ \
@@ -272,7 +275,7 @@ else
 endif
 
 .PHONY: clean
-clean: undeploy-kafka undeploy-prometheus-operator undeploy-istio
+clean: undeploy-kafka undeploy-prometheus-operator undeploy-istio undeploy-cert-manager
 	$(VECHO)kubectl delete namespace $(KAFKA_NAMESPACE) --ignore-not-found=true 2>&1 || true
 	$(VECHO)if [ -d tests/_build ]; then rm -rf tests/_build ; fi
 	$(VECHO)kubectl delete -f ./tests/cassandra.yml --ignore-not-found=true -n $(STORAGE_NAMESPACE) || true

diff --git a/hack/run-e2e-tests.sh b/hack/run-e2e-tests.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+current_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+set -e
+
+# Enable verbosity
+if [ "$VERBOSE" = true ]; then
+    set -o xtrace
+fi
+
+if [ "$#" -ne 3 ]; then
+    echo "$0 <test_suite_name> <use_kind_cluster> <olm>"
+    exit 1
+fi
+
+test_suite_name=$1
+use_kind_cluster=$2
+olm=$3
+
+root_dir=$current_dir/../
+reports_dir=$root_dir/reports
+
+# Ensure KUTTL is installed
+$current_dir/install/install-kuttl.sh
+export KUTTL=$root_dir/bin/kubectl-kuttl
+
+mkdir -p $reports_dir
+
+cd $root_dir
+make render-e2e-tests-$test_suite_name
+
+if [ "$use_kind_cluster" == true ]; then
+	kubectl wait --timeout=5m --for=condition=available deployment ingress-nginx-controller -n ingress-nginx
+	kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=5m
+	make cert-manager
+fi
+
+if [ "$olm" = true ]; then
+    echo "Skipping Jaeger Operator installation because OLM=true"
+else
+	echo Installing Jaeger Operator...
+	kubectl create namespace observability 2>&1 | grep -v "already exists" || true
+	kubectl apply -f ./tests/_build/manifests/01-jaeger-operator.yaml -n observability
+	kubectl wait --timeout=5m --for=condition=available deployment jaeger-operator -n observability
+fi
+
+echo Running $test_suite_name E2E tests
+cd tests/e2e/$test_suite_name/_build
+
+# Don't stop if something fails because we want to process the
+# report anyway
+set +e
+
+$KUTTL test $KUTTL_OPTIONS --report xml
+exit_code=$?
+
+set -e
+
+# The output XML needs some work because it adds "artifacts" as a test case.
+# Also, the suites doesn't have a name so, we need to add one.
+go install github.com/iblancasa/junitcli/cmd/[email protected]
+junitcli --suite-name $test_suite_name --report --output $reports_dir/$test_suite_name.xml ./artifacts/kuttl-test.xml
+
+if [ "$KIND_KEEP_CLUSTER" != true ] && [ "$use_kind_cluster" == true ]; then
+	cd $root_dir
+	make stop-kind
+fi
+
+exit $exit_code
diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile
@@ -7,7 +7,7 @@ export ASSERT_IMG ?= ${IMG_PREFIX}/asserts-e2e:$(shell date +%s)
 USE_KIND_CLUSTER ?= true
 
 .PHONY: prepare-e2e-tests
-prepare-e2e-tests: kuttl build prepare-e2e-images generate-e2e-files
+prepare-e2e-tests: kuttl build generate-e2e-files
 
 .PHONY: set-assert-e2e-img-name
 set-assert-e2e-img-name:
@@ -76,8 +76,11 @@ endif
 .PHONY: run-e2e-tests
 run-e2e-tests: list-test-suites
 	$(VECHO) for suite in $(TEST_SUITES); do \
-		$(MAKE) run-e2e-tests-$$suite ; \
+		$(MAKE) run-e2e-tests-$$suite -i; \
 	done
+	# Show a final report
+	junitcli --report reports
+	cd reports && ! grep -nr failed
 
 .PHONY: render-e2e-tests
 render-e2e-tests: generate-e2e-files list-test-suites
@@ -98,26 +101,5 @@ e2e-test-suites: list-test-suites
 
 	@echo "You can run a test suite with make run-e2e-tests-<suite name>. E.g: make run-e2e-tests-smoke"
 
-run-suite-tests: start-kind kuttl build load-operator-image generate-e2e-files
-	$(VECHO)PATH=${PATH}:$(shell pwd)/bin $(MAKE) render-e2e-tests-$(TEST_SUITE_NAME)
-
-ifeq ($(USE_KIND_CLUSTER),true)
-	$(VECHO)kubectl wait --timeout=5m --for=condition=available deployment ingress-nginx-controller -n ingress-nginx
-	$(VECHO)kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=5m
-	$(VECHO)$(MAKE) cert-manager
-endif
-
-ifeq ($(OLM),true)
-	$(ECHO) Skipping Jaeger Operator installation because OLM=true
-else
-	$(ECHO) Installing Jaeger Operator...
-	$(VECHO)kubectl create namespace observability 2>&1 | grep -v "already exists" || true
-	$(VECHO)kubectl apply -f ./tests/_build/manifests/01-jaeger-operator.yaml -n observability
-	$(VECHO)kubectl wait --timeout=5m --for=condition=available deployment jaeger-operator -n observability
-endif
-	$(ECHO) Running $(TEST_SUITE_NAME) E2E tests
-	$(VECHO)cd tests/e2e/$(TEST_SUITE_NAME)/_build && PATH=${PATH}:$(shell pwd)/bin $(KUTTL) test $(KUTTL_OPTIONS)
-
-ifeq ($(USE_KIND_CLUSTER),true)
-	$(VECHO)$(MAKE) stop-kind
-endif
+run-suite-tests: start-kind prepare-e2e-tests load-operator-image
+	./hack/run-e2e-tests.sh $(TEST_SUITE_NAME) $(USE_KIND_CLUSTER) $(OLM)
diff --git a/tests/e2e/render-utils.sh b/tests/e2e/render-utils.sh
@@ -412,7 +412,7 @@ function get_jaeger_name() {
 
     deployment_file=$1
 
-    jaeger_name=$(yq e '.metadata.name' $deployment_file)
+    jaeger_name=$($YQ e '.metadata.name' $deployment_file)
 
     if [ -z "$jaeger_name" ]; then
         error "No name for Jaeger deployment in file $deployment_file"
@@ -439,14 +439,14 @@ function get_jaeger_strategy() {
 
     deployment_file=$1
 
-    strategy=$(yq e '.spec.strategy' $deployment_file)
+    strategy=$($YQ e '.spec.strategy' $deployment_file)
 
     if [ "$strategy" != "null" ]; then
         echo $strategy
         return 0
     fi
 
-    strategy=$(yq e '.spec.agent.strategy' $deployment_file)
+    strategy=$($YQ e '.spec.agent.strategy' $deployment_file)
     if [ "$strategy" = "null" ]; then
         echo "allInOne"
         return 0
@@ -678,3 +678,4 @@ fi
 export CRD_DIR
 
 $GOMPLATE -f ../../../templates/kuttl-test.yaml -o ./kuttl-test.yaml
+mkdir -p artifacts
diff --git a/tests/e2e/smoke/simplest/README.md b/tests/e2e/smoke/simplest/README.md
diff --git a/tests/e2e/streaming/render.sh b/tests/e2e/streaming/render.sh
@@ -17,7 +17,6 @@ render_smoke_test "tls-streaming" "allInOne" "05"
 start_test "streaming-with-autoprovisioning"
 export CLUSTER_NAME="auto-provisioned"
 jaeger_name="auto-provisioned"
-
 if [ $IS_OPENSHIFT = true ]; then
     # Remove the installation of the operator
     rm ./00-install.yaml ./00-assert.yaml

diff --git a/tests/templates/kuttl-test.yaml b/tests/templates/kuttl-test.yaml
@@ -1,7 +1,7 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 crdDir: {{ .Env.CRD_DIR }}
-artifactsDir: ../../../_build/artifacts/
+artifactsDir: ./artifacts/
 testDirs:
   - .
 timeout: 330

diff --git a/tests/templates/openshift/configure-api-query-oauth.yaml.template b/tests/templates/openshift/configure-api-query-oauth.yaml.template
@@ -39,5 +39,5 @@ spec:
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  - script: "gomplate -f {{ .Env.TEMPLATES_DIR }}/openshift/api-query-cluster-role-binding.template.yaml -o cluster-role.yaml"
+  - script: "{{ .Env.GOMPLATE }} -f {{ .Env.TEMPLATES_DIR }}/openshift/api-query-cluster-role-binding.template.yaml -o cluster-role.yaml"
   - script: "kubectl apply -f cluster-role.yaml -n $NAMESPACE"
diff --git a/tests/templates/openshift/report-spans.yaml.template b/tests/templates/openshift/report-spans.yaml.template
@@ -40,7 +40,7 @@ spec:
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  - script: "gomplate -f {{ .Env.TEMPLATES_DIR }}/openshift/api-query-cluster-role-binding.template.yaml -o smoke-cluster-role.yaml"
+  - script: "{{ .Env.GOMPLATE }} -f {{ .Env.TEMPLATES_DIR }}/openshift/api-query-cluster-role-binding.template.yaml -o smoke-cluster-role.yaml"
   - script: "kubectl apply -f smoke-cluster-role.yaml -n $NAMESPACE"
-  - script: "DAYS={{ .Env.DAYS }} ASSERT_IMG={{ .Env.ASSERT_IMG }} {{if getenv "JOB_NUMBER"}}JOB_NUMBER={{ .Env.JOB_NUMBER }}{{end}} JAEGER_COLLECTOR_ENDPOINT={{ .Env.JAEGER_COLLECTOR_ENDPOINT }} {{if getenv "JAEGER_QUERY_ENDPOINT"}}JAEGER_QUERY_ENDPOINT={{ .Env.JAEGER_QUERY_ENDPOINT }} MOUNT_SECRET=$(kubectl get sa e2e-token-api-user -n $NAMESPACE -o json | jq -r '.secrets[] |  select( .name | test(\"e2e-token-api-user-token-\")).name'){{end}} gomplate -f {{ .Env.TEMPLATES_DIR }}/report-spans.yaml.template -o report-span-{{ .Env.JOB_NUMBER }}-job.yaml"
+  - script: "DAYS={{ .Env.DAYS }} ASSERT_IMG={{ .Env.ASSERT_IMG }} {{if getenv "JOB_NUMBER"}}JOB_NUMBER={{ .Env.JOB_NUMBER }}{{end}} JAEGER_COLLECTOR_ENDPOINT={{ .Env.JAEGER_COLLECTOR_ENDPOINT }} {{if getenv "JAEGER_QUERY_ENDPOINT"}}JAEGER_QUERY_ENDPOINT={{ .Env.JAEGER_QUERY_ENDPOINT }} MOUNT_SECRET=$(kubectl get sa e2e-token-api-user -n $NAMESPACE -o json | jq -r '.secrets[] |  select( .name | test(\"e2e-token-api-user-token-\")).name'){{end}} {{ .Env.GOMPLATE }} -f {{ .Env.TEMPLATES_DIR }}/report-spans.yaml.template -o report-span-{{ .Env.JOB_NUMBER }}-job.yaml"
   - script: "kubectl create -f report-span{{if getenv "JOB_NUMBER"}}-{{ .Env.JOB_NUMBER }}{{end}}-job.yaml -n $NAMESPACE"
diff --git a/tests/templates/openshift/smoke-test.yaml.template b/tests/templates/openshift/smoke-test.yaml.template
@@ -40,7 +40,7 @@ spec:
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  - script: "gomplate -f {{ .Env.TEMPLATES_DIR }}/openshift/api-query-cluster-role-binding.template.yaml -o smoke-cluster-role.yaml"
+  - script: "{{ .Env.GOMPLATE }} -f {{ .Env.TEMPLATES_DIR }}/openshift/api-query-cluster-role-binding.template.yaml -o smoke-cluster-role.yaml"
   - script: "kubectl apply -f smoke-cluster-role.yaml -n $NAMESPACE"
-  - script: "ASSERT_IMG={{ .Env.ASSERT_IMG }} JAEGER_COLLECTOR_ENDPOINT={{ .Env.JAEGER_COLLECTOR_ENDPOINT }} JAEGER_QUERY_ENDPOINT={{ .Env.JAEGER_QUERY_ENDPOINT }} MOUNT_SECRET=$(kubectl get sa e2e-token-api-user -o yaml -n $NAMESPACE | yq eval '.secrets[] | select( .name == \"e2e-token-api-user-token-*\")'.name) gomplate -f {{ .Env.TEMPLATES_DIR }}/smoke-test.yaml.template -o smoke-test-job.yaml"
+  - script: "ASSERT_IMG={{ .Env.ASSERT_IMG }} JAEGER_COLLECTOR_ENDPOINT={{ .Env.JAEGER_COLLECTOR_ENDPOINT }} JAEGER_QUERY_ENDPOINT={{ .Env.JAEGER_QUERY_ENDPOINT }} MOUNT_SECRET=$(kubectl get sa e2e-token-api-user -o yaml -n $NAMESPACE | yq eval '.secrets[] | select( .name == \"e2e-token-api-user-token-*\")'.name) {{ .Env.GOMPLATE }} -f {{ .Env.TEMPLATES_DIR }}/smoke-test.yaml.template -o smoke-test-job.yaml"
   - script: "kubectl create -f smoke-test-job.yaml -n $NAMESPACE"
diff --git a/tests/templates/render-test-ui.yaml.template b/tests/templates/render-test-ui.yaml.template
@@ -4,5 +4,5 @@ commands:
   # Sometimes, the Ingress route is there but not 100% ready so, when kubectl tries to get the
   # hostname, it returns an empty string
   - command: ./ensure-ingress-host.sh
-  - script: "QUERY_BASE_PATH={{ .Env.QUERY_BASE_PATH }} ASSERT_IMG={{ .Env.ASSERT_IMG }} TRACKING_ID={{ .Env.TRACKING_ID }} QUERY_HOSTNAME={{ .Env.GET_URL_COMMAND }} gomplate -f {{ .Env.TEMPLATES_DIR }}/test-uiconfig.yaml.template -o run-assert.yaml"
+  - script: "QUERY_BASE_PATH={{ .Env.QUERY_BASE_PATH }} ASSERT_IMG={{ .Env.ASSERT_IMG }} TRACKING_ID={{ .Env.TRACKING_ID }} QUERY_HOSTNAME={{ .Env.GET_URL_COMMAND }} {{ .Env.GOMPLATE }} -f {{ .Env.TEMPLATES_DIR }}/test-uiconfig.yaml.template -o run-assert.yaml"
   - script: "kubectl create -f run-assert.yaml -n $NAMESPACE"