cloud, plz: check for abort at the start of each reconcile

api/v1alpha1/k6conditions.go

@@ -73,13 +73,14 @@ func Initialize(k6 TestRunI) {
 		},
 	}
 
+	UpdateCondition(k6, CloudTestRunAborted, metav1.ConditionFalse)
+
 	// PLZ test run case
 	if len(k6.GetSpec().TestRunID) > 0 {
 		UpdateCondition(k6, CloudTestRun, metav1.ConditionTrue)
 		UpdateCondition(k6, CloudPLZTestRun, metav1.ConditionTrue)
 		UpdateCondition(k6, CloudTestRunCreated, metav1.ConditionTrue)
 		UpdateCondition(k6, CloudTestRunFinalized, metav1.ConditionFalse)
-		UpdateCondition(k6, CloudTestRunAborted, metav1.ConditionFalse)
 
 		k6.GetStatus().TestRunID = k6.GetSpec().TestRunID
 	} else {
@@ -151,17 +152,23 @@ func (k6status *TestRunStatus) SetIfNewer(proposedStatus TestRunStatus) (isNewer
 				isNewer = true
 			}
 		case "created":
-			if proposedStatus.Stage == "started" || proposedStatus.Stage == "finished" || proposedStatus.Stage == "error" {
+			if proposedStatus.Stage == "started" ||
+				proposedStatus.Stage == "finished" ||
+				proposedStatus.Stage == "error" ||
+				proposedStatus.Stage == "stopped" {
 				k6status.Stage = proposedStatus.Stage
 				isNewer = true
 			}
 		case "started":
-			if proposedStatus.Stage == "stopped" || proposedStatus.Stage == "finished" || proposedStatus.Stage == "error" {
+			if proposedStatus.Stage == "stopped" ||
+				proposedStatus.Stage == "finished" ||
+				proposedStatus.Stage == "error" {
 				k6status.Stage = proposedStatus.Stage
 				isNewer = true
 			}
 		case "stopped":
-			if proposedStatus.Stage == "finished" || proposedStatus.Stage == "error" {
+			if proposedStatus.Stage == "finished" ||
+				proposedStatus.Stage == "error" {
 				k6status.Stage = proposedStatus.Stage
 				isNewer = true
 			}

controllers/k6_start.go

@@ -70,9 +70,12 @@ func StartJobs(ctx context.Context, log logr.Logger, k6 v1alpha1.TestRunI, r *Te
 		} else {
 			// let's try this approach
 			if time.Since(t).Minutes() > 5 {
+				msg := "Creation of runner pods takes too long: your configuration might be off. Check if runner jobs and pods were created successfully."
+				log.Info(msg)
+
 				if v1alpha1.IsTrue(k6, v1alpha1.CloudTestRun) {
 					events := cloud.ErrorEvent(cloud.K6OperatorStartError).
-						WithDetail("Creation of runner pods takes too long: your configuration might be off. Check if runner jobs and pods were created successfully.").
+						WithDetail(msg).
 						WithAbort()
 					cloud.SendTestRunEvents(r.k6CloudClient, v1alpha1.TestRunID(k6), log, events)
 				}
@@ -116,7 +119,7 @@ func StartJobs(ctx context.Context, log logr.Logger, k6 v1alpha1.TestRunI, r *Te
 
 	log.Info("Created starter job")
 
-	log.Info("Changing stage of K6 status to started")
+	log.Info("Changing stage of TestRun status to started")
 	k6.GetStatus().Stage = "started"
 	v1alpha1.UpdateCondition(k6, v1alpha1.TestRunRunning, metav1.ConditionTrue)
 

controllers/k6_stop.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"context"
+	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/grafana/k6-operator/api/v1alpha1"
@@ -56,15 +57,15 @@ func StopJobs(ctx context.Context, log logr.Logger, k6 v1alpha1.TestRunI, r *Tes
 
 	log.Info("Created stop job")
 
-	log.Info("Changing stage of K6 status to stopped")
+	log.Info("Changing stage of TestRun status to stopped")
 	k6.GetStatus().Stage = "stopped"
 	v1alpha1.UpdateCondition(k6, v1alpha1.TestRunRunning, metav1.ConditionFalse)
 	v1alpha1.UpdateCondition(k6, v1alpha1.CloudTestRunAborted, metav1.ConditionTrue)
 
 	if updateHappened, err := r.UpdateStatus(ctx, k6, log); err != nil {
 		return ctrl.Result{}, err
 	} else if updateHappened {
-		return ctrl.Result{Requeue: true}, nil
+		return ctrl.Result{RequeueAfter: time.Second}, nil
 	}
 	return ctrl.Result{}, nil
 }

controllers/testrun_controller.go

@@ -16,7 +16,6 @@ package controllers
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -100,6 +99,14 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, req ctrl.Request, log
 	// Decision making here is now a mix between stages and conditions.
 	// TODO: refactor further.
 
+	if v1alpha1.IsTrue(k6, v1alpha1.CloudTestRun) && v1alpha1.IsFalse(k6, v1alpha1.CloudTestRunAborted) {
+		// check in with the BE for status
+		if r.ShouldAbort(ctx, k6, log) {
+			log.Info("Received an abort signal from the k6 Cloud: stopping the test.")
+			return StopJobs(ctx, log, k6, r)
+		}
+	}
+
 	switch k6.GetStatus().Stage {
 	case "":
 		log.Info("Initialize test")
@@ -110,7 +117,7 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, req ctrl.Request, log
 			return ctrl.Result{}, err
 		}
 
-		log.Info("Changing stage of K6 status to initialization")
+		log.Info("Changing stage of TestRun status to initialization")
 		k6.GetStatus().Stage = "initialization"
 
 		if updateHappened, err := r.UpdateStatus(ctx, k6, log); err != nil {
@@ -129,7 +136,7 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, req ctrl.Request, log
 		if v1alpha1.IsFalse(k6, v1alpha1.CloudTestRun) {
 			// RunValidations has already happened and this is not a
 			// cloud test: we can move on
-			log.Info("Changing stage of K6 status to initialized")
+			log.Info("Changing stage of TestRun status to initialized")
 
 			k6.GetStatus().Stage = "initialized"
 
@@ -147,7 +154,7 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, req ctrl.Request, log
 
 			} else {
 				// if test run was created, then only changing status is left
-				log.Info("Changing stage of K6 status to initialized")
+				log.Info("Changing stage of TestRun status to initialized")
 
 				k6.GetStatus().Stage = "initialized"
 
@@ -179,7 +186,8 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, req ctrl.Request, log
 		// wait for the test to finish
 		if !FinishJobs(ctx, log, k6, r) {
 
-			if v1alpha1.IsTrue(k6, v1alpha1.CloudPLZTestRun) && v1alpha1.IsFalse(k6, v1alpha1.CloudTestRunAborted) {
+			// TODO: confirm if this check is needed given the check in the beginning of reconcile
+			if v1alpha1.IsTrue(k6, v1alpha1.CloudTestRun) && v1alpha1.IsFalse(k6, v1alpha1.CloudTestRunAborted) {
 				// check in with the BE for status
 				if r.ShouldAbort(ctx, k6, log) {
 					log.Info("Received an abort signal from the k6 Cloud: stopping the test.")
@@ -201,7 +209,7 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, req ctrl.Request, log
 		if v1alpha1.IsTrue(k6, v1alpha1.TestRunRunning) {
 			v1alpha1.UpdateCondition(k6, v1alpha1.TestRunRunning, metav1.ConditionFalse)
 
-			log.Info("Changing stage of K6 status to stopped")
+			log.Info("Changing stage of TestRun status to stopped")
 			k6.GetStatus().Stage = "stopped"
 
 			_, err := r.UpdateStatus(ctx, k6, log)
@@ -254,7 +262,7 @@ func (r *TestRunReconciler) reconcile(ctx context.Context, req ctrl.Request, log
 			}
 		}
 
-		log.Info("Changing stage of K6 status to finished")
+		log.Info("Changing stage of TestRun status to finished")
 		k6.GetStatus().Stage = "finished"
 
 		_, err := r.UpdateStatus(ctx, k6, log)
@@ -355,12 +363,12 @@ func (r *TestRunReconciler) UpdateStatus(ctx context.Context, k6 v1alpha1.TestRu
 // cause a forced stop. It is meant to be used only by PLZ test runs.
 func (r *TestRunReconciler) ShouldAbort(ctx context.Context, k6 v1alpha1.TestRunI, log logr.Logger) bool {
 	// sanity check
-	if len(k6.GetStatus().TestRunID) == 0 {
-		log.Error(errors.New("empty test run ID"), "Trying to get state of test run with empty test run ID")
+	if len(v1alpha1.TestRunID(k6)) == 0 {
+		// log.Error(errors.New("empty test run ID"), "Trying to get state of test run with empty test run ID")
 		return false
 	}
 
-	status, err := cloud.GetTestRunState(r.k6CloudClient, k6.GetStatus().TestRunID, log)
+	status, err := cloud.GetTestRunState(r.k6CloudClient, v1alpha1.TestRunID(k6), log)
 	if err != nil {
 		log.Error(err, "Failed to get test run state.")
 		return false

pkg/cloud/test_runs.go

@@ -136,6 +136,8 @@ func SendTestRunEvents(client *cloudapi.Client, refID string, log logr.Logger, e
 		return
 	}
 
+	log.Info(fmt.Sprintf("Sending events to k6 Cloud %+v", *events))
+
 	// status code is checked in Do
 	if err = client.Do(req, nil); err != nil {
 		log.Error(err, fmt.Sprintf("Failed to send events %+v", events))