cloud, plz: add events on failed creation with simple timeout

grafana · Oct 31, 2023 · 6698739 · 6698739
1 parent 59e62ed
commit 6698739
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 5 deletions.
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/controllers/k6_create.go b/controllers/k6_create.go
@@ -43,9 +43,9 @@ func CreateJobs(ctx context.Context, log logr.Logger, k6 v1alpha1.TestRunI, r *T
 			// Consider updating status to error to let a user know quicker?
 			log.Error(err, "A problem while getting token.")
 
-			if k6.IsTrue(v1alpha1.CloudTestRun) {
+			if v1alpha1.IsTrue(k6, v1alpha1.CloudTestRun) {
 				events := cloud.ErrorEvent(cloud.K6OperatorStartError).WithDetail(fmt.Sprintf("Failed to retrieve token: %v", err))
-				cloud.SendTestRunEvents(r.k6CloudClient, k6.Spec.TestRunID, log, events)
+				cloud.SendTestRunEvents(r.k6CloudClient, k6.GetSpec().TestRunID, log, events)
 			}
 
 			return ctrl.Result{}, nil
@@ -58,6 +58,12 @@ func CreateJobs(ctx context.Context, log logr.Logger, k6 v1alpha1.TestRunI, r *T
 	log.Info("Creating test jobs")
 
 	if res, err = createJobSpecs(ctx, log, k6, r, token); err != nil {
+
+		if v1alpha1.IsTrue(k6, v1alpha1.CloudTestRun) {
+			events := cloud.ErrorEvent(cloud.K6OperatorStartError).WithDetail(fmt.Sprintf("Failed to create runner jobs: %v", err))
+			cloud.SendTestRunEvents(r.k6CloudClient, k6.GetSpec().TestRunID, log, events)
+		}
+
 		return res, err
 	}
 

diff --git a/controllers/k6_start.go b/controllers/k6_start.go
@@ -2,12 +2,14 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/grafana/k6-operator/api/v1alpha1"
+	"github.com/grafana/k6-operator/pkg/cloud"
 	"github.com/grafana/k6-operator/pkg/resources/jobs"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -62,6 +64,19 @@ func StartJobs(ctx context.Context, log logr.Logger, k6 v1alpha1.TestRunI, r *Te
 	log.Info(fmt.Sprintf("%d/%d runner pods ready", count, k6.GetSpec().Parallelism))
 
 	if count != int(k6.GetSpec().Parallelism) {
+		if t, ok := v1alpha1.LastUpdate(k6, v1alpha1.TestRunRunning); !ok {
+			// this should never happen
+			return res, errors.New("Cannot find condition TestRunRunning")
+		} else {
+			// let's try this approach
+			if time.Since(t).Minutes() > 5 {
+				if v1alpha1.IsTrue(k6, v1alpha1.CloudTestRun) {
+					events := cloud.ErrorEvent(cloud.K6OperatorStartError).WithDetail("Creation of runner pods takes too long: perhaps, something is off with your configuration. Check if runner jobs and pods were created successfully.")
+					cloud.SendTestRunEvents(r.k6CloudClient, k6.GetSpec().TestRunID, log, events)
+				}
+			}
+		}
+
 		return res, nil
 	}
 

diff --git a/pkg/cloud/test_runs.go b/pkg/cloud/test_runs.go
@@ -128,9 +128,9 @@ func SendTestRunEvents(client *cloudapi.Client, refID string, log logr.Logger, e
 		return
 	}
 
-	url := fmt.Sprintf("%s/orchestrator/v1/testruns/%s/events", client.BaseURL(), refID)
-
+	url := fmt.Sprintf("%s/orchestrator/v1/testruns/%s/events", strings.TrimSuffix(client.BaseURL(), "/v1"), refID)
 	req, err := client.NewRequest("POST", url, events)
+
 	if err != nil {
 		log.Error(err, fmt.Sprintf("Failed to create events HTTP request %+v", events))
 		return