From b6a53b1648726b49d85d5715b7143421075dea96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= <ege.gunes@percona.com>
Date: Fri, 15 Sep 2023 12:06:27 +0300
Subject: [PATCH] K8SPSMDB-971: Ensure restore status is updated (#1311)

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
---
 .../perconaservermongodbrestore_controller.go | 28 +++++++++++++++++--
 .../perconaservermongodbrestore/physical.go   |  2 +-
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go b/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go
index fd8ef0f0af..29c08e23a9 100644
--- a/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go
+++ b/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go
@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -236,7 +237,14 @@ func (r *ReconcilePerconaServerMongoDBRestore) getBackup(ctx context.Context, cr
 }
 
 func (r *ReconcilePerconaServerMongoDBRestore) updateStatus(ctx context.Context, cr *psmdbv1.PerconaServerMongoDBRestore) error {
-	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+	var backoff = wait.Backoff{
+		Steps:    5,
+		Duration: 500 * time.Millisecond,
+		Factor:   5.0,
+		Jitter:   0.1,
+	}
+
+	err := retry.OnError(backoff, func(error) bool { return true }, func() error {
 		c := &psmdbv1.PerconaServerMongoDBRestore{}
 
 		err := r.client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, c)
@@ -246,7 +254,23 @@ func (r *ReconcilePerconaServerMongoDBRestore) updateStatus(ctx context.Context,
 
 		c.Status = cr.Status
 
-		return r.client.Status().Update(ctx, c)
+		err = r.client.Status().Update(ctx, c)
+		if err != nil {
+			return err
+		}
+
+		// ensure status is updated
+		c = &psmdbv1.PerconaServerMongoDBRestore{}
+		err = r.client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, c)
+		if err != nil {
+			return err
+		}
+
+		if c.Status.State != cr.Status.State {
+			return errors.New("status not updated")
+		}
+
+		return nil
 	})
 
 	if k8serrors.IsNotFound(err) {
diff --git a/pkg/controller/perconaservermongodbrestore/physical.go b/pkg/controller/perconaservermongodbrestore/physical.go
index 0c2f447201..7a41d14ab2 100644
--- a/pkg/controller/perconaservermongodbrestore/physical.go
+++ b/pkg/controller/perconaservermongodbrestore/physical.go
@@ -276,7 +276,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont
 		log.V(1).Info("Check restore status", "command", command, "pod", pod.Name)
 
 		if err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil {
-			return errors.Wrap(err, "describe restore")
+			return errors.Wrapf(err, "describe restore stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String())
 		}
 
 		return nil