From 4d9eb3f0d5440cb2a0cf58f83b3d1932a517ed91 Mon Sep 17 00:00:00 2001
From: pratikmahajan <pratik@mahajan.xyz>
Date: Thu, 24 Oct 2024 14:46:05 +0000
Subject: [PATCH] add retry logic to fetching blob from container registry

retries fetching the blob instead of erroring out and erasing
the progress till the point
---
 .../release_scrape_dockerv2/registry/mod.rs   | 46 ++++++++++++++-----
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs
index 9d7b6ec64..bc686c1e9 100644
--- a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs
+++ b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs
@@ -588,17 +588,41 @@ async fn find_first_release_metadata(
         trace!("[{}] Downloading layer {}", &tag, &layer_digest);
         let (repo, tag) = (repo.clone(), tag.clone());
 
-        let blob = registry_client
-            .get_blob(&repo, &layer_digest)
-            .map_err(|e| {
-                format_err!(
-                    "fetching blob for repo {} with layer_digest {}: {}",
-                    &repo,
-                    &layer_digest,
-                    e
-                )
-            })
-            .await?;
+        let blob = {
+            let mut attempts = 0;
+            const MAX_ATTEMPTS: u32 = 3;
+            const RETRY_DELAY: Duration = Duration::from_secs(1);
+            loop {
+                attempts += 1;
+                match registry_client
+                    .get_blob(&repo, &layer_digest)
+                    .map_err(|e| {
+                        format_err!(
+                            "fetching blob for repo {} with layer_digest {}: {}",
+                            &repo,
+                            &layer_digest,
+                            e
+                        )
+                    })
+                    .await
+                {
+                    Ok(layer_blob) => break layer_blob,
+                    Err(e) => {
+                        if attempts >= MAX_ATTEMPTS {
+                            return Err(e);
+                        }
+
+                        warn!(
+                            "getting blob failed (attempt {}/{}): {}, retrying...",
+                            attempts, MAX_ATTEMPTS, e
+                        );
+
+                        // Wait before retrying
+                        sleep(RETRY_DELAY).await;
+                    }
+                }
+            }
+        };
 
         let metadata_filename = "release-manifests/release-metadata";