INCIDENT-37491: ocitool copy content: add retry mechanism

MODULE.bazel

@@ -28,6 +28,7 @@ use_repo(
     "com_github_mitchellh_go_homedir",
     "com_github_opencontainers_go_digest",
     "com_github_opencontainers_image_spec",
+    "com_github_sethvargo_go_retry",
     "com_github_sirupsen_logrus",
     "com_github_stretchr_testify",
     "com_github_urfave_cli_v2",

go.mod

@@ -50,6 +50,7 @@ require (
 	github.com/prometheus/common v0.53.0 // indirect
 	github.com/prometheus/procfs v0.14.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sethvargo/go-retry v0.3.0
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect

go.sum

@@ -162,6 +162,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
+github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

go/cmd/ocitool/imagelayout_cmd.go

@@ -98,7 +98,7 @@ func CreateOciImageLayoutCmd(c *cli.Context) error {
 	}
 
 	// copy the parent last (in case of image index)
-	err = ociutil.CopyContent(c.Context, multiProvider, &ociIngester, baseDesc)
+	err = ociutil.CopyContentWithRetries(c.Context, multiProvider, &ociIngester, baseDesc)
 	if err != nil {
 		return fmt.Errorf("failed to copy parent content to OCI Image Layout: %w", err)
 	}

go/cmd/ocitool/push_cmd.go

@@ -70,7 +70,7 @@ func PushCmd(c *cli.Context) error {
 	}
 
 	// push the parent last (in case of image index)
-	err = ociutil.CopyContent(c.Context, allProviders, regIng, baseDesc)
+	err = ociutil.CopyContentWithRetries(c.Context, allProviders, regIng, baseDesc)
 	if err != nil {
 		return fmt.Errorf("failed to push parent content to registry: %w", err)
 	}

go/pkg/ociutil/BUILD.bazel

@@ -36,6 +36,7 @@ go_library(
         "@com_github_containerd_containerd//remotes/docker:go_default_library",
         "@com_github_opencontainers_go_digest//:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
+        "@com_github_sethvargo_go_retry//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@gazelle//rule:go_default_library",
         "@land_oras_oras_go//pkg/oras:go_default_library",

go/pkg/ociutil/handler.go

@@ -14,7 +14,7 @@ import (
 // ingestor
 func CopyContentHandler(handler images.HandlerFunc, from content.Provider, to content.Ingester) images.HandlerFunc {
 	return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-		err := CopyContent(ctx, from, to, desc)
+		err := CopyContentWithRetries(ctx, from, to, desc)
 		if err != nil {
 			return nil, err
 		}
@@ -46,7 +46,7 @@ func copyContentFromHandler(ctx context.Context, handler images.HandlerFunc, fro
 		return err
 	}
 
-	err = CopyContent(ctx, from, to, desc)
+	err = CopyContentWithRetries(ctx, from, to, desc)
 	if err != nil {
 		return err
 	}

go/pkg/ociutil/push.go

@@ -8,8 +8,10 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/DataDog/rules_oci/go/pkg/credhelper"
+	"github.com/sethvargo/go-retry"
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
@@ -305,3 +307,16 @@ func CopyContent(ctx context.Context, from content.Provider, to content.Ingester
 
 	return nil
 }
+
+func CopyContentWithRetries(ctx context.Context, from content.Provider, to content.Ingester, desc ocispec.Descriptor) error {
+	attempt := 0
+	const maxRetries uint64 = 2
+	return retry.Do(ctx, retry.WithMaxRetries(maxRetries, retry.NewFibonacci(1*time.Second)), func(_ context.Context) error {
+		attempt++
+		err := CopyContent(ctx, from, to, desc)
+		if err != nil {
+			fmt.Printf("Attempt %d of %d failed: %v\n", attempt, maxRetries+1, err)
+		}
+		return err
+	})
+}

oci/BUILD.bazel

@@ -99,16 +99,16 @@ bzl_library(
 )
 
 bzl_library(
-    name = "debug_flag",
-    srcs = ["debug_flag.bzl"],
+    name = "layer",
+    srcs = ["layer.bzl"],
     visibility = ["//visibility:public"],
+    deps = ["@com_github_datadog_rules_oci//oci:providers"],
 )
 
 bzl_library(
-    name = "layer",
-    srcs = ["layer.bzl"],
+    name = "debug_flag",
+    srcs = ["debug_flag.bzl"],
     visibility = ["//visibility:public"],
-    deps = ["@com_github_datadog_rules_oci//oci:providers"],
 )
 
 bzl_library(