WIP Limit registry connections to prevent 503 errors during push

cmd/nerdctl/image/image_push.go

@@ -69,6 +69,17 @@ func PushCommand() *cobra.Command {
 
 	cmd.Flags().Bool(allowNonDistFlag, false, "Allow pushing images with non-distributable blobs")
 
+	// #region connection limit flags
+	cmd.Flags().Int("max-conns-per-host", 5, "Maximum number of connections per registry host")
+	cmd.Flags().Int("max-idle-conns", 50, "Maximum number of idle connections")
+	cmd.Flags().Int("request-timeout", 300, "Request timeout in seconds")
+	// #endregion
+
+	// #region retry flags
+	cmd.Flags().Int("max-retries", 3, "Maximum number of retry attempts for 503 errors")
+	cmd.Flags().Int("retry-initial-delay", 1000, "Initial delay before first retry in milliseconds")
+	// #endregion
+
 	return cmd
 }
 
@@ -113,6 +124,26 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 	if err != nil {
 		return types.ImagePushOptions{}, err
 	}
+	maxConnsPerHost, err := cmd.Flags().GetInt("max-conns-per-host")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	maxIdleConns, err := cmd.Flags().GetInt("max-idle-conns")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	requestTimeout, err := cmd.Flags().GetInt("request-timeout")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	maxRetries, err := cmd.Flags().GetInt("max-retries")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
+	retryInitialDelay, err := cmd.Flags().GetInt("retry-initial-delay")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
 	return types.ImagePushOptions{
 		GOptions:                       globalOptions,
 		SignOptions:                    signOptions,
@@ -124,6 +155,11 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 		IpfsAddress:                    ipfsAddress,
 		Quiet:                          quiet,
 		AllowNondistributableArtifacts: allowNonDist,
+		MaxConnsPerHost:                maxConnsPerHost,
+		MaxIdleConns:                   maxIdleConns,
+		RequestTimeout:                 requestTimeout,
+		MaxRetries:                     maxRetries,
+		RetryInitialDelay:              retryInitialDelay,
 		Stdout:                         cmd.OutOrStdout(),
 	}, nil
 }

pkg/api/types/image_types.go

@@ -202,6 +202,16 @@ type ImagePushOptions struct {
 	Quiet bool
 	// AllowNondistributableArtifacts allow pushing non-distributable artifacts
 	AllowNondistributableArtifacts bool
+	// MaxConnsPerHost maximum number of connections per registry host (default: 5)
+	MaxConnsPerHost int
+	// MaxIdleConns maximum number of idle connections (default: 50)
+	MaxIdleConns int
+	// RequestTimeout timeout for registry requests in seconds (default: 300)
+	RequestTimeout int
+	// MaxRetries maximum number of retry attempts for 503 errors (default: 3)
+	MaxRetries int
+	// RetryInitialDelay initial delay before first retry in milliseconds (default: 1000)
+	RetryInitialDelay int
 }
 
 // RemoteSnapshotterFlags are used for pulling with remote snapshotters

pkg/cmd/image/push.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -113,6 +114,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		return err
 	}
 	ref := parsedReference.String()
+	refDomain := parsedReference.Domain
 
 	platMC, err := platformutil.NewMatchComparer(options.AllPlatforms, options.Platforms)
 	if err != nil {
@@ -171,6 +173,76 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		}
 	}
 
+	// In order to push images where most layers are the same but the
+	// repository name is different, it is necessary to refresh the
+	// PushTracker. Otherwise, the MANIFEST_BLOB_UNKNOWN error will occur due
+	// to the registry not creating the corresponding layer link file,
+	// resulting in the failure of the entire image push.
+	pushTracker := docker.NewInMemoryTracker()
+
+	pushFunc := func(r remotes.Resolver) error {
+		return push.Push(ctx, client, r, pushTracker, options.Stdout, pushRef, ref, platMC, options.AllowNondistributableArtifacts, options.Quiet)
+	}
+
+	var dOpts []dockerconfigresolver.Opt
+	if options.GOptions.InsecureRegistry {
+		log.G(ctx).Warnf("skipping verifying HTTPS certs for %q", refDomain)
+		dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true))
+	}
+	dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(options.GOptions.HostsDir))
+
+	// Configure connection limits to prevent registry overload (503 errors)
+	if options.MaxConnsPerHost > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxConnsPerHost(options.MaxConnsPerHost))
+	}
+	if options.MaxIdleConns > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxIdleConns(options.MaxIdleConns))
+	}
+	if options.RequestTimeout > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithRequestTimeout(time.Duration(options.RequestTimeout)*time.Second))
+	}
+	if options.MaxRetries > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithMaxRetries(options.MaxRetries))
+	}
+	if options.RetryInitialDelay > 0 {
+		dOpts = append(dOpts, dockerconfigresolver.WithRetryInitialDelay(time.Duration(options.RetryInitialDelay)*time.Millisecond))
+	}
+	// Use the local push tracker for this operation
+	dOpts = append(dOpts, dockerconfigresolver.WithTracker(pushTracker))
+
+	if options.GOptions.InsecureRegistry {
+		log.G(ctx).WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain)
+		dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true))
+		// Apply same connection limits for HTTP fallback
+		if options.MaxConnsPerHost > 0 {
+			dOpts = append(dOpts, dockerconfigresolver.WithMaxConnsPerHost(options.MaxConnsPerHost))
+		}
+		if options.MaxIdleConns > 0 {
+			dOpts = append(dOpts, dockerconfigresolver.WithMaxIdleConns(options.MaxIdleConns))
+		}
+		if options.RequestTimeout > 0 {
+			dOpts = append(dOpts, dockerconfigresolver.WithRequestTimeout(time.Duration(options.RequestTimeout)*time.Second))
+		}
+		if options.MaxRetries > 0 {
+			dOpts = append(dOpts, dockerconfigresolver.WithMaxRetries(options.MaxRetries))
+		}
+		if options.RetryInitialDelay > 0 {
+			dOpts = append(dOpts, dockerconfigresolver.WithRetryInitialDelay(time.Duration(options.RetryInitialDelay)*time.Millisecond))
+		}
+	}
+
+	resolver, err := dockerconfigresolver.New(ctx, refDomain, dOpts...)
+	if err != nil {
+		return err
+	}
+
+	if err = pushFunc(resolver); err != nil {
+		// In some circumstance (e.g. people just use 80 port to support pure http), the error will contain message like "dial tcp <port>: connection refused"
+		if !errors.Is(err, http.ErrSchemeMismatch) && !errutil.IsErrConnectionRefused(err) {
+			return err
+		}
+	}
+
 	img, err := client.ImageService().Get(ctx, pushRef)
 	if err != nil {
 		return err