Retry transient uploads and continue backups

internal/baidu/upload.go

@@ -21,10 +21,11 @@ import (
 )
 
 const (
-	defaultPartSize = int64(4 << 20)
-	vipPartSize     = int64(16 << 20)
-	svipPartSize    = int64(32 << 20)
-	maxPartCount    = 2048
+	defaultPartSize   = int64(4 << 20)
+	vipPartSize       = int64(16 << 20)
+	svipPartSize      = int64(32 << 20)
+	maxPartCount      = 2048
+	maxUploadAttempts = 6
 )
 
 type UploadProgress func(uploaded, total int64)
@@ -247,7 +248,7 @@ func (c *Client) uploadPartsParallel(ctx context.Context, file *os.File, remote,
 
 func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error {
 	var lastErr error
-	for attempt := 0; attempt < 3; attempt++ {
+	for attempt := 0; attempt < maxUploadAttempts; attempt++ {
 		if err := ctx.Err(); err != nil {
 			return err
 		}
@@ -256,13 +257,18 @@ func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote,
 		if lastErr == nil {
 			return nil
 		}
-		if attempt < 2 {
-			if err := sleepContext(ctx, time.Duration(1<<attempt)*time.Second); err != nil {
+		if attempt < maxUploadAttempts-1 {
+			if err := sleepContext(ctx, c.uploadRetryDelay(attempt)); err != nil {
 				return err
 			}
 		}
 	}
-	return lastErr
+	return fmt.Errorf("upload failed after %d attempts: %w", maxUploadAttempts, lastErr)
+}
+
+func defaultUploadRetryDelay(attempt int) time.Duration {
+	delay := time.Second * time.Duration(1<<attempt)
+	return min(delay, 30*time.Second)
 }
 
 func (c *Client) uploadPart(ctx context.Context, section *io.SectionReader, filename, remote, uploadID string, part int) error {