Retry transient uploads and continue backups

internal/baidu/client_test.go

@@ -12,6 +12,7 @@ import (
 	"path"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -205,6 +206,60 @@ func TestUploadMultipartFlow(t *testing.T) {
 	}
 }
 
+func TestUploadPartRetriesGatewayTimeout(t *testing.T) {
+	file, err := os.CreateTemp(t.TempDir(), "retry-upload-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	content := []byte("retry payload")
+	if _, err := file.Write(content); err != nil {
+		t.Fatal(err)
+	}
+
+	attempts := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/rest/2.0/pcs/superfile2" {
+			http.NotFound(w, r)
+			return
+		}
+		attempts++
+		_, _ = io.Copy(io.Discard, r.Body)
+		if attempts < 4 {
+			http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
+			return
+		}
+		fmt.Fprint(w, `{"md5":"ok"}`)
+	}))
+	defer server.Close()
+
+	client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL))
+	client.uploadRetryDelay = func(int) time.Duration { return 0 }
+	if err := client.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-1", 0, 0, int64(len(content))); err != nil {
+		t.Fatal(err)
+	}
+	if attempts != 4 {
+		t.Fatalf("attempts = %d, want 4", attempts)
+	}
+
+	failureAttempts := 0
+	failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		failureAttempts++
+		_, _ = io.Copy(io.Discard, r.Body)
+		http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
+	}))
+	defer failureServer.Close()
+	failureClient := New(testConfig("token"), WithEndpoints(failureServer.URL, failureServer.URL, failureServer.URL))
+	failureClient.uploadRetryDelay = func(int) time.Duration { return 0 }
+	err = failureClient.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-2", 0, 0, int64(len(content)))
+	if err == nil || !strings.Contains(err.Error(), "after 6 attempts") {
+		t.Fatalf("error = %v", err)
+	}
+	if failureAttempts != maxUploadAttempts {
+		t.Fatalf("attempts = %d, want %d", failureAttempts, maxUploadAttempts)
+	}
+}
+
 func TestMkdirAllCreatesMissingRemoteDirectories(t *testing.T) {
 	directories := map[string]bool{"/apps/bdrclone": true}
 	var created []string