Retry transient uploads and continue backups

README.md

@@ -89,7 +89,10 @@ OOB 模式使用 `redirect_uri=oob`，无需在本机监听端口。
 # 递归上传目录内容，保留相对目录结构
 ./bdrclone upload ~/ /备份/mac/
 # 排除受保护目录和缓存；--exclude 可重复，支持 **
-./bdrclone upload ~/ /备份/mac/ --exclude .Trash --exclude 'Library/Caches/**'
+./bdrclone upload ~/ /备份/mac/ \
+  --exclude .Trash \
+  --exclude 'Library/Caches/**' \
+  --exclude '.cache/codex-runtimes/**'
 ./bdrclone mkdir /备份/新目录
 ./bdrclone mv /备份/a.txt /备份/b.txt
 ./bdrclone cp /备份/b.txt /副本/b.txt
@@ -127,6 +130,7 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf
 - 百度网盘官方接口不允许创建空文件；挂载下的空文件在 flush 时会失败。
 - 目录上传会创建远端空目录，但会跳过无法读取的路径、符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。
 - `upload --exclude` 使用相对于源目录的 glob；无 `/` 的规则匹配任意层级的文件名，`**` 可跨目录匹配。
+- 分片上传遇到临时网络或网关错误会退避重试 6 次；目录中单个文件最终失败时会记录错误并继续，结束时返回失败状态。
 - 大文件下载必须使用 `User-Agent: pan.baidu.com`，客户端已统一设置。
 - 分片大小由会员等级决定：普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB；百度限制分片数，
   因此不同等级的单文件上限不同。

cmd/bdrclone/main.go

@@ -276,12 +276,13 @@ func (a *application) uploadCommand() *cobra.Command {
 		}
 		stderr := cmd.ErrOrStderr()
 		lineOpen := false
-		summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(localPath, remotePath string, done, total int64) {
+		summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(_, remotePath string, done, total int64) {
+			displayPath := uploadProgressPath(args[1], remotePath)
 			percent := int64(0)
 			if total > 0 {
 				percent = done * 100 / total
 			}
-			fmt.Fprintf(stderr, "\r上传 %s -> %s: %d/%d bytes (%d%%)", localPath, remotePath, done, total, percent)
+			fmt.Fprintf(stderr, "\r上传 %s: %d/%d bytes (%d%%)", displayPath, done, total, percent)
 			lineOpen = done < total
 			if !lineOpen {
 				fmt.Fprintln(stderr)
@@ -292,15 +293,20 @@ func (a *application) uploadCommand() *cobra.Command {
 				lineOpen = false
 			}
 			fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason)
+		}, Failed: func(_, _ string, uploadErr error) {
+			if lineOpen {
+				fmt.Fprintln(stderr)
+				lineOpen = false
+			}
+			fmt.Fprintf(stderr, "失败: %v\n", uploadErr)
 		}})
 		if lineOpen {
 			fmt.Fprintln(stderr)
 		}
-		if err != nil {
+		if summary.Directories > 0 || summary.Files > 0 || summary.Skipped > 0 || summary.Failed > 0 {
-			return err
+			fmt.Fprintf(stderr, "完成：成功上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项，失败 %d 个文件\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped, summary.Failed)
 		}
-		fmt.Fprintf(stderr, "完成：上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped)
+		return err
-		return nil
 	}}
 	cmd.Flags().StringArrayVar(&excludes, "exclude", nil, "排除源目录中的 glob（可重复，支持 **）")
 	return cmd

cmd/bdrclone/upload_path.go

@@ -23,16 +23,19 @@ type uploadPathSummary struct {
 	Files       int
 	Directories int
 	Skipped     int
+	Failed      int
 	Bytes       int64
 }
 
 type uploadPathProgress func(localPath, remotePath string, uploaded, total int64)
 type uploadPathSkipped func(localPath, reason string)
+type uploadPathFailed func(localPath, remotePath string, err error)
 
 type uploadPathOptions struct {
 	Excludes []string
 	Progress uploadPathProgress
 	Skipped  uploadPathSkipped
+	Failed   uploadPathFailed
 }
 
 type uploadWalkDir func(string, fs.WalkDirFunc) error
@@ -77,6 +80,7 @@ func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, re
 		return uploadPathSummary{}, err
 	}
 	summary := uploadPathSummary{Directories: 1}
+	var firstUploadErr error
 	err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
 		if walkErr != nil {
 			if err := ctx.Err(); err != nil {
@@ -131,11 +135,23 @@ func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, re
 			reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件")
 			return nil
 		}
-		return uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary)
+		if err := uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary); err != nil {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+			if firstUploadErr == nil {
+				firstUploadErr = err
+			}
+			reportUploadFailure(&summary, options, localPath, remotePath, err)
+		}
+		return nil
 	})
 	if err != nil {
 		return summary, err
 	}
+	if summary.Failed > 0 {
+		return summary, fmt.Errorf("%d files failed to upload; first failure: %w", summary.Failed, firstUploadErr)
+	}
 	return summary, nil
 }
 
@@ -146,6 +162,13 @@ func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, loc
 	}
 }
 
+func reportUploadFailure(summary *uploadPathSummary, options uploadPathOptions, localPath, remotePath string, err error) {
+	summary.Failed++
+	if options.Failed != nil {
+		options.Failed(localPath, remotePath, err)
+	}
+}
+
 func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error {
 	completeReported := false
 	if progress != nil && info.Size() > 0 {
@@ -172,6 +195,14 @@ func cleanUploadRemotePath(name string) string {
 	return pathpkg.Clean("/" + strings.TrimPrefix(name, "/"))
 }
 
+func uploadProgressPath(remoteDestination, remotePath string) string {
+	root := cleanUploadRemotePath(remoteDestination)
+	if relative := strings.TrimPrefix(remotePath, root+"/"); relative != remotePath {
+		return relative
+	}
+	return pathpkg.Base(remotePath)
+}
+
 func normalizeUploadExcludes(patterns []string) ([]string, error) {
 	result := make([]string, 0, len(patterns))
 	for _, pattern := range patterns {

cmd/bdrclone/upload_path_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -16,6 +17,7 @@ import (
 type recordingUploadClient struct {
 	directories []string
 	files       map[string][]byte
+	failures    map[string]error
 }
 
 func (c *recordingUploadClient) MkdirAll(_ context.Context, remotePath string) error {
@@ -36,6 +38,9 @@ func (c *recordingUploadClient) UploadFile(_ context.Context, localPath, remoteP
 	if c.files == nil {
 		c.files = make(map[string][]byte)
 	}
+	if err := c.failures[remotePath]; err != nil {
+		return baidu.File{}, err
+	}
 	c.files[remotePath] = content
 	if progress != nil {
 		progress(int64(len(content)), int64(len(content)))
@@ -197,6 +202,35 @@ func TestUploadLocalDirectoryContinuesAfterReadError(t *testing.T) {
 	}
 }
 
+func TestUploadLocalDirectoryContinuesAfterUploadFailure(t *testing.T) {
+	root := t.TempDir()
+	if err := os.WriteFile(filepath.Join(root, "a-fails.txt"), []byte("failed"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "b-succeeds.txt"), []byte("success"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	client := &recordingUploadClient{failures: map[string]error{
+		"/backup/mac/a-fails.txt": errors.New("gateway timeout"),
+	}}
+	var failed []string
+	summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{Failed: func(localPath, remotePath string, err error) {
+		failed = append(failed, filepath.Base(localPath)+":"+remotePath+":"+err.Error())
+	}})
+	if err == nil || !strings.Contains(err.Error(), "1 files failed to upload") {
+		t.Fatalf("error = %v", err)
+	}
+	if summary.Files != 1 || summary.Failed != 1 || summary.Bytes != 7 {
+		t.Fatalf("summary = %+v", summary)
+	}
+	if len(failed) != 1 || !strings.Contains(failed[0], "a-fails.txt:/backup/mac/a-fails.txt:") || !strings.Contains(failed[0], "gateway timeout") {
+		t.Fatalf("failed = %q", failed)
+	}
+	if !bytes.Equal(client.files["/backup/mac/b-succeeds.txt"], []byte("success")) {
+		t.Fatalf("b-succeeds.txt was not uploaded: %v", client.files)
+	}
+}
+
 func TestUploadExcludePatternMatching(t *testing.T) {
 	patterns, err := normalizeUploadExcludes([]string{".Trash/", "Library/Caches/**", "**/*.tmp"})
 	if err != nil {
@@ -220,3 +254,12 @@ func TestUploadExcludePatternMatching(t *testing.T) {
 		t.Fatal("invalid exclude pattern was accepted")
 	}
 }
+
+func TestUploadProgressPath(t *testing.T) {
+	if got := uploadProgressPath("/backup/mac/", "/backup/mac/Library/file.txt"); got != "Library/file.txt" {
+		t.Fatalf("progress path = %q", got)
+	}
+	if got := uploadProgressPath("/backup/file.txt", "/backup/file.txt"); got != "file.txt" {
+		t.Fatalf("single-file progress path = %q", got)
+	}
+}

internal/baidu/client.go

@@ -25,16 +25,17 @@ const (
 )
 
 type Client struct {
-	httpClient  *http.Client
+	httpClient       *http.Client
-	apiBase     string
+	apiBase          string
-	oauthBase   string
+	oauthBase        string
-	uploadBase  string
+	uploadBase       string
-	userAgent   string
+	userAgent        string
-	root        string
+	root             string
-	uploadParts int
+	uploadParts      int
-	partSize    int64
+	partSize         int64
-	downloadMu  sync.Mutex
+	uploadRetryDelay func(int) time.Duration
-	downloadURL map[int64]cachedDownloadURL
+	downloadMu       sync.Mutex
+	downloadURL      map[int64]cachedDownloadURL
 
 	mu          sync.Mutex
 	clientID    string
@@ -75,8 +76,9 @@ func New(cfg *config.Config, opts ...Option) *Client {
 		apiBase:    defaultAPIBase, oauthBase: defaultOAuthBase, uploadBase: defaultUploadBase,
 		userAgent: cfg.UserAgent, root: cfg.Root,
 		uploadParts: cfg.UploadParts, partSize: cfg.PartSize,
-		downloadURL: make(map[int64]cachedDownloadURL),
+		uploadRetryDelay: defaultUploadRetryDelay,
-		clientID:    cfg.ClientID, secret: cfg.ClientSecret,
+		downloadURL:      make(map[int64]cachedDownloadURL),
+		clientID:         cfg.ClientID, secret: cfg.ClientSecret,
 		accessToken: cfg.AccessToken, refresh: cfg.RefreshToken, expiresAt: cfg.ExpiresAt,
 	}
 	for _, opt := range opts {

internal/baidu/client_test.go

@@ -12,6 +12,7 @@ import (
 	"path"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -205,6 +206,60 @@ func TestUploadMultipartFlow(t *testing.T) {
 	}
 }
 
+func TestUploadPartRetriesGatewayTimeout(t *testing.T) {
+	file, err := os.CreateTemp(t.TempDir(), "retry-upload-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	content := []byte("retry payload")
+	if _, err := file.Write(content); err != nil {
+		t.Fatal(err)
+	}
+
+	attempts := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/rest/2.0/pcs/superfile2" {
+			http.NotFound(w, r)
+			return
+		}
+		attempts++
+		_, _ = io.Copy(io.Discard, r.Body)
+		if attempts < 4 {
+			http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
+			return
+		}
+		fmt.Fprint(w, `{"md5":"ok"}`)
+	}))
+	defer server.Close()
+
+	client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL))
+	client.uploadRetryDelay = func(int) time.Duration { return 0 }
+	if err := client.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-1", 0, 0, int64(len(content))); err != nil {
+		t.Fatal(err)
+	}
+	if attempts != 4 {
+		t.Fatalf("attempts = %d, want 4", attempts)
+	}
+
+	failureAttempts := 0
+	failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		failureAttempts++
+		_, _ = io.Copy(io.Discard, r.Body)
+		http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
+	}))
+	defer failureServer.Close()
+	failureClient := New(testConfig("token"), WithEndpoints(failureServer.URL, failureServer.URL, failureServer.URL))
+	failureClient.uploadRetryDelay = func(int) time.Duration { return 0 }
+	err = failureClient.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-2", 0, 0, int64(len(content)))
+	if err == nil || !strings.Contains(err.Error(), "after 6 attempts") {
+		t.Fatalf("error = %v", err)
+	}
+	if failureAttempts != maxUploadAttempts {
+		t.Fatalf("attempts = %d, want %d", failureAttempts, maxUploadAttempts)
+	}
+}
+
 func TestMkdirAllCreatesMissingRemoteDirectories(t *testing.T) {
 	directories := map[string]bool{"/apps/bdrclone": true}
 	var created []string

internal/baidu/upload.go

@@ -21,10 +21,11 @@ import (
 )
 
 const (
-	defaultPartSize = int64(4 << 20)
+	defaultPartSize   = int64(4 << 20)
-	vipPartSize     = int64(16 << 20)
+	vipPartSize       = int64(16 << 20)
-	svipPartSize    = int64(32 << 20)
+	svipPartSize      = int64(32 << 20)
-	maxPartCount    = 2048
+	maxPartCount      = 2048
+	maxUploadAttempts = 6
 )
 
 type UploadProgress func(uploaded, total int64)
@@ -247,7 +248,7 @@ func (c *Client) uploadPartsParallel(ctx context.Context, file *os.File, remote,
 
 func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error {
 	var lastErr error
-	for attempt := 0; attempt < 3; attempt++ {
+	for attempt := 0; attempt < maxUploadAttempts; attempt++ {
 		if err := ctx.Err(); err != nil {
 			return err
 		}
@@ -256,13 +257,18 @@ func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote,
 		if lastErr == nil {
 			return nil
 		}
-		if attempt < 2 {
+		if attempt < maxUploadAttempts-1 {
-			if err := sleepContext(ctx, time.Duration(1<<attempt)*time.Second); err != nil {
+			if err := sleepContext(ctx, c.uploadRetryDelay(attempt)); err != nil {
 				return err
 			}
 		}
 	}
-	return lastErr
+	return fmt.Errorf("upload failed after %d attempts: %w", maxUploadAttempts, lastErr)
+}
+
+func defaultUploadRetryDelay(attempt int) time.Duration {
+	delay := time.Second * time.Duration(1<<attempt)
+	return min(delay, 30*time.Second)
 }
 
 func (c *Client) uploadPart(ctx context.Context, section *io.SectionReader, filename, remote, uploadID string, part int) error {