From c602698ce955d6d1f06e98072564f2cb53f696a3 Mon Sep 17 00:00:00 2001 From: youbin Date: Sat, 15 Aug 2026 15:32:43 +0800 Subject: [PATCH] Retry transient uploads and continue backups --- README.md | 6 +++- cmd/bdrclone/main.go | 18 +++++++---- cmd/bdrclone/upload_path.go | 33 ++++++++++++++++++- cmd/bdrclone/upload_path_test.go | 43 +++++++++++++++++++++++++ internal/baidu/client.go | 26 ++++++++------- internal/baidu/client_test.go | 55 ++++++++++++++++++++++++++++++++ internal/baidu/upload.go | 22 ++++++++----- 7 files changed, 175 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 6703618..366ad72 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,10 @@ OOB 模式使用 `redirect_uri=oob`，无需在本机监听端口。 # 递归上传目录内容，保留相对目录结构 ./bdrclone upload ~/ /备份/mac/ # 排除受保护目录和缓存；--exclude 可重复，支持 ** -./bdrclone upload ~/ /备份/mac/ --exclude .Trash --exclude 'Library/Caches/**' +./bdrclone upload ~/ /备份/mac/ \ + --exclude .Trash \ + --exclude 'Library/Caches/**' \ + --exclude '.cache/codex-runtimes/**' ./bdrclone mkdir /备份/新目录 ./bdrclone mv /备份/a.txt /备份/b.txt ./bdrclone cp /备份/b.txt /副本/b.txt @@ -127,6 +130,7 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf - 百度网盘官方接口不允许创建空文件；挂载下的空文件在 flush 时会失败。 - 目录上传会创建远端空目录，但会跳过无法读取的路径、符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。 - `upload --exclude` 使用相对于源目录的 glob；无 `/` 的规则匹配任意层级的文件名，`**` 可跨目录匹配。 +- 分片上传遇到临时网络或网关错误会退避重试 6 次；目录中单个文件最终失败时会记录错误并继续，结束时返回失败状态。 - 大文件下载必须使用 `User-Agent: pan.baidu.com`，客户端已统一设置。 - 分片大小由会员等级决定：普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB；百度限制分片数， 因此不同等级的单文件上限不同。 diff --git a/cmd/bdrclone/main.go b/cmd/bdrclone/main.go index 5e67b6c..e986412 100644 --- a/cmd/bdrclone/main.go +++ b/cmd/bdrclone/main.go @@ -276,12 +276,13 @@ func (a *application) uploadCommand() *cobra.Command { } stderr := cmd.ErrOrStderr() lineOpen := false - summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(localPath, remotePath string, done, total int64) { + summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(_, remotePath string, done, total int64) { + displayPath := uploadProgressPath(args[1], remotePath) percent := int64(0) if total > 0 { percent = done * 100 / total } - fmt.Fprintf(stderr, "\r上传 %s -> %s: %d/%d bytes (%d%%)", localPath, remotePath, done, total, percent) + fmt.Fprintf(stderr, "\r上传 %s: %d/%d bytes (%d%%)", displayPath, done, total, percent) lineOpen = done < total if !lineOpen { fmt.Fprintln(stderr) @@ -292,15 +293,20 @@ func (a *application) uploadCommand() *cobra.Command { lineOpen = false } fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason) + }, Failed: func(_, _ string, uploadErr error) { + if lineOpen { + fmt.Fprintln(stderr) + lineOpen = false + } + fmt.Fprintf(stderr, "失败: %v\n", uploadErr) }}) if lineOpen { fmt.Fprintln(stderr) } - if err != nil { - return err + if summary.Directories > 0 || summary.Files > 0 || summary.Skipped > 0 || summary.Failed > 0 { + fmt.Fprintf(stderr, "完成：成功上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项，失败 %d 个文件\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped, summary.Failed) } - fmt.Fprintf(stderr, "完成：上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped) - return nil + return err }} cmd.Flags().StringArrayVar(&excludes, "exclude", nil, "排除源目录中的 glob（可重复，支持 **）") return cmd diff --git a/cmd/bdrclone/upload_path.go b/cmd/bdrclone/upload_path.go index b4da29e..7656584 100644 --- a/cmd/bdrclone/upload_path.go +++ b/cmd/bdrclone/upload_path.go @@ -23,16 +23,19 @@ type uploadPathSummary struct { Files int Directories int Skipped int + Failed int Bytes int64 } type uploadPathProgress func(localPath, remotePath string, uploaded, total int64) type uploadPathSkipped func(localPath, reason string) +type uploadPathFailed func(localPath, remotePath string, err error) type uploadPathOptions struct { Excludes []string Progress uploadPathProgress Skipped uploadPathSkipped + Failed uploadPathFailed } type uploadWalkDir func(string, fs.WalkDirFunc) error @@ -77,6 +80,7 @@ func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, re return uploadPathSummary{}, err } summary := uploadPathSummary{Directories: 1} + var firstUploadErr error err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error { if walkErr != nil { if err := ctx.Err(); err != nil { @@ -131,11 +135,23 @@ func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, re reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件") return nil } - return uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary) + if err := uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + if firstUploadErr == nil { + firstUploadErr = err + } + reportUploadFailure(&summary, options, localPath, remotePath, err) + } + return nil }) if err != nil { return summary, err } + if summary.Failed > 0 { + return summary, fmt.Errorf("%d files failed to upload; first failure: %w", summary.Failed, firstUploadErr) + } return summary, nil } @@ -146,6 +162,13 @@ func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, loc } } +func reportUploadFailure(summary *uploadPathSummary, options uploadPathOptions, localPath, remotePath string, err error) { + summary.Failed++ + if options.Failed != nil { + options.Failed(localPath, remotePath, err) + } +} + func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error { completeReported := false if progress != nil && info.Size() > 0 { @@ -172,6 +195,14 @@ func cleanUploadRemotePath(name string) string { return pathpkg.Clean("/" + strings.TrimPrefix(name, "/")) } +func uploadProgressPath(remoteDestination, remotePath string) string { + root := cleanUploadRemotePath(remoteDestination) + if relative := strings.TrimPrefix(remotePath, root+"/"); relative != remotePath { + return relative + } + return pathpkg.Base(remotePath) +} + func normalizeUploadExcludes(patterns []string) ([]string, error) { result := make([]string, 0, len(patterns)) for _, pattern := range patterns { diff --git a/cmd/bdrclone/upload_path_test.go b/cmd/bdrclone/upload_path_test.go index ebdd936..c6093be 100644 --- a/cmd/bdrclone/upload_path_test.go +++ b/cmd/bdrclone/upload_path_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "errors" "io/fs" "os" "path/filepath" @@ -16,6 +17,7 @@ import ( type recordingUploadClient struct { directories []string files map[string][]byte + failures map[string]error } func (c *recordingUploadClient) MkdirAll(_ context.Context, remotePath string) error { @@ -36,6 +38,9 @@ func (c *recordingUploadClient) UploadFile(_ context.Context, localPath, remoteP if c.files == nil { c.files = make(map[string][]byte) } + if err := c.failures[remotePath]; err != nil { + return baidu.File{}, err + } c.files[remotePath] = content if progress != nil { progress(int64(len(content)), int64(len(content))) @@ -197,6 +202,35 @@ func TestUploadLocalDirectoryContinuesAfterReadError(t *testing.T) { } } +func TestUploadLocalDirectoryContinuesAfterUploadFailure(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "a-fails.txt"), []byte("failed"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "b-succeeds.txt"), []byte("success"), 0o600); err != nil { + t.Fatal(err) + } + client := &recordingUploadClient{failures: map[string]error{ + "/backup/mac/a-fails.txt": errors.New("gateway timeout"), + }} + var failed []string + summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{Failed: func(localPath, remotePath string, err error) { + failed = append(failed, filepath.Base(localPath)+":"+remotePath+":"+err.Error()) + }}) + if err == nil || !strings.Contains(err.Error(), "1 files failed to upload") { + t.Fatalf("error = %v", err) + } + if summary.Files != 1 || summary.Failed != 1 || summary.Bytes != 7 { + t.Fatalf("summary = %+v", summary) + } + if len(failed) != 1 || !strings.Contains(failed[0], "a-fails.txt:/backup/mac/a-fails.txt:") || !strings.Contains(failed[0], "gateway timeout") { + t.Fatalf("failed = %q", failed) + } + if !bytes.Equal(client.files["/backup/mac/b-succeeds.txt"], []byte("success")) { + t.Fatalf("b-succeeds.txt was not uploaded: %v", client.files) + } +} + func TestUploadExcludePatternMatching(t *testing.T) { patterns, err := normalizeUploadExcludes([]string{".Trash/", "Library/Caches/**", "**/*.tmp"}) if err != nil { @@ -220,3 +254,12 @@ func TestUploadExcludePatternMatching(t *testing.T) { t.Fatal("invalid exclude pattern was accepted") } } + +func TestUploadProgressPath(t *testing.T) { + if got := uploadProgressPath("/backup/mac/", "/backup/mac/Library/file.txt"); got != "Library/file.txt" { + t.Fatalf("progress path = %q", got) + } + if got := uploadProgressPath("/backup/file.txt", "/backup/file.txt"); got != "file.txt" { + t.Fatalf("single-file progress path = %q", got) + } +} diff --git a/internal/baidu/client.go b/internal/baidu/client.go index 3386b3b..da33b7f 100644 --- a/internal/baidu/client.go +++ b/internal/baidu/client.go @@ -25,16 +25,17 @@ const ( ) type Client struct { - httpClient *http.Client - apiBase string - oauthBase string - uploadBase string - userAgent string - root string - uploadParts int - partSize int64 - downloadMu sync.Mutex - downloadURL map[int64]cachedDownloadURL + httpClient *http.Client + apiBase string + oauthBase string + uploadBase string + userAgent string + root string + uploadParts int + partSize int64 + uploadRetryDelay func(int) time.Duration + downloadMu sync.Mutex + downloadURL map[int64]cachedDownloadURL mu sync.Mutex clientID string @@ -75,8 +76,9 @@ func New(cfg *config.Config, opts ...Option) *Client { apiBase: defaultAPIBase, oauthBase: defaultOAuthBase, uploadBase: defaultUploadBase, userAgent: cfg.UserAgent, root: cfg.Root, uploadParts: cfg.UploadParts, partSize: cfg.PartSize, - downloadURL: make(map[int64]cachedDownloadURL), - clientID: cfg.ClientID, secret: cfg.ClientSecret, + uploadRetryDelay: defaultUploadRetryDelay, + downloadURL: make(map[int64]cachedDownloadURL), + clientID: cfg.ClientID, secret: cfg.ClientSecret, accessToken: cfg.AccessToken, refresh: cfg.RefreshToken, expiresAt: cfg.ExpiresAt, } for _, opt := range opts { diff --git a/internal/baidu/client_test.go b/internal/baidu/client_test.go index 8788292..bb92883 100644 --- a/internal/baidu/client_test.go +++ b/internal/baidu/client_test.go @@ -12,6 +12,7 @@ import ( "path" "reflect" "strconv" + "strings" "sync" "testing" "time" @@ -205,6 +206,60 @@ func TestUploadMultipartFlow(t *testing.T) { } } +func TestUploadPartRetriesGatewayTimeout(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "retry-upload-*") + if err != nil { + t.Fatal(err) + } + defer file.Close() + content := []byte("retry payload") + if _, err := file.Write(content); err != nil { + t.Fatal(err) + } + + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rest/2.0/pcs/superfile2" { + http.NotFound(w, r) + return + } + attempts++ + _, _ = io.Copy(io.Discard, r.Body) + if attempts < 4 { + http.Error(w, "gateway timeout", http.StatusGatewayTimeout) + return + } + fmt.Fprint(w, `{"md5":"ok"}`) + })) + defer server.Close() + + client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL)) + client.uploadRetryDelay = func(int) time.Duration { return 0 } + if err := client.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-1", 0, 0, int64(len(content))); err != nil { + t.Fatal(err) + } + if attempts != 4 { + t.Fatalf("attempts = %d, want 4", attempts) + } + + failureAttempts := 0 + failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + failureAttempts++ + _, _ = io.Copy(io.Discard, r.Body) + http.Error(w, "gateway timeout", http.StatusGatewayTimeout) + })) + defer failureServer.Close() + failureClient := New(testConfig("token"), WithEndpoints(failureServer.URL, failureServer.URL, failureServer.URL)) + failureClient.uploadRetryDelay = func(int) time.Duration { return 0 } + err = failureClient.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-2", 0, 0, int64(len(content))) + if err == nil || !strings.Contains(err.Error(), "after 6 attempts") { + t.Fatalf("error = %v", err) + } + if failureAttempts != maxUploadAttempts { + t.Fatalf("attempts = %d, want %d", failureAttempts, maxUploadAttempts) + } +} + func TestMkdirAllCreatesMissingRemoteDirectories(t *testing.T) { directories := map[string]bool{"/apps/bdrclone": true} var created []string diff --git a/internal/baidu/upload.go b/internal/baidu/upload.go index f35783f..3f58beb 100644 --- a/internal/baidu/upload.go +++ b/internal/baidu/upload.go @@ -21,10 +21,11 @@ import ( ) const ( - defaultPartSize = int64(4 << 20) - vipPartSize = int64(16 << 20) - svipPartSize = int64(32 << 20) - maxPartCount = 2048 + defaultPartSize = int64(4 << 20) + vipPartSize = int64(16 << 20) + svipPartSize = int64(32 << 20) + maxPartCount = 2048 + maxUploadAttempts = 6 ) type UploadProgress func(uploaded, total int64) @@ -247,7 +248,7 @@ func (c *Client) uploadPartsParallel(ctx context.Context, file *os.File, remote, func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error { var lastErr error - for attempt := 0; attempt < 3; attempt++ { + for attempt := 0; attempt < maxUploadAttempts; attempt++ { if err := ctx.Err(); err != nil { return err } @@ -256,13 +257,18 @@ func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, if lastErr == nil { return nil } - if attempt < 2 { - if err := sleepContext(ctx, time.Duration(1<