youbin/bdrclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retry transient uploads and continue backups
All checks were successful
Build / Test and build (push) Successful in 5m53s
Details

Browse Source
This commit is contained in:
youbin
2026-08-15 15:32:43 +08:00
parent 977fcefc2f
commit c602698ce9
7 changed files with 175 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -89,7 +89,10 @@ OOB 模式使用 `redirect_uri=oob`，无需在本机监听端口。
# 递归上传目录内容，保留相对目录结构
./bdrclone upload ~/ /备份/mac/
# 排除受保护目录和缓存；--exclude 可重复，支持 **
./bdrclone upload ~/ /备份/mac/ --exclude .Trash --exclude 'Library/Caches/**'
./bdrclone upload ~/ /备份/mac/ \
--exclude .Trash \
--exclude 'Library/Caches/**' \
--exclude '.cache/codex-runtimes/**'
./bdrclone mkdir /备份/新目录
./bdrclone mv /备份/a.txt /备份/b.txt
./bdrclone cp /备份/b.txt /副本/b.txt
@@ -127,6 +130,7 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf
- 百度网盘官方接口不允许创建空文件；挂载下的空文件在 flush 时会失败。
- 目录上传会创建远端空目录，但会跳过无法读取的路径、符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。
- `upload --exclude` 使用相对于源目录的 glob；无 `/` 的规则匹配任意层级的文件名，`**` 可跨目录匹配。
- 分片上传遇到临时网络或网关错误会退避重试 6 次；目录中单个文件最终失败时会记录错误并继续，结束时返回失败状态。
- 大文件下载必须使用 `User-Agent: pan.baidu.com`，客户端已统一设置。
- 分片大小由会员等级决定：普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB；百度限制分片数，
因此不同等级的单文件上限不同。

18
cmd/bdrclone/main.go
View File

@@ -276,12 +276,13 @@ func (a *application) uploadCommand() *cobra.Command {
}
stderr := cmd.ErrOrStderr()
lineOpen := false
summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(localPath, remotePath string, done, total int64) {
summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(_, remotePath string, done, total int64) {
displayPath := uploadProgressPath(args[1], remotePath)
percent := int64(0)
if total > 0 {
percent = done * 100 / total
}
fmt.Fprintf(stderr, "\r上传 %s -> %s: %d/%d bytes (%d%%)", localPath, remotePath, done, total, percent)
fmt.Fprintf(stderr, "\r上传 %s: %d/%d bytes (%d%%)", displayPath, done, total, percent)
lineOpen = done < total
if !lineOpen {
fmt.Fprintln(stderr)
@@ -292,15 +293,20 @@ func (a *application) uploadCommand() *cobra.Command {
lineOpen = false
}
fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason)
}, Failed: func(_, _ string, uploadErr error) {
if lineOpen {
fmt.Fprintln(stderr)
lineOpen = false
}
fmt.Fprintf(stderr, "失败: %v\n", uploadErr)
}})
if lineOpen {
fmt.Fprintln(stderr)
}
if err != nil {
return err
if summary.Directories > 0 || summary.Files > 0 || summary.Skipped > 0 || summary.Failed > 0 {
fmt.Fprintf(stderr, "完成：成功上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项，失败 %d 个文件\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped, summary.Failed)
}
fmt.Fprintf(stderr, "完成：上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped)
return nil
return err
}}
cmd.Flags().StringArrayVar(&excludes, "exclude", nil, "排除源目录中的 glob（可重复，支持 **）")
return cmd

33
cmd/bdrclone/upload_path.go
View File

@@ -23,16 +23,19 @@ type uploadPathSummary struct {
Files int
Directories int
Skipped int
Failed int
Bytes int64
}
type uploadPathProgress func(localPath, remotePath string, uploaded, total int64)
type uploadPathSkipped func(localPath, reason string)
type uploadPathFailed func(localPath, remotePath string, err error)
type uploadPathOptions struct {
Excludes []string
Progress uploadPathProgress
Skipped uploadPathSkipped
Failed uploadPathFailed
}
type uploadWalkDir func(string, fs.WalkDirFunc) error
@@ -77,6 +80,7 @@ func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, re
return uploadPathSummary{}, err
}
summary := uploadPathSummary{Directories: 1}
var firstUploadErr error
err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
if err := ctx.Err(); err != nil {
@@ -131,11 +135,23 @@ func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, re
reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件")
return nil
}
return uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary)
if err := uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
if firstUploadErr == nil {
firstUploadErr = err
}
reportUploadFailure(&summary, options, localPath, remotePath, err)
}
return nil
})
if err != nil {
return summary, err
}
if summary.Failed > 0 {
return summary, fmt.Errorf("%d files failed to upload; first failure: %w", summary.Failed, firstUploadErr)
}
return summary, nil
}
@@ -146,6 +162,13 @@ func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, loc
}
}
func reportUploadFailure(summary *uploadPathSummary, options uploadPathOptions, localPath, remotePath string, err error) {
summary.Failed++
if options.Failed != nil {
options.Failed(localPath, remotePath, err)
}
}
func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error {
completeReported := false
if progress != nil && info.Size() > 0 {
@@ -172,6 +195,14 @@ func cleanUploadRemotePath(name string) string {
return pathpkg.Clean("/" + strings.TrimPrefix(name, "/"))
}
func uploadProgressPath(remoteDestination, remotePath string) string {
root := cleanUploadRemotePath(remoteDestination)
if relative := strings.TrimPrefix(remotePath, root+"/"); relative != remotePath {
return relative
}
return pathpkg.Base(remotePath)
}
func normalizeUploadExcludes(patterns []string) ([]string, error) {
result := make([]string, 0, len(patterns))
for _, pattern := range patterns {

43
cmd/bdrclone/upload_path_test.go
View File

@@ -3,6 +3,7 @@ package main
import (
"bytes"
"context"
"errors"
"io/fs"
"os"
"path/filepath"
@@ -16,6 +17,7 @@ import (
type recordingUploadClient struct {
directories []string
files map[string][]byte
failures map[string]error
}
func (c *recordingUploadClient) MkdirAll(_ context.Context, remotePath string) error {
@@ -36,6 +38,9 @@ func (c *recordingUploadClient) UploadFile(_ context.Context, localPath, remoteP
if c.files == nil {
c.files = make(map[string][]byte)
}
if err := c.failures[remotePath]; err != nil {
return baidu.File{}, err
}
c.files[remotePath] = content
if progress != nil {
progress(int64(len(content)), int64(len(content)))
@@ -197,6 +202,35 @@ func TestUploadLocalDirectoryContinuesAfterReadError(t *testing.T) {
}
}
func TestUploadLocalDirectoryContinuesAfterUploadFailure(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "a-fails.txt"), []byte("failed"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "b-succeeds.txt"), []byte("success"), 0o600); err != nil {
t.Fatal(err)
}
client := &recordingUploadClient{failures: map[string]error{
"/backup/mac/a-fails.txt": errors.New("gateway timeout"),
}}
var failed []string
summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{Failed: func(localPath, remotePath string, err error) {
failed = append(failed, filepath.Base(localPath)+":"+remotePath+":"+err.Error())
}})
if err == nil || !strings.Contains(err.Error(), "1 files failed to upload") {
t.Fatalf("error = %v", err)
}
if summary.Files != 1 || summary.Failed != 1 || summary.Bytes != 7 {
t.Fatalf("summary = %+v", summary)
}
if len(failed) != 1 || !strings.Contains(failed[0], "a-fails.txt:/backup/mac/a-fails.txt:") || !strings.Contains(failed[0], "gateway timeout") {
t.Fatalf("failed = %q", failed)
}
if !bytes.Equal(client.files["/backup/mac/b-succeeds.txt"], []byte("success")) {
t.Fatalf("b-succeeds.txt was not uploaded: %v", client.files)
}
}
func TestUploadExcludePatternMatching(t *testing.T) {
patterns, err := normalizeUploadExcludes([]string{".Trash/", "Library/Caches/**", "**/*.tmp"})
if err != nil {
@@ -220,3 +254,12 @@ func TestUploadExcludePatternMatching(t *testing.T) {
t.Fatal("invalid exclude pattern was accepted")
}
}
func TestUploadProgressPath(t *testing.T) {
if got := uploadProgressPath("/backup/mac/", "/backup/mac/Library/file.txt"); got != "Library/file.txt" {
t.Fatalf("progress path = %q", got)
}
if got := uploadProgressPath("/backup/file.txt", "/backup/file.txt"); got != "file.txt" {
t.Fatalf("single-file progress path = %q", got)
}
}

26
internal/baidu/client.go
View File

@@ -25,16 +25,17 @@ const (
)
type Client struct {
httpClient *http.Client
apiBase string
oauthBase string
uploadBase string
userAgent string
root string
uploadParts int
partSize int64
downloadMu sync.Mutex
downloadURL map[int64]cachedDownloadURL
httpClient *http.Client
apiBase string
oauthBase string
uploadBase string
userAgent string
root string
uploadParts int
partSize int64
uploadRetryDelay func(int) time.Duration
downloadMu sync.Mutex
downloadURL map[int64]cachedDownloadURL
mu sync.Mutex
clientID string
@@ -75,8 +76,9 @@ func New(cfg *config.Config, opts ...Option) *Client {
apiBase: defaultAPIBase, oauthBase: defaultOAuthBase, uploadBase: defaultUploadBase,
userAgent: cfg.UserAgent, root: cfg.Root,
uploadParts: cfg.UploadParts, partSize: cfg.PartSize,
downloadURL: make(map[int64]cachedDownloadURL),
clientID: cfg.ClientID, secret: cfg.ClientSecret,
uploadRetryDelay: defaultUploadRetryDelay,
downloadURL: make(map[int64]cachedDownloadURL),
clientID: cfg.ClientID, secret: cfg.ClientSecret,
accessToken: cfg.AccessToken, refresh: cfg.RefreshToken, expiresAt: cfg.ExpiresAt,
}
for _, opt := range opts {

55
internal/baidu/client_test.go
View File

@@ -12,6 +12,7 @@ import (
"path"
"reflect"
"strconv"
"strings"
"sync"
"testing"
"time"
@@ -205,6 +206,60 @@ func TestUploadMultipartFlow(t *testing.T) {
}
}
func TestUploadPartRetriesGatewayTimeout(t *testing.T) {
file, err := os.CreateTemp(t.TempDir(), "retry-upload-*")
if err != nil {
t.Fatal(err)
}
defer file.Close()
content := []byte("retry payload")
if _, err := file.Write(content); err != nil {
t.Fatal(err)
}
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/rest/2.0/pcs/superfile2" {
http.NotFound(w, r)
return
}
attempts++
_, _ = io.Copy(io.Discard, r.Body)
if attempts < 4 {
http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
return
}
fmt.Fprint(w, `{"md5":"ok"}`)
}))
defer server.Close()
client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL))
client.uploadRetryDelay = func(int) time.Duration { return 0 }
if err := client.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-1", 0, 0, int64(len(content))); err != nil {
t.Fatal(err)
}
if attempts != 4 {
t.Fatalf("attempts = %d, want 4", attempts)
}
failureAttempts := 0
failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
failureAttempts++
_, _ = io.Copy(io.Discard, r.Body)
http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
}))
defer failureServer.Close()
failureClient := New(testConfig("token"), WithEndpoints(failureServer.URL, failureServer.URL, failureServer.URL))
failureClient.uploadRetryDelay = func(int) time.Duration { return 0 }
err = failureClient.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-2", 0, 0, int64(len(content)))
if err == nil || !strings.Contains(err.Error(), "after 6 attempts") {
t.Fatalf("error = %v", err)
}
if failureAttempts != maxUploadAttempts {
t.Fatalf("attempts = %d, want %d", failureAttempts, maxUploadAttempts)
}
}
func TestMkdirAllCreatesMissingRemoteDirectories(t *testing.T) {
directories := map[string]bool{"/apps/bdrclone": true}
var created []string

22
internal/baidu/upload.go
View File

@@ -21,10 +21,11 @@ import (
)
const (
defaultPartSize = int64(4 << 20)
vipPartSize = int64(16 << 20)
svipPartSize = int64(32 << 20)
maxPartCount = 2048
defaultPartSize = int64(4 << 20)
vipPartSize = int64(16 << 20)
svipPartSize = int64(32 << 20)
maxPartCount = 2048
maxUploadAttempts = 6
)
type UploadProgress func(uploaded, total int64)
@@ -247,7 +248,7 @@ func (c *Client) uploadPartsParallel(ctx context.Context, file *os.File, remote,
func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error {
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
for attempt := 0; attempt < maxUploadAttempts; attempt++ {
if err := ctx.Err(); err != nil {
return err
}
@@ -256,13 +257,18 @@ func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote,
if lastErr == nil {
return nil
}
if attempt < 2 {
if err := sleepContext(ctx, time.Duration(1<<attempt)*time.Second); err != nil {
if attempt < maxUploadAttempts-1 {
if err := sleepContext(ctx, c.uploadRetryDelay(attempt)); err != nil {
return err
}
}
}
return lastErr
return fmt.Errorf("upload failed after %d attempts: %w", maxUploadAttempts, lastErr)
}
func defaultUploadRetryDelay(attempt int) time.Duration {
delay := time.Second * time.Duration(1<<attempt)
return min(delay, 30*time.Second)
}
func (c *Client) uploadPart(ctx context.Context, section *io.SectionReader, filename, remote, uploadID string, part int) error {