Continue directory uploads past unreadable paths

2026-08-15 14:45:02 +08:00
parent c1758cf29c
commit 977fcefc2f
4 changed files with 251 additions and 27 deletions
--- a/README.md
+++ b/README.md
@@ -88,6 +88,8 @@ OOB 模式使用 `redirect_uri=oob`，无需在本机监听端口。
 ./bdrclone upload ./photo.jpg /备份/photo.jpg
 # 递归上传目录内容，保留相对目录结构
 ./bdrclone upload ~/ /备份/mac/
+# 排除受保护目录和缓存；--exclude 可重复，支持 **
+./bdrclone upload ~/ /备份/mac/ --exclude .Trash --exclude 'Library/Caches/**'
 ./bdrclone mkdir /备份/新目录
 ./bdrclone mv /备份/a.txt /备份/b.txt
 ./bdrclone cp /备份/b.txt /副本/b.txt
@@ -123,7 +125,8 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf
 ## 百度 API 限制

 - 百度网盘官方接口不允许创建空文件；挂载下的空文件在 flush 时会失败。
- 目录上传会创建远端空目录，但会跳过符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。
+- 目录上传会创建远端空目录，但会跳过无法读取的路径、符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。
+- `upload --exclude` 使用相对于源目录的 glob；无 `/` 的规则匹配任意层级的文件名，`**` 可跨目录匹配。
 - 大文件下载必须使用 `User-Agent: pan.baidu.com`，客户端已统一设置。
 - 分片大小由会员等级决定：普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB；百度限制分片数，
  因此不同等级的单文件上限不同。
--- a/cmd/bdrclone/main.go
+++ b/cmd/bdrclone/main.go
@@ -268,14 +268,15 @@ func (a *application) manageCommand(use, short string, fn func(context.Context,
 }

 func (a *application) uploadCommand() *cobra.Command {
-	return &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error {
+	var excludes []string
+	cmd := &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error {
 		client, _, err := a.client()
 		if err != nil {
 			return err
 		}
 		stderr := cmd.ErrOrStderr()
 		lineOpen := false
-		summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], func(localPath, remotePath string, done, total int64) {
+		summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(localPath, remotePath string, done, total int64) {
 			percent := int64(0)
 			if total > 0 {
 				percent = done * 100 / total
@@ -285,13 +286,13 @@ func (a *application) uploadCommand() *cobra.Command {
 			if !lineOpen {
 				fmt.Fprintln(stderr)
 			}
-		}, func(localPath, reason string) {
+		}, Skipped: func(localPath, reason string) {
 			if lineOpen {
 				fmt.Fprintln(stderr)
 				lineOpen = false
 			}
 			fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason)
-		})
+		}})
 		if lineOpen {
 			fmt.Fprintln(stderr)
 		}
@@ -301,6 +302,8 @@ func (a *application) uploadCommand() *cobra.Command {
 		fmt.Fprintf(stderr, "完成：上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped)
 		return nil
 	}}
+	cmd.Flags().StringArrayVar(&excludes, "exclude", nil, "排除源目录中的 glob（可重复，支持 **）")
+	return cmd
 }

 func (a *application) downloadCommand() *cobra.Command {
--- a/cmd/bdrclone/upload_path.go
+++ b/cmd/bdrclone/upload_path.go
@@ -2,6 +2,7 @@ package main

 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -28,7 +29,19 @@ type uploadPathSummary struct {
 type uploadPathProgress func(localPath, remotePath string, uploaded, total int64)
 type uploadPathSkipped func(localPath, reason string)

-func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, progress uploadPathProgress, skipped uploadPathSkipped) (uploadPathSummary, error) {
+type uploadPathOptions struct {
+	Excludes []string
+	Progress uploadPathProgress
+	Skipped  uploadPathSkipped
+}
+
+type uploadWalkDir func(string, fs.WalkDirFunc) error
+
+func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, options uploadPathOptions) (uploadPathSummary, error) {
+	excludes, err := normalizeUploadExcludes(options.Excludes)
+	if err != nil {
+		return uploadPathSummary{}, err
+	}
 	info, err := os.Stat(localSource)
 	if err != nil {
 		return uploadPathSummary{}, fmt.Errorf("stat local source: %w", err)
@@ -39,7 +52,11 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource,
 			return uploadPathSummary{}, fmt.Errorf("unsupported local file type: %s", localSource)
 		}
 		var summary uploadPathSummary
-		err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, progress, &summary)
+		if matchesUploadExclude(filepath.Base(localSource), excludes) {
+			reportUploadSkip(&summary, options, localSource, "已被 --exclude 规则排除")
+			return summary, nil
+		}
+		err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, options.Progress, &summary)
 		return summary, err
 	}

@@ -51,13 +68,25 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource,
 	if err != nil {
 		return uploadPathSummary{}, fmt.Errorf("resolve local source links: %w", err)
 	}
+	options.Excludes = excludes
+	return uploadLocalDirectory(ctx, client, root, remoteDestination, options, filepath.WalkDir)
+}
+
+func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, remoteDestination string, options uploadPathOptions, walk uploadWalkDir) (uploadPathSummary, error) {
 	if err := client.MkdirAll(ctx, remoteDestination); err != nil {
 		return uploadPathSummary{}, err
 	}
 	summary := uploadPathSummary{Directories: 1}
-	err = filepath.WalkDir(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
+	err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
 		if walkErr != nil {
-			return fmt.Errorf("read local path %s: %w", localPath, walkErr)
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+			reportUploadSkip(&summary, options, localPath, "无法读取: "+walkErr.Error())
+			if entry != nil && entry.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
 		}
 		if err := ctx.Err(); err != nil {
 			return err
@@ -69,7 +98,15 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource,
 		if err != nil {
 			return fmt.Errorf("make local path relative: %w", err)
 		}
-		remotePath := pathpkg.Join(remoteDestination, filepath.ToSlash(relative))
+		relative = filepath.ToSlash(relative)
+		if matchesUploadExclude(relative, options.Excludes) {
+			reportUploadSkip(&summary, options, localPath, "已被 --exclude 规则排除")
+			if entry.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		remotePath := pathpkg.Join(remoteDestination, relative)
 		if entry.IsDir() {
 			if err := client.EnsureDirectory(ctx, remotePath); err != nil {
 				return err
@@ -78,31 +115,23 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource,
 			return nil
 		}
 		if entry.Type()&os.ModeSymlink != 0 {
-			summary.Skipped++
-			if skipped != nil {
-				skipped(localPath, "符号链接不会被跟随")
-			}
+			reportUploadSkip(&summary, options, localPath, "符号链接不会被跟随")
 			return nil
 		}
 		info, err := entry.Info()
 		if err != nil {
-			return fmt.Errorf("stat local path %s: %w", localPath, err)
+			reportUploadSkip(&summary, options, localPath, "无法读取文件信息: "+err.Error())
+			return nil
 		}
 		if !info.Mode().IsRegular() {
-			summary.Skipped++
-			if skipped != nil {
-				skipped(localPath, "不支持的本地文件类型 "+info.Mode().Type().String())
-			}
+			reportUploadSkip(&summary, options, localPath, "不支持的本地文件类型 "+info.Mode().Type().String())
 			return nil
 		}
 		if info.Size() == 0 {
-			summary.Skipped++
-			if skipped != nil {
-				skipped(localPath, "百度开放 API 不支持空文件")
-			}
+			reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件")
 			return nil
 		}
-		return uploadRegularFile(ctx, client, localPath, remotePath, info, progress, &summary)
+		return uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary)
 	})
 	if err != nil {
 		return summary, err
@@ -110,6 +139,13 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource,
 	return summary, nil
 }

+func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, localPath, reason string) {
+	summary.Skipped++
+	if options.Skipped != nil {
+		options.Skipped(localPath, reason)
+	}
+}
+
 func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error {
 	completeReported := false
 	if progress != nil && info.Size() > 0 {
@@ -135,3 +171,69 @@ func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath,
 func cleanUploadRemotePath(name string) string {
 	return pathpkg.Clean("/" + strings.TrimPrefix(name, "/"))
 }
+
+func normalizeUploadExcludes(patterns []string) ([]string, error) {
+	result := make([]string, 0, len(patterns))
+	for _, pattern := range patterns {
+		pattern = filepath.ToSlash(pattern)
+		pattern = strings.TrimPrefix(pattern, "./")
+		pattern = strings.Trim(pattern, "/")
+		if pattern == "" {
+			return nil, errors.New("exclude pattern cannot be empty")
+		}
+		for _, component := range strings.Split(pattern, "/") {
+			if component == "**" {
+				continue
+			}
+			if _, err := pathpkg.Match(component, ""); err != nil {
+				return nil, fmt.Errorf("invalid exclude pattern %q: %w", pattern, err)
+			}
+		}
+		result = append(result, pattern)
+	}
+	return result, nil
+}
+
+func matchesUploadExclude(relative string, patterns []string) bool {
+	relative = strings.Trim(filepath.ToSlash(relative), "/")
+	for _, pattern := range patterns {
+		if !strings.Contains(pattern, "/") {
+			matched, _ := pathpkg.Match(pattern, pathpkg.Base(relative))
+			if matched {
+				return true
+			}
+			continue
+		}
+		if matchUploadPathSegments(strings.Split(pattern, "/"), strings.Split(relative, "/")) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchUploadPathSegments(pattern, name []string) bool {
+	type state struct{ pattern, name int }
+	memo := make(map[state]bool)
+	seen := make(map[state]bool)
+	var match func(int, int) bool
+	match = func(patternIndex, nameIndex int) bool {
+		key := state{patternIndex, nameIndex}
+		if seen[key] {
+			return memo[key]
+		}
+		seen[key] = true
+		var result bool
+		switch {
+		case patternIndex == len(pattern):
+			result = nameIndex == len(name)
+		case pattern[patternIndex] == "**":
+			result = match(patternIndex+1, nameIndex) || (nameIndex < len(name) && match(patternIndex, nameIndex+1))
+		case nameIndex < len(name):
+			segmentMatches, _ := pathpkg.Match(pattern[patternIndex], name[nameIndex])
+			result = segmentMatches && match(patternIndex+1, nameIndex+1)
+		}
+		memo[key] = result
+		return result
+	}
+	return match(0, 0)
+}
--- a/cmd/bdrclone/upload_path_test.go
+++ b/cmd/bdrclone/upload_path_test.go
@@ -3,9 +3,11 @@ package main
 import (
 	"bytes"
 	"context"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"

 	"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
@@ -64,9 +66,9 @@ func TestUploadLocalDirectoryPreservesRelativePaths(t *testing.T) {

 	client := &recordingUploadClient{}
 	var skipped []string
-	summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", nil, func(localPath, _ string) {
+	summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", uploadPathOptions{Skipped: func(localPath, _ string) {
 		skipped = append(skipped, filepath.Base(localPath))
-	})
+	}})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,7 +95,7 @@ func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) {
 		t.Fatal(err)
 	}
 	client := &recordingUploadClient{}
-	summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", nil, nil)
+	summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", uploadPathOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -104,3 +106,117 @@ func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) {
 		t.Fatalf("uploaded files = %v", client.files)
 	}
 }
+
+func TestUploadLocalDirectoryHonorsExcludePatterns(t *testing.T) {
+	root := t.TempDir()
+	files := map[string]string{
+		"keep.txt":                                     "keep",
+		"scratch.tmp":                                  "temporary",
+		filepath.Join(".Trash", "old"):                 "trash",
+		filepath.Join("Library", "keep"):               "library",
+		filepath.Join("Library", "Caches", "cache.db"): "cache",
+	}
+	for name, content := range files {
+		localPath := filepath.Join(root, name)
+		if err := os.MkdirAll(filepath.Dir(localPath), 0o700); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(localPath, []byte(content), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	client := &recordingUploadClient{}
+	summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{
+		Excludes: []string{".Trash", "Library/Caches/**", "*.tmp"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.Files != 2 || summary.Directories != 2 || summary.Skipped != 3 {
+		t.Fatalf("summary = %+v", summary)
+	}
+	if want := []string{"/backup/mac", "/backup/mac/Library"}; !reflect.DeepEqual(client.directories, want) {
+		t.Fatalf("directories = %q, want %q", client.directories, want)
+	}
+	if _, ok := client.files["/backup/mac/keep.txt"]; !ok {
+		t.Fatalf("keep.txt was not uploaded: %v", client.files)
+	}
+	if _, ok := client.files["/backup/mac/Library/keep"]; !ok {
+		t.Fatalf("Library/keep was not uploaded: %v", client.files)
+	}
+}
+
+func TestUploadLocalDirectoryContinuesAfterReadError(t *testing.T) {
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "blocked"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "good.txt"), []byte("good"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	entryByName := make(map[string]os.DirEntry, len(entries))
+	for _, entry := range entries {
+		entryByName[entry.Name()] = entry
+	}
+	rootInfo, err := os.Stat(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	walk := func(_ string, visit fs.WalkDirFunc) error {
+		if err := visit(root, fs.FileInfoToDirEntry(rootInfo), nil); err != nil {
+			return err
+		}
+		blockedPath := filepath.Join(root, "blocked")
+		if err := visit(blockedPath, entryByName["blocked"], fs.ErrPermission); err != nil && err != fs.SkipDir {
+			return err
+		}
+		return visit(filepath.Join(root, "good.txt"), entryByName["good.txt"], nil)
+	}
+
+	client := &recordingUploadClient{}
+	var skipped []string
+	summary, err := uploadLocalDirectory(context.Background(), client, root, "/backup/mac", uploadPathOptions{Skipped: func(localPath, reason string) {
+		skipped = append(skipped, filepath.Base(localPath)+":"+reason)
+	}}, walk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.Files != 1 || summary.Skipped != 1 {
+		t.Fatalf("summary = %+v", summary)
+	}
+	if len(skipped) != 1 || !strings.Contains(skipped[0], "blocked:无法读取: permission denied") {
+		t.Fatalf("skipped = %q", skipped)
+	}
+	if !bytes.Equal(client.files["/backup/mac/good.txt"], []byte("good")) {
+		t.Fatalf("good.txt was not uploaded: %v", client.files)
+	}
+}
+
+func TestUploadExcludePatternMatching(t *testing.T) {
+	patterns, err := normalizeUploadExcludes([]string{".Trash/", "Library/Caches/**", "**/*.tmp"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, want := range map[string]bool{
+		".Trash":                   true,
+		"nested/.Trash":            true,
+		"Library/Caches":           true,
+		"Library/Caches/data/file": true,
+		"root.tmp":                 true,
+		"nested/file.tmp":          true,
+		"Library/Application/file": false,
+		"nested/file.tmp.keep":     false,
+	} {
+		if got := matchesUploadExclude(name, patterns); got != want {
+			t.Errorf("matchesUploadExclude(%q) = %v, want %v", name, got, want)
+		}
+	}
+	if _, err := normalizeUploadExcludes([]string{"bad["}); err == nil {
+		t.Fatal("invalid exclude pattern was accepted")
+	}
+}