diff --git a/README.md b/README.md index 7347e7f..6703618 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,8 @@ OOB 模式使用 `redirect_uri=oob`，无需在本机监听端口。 ./bdrclone upload ./photo.jpg /备份/photo.jpg # 递归上传目录内容，保留相对目录结构 ./bdrclone upload ~/ /备份/mac/ +# 排除受保护目录和缓存；--exclude 可重复，支持 ** +./bdrclone upload ~/ /备份/mac/ --exclude .Trash --exclude 'Library/Caches/**' ./bdrclone mkdir /备份/新目录 ./bdrclone mv /备份/a.txt /备份/b.txt ./bdrclone cp /备份/b.txt /副本/b.txt @@ -123,7 +125,8 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf ## 百度 API 限制 - 百度网盘官方接口不允许创建空文件；挂载下的空文件在 flush 时会失败。 -- 目录上传会创建远端空目录，但会跳过符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。 +- 目录上传会创建远端空目录，但会跳过无法读取的路径、符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。 +- `upload --exclude` 使用相对于源目录的 glob；无 `/` 的规则匹配任意层级的文件名，`**` 可跨目录匹配。 - 大文件下载必须使用 `User-Agent: pan.baidu.com`，客户端已统一设置。 - 分片大小由会员等级决定：普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB；百度限制分片数， 因此不同等级的单文件上限不同。 diff --git a/cmd/bdrclone/main.go b/cmd/bdrclone/main.go index 271df89..5e67b6c 100644 --- a/cmd/bdrclone/main.go +++ b/cmd/bdrclone/main.go @@ -268,14 +268,15 @@ func (a *application) manageCommand(use, short string, fn func(context.Context, } func (a *application) uploadCommand() *cobra.Command { - return &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error { + var excludes []string + cmd := &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error { client, _, err := a.client() if err != nil { return err } stderr := cmd.ErrOrStderr() lineOpen := false - summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], func(localPath, remotePath string, done, total int64) { + summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(localPath, remotePath string, done, total int64) { percent := int64(0) if total > 0 { percent = done * 100 / total @@ -285,13 +286,13 @@ func (a *application) uploadCommand() *cobra.Command { if !lineOpen { fmt.Fprintln(stderr) } - }, func(localPath, reason string) { + }, Skipped: func(localPath, reason string) { if lineOpen { fmt.Fprintln(stderr) lineOpen = false } fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason) - }) + }}) if lineOpen { fmt.Fprintln(stderr) } @@ -301,6 +302,8 @@ func (a *application) uploadCommand() *cobra.Command { fmt.Fprintf(stderr, "完成：上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped) return nil }} + cmd.Flags().StringArrayVar(&excludes, "exclude", nil, "排除源目录中的 glob（可重复，支持 **）") + return cmd } func (a *application) downloadCommand() *cobra.Command { diff --git a/cmd/bdrclone/upload_path.go b/cmd/bdrclone/upload_path.go index 9c70e11..b4da29e 100644 --- a/cmd/bdrclone/upload_path.go +++ b/cmd/bdrclone/upload_path.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "io/fs" "os" @@ -28,7 +29,19 @@ type uploadPathSummary struct { type uploadPathProgress func(localPath, remotePath string, uploaded, total int64) type uploadPathSkipped func(localPath, reason string) -func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, progress uploadPathProgress, skipped uploadPathSkipped) (uploadPathSummary, error) { +type uploadPathOptions struct { + Excludes []string + Progress uploadPathProgress + Skipped uploadPathSkipped +} + +type uploadWalkDir func(string, fs.WalkDirFunc) error + +func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, options uploadPathOptions) (uploadPathSummary, error) { + excludes, err := normalizeUploadExcludes(options.Excludes) + if err != nil { + return uploadPathSummary{}, err + } info, err := os.Stat(localSource) if err != nil { return uploadPathSummary{}, fmt.Errorf("stat local source: %w", err) @@ -39,7 +52,11 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, return uploadPathSummary{}, fmt.Errorf("unsupported local file type: %s", localSource) } var summary uploadPathSummary - err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, progress, &summary) + if matchesUploadExclude(filepath.Base(localSource), excludes) { + reportUploadSkip(&summary, options, localSource, "已被 --exclude 规则排除") + return summary, nil + } + err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, options.Progress, &summary) return summary, err } @@ -51,13 +68,25 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, if err != nil { return uploadPathSummary{}, fmt.Errorf("resolve local source links: %w", err) } + options.Excludes = excludes + return uploadLocalDirectory(ctx, client, root, remoteDestination, options, filepath.WalkDir) +} + +func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, remoteDestination string, options uploadPathOptions, walk uploadWalkDir) (uploadPathSummary, error) { if err := client.MkdirAll(ctx, remoteDestination); err != nil { return uploadPathSummary{}, err } summary := uploadPathSummary{Directories: 1} - err = filepath.WalkDir(root, func(localPath string, entry fs.DirEntry, walkErr error) error { + err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error { if walkErr != nil { - return fmt.Errorf("read local path %s: %w", localPath, walkErr) + if err := ctx.Err(); err != nil { + return err + } + reportUploadSkip(&summary, options, localPath, "无法读取: "+walkErr.Error()) + if entry != nil && entry.IsDir() { + return fs.SkipDir + } + return nil } if err := ctx.Err(); err != nil { return err @@ -69,7 +98,15 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, if err != nil { return fmt.Errorf("make local path relative: %w", err) } - remotePath := pathpkg.Join(remoteDestination, filepath.ToSlash(relative)) + relative = filepath.ToSlash(relative) + if matchesUploadExclude(relative, options.Excludes) { + reportUploadSkip(&summary, options, localPath, "已被 --exclude 规则排除") + if entry.IsDir() { + return fs.SkipDir + } + return nil + } + remotePath := pathpkg.Join(remoteDestination, relative) if entry.IsDir() { if err := client.EnsureDirectory(ctx, remotePath); err != nil { return err @@ -78,31 +115,23 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, return nil } if entry.Type()&os.ModeSymlink != 0 { - summary.Skipped++ - if skipped != nil { - skipped(localPath, "符号链接不会被跟随") - } + reportUploadSkip(&summary, options, localPath, "符号链接不会被跟随") return nil } info, err := entry.Info() if err != nil { - return fmt.Errorf("stat local path %s: %w", localPath, err) + reportUploadSkip(&summary, options, localPath, "无法读取文件信息: "+err.Error()) + return nil } if !info.Mode().IsRegular() { - summary.Skipped++ - if skipped != nil { - skipped(localPath, "不支持的本地文件类型 "+info.Mode().Type().String()) - } + reportUploadSkip(&summary, options, localPath, "不支持的本地文件类型 "+info.Mode().Type().String()) return nil } if info.Size() == 0 { - summary.Skipped++ - if skipped != nil { - skipped(localPath, "百度开放 API 不支持空文件") - } + reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件") return nil } - return uploadRegularFile(ctx, client, localPath, remotePath, info, progress, &summary) + return uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary) }) if err != nil { return summary, err @@ -110,6 +139,13 @@ func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, return summary, nil } +func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, localPath, reason string) { + summary.Skipped++ + if options.Skipped != nil { + options.Skipped(localPath, reason) + } +} + func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error { completeReported := false if progress != nil && info.Size() > 0 { @@ -135,3 +171,69 @@ func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, func cleanUploadRemotePath(name string) string { return pathpkg.Clean("/" + strings.TrimPrefix(name, "/")) } + +func normalizeUploadExcludes(patterns []string) ([]string, error) { + result := make([]string, 0, len(patterns)) + for _, pattern := range patterns { + pattern = filepath.ToSlash(pattern) + pattern = strings.TrimPrefix(pattern, "./") + pattern = strings.Trim(pattern, "/") + if pattern == "" { + return nil, errors.New("exclude pattern cannot be empty") + } + for _, component := range strings.Split(pattern, "/") { + if component == "**" { + continue + } + if _, err := pathpkg.Match(component, ""); err != nil { + return nil, fmt.Errorf("invalid exclude pattern %q: %w", pattern, err) + } + } + result = append(result, pattern) + } + return result, nil +} + +func matchesUploadExclude(relative string, patterns []string) bool { + relative = strings.Trim(filepath.ToSlash(relative), "/") + for _, pattern := range patterns { + if !strings.Contains(pattern, "/") { + matched, _ := pathpkg.Match(pattern, pathpkg.Base(relative)) + if matched { + return true + } + continue + } + if matchUploadPathSegments(strings.Split(pattern, "/"), strings.Split(relative, "/")) { + return true + } + } + return false +} + +func matchUploadPathSegments(pattern, name []string) bool { + type state struct{ pattern, name int } + memo := make(map[state]bool) + seen := make(map[state]bool) + var match func(int, int) bool + match = func(patternIndex, nameIndex int) bool { + key := state{patternIndex, nameIndex} + if seen[key] { + return memo[key] + } + seen[key] = true + var result bool + switch { + case patternIndex == len(pattern): + result = nameIndex == len(name) + case pattern[patternIndex] == "**": + result = match(patternIndex+1, nameIndex) || (nameIndex < len(name) && match(patternIndex, nameIndex+1)) + case nameIndex < len(name): + segmentMatches, _ := pathpkg.Match(pattern[patternIndex], name[nameIndex]) + result = segmentMatches && match(patternIndex+1, nameIndex+1) + } + memo[key] = result + return result + } + return match(0, 0) +} diff --git a/cmd/bdrclone/upload_path_test.go b/cmd/bdrclone/upload_path_test.go index 24fa2aa..ebdd936 100644 --- a/cmd/bdrclone/upload_path_test.go +++ b/cmd/bdrclone/upload_path_test.go @@ -3,9 +3,11 @@ package main import ( "bytes" "context" + "io/fs" "os" "path/filepath" "reflect" + "strings" "testing" "gitea.dddbg.com/youbin/bdrclone/internal/baidu" @@ -64,9 +66,9 @@ func TestUploadLocalDirectoryPreservesRelativePaths(t *testing.T) { client := &recordingUploadClient{} var skipped []string - summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", nil, func(localPath, _ string) { + summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", uploadPathOptions{Skipped: func(localPath, _ string) { skipped = append(skipped, filepath.Base(localPath)) - }) + }}) if err != nil { t.Fatal(err) } @@ -93,7 +95,7 @@ func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) { t.Fatal(err) } client := &recordingUploadClient{} - summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", nil, nil) + summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", uploadPathOptions{}) if err != nil { t.Fatal(err) } @@ -104,3 +106,117 @@ func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) { t.Fatalf("uploaded files = %v", client.files) } } + +func TestUploadLocalDirectoryHonorsExcludePatterns(t *testing.T) { + root := t.TempDir() + files := map[string]string{ + "keep.txt": "keep", + "scratch.tmp": "temporary", + filepath.Join(".Trash", "old"): "trash", + filepath.Join("Library", "keep"): "library", + filepath.Join("Library", "Caches", "cache.db"): "cache", + } + for name, content := range files { + localPath := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(localPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(localPath, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + + client := &recordingUploadClient{} + summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{ + Excludes: []string{".Trash", "Library/Caches/**", "*.tmp"}, + }) + if err != nil { + t.Fatal(err) + } + if summary.Files != 2 || summary.Directories != 2 || summary.Skipped != 3 { + t.Fatalf("summary = %+v", summary) + } + if want := []string{"/backup/mac", "/backup/mac/Library"}; !reflect.DeepEqual(client.directories, want) { + t.Fatalf("directories = %q, want %q", client.directories, want) + } + if _, ok := client.files["/backup/mac/keep.txt"]; !ok { + t.Fatalf("keep.txt was not uploaded: %v", client.files) + } + if _, ok := client.files["/backup/mac/Library/keep"]; !ok { + t.Fatalf("Library/keep was not uploaded: %v", client.files) + } +} + +func TestUploadLocalDirectoryContinuesAfterReadError(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "blocked"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "good.txt"), []byte("good"), 0o600); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + entryByName := make(map[string]os.DirEntry, len(entries)) + for _, entry := range entries { + entryByName[entry.Name()] = entry + } + rootInfo, err := os.Stat(root) + if err != nil { + t.Fatal(err) + } + walk := func(_ string, visit fs.WalkDirFunc) error { + if err := visit(root, fs.FileInfoToDirEntry(rootInfo), nil); err != nil { + return err + } + blockedPath := filepath.Join(root, "blocked") + if err := visit(blockedPath, entryByName["blocked"], fs.ErrPermission); err != nil && err != fs.SkipDir { + return err + } + return visit(filepath.Join(root, "good.txt"), entryByName["good.txt"], nil) + } + + client := &recordingUploadClient{} + var skipped []string + summary, err := uploadLocalDirectory(context.Background(), client, root, "/backup/mac", uploadPathOptions{Skipped: func(localPath, reason string) { + skipped = append(skipped, filepath.Base(localPath)+":"+reason) + }}, walk) + if err != nil { + t.Fatal(err) + } + if summary.Files != 1 || summary.Skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + if len(skipped) != 1 || !strings.Contains(skipped[0], "blocked:无法读取: permission denied") { + t.Fatalf("skipped = %q", skipped) + } + if !bytes.Equal(client.files["/backup/mac/good.txt"], []byte("good")) { + t.Fatalf("good.txt was not uploaded: %v", client.files) + } +} + +func TestUploadExcludePatternMatching(t *testing.T) { + patterns, err := normalizeUploadExcludes([]string{".Trash/", "Library/Caches/**", "**/*.tmp"}) + if err != nil { + t.Fatal(err) + } + for name, want := range map[string]bool{ + ".Trash": true, + "nested/.Trash": true, + "Library/Caches": true, + "Library/Caches/data/file": true, + "root.tmp": true, + "nested/file.tmp": true, + "Library/Application/file": false, + "nested/file.tmp.keep": false, + } { + if got := matchesUploadExclude(name, patterns); got != want { + t.Errorf("matchesUploadExclude(%q) = %v, want %v", name, got, want) + } + } + if _, err := normalizeUploadExcludes([]string{"bad["}); err == nil { + t.Fatal("invalid exclude pattern was accepted") + } +}