From c1758cf29c02a59e37fcaf6ef1885ffae6996409 Mon Sep 17 00:00:00 2001 From: youbin Date: Fri, 14 Aug 2026 23:47:54 +0800 Subject: [PATCH] Support recursive directory uploads --- README.md | 3 + cmd/bdrclone/main.go | 32 ++++++-- cmd/bdrclone/upload_path.go | 137 +++++++++++++++++++++++++++++++ cmd/bdrclone/upload_path_test.go | 106 ++++++++++++++++++++++++ internal/baidu/client_test.go | 51 ++++++++++++ internal/baidu/files.go | 32 ++++++++ 6 files changed, 355 insertions(+), 6 deletions(-) create mode 100644 cmd/bdrclone/upload_path.go create mode 100644 cmd/bdrclone/upload_path_test.go diff --git a/README.md b/README.md index b40653d..7347e7f 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,8 @@ OOB 模式使用 `redirect_uri=oob`，无需在本机监听端口。 ./bdrclone stat /文档/report.pdf ./bdrclone download /文档/report.pdf ./report.pdf ./bdrclone upload ./photo.jpg /备份/photo.jpg +# 递归上传目录内容，保留相对目录结构 +./bdrclone upload ~/ /备份/mac/ ./bdrclone mkdir /备份/新目录 ./bdrclone mv /备份/a.txt /备份/b.txt ./bdrclone cp /备份/b.txt /副本/b.txt @@ -121,6 +123,7 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf ## 百度 API 限制 - 百度网盘官方接口不允许创建空文件；挂载下的空文件在 flush 时会失败。 +- 目录上传会创建远端空目录，但会跳过符号链接、特殊文件及百度接口无法表示的空文件，并逐项提示。 - 大文件下载必须使用 `User-Agent: pan.baidu.com`，客户端已统一设置。 - 分片大小由会员等级决定：普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB；百度限制分片数， 因此不同等级的单文件上限不同。 diff --git a/cmd/bdrclone/main.go b/cmd/bdrclone/main.go index a3e2e0c..271df89 100644 --- a/cmd/bdrclone/main.go +++ b/cmd/bdrclone/main.go @@ -268,18 +268,38 @@ func (a *application) manageCommand(use, short string, fn func(context.Context, } func (a *application) uploadCommand() *cobra.Command { - return &cobra.Command{Use: "upload <本地文件> <远端文件>", Args: cobra.ExactArgs(2), Short: "分片上传本地文件", RunE: func(cmd *cobra.Command, args []string) error { + return &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error { client, _, err := a.client() if err != nil { return err } - _, err = client.UploadFile(cmd.Context(), args[0], args[1], func(done, total int64) { - fmt.Fprintf(os.Stderr, "\r上传 %d/%d bytes (%d%%)", done, total, done*100/total) + stderr := cmd.ErrOrStderr() + lineOpen := false + summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], func(localPath, remotePath string, done, total int64) { + percent := int64(0) + if total > 0 { + percent = done * 100 / total + } + fmt.Fprintf(stderr, "\r上传 %s -> %s: %d/%d bytes (%d%%)", localPath, remotePath, done, total, percent) + lineOpen = done < total + if !lineOpen { + fmt.Fprintln(stderr) + } + }, func(localPath, reason string) { + if lineOpen { + fmt.Fprintln(stderr) + lineOpen = false + } + fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason) }) - if err == nil { - fmt.Fprintln(os.Stderr) + if lineOpen { + fmt.Fprintln(stderr) } - return err + if err != nil { + return err + } + fmt.Fprintf(stderr, "完成：上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped) + return nil }} } diff --git a/cmd/bdrclone/upload_path.go b/cmd/bdrclone/upload_path.go new file mode 100644 index 0000000..9c70e11 --- /dev/null +++ b/cmd/bdrclone/upload_path.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "fmt" + "io/fs" + "os" + pathpkg "path" + "path/filepath" + "strings" + + "gitea.dddbg.com/youbin/bdrclone/internal/baidu" +) + +type uploadPathClient interface { + MkdirAll(context.Context, string) error + EnsureDirectory(context.Context, string) error + UploadFile(context.Context, string, string, baidu.UploadProgress) (baidu.File, error) +} + +type uploadPathSummary struct { + Files int + Directories int + Skipped int + Bytes int64 +} + +type uploadPathProgress func(localPath, remotePath string, uploaded, total int64) +type uploadPathSkipped func(localPath, reason string) + +func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, progress uploadPathProgress, skipped uploadPathSkipped) (uploadPathSummary, error) { + info, err := os.Stat(localSource) + if err != nil { + return uploadPathSummary{}, fmt.Errorf("stat local source: %w", err) + } + remoteDestination = cleanUploadRemotePath(remoteDestination) + if !info.IsDir() { + if !info.Mode().IsRegular() { + return uploadPathSummary{}, fmt.Errorf("unsupported local file type: %s", localSource) + } + var summary uploadPathSummary + err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, progress, &summary) + return summary, err + } + + root, err := filepath.Abs(localSource) + if err != nil { + return uploadPathSummary{}, fmt.Errorf("resolve local source: %w", err) + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return uploadPathSummary{}, fmt.Errorf("resolve local source links: %w", err) + } + if err := client.MkdirAll(ctx, remoteDestination); err != nil { + return uploadPathSummary{}, err + } + summary := uploadPathSummary{Directories: 1} + err = filepath.WalkDir(root, func(localPath string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return fmt.Errorf("read local path %s: %w", localPath, walkErr) + } + if err := ctx.Err(); err != nil { + return err + } + if localPath == root { + return nil + } + relative, err := filepath.Rel(root, localPath) + if err != nil { + return fmt.Errorf("make local path relative: %w", err) + } + remotePath := pathpkg.Join(remoteDestination, filepath.ToSlash(relative)) + if entry.IsDir() { + if err := client.EnsureDirectory(ctx, remotePath); err != nil { + return err + } + summary.Directories++ + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + summary.Skipped++ + if skipped != nil { + skipped(localPath, "符号链接不会被跟随") + } + return nil + } + info, err := entry.Info() + if err != nil { + return fmt.Errorf("stat local path %s: %w", localPath, err) + } + if !info.Mode().IsRegular() { + summary.Skipped++ + if skipped != nil { + skipped(localPath, "不支持的本地文件类型 "+info.Mode().Type().String()) + } + return nil + } + if info.Size() == 0 { + summary.Skipped++ + if skipped != nil { + skipped(localPath, "百度开放 API 不支持空文件") + } + return nil + } + return uploadRegularFile(ctx, client, localPath, remotePath, info, progress, &summary) + }) + if err != nil { + return summary, err + } + return summary, nil +} + +func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error { + completeReported := false + if progress != nil && info.Size() > 0 { + progress(localPath, remotePath, 0, info.Size()) + } + _, err := client.UploadFile(ctx, localPath, remotePath, func(uploaded, total int64) { + completeReported = uploaded >= total + if progress != nil { + progress(localPath, remotePath, uploaded, total) + } + }) + if err != nil { + return fmt.Errorf("upload %s to %s: %w", localPath, remotePath, err) + } + if progress != nil && info.Size() > 0 && !completeReported { + progress(localPath, remotePath, info.Size(), info.Size()) + } + summary.Files++ + summary.Bytes += info.Size() + return nil +} + +func cleanUploadRemotePath(name string) string { + return pathpkg.Clean("/" + strings.TrimPrefix(name, "/")) +} diff --git a/cmd/bdrclone/upload_path_test.go b/cmd/bdrclone/upload_path_test.go new file mode 100644 index 0000000..24fa2aa --- /dev/null +++ b/cmd/bdrclone/upload_path_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "reflect" + "testing" + + "gitea.dddbg.com/youbin/bdrclone/internal/baidu" +) + +type recordingUploadClient struct { + directories []string + files map[string][]byte +} + +func (c *recordingUploadClient) MkdirAll(_ context.Context, remotePath string) error { + c.directories = append(c.directories, remotePath) + return nil +} + +func (c *recordingUploadClient) EnsureDirectory(_ context.Context, remotePath string) error { + c.directories = append(c.directories, remotePath) + return nil +} + +func (c *recordingUploadClient) UploadFile(_ context.Context, localPath, remotePath string, progress baidu.UploadProgress) (baidu.File, error) { + content, err := os.ReadFile(localPath) + if err != nil { + return baidu.File{}, err + } + if c.files == nil { + c.files = make(map[string][]byte) + } + c.files[remotePath] = content + if progress != nil { + progress(int64(len(content)), int64(len(content))) + } + return baidu.File{Path: remotePath, Size: int64(len(content))}, nil +} + +func TestUploadLocalDirectoryPreservesRelativePaths(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "top.txt"), []byte("top"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "empty-dir"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "nested"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "nested", "file.bin"), []byte("nested"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "zero.txt"), nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink("top.txt", filepath.Join(root, "top-link")); err != nil { + t.Fatal(err) + } + + client := &recordingUploadClient{} + var skipped []string + summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", nil, func(localPath, _ string) { + skipped = append(skipped, filepath.Base(localPath)) + }) + if err != nil { + t.Fatal(err) + } + if summary.Files != 2 || summary.Directories != 3 || summary.Skipped != 2 || summary.Bytes != 9 { + t.Fatalf("summary = %+v", summary) + } + if want := []string{"/backup/mac", "/backup/mac/empty-dir", "/backup/mac/nested"}; !reflect.DeepEqual(client.directories, want) { + t.Fatalf("directories = %q, want %q", client.directories, want) + } + if !bytes.Equal(client.files["/backup/mac/top.txt"], []byte("top")) { + t.Fatalf("top.txt = %q", client.files["/backup/mac/top.txt"]) + } + if !bytes.Equal(client.files["/backup/mac/nested/file.bin"], []byte("nested")) { + t.Fatalf("nested/file.bin = %q", client.files["/backup/mac/nested/file.bin"]) + } + if want := []string{"top-link", "zero.txt"}; !reflect.DeepEqual(skipped, want) { + t.Fatalf("skipped = %q, want %q", skipped, want) + } +} + +func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) { + localPath := filepath.Join(t.TempDir(), "source.txt") + if err := os.WriteFile(localPath, []byte("content"), 0o600); err != nil { + t.Fatal(err) + } + client := &recordingUploadClient{} + summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", nil, nil) + if err != nil { + t.Fatal(err) + } + if summary.Files != 1 || summary.Directories != 0 || summary.Bytes != 7 { + t.Fatalf("summary = %+v", summary) + } + if !bytes.Equal(client.files["/backup/renamed.txt"], []byte("content")) { + t.Fatalf("uploaded files = %v", client.files) + } +} diff --git a/internal/baidu/client_test.go b/internal/baidu/client_test.go index d33a17f..8788292 100644 --- a/internal/baidu/client_test.go +++ b/internal/baidu/client_test.go @@ -9,6 +9,8 @@ import ( "net/http" "net/http/httptest" "os" + "path" + "reflect" "strconv" "sync" "testing" @@ -203,6 +205,55 @@ func TestUploadMultipartFlow(t *testing.T) { } } +func TestMkdirAllCreatesMissingRemoteDirectories(t *testing.T) { + directories := map[string]bool{"/apps/bdrclone": true} + var created []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rest/2.0/xpan/file" { + http.NotFound(w, r) + return + } + switch r.URL.Query().Get("method") { + case "list": + parent := r.URL.Query().Get("dir") + entries := make([]File, 0) + for directory := range directories { + if directory != parent && path.Dir(directory) == parent { + entries = append(entries, File{Path: directory, ServerFilename: path.Base(directory), IsDir: 1}) + } + } + _ = json.NewEncoder(w).Encode(map[string]any{"errno": 0, "list": entries}) + case "create": + if err := r.ParseForm(); err != nil { + t.Error(err) + return + } + directory := r.Form.Get("path") + if r.Form.Get("isdir") != "1" { + t.Errorf("isdir = %q", r.Form.Get("isdir")) + } + directories[directory] = true + created = append(created, directory) + _ = json.NewEncoder(w).Encode(File{Path: directory, ServerFilename: path.Base(directory), IsDir: 1}) + default: + http.Error(w, "unexpected method", http.StatusBadRequest) + } + })) + defer server.Close() + + client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL)) + if err := client.MkdirAll(context.Background(), "/backup/mac/photos"); err != nil { + t.Fatal(err) + } + if err := client.MkdirAll(context.Background(), "/backup/mac/photos"); err != nil { + t.Fatal(err) + } + want := []string{"/apps/bdrclone/backup", "/apps/bdrclone/backup/mac", "/apps/bdrclone/backup/mac/photos"} + if !reflect.DeepEqual(created, want) { + t.Fatalf("created = %q, want %q", created, want) + } +} + func testConfig(token string) *config.Config { return &config.Config{ ClientID: "client", ClientSecret: "secret", AccessToken: token, diff --git a/internal/baidu/files.go b/internal/baidu/files.go index db8f661..38876d0 100644 --- a/internal/baidu/files.go +++ b/internal/baidu/files.go @@ -66,6 +66,38 @@ func (c *Client) Mkdir(ctx context.Context, name string) (File, error) { return result, nil } +func (c *Client) MkdirAll(ctx context.Context, name string) error { + target := path.Clean("/" + strings.TrimPrefix(name, "/")) + if target == "/" { + return nil + } + current := "/" + for _, component := range strings.Split(strings.TrimPrefix(target, "/"), "/") { + current = path.Join(current, component) + if err := c.EnsureDirectory(ctx, current); err != nil { + return err + } + } + return nil +} + +func (c *Client) EnsureDirectory(ctx context.Context, name string) error { + entry, err := c.Stat(ctx, name) + if err == nil { + if !entry.IsDirectory() { + return fmt.Errorf("remote path exists and is not a directory: %s", name) + } + return nil + } + if !errors.Is(err, ErrNotFound) { + return fmt.Errorf("check remote directory %s: %w", name, err) + } + if _, err := c.Mkdir(ctx, name); err != nil { + return fmt.Errorf("create remote directory %s: %w", name, err) + } + return nil +} + func (c *Client) Delete(ctx context.Context, name string) error { remote := c.RemotePath(name) if remote == c.root {