Support recursive directory uploads

cmd/bdrclone/main.go

@@ -268,18 +268,38 @@ func (a *application) manageCommand(use, short string, fn func(context.Context,
 }
 
 func (a *application) uploadCommand() *cobra.Command {
-	return &cobra.Command{Use: "upload <本地文件> <远端文件>", Args: cobra.ExactArgs(2), Short: "分片上传本地文件", RunE: func(cmd *cobra.Command, args []string) error {
+	return &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error {
 		client, _, err := a.client()
 		if err != nil {
 			return err
 		}
-		_, err = client.UploadFile(cmd.Context(), args[0], args[1], func(done, total int64) {
-			fmt.Fprintf(os.Stderr, "\r上传 %d/%d bytes (%d%%)", done, total, done*100/total)
+		stderr := cmd.ErrOrStderr()
+		lineOpen := false
+		summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], func(localPath, remotePath string, done, total int64) {
+			percent := int64(0)
+			if total > 0 {
+				percent = done * 100 / total
+			}
+			fmt.Fprintf(stderr, "\r上传 %s -> %s: %d/%d bytes (%d%%)", localPath, remotePath, done, total, percent)
+			lineOpen = done < total
+			if !lineOpen {
+				fmt.Fprintln(stderr)
+			}
+		}, func(localPath, reason string) {
+			if lineOpen {
+				fmt.Fprintln(stderr)
+				lineOpen = false
+			}
+			fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason)
 		})
-		if err == nil {
-			fmt.Fprintln(os.Stderr)
+		if lineOpen {
+			fmt.Fprintln(stderr)
 		}
-		return err
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(stderr, "完成：上传 %d 个文件（%s），处理 %d 个目录，跳过 %d 项\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped)
+		return nil
 	}}
 }
 

cmd/bdrclone/upload_path.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	pathpkg "path"
+	"path/filepath"
+	"strings"
+
+	"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
+)
+
+type uploadPathClient interface {
+	MkdirAll(context.Context, string) error
+	EnsureDirectory(context.Context, string) error
+	UploadFile(context.Context, string, string, baidu.UploadProgress) (baidu.File, error)
+}
+
+type uploadPathSummary struct {
+	Files       int
+	Directories int
+	Skipped     int
+	Bytes       int64
+}
+
+type uploadPathProgress func(localPath, remotePath string, uploaded, total int64)
+type uploadPathSkipped func(localPath, reason string)
+
+func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, progress uploadPathProgress, skipped uploadPathSkipped) (uploadPathSummary, error) {
+	info, err := os.Stat(localSource)
+	if err != nil {
+		return uploadPathSummary{}, fmt.Errorf("stat local source: %w", err)
+	}
+	remoteDestination = cleanUploadRemotePath(remoteDestination)
+	if !info.IsDir() {
+		if !info.Mode().IsRegular() {
+			return uploadPathSummary{}, fmt.Errorf("unsupported local file type: %s", localSource)
+		}
+		var summary uploadPathSummary
+		err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, progress, &summary)
+		return summary, err
+	}
+
+	root, err := filepath.Abs(localSource)
+	if err != nil {
+		return uploadPathSummary{}, fmt.Errorf("resolve local source: %w", err)
+	}
+	root, err = filepath.EvalSymlinks(root)
+	if err != nil {
+		return uploadPathSummary{}, fmt.Errorf("resolve local source links: %w", err)
+	}
+	if err := client.MkdirAll(ctx, remoteDestination); err != nil {
+		return uploadPathSummary{}, err
+	}
+	summary := uploadPathSummary{Directories: 1}
+	err = filepath.WalkDir(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return fmt.Errorf("read local path %s: %w", localPath, walkErr)
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if localPath == root {
+			return nil
+		}
+		relative, err := filepath.Rel(root, localPath)
+		if err != nil {
+			return fmt.Errorf("make local path relative: %w", err)
+		}
+		remotePath := pathpkg.Join(remoteDestination, filepath.ToSlash(relative))
+		if entry.IsDir() {
+			if err := client.EnsureDirectory(ctx, remotePath); err != nil {
+				return err
+			}
+			summary.Directories++
+			return nil
+		}
+		if entry.Type()&os.ModeSymlink != 0 {
+			summary.Skipped++
+			if skipped != nil {
+				skipped(localPath, "符号链接不会被跟随")
+			}
+			return nil
+		}
+		info, err := entry.Info()
+		if err != nil {
+			return fmt.Errorf("stat local path %s: %w", localPath, err)
+		}
+		if !info.Mode().IsRegular() {
+			summary.Skipped++
+			if skipped != nil {
+				skipped(localPath, "不支持的本地文件类型 "+info.Mode().Type().String())
+			}
+			return nil
+		}
+		if info.Size() == 0 {
+			summary.Skipped++
+			if skipped != nil {
+				skipped(localPath, "百度开放 API 不支持空文件")
+			}
+			return nil
+		}
+		return uploadRegularFile(ctx, client, localPath, remotePath, info, progress, &summary)
+	})
+	if err != nil {
+		return summary, err
+	}
+	return summary, nil
+}
+
+func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error {
+	completeReported := false
+	if progress != nil && info.Size() > 0 {
+		progress(localPath, remotePath, 0, info.Size())
+	}
+	_, err := client.UploadFile(ctx, localPath, remotePath, func(uploaded, total int64) {
+		completeReported = uploaded >= total
+		if progress != nil {
+			progress(localPath, remotePath, uploaded, total)
+		}
+	})
+	if err != nil {
+		return fmt.Errorf("upload %s to %s: %w", localPath, remotePath, err)
+	}
+	if progress != nil && info.Size() > 0 && !completeReported {
+		progress(localPath, remotePath, info.Size(), info.Size())
+	}
+	summary.Files++
+	summary.Bytes += info.Size()
+	return nil
+}
+
+func cleanUploadRemotePath(name string) string {
+	return pathpkg.Clean("/" + strings.TrimPrefix(name, "/"))
+}

cmd/bdrclone/upload_path_test.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
+)
+
+type recordingUploadClient struct {
+	directories []string
+	files       map[string][]byte
+}
+
+func (c *recordingUploadClient) MkdirAll(_ context.Context, remotePath string) error {
+	c.directories = append(c.directories, remotePath)
+	return nil
+}
+
+func (c *recordingUploadClient) EnsureDirectory(_ context.Context, remotePath string) error {
+	c.directories = append(c.directories, remotePath)
+	return nil
+}
+
+func (c *recordingUploadClient) UploadFile(_ context.Context, localPath, remotePath string, progress baidu.UploadProgress) (baidu.File, error) {
+	content, err := os.ReadFile(localPath)
+	if err != nil {
+		return baidu.File{}, err
+	}
+	if c.files == nil {
+		c.files = make(map[string][]byte)
+	}
+	c.files[remotePath] = content
+	if progress != nil {
+		progress(int64(len(content)), int64(len(content)))
+	}
+	return baidu.File{Path: remotePath, Size: int64(len(content))}, nil
+}
+
+func TestUploadLocalDirectoryPreservesRelativePaths(t *testing.T) {
+	root := t.TempDir()
+	if err := os.WriteFile(filepath.Join(root, "top.txt"), []byte("top"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Join(root, "empty-dir"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Join(root, "nested"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "nested", "file.bin"), []byte("nested"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "zero.txt"), nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("top.txt", filepath.Join(root, "top-link")); err != nil {
+		t.Fatal(err)
+	}
+
+	client := &recordingUploadClient{}
+	var skipped []string
+	summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", nil, func(localPath, _ string) {
+		skipped = append(skipped, filepath.Base(localPath))
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.Files != 2 || summary.Directories != 3 || summary.Skipped != 2 || summary.Bytes != 9 {
+		t.Fatalf("summary = %+v", summary)
+	}
+	if want := []string{"/backup/mac", "/backup/mac/empty-dir", "/backup/mac/nested"}; !reflect.DeepEqual(client.directories, want) {
+		t.Fatalf("directories = %q, want %q", client.directories, want)
+	}
+	if !bytes.Equal(client.files["/backup/mac/top.txt"], []byte("top")) {
+		t.Fatalf("top.txt = %q", client.files["/backup/mac/top.txt"])
+	}
+	if !bytes.Equal(client.files["/backup/mac/nested/file.bin"], []byte("nested")) {
+		t.Fatalf("nested/file.bin = %q", client.files["/backup/mac/nested/file.bin"])
+	}
+	if want := []string{"top-link", "zero.txt"}; !reflect.DeepEqual(skipped, want) {
+		t.Fatalf("skipped = %q, want %q", skipped, want)
+	}
+}
+
+func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) {
+	localPath := filepath.Join(t.TempDir(), "source.txt")
+	if err := os.WriteFile(localPath, []byte("content"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	client := &recordingUploadClient{}
+	summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.Files != 1 || summary.Directories != 0 || summary.Bytes != 7 {
+		t.Fatalf("summary = %+v", summary)
+	}
+	if !bytes.Equal(client.files["/backup/renamed.txt"], []byte("content")) {
+		t.Fatalf("uploaded files = %v", client.files)
+	}
+}