11 Commits

Author SHA1 Message Date
0b4c2abf65 Release v1.0.1 [skip ci] 2026-08-15 15:50:58 +08:00
c602698ce9 Retry transient uploads and continue backups
All checks were successful
Build / Test and build (push) Successful in 5m53s
2026-08-15 15:32:43 +08:00
977fcefc2f Continue directory uploads past unreadable paths
All checks were successful
Build / Test and build (push) Successful in 5m29s
2026-08-15 14:45:02 +08:00
c1758cf29c Support recursive directory uploads
All checks were successful
Build / Test and build (push) Successful in 5m41s
2026-08-14 23:47:54 +08:00
c2eecdebbb Keep release builds in package registry [skip ci] 2026-08-14 15:27:49 +08:00
f6111c9715 Use package token for registry publishing [skip ci] 2026-08-14 15:20:31 +08:00
2de606c27d Use internal Gitea endpoint for package uploads [skip ci] 2026-08-14 15:12:13 +08:00
8cde8109e9 Make package publishing resilient [skip ci] 2026-08-14 15:03:17 +08:00
a2ecfcf550 Publish release builds to Gitea packages
All checks were successful
Build / Test and build (push) Successful in 5m37s
2026-08-14 14:50:02 +08:00
46c25768e4 Use Gitea-compatible artifact upload
All checks were successful
Build / Test and build (push) Successful in 6m30s
2026-08-14 14:09:46 +08:00
f23f1a4e5a Add automated cross-platform builds
Some checks failed
Build / Test and build (push) Failing after 12m31s
2026-08-14 13:55:30 +08:00
9 changed files with 890 additions and 27 deletions

143
.gitea/workflows/build.yml Normal file
View File

@@ -0,0 +1,143 @@
name: Build
on:
push:
branches:
- "**"
tags:
- "v*"
pull_request:
workflow_dispatch:
inputs:
publish_version:
description: Version to publish to the Gitea package registry
required: false
type: string
permissions:
contents: read
packages: write
jobs:
build:
name: Test and build
runs-on: ubuntu-latest
steps:
- name: Check out source
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: false
- name: Test
run: go test -race ./...
- name: Vet
run: go vet ./...
- name: Build release archives
shell: bash
env:
BUILD_REF: ${{ github.ref }}
BUILD_REF_NAME: ${{ github.ref_name }}
BUILD_SHA: ${{ github.sha }}
PUBLISH_VERSION: ${{ inputs.publish_version }}
run: |
set -euo pipefail
package_version="snapshot-${BUILD_SHA:0:8}"
version_flags=""
if [[ "$BUILD_REF" == refs/tags/v* ]]; then
package_version="$BUILD_REF_NAME"
elif [[ -n "$PUBLISH_VERSION" ]]; then
package_version="$PUBLISH_VERSION"
fi
if [[ "$package_version" != snapshot-* ]]; then
if [[ ! "$package_version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]]; then
echo "Invalid release version: $package_version" >&2
exit 1
fi
version_flags="-X main.version=$package_version"
fi
mkdir -p dist
for target in \
linux/amd64 linux/arm64 \
darwin/amd64 darwin/arm64 \
windows/amd64 windows/arm64
do
target_os="${target%/*}"
target_arch="${target#*/}"
archive="bdrclone-${package_version}-${target_os}-${target_arch}"
executable="bdrclone"
if [[ "$target_os" == "windows" ]]; then
executable="bdrclone.exe"
fi
package_dir="dist/$archive"
mkdir -p "$package_dir"
CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" \
go build -trimpath -ldflags "-s -w $version_flags" \
-o "$package_dir/$executable" ./cmd/bdrclone
cp README.md "$package_dir/"
tar -C dist -czf "dist/$archive.tar.gz" "$archive"
rm -rf "$package_dir"
done
cd dist
sha256sum ./*.tar.gz > SHA256SUMS
- name: Publish Gitea package
if: startsWith(github.ref, 'refs/tags/v') || inputs.publish_version != ''
shell: bash
env:
GITEA_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
GITEA_SERVER_URL: ${{ github.server_url }}
GITEA_OWNER: ${{ github.repository_owner }}
GITEA_REPOSITORY: ${{ github.repository }}
PACKAGE_INTERNAL_URL: ${{ vars.PACKAGE_INTERNAL_URL }}
BUILD_REF_NAME: ${{ github.ref_name }}
PUBLISH_VERSION: ${{ inputs.publish_version }}
run: |
set -euo pipefail
package_version="${PUBLISH_VERSION:-$BUILD_REF_NAME}"
repository_name="${GITEA_REPOSITORY#*/}"
package_server="${PACKAGE_INTERNAL_URL:-$GITEA_SERVER_URL}"
gitea_host="${GITEA_SERVER_URL#*://}"
gitea_host="${gitea_host%%/*}"
package_url="$package_server/api/packages/$GITEA_OWNER/generic/bdrclone/$package_version"
for package_file in dist/*.tar.gz dist/SHA256SUMS; do
curl --fail-with-body --silent --show-error \
--retry 5 --retry-all-errors --retry-delay 3 \
--connect-timeout 20 --max-time 300 \
--request PUT \
--header "Authorization: token $GITEA_TOKEN" \
--header "Host: $gitea_host" \
--upload-file "$package_file" \
"$package_url/$(basename "$package_file")"
done
curl --fail-with-body --silent --show-error \
--retry 5 --retry-all-errors --retry-delay 3 \
--connect-timeout 20 --max-time 300 \
--request POST \
--header "Authorization: token $GITEA_TOKEN" \
--header "Host: $gitea_host" \
"$package_server/api/v1/packages/$GITEA_OWNER/generic/bdrclone/-/link/$repository_name"
- name: Upload temporary build artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/v') && inputs.publish_version == '' }}
continue-on-error: true
uses: actions/upload-artifact@v3
with:
name: bdrclone-build-${{ github.run_number }}
path: |
dist/*.tar.gz
dist/SHA256SUMS
if-no-files-found: error
retention-days: 30

View File

@@ -86,6 +86,13 @@ OOB 模式使用 `redirect_uri=oob`,无需在本机监听端口。
./bdrclone stat /文档/report.pdf ./bdrclone stat /文档/report.pdf
./bdrclone download /文档/report.pdf ./report.pdf ./bdrclone download /文档/report.pdf ./report.pdf
./bdrclone upload ./photo.jpg /备份/photo.jpg ./bdrclone upload ./photo.jpg /备份/photo.jpg
# 递归上传目录内容,保留相对目录结构
./bdrclone upload ~/ /备份/mac/
# 排除受保护目录和缓存;--exclude 可重复,支持 **
./bdrclone upload ~/ /备份/mac/ \
--exclude .Trash \
--exclude 'Library/Caches/**' \
--exclude '.cache/codex-runtimes/**'
./bdrclone mkdir /备份/新目录 ./bdrclone mkdir /备份/新目录
./bdrclone mv /备份/a.txt /备份/b.txt ./bdrclone mv /备份/a.txt /备份/b.txt
./bdrclone cp /备份/b.txt /副本/b.txt ./bdrclone cp /备份/b.txt /副本/b.txt
@@ -121,6 +128,9 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf
## 百度 API 限制 ## 百度 API 限制
- 百度网盘官方接口不允许创建空文件;挂载下的空文件在 flush 时会失败。 - 百度网盘官方接口不允许创建空文件;挂载下的空文件在 flush 时会失败。
- 目录上传会创建远端空目录,但会跳过无法读取的路径、符号链接、特殊文件及百度接口无法表示的空文件,并逐项提示。
- `upload --exclude` 使用相对于源目录的 glob;无 `/` 的规则匹配任意层级的文件名,`**` 可跨目录匹配。
- 分片上传遇到临时网络或网关错误会退避重试 6 次;目录中单个文件最终失败时会记录错误并继续,结束时返回失败状态。
- 大文件下载必须使用 `User-Agent: pan.baidu.com`,客户端已统一设置。 - 大文件下载必须使用 `User-Agent: pan.baidu.com`,客户端已统一设置。
- 分片大小由会员等级决定:普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB;百度限制分片数, - 分片大小由会员等级决定:普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB;百度限制分片数,
因此不同等级的单文件上限不同。 因此不同等级的单文件上限不同。

View File

@@ -21,7 +21,7 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
var version = "v1.0.0" var version = "v1.0.1"
type application struct { type application struct {
configPath string configPath string
@@ -268,19 +268,48 @@ func (a *application) manageCommand(use, short string, fn func(context.Context,
} }
func (a *application) uploadCommand() *cobra.Command { func (a *application) uploadCommand() *cobra.Command {
return &cobra.Command{Use: "upload <本地文件> <远端文件>", Args: cobra.ExactArgs(2), Short: "分片上传本地文件", RunE: func(cmd *cobra.Command, args []string) error { var excludes []string
cmd := &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error {
client, _, err := a.client() client, _, err := a.client()
if err != nil { if err != nil {
return err return err
} }
_, err = client.UploadFile(cmd.Context(), args[0], args[1], func(done, total int64) { stderr := cmd.ErrOrStderr()
fmt.Fprintf(os.Stderr, "\r上传 %d/%d bytes (%d%%)", done, total, done*100/total) lineOpen := false
}) summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(_, remotePath string, done, total int64) {
if err == nil { displayPath := uploadProgressPath(args[1], remotePath)
fmt.Fprintln(os.Stderr) percent := int64(0)
if total > 0 {
percent = done * 100 / total
}
fmt.Fprintf(stderr, "\r上传 %s: %d/%d bytes (%d%%)", displayPath, done, total, percent)
lineOpen = done < total
if !lineOpen {
fmt.Fprintln(stderr)
}
}, Skipped: func(localPath, reason string) {
if lineOpen {
fmt.Fprintln(stderr)
lineOpen = false
}
fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason)
}, Failed: func(_, _ string, uploadErr error) {
if lineOpen {
fmt.Fprintln(stderr)
lineOpen = false
}
fmt.Fprintf(stderr, "失败: %v\n", uploadErr)
}})
if lineOpen {
fmt.Fprintln(stderr)
}
if summary.Directories > 0 || summary.Files > 0 || summary.Skipped > 0 || summary.Failed > 0 {
fmt.Fprintf(stderr, "完成:成功上传 %d 个文件(%s),处理 %d 个目录,跳过 %d 项,失败 %d 个文件\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped, summary.Failed)
} }
return err return err
}} }}
cmd.Flags().StringArrayVar(&excludes, "exclude", nil, "排除源目录中的 glob(可重复,支持 **)")
return cmd
} }
func (a *application) downloadCommand() *cobra.Command { func (a *application) downloadCommand() *cobra.Command {

270
cmd/bdrclone/upload_path.go Normal file
View File

@@ -0,0 +1,270 @@
package main
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
pathpkg "path"
"path/filepath"
"strings"
"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
)
type uploadPathClient interface {
MkdirAll(context.Context, string) error
EnsureDirectory(context.Context, string) error
UploadFile(context.Context, string, string, baidu.UploadProgress) (baidu.File, error)
}
type uploadPathSummary struct {
Files int
Directories int
Skipped int
Failed int
Bytes int64
}
type uploadPathProgress func(localPath, remotePath string, uploaded, total int64)
type uploadPathSkipped func(localPath, reason string)
type uploadPathFailed func(localPath, remotePath string, err error)
type uploadPathOptions struct {
Excludes []string
Progress uploadPathProgress
Skipped uploadPathSkipped
Failed uploadPathFailed
}
type uploadWalkDir func(string, fs.WalkDirFunc) error
func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, options uploadPathOptions) (uploadPathSummary, error) {
excludes, err := normalizeUploadExcludes(options.Excludes)
if err != nil {
return uploadPathSummary{}, err
}
info, err := os.Stat(localSource)
if err != nil {
return uploadPathSummary{}, fmt.Errorf("stat local source: %w", err)
}
remoteDestination = cleanUploadRemotePath(remoteDestination)
if !info.IsDir() {
if !info.Mode().IsRegular() {
return uploadPathSummary{}, fmt.Errorf("unsupported local file type: %s", localSource)
}
var summary uploadPathSummary
if matchesUploadExclude(filepath.Base(localSource), excludes) {
reportUploadSkip(&summary, options, localSource, "已被 --exclude 规则排除")
return summary, nil
}
err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, options.Progress, &summary)
return summary, err
}
root, err := filepath.Abs(localSource)
if err != nil {
return uploadPathSummary{}, fmt.Errorf("resolve local source: %w", err)
}
root, err = filepath.EvalSymlinks(root)
if err != nil {
return uploadPathSummary{}, fmt.Errorf("resolve local source links: %w", err)
}
options.Excludes = excludes
return uploadLocalDirectory(ctx, client, root, remoteDestination, options, filepath.WalkDir)
}
func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, remoteDestination string, options uploadPathOptions, walk uploadWalkDir) (uploadPathSummary, error) {
if err := client.MkdirAll(ctx, remoteDestination); err != nil {
return uploadPathSummary{}, err
}
summary := uploadPathSummary{Directories: 1}
var firstUploadErr error
err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
if err := ctx.Err(); err != nil {
return err
}
reportUploadSkip(&summary, options, localPath, "无法读取: "+walkErr.Error())
if entry != nil && entry.IsDir() {
return fs.SkipDir
}
return nil
}
if err := ctx.Err(); err != nil {
return err
}
if localPath == root {
return nil
}
relative, err := filepath.Rel(root, localPath)
if err != nil {
return fmt.Errorf("make local path relative: %w", err)
}
relative = filepath.ToSlash(relative)
if matchesUploadExclude(relative, options.Excludes) {
reportUploadSkip(&summary, options, localPath, "已被 --exclude 规则排除")
if entry.IsDir() {
return fs.SkipDir
}
return nil
}
remotePath := pathpkg.Join(remoteDestination, relative)
if entry.IsDir() {
if err := client.EnsureDirectory(ctx, remotePath); err != nil {
return err
}
summary.Directories++
return nil
}
if entry.Type()&os.ModeSymlink != 0 {
reportUploadSkip(&summary, options, localPath, "符号链接不会被跟随")
return nil
}
info, err := entry.Info()
if err != nil {
reportUploadSkip(&summary, options, localPath, "无法读取文件信息: "+err.Error())
return nil
}
if !info.Mode().IsRegular() {
reportUploadSkip(&summary, options, localPath, "不支持的本地文件类型 "+info.Mode().Type().String())
return nil
}
if info.Size() == 0 {
reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件")
return nil
}
if err := uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
if firstUploadErr == nil {
firstUploadErr = err
}
reportUploadFailure(&summary, options, localPath, remotePath, err)
}
return nil
})
if err != nil {
return summary, err
}
if summary.Failed > 0 {
return summary, fmt.Errorf("%d files failed to upload; first failure: %w", summary.Failed, firstUploadErr)
}
return summary, nil
}
func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, localPath, reason string) {
summary.Skipped++
if options.Skipped != nil {
options.Skipped(localPath, reason)
}
}
func reportUploadFailure(summary *uploadPathSummary, options uploadPathOptions, localPath, remotePath string, err error) {
summary.Failed++
if options.Failed != nil {
options.Failed(localPath, remotePath, err)
}
}
func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error {
completeReported := false
if progress != nil && info.Size() > 0 {
progress(localPath, remotePath, 0, info.Size())
}
_, err := client.UploadFile(ctx, localPath, remotePath, func(uploaded, total int64) {
completeReported = uploaded >= total
if progress != nil {
progress(localPath, remotePath, uploaded, total)
}
})
if err != nil {
return fmt.Errorf("upload %s to %s: %w", localPath, remotePath, err)
}
if progress != nil && info.Size() > 0 && !completeReported {
progress(localPath, remotePath, info.Size(), info.Size())
}
summary.Files++
summary.Bytes += info.Size()
return nil
}
func cleanUploadRemotePath(name string) string {
return pathpkg.Clean("/" + strings.TrimPrefix(name, "/"))
}
func uploadProgressPath(remoteDestination, remotePath string) string {
root := cleanUploadRemotePath(remoteDestination)
if relative := strings.TrimPrefix(remotePath, root+"/"); relative != remotePath {
return relative
}
return pathpkg.Base(remotePath)
}
func normalizeUploadExcludes(patterns []string) ([]string, error) {
result := make([]string, 0, len(patterns))
for _, pattern := range patterns {
pattern = filepath.ToSlash(pattern)
pattern = strings.TrimPrefix(pattern, "./")
pattern = strings.Trim(pattern, "/")
if pattern == "" {
return nil, errors.New("exclude pattern cannot be empty")
}
for _, component := range strings.Split(pattern, "/") {
if component == "**" {
continue
}
if _, err := pathpkg.Match(component, ""); err != nil {
return nil, fmt.Errorf("invalid exclude pattern %q: %w", pattern, err)
}
}
result = append(result, pattern)
}
return result, nil
}
func matchesUploadExclude(relative string, patterns []string) bool {
relative = strings.Trim(filepath.ToSlash(relative), "/")
for _, pattern := range patterns {
if !strings.Contains(pattern, "/") {
matched, _ := pathpkg.Match(pattern, pathpkg.Base(relative))
if matched {
return true
}
continue
}
if matchUploadPathSegments(strings.Split(pattern, "/"), strings.Split(relative, "/")) {
return true
}
}
return false
}
func matchUploadPathSegments(pattern, name []string) bool {
type state struct{ pattern, name int }
memo := make(map[state]bool)
seen := make(map[state]bool)
var match func(int, int) bool
match = func(patternIndex, nameIndex int) bool {
key := state{patternIndex, nameIndex}
if seen[key] {
return memo[key]
}
seen[key] = true
var result bool
switch {
case patternIndex == len(pattern):
result = nameIndex == len(name)
case pattern[patternIndex] == "**":
result = match(patternIndex+1, nameIndex) || (nameIndex < len(name) && match(patternIndex, nameIndex+1))
case nameIndex < len(name):
segmentMatches, _ := pathpkg.Match(pattern[patternIndex], name[nameIndex])
result = segmentMatches && match(patternIndex+1, nameIndex+1)
}
memo[key] = result
return result
}
return match(0, 0)
}

View File

@@ -0,0 +1,265 @@
package main
import (
"bytes"
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
)
type recordingUploadClient struct {
directories []string
files map[string][]byte
failures map[string]error
}
func (c *recordingUploadClient) MkdirAll(_ context.Context, remotePath string) error {
c.directories = append(c.directories, remotePath)
return nil
}
func (c *recordingUploadClient) EnsureDirectory(_ context.Context, remotePath string) error {
c.directories = append(c.directories, remotePath)
return nil
}
func (c *recordingUploadClient) UploadFile(_ context.Context, localPath, remotePath string, progress baidu.UploadProgress) (baidu.File, error) {
content, err := os.ReadFile(localPath)
if err != nil {
return baidu.File{}, err
}
if c.files == nil {
c.files = make(map[string][]byte)
}
if err := c.failures[remotePath]; err != nil {
return baidu.File{}, err
}
c.files[remotePath] = content
if progress != nil {
progress(int64(len(content)), int64(len(content)))
}
return baidu.File{Path: remotePath, Size: int64(len(content))}, nil
}
func TestUploadLocalDirectoryPreservesRelativePaths(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "top.txt"), []byte("top"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(root, "empty-dir"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(root, "nested"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "nested", "file.bin"), []byte("nested"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "zero.txt"), nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink("top.txt", filepath.Join(root, "top-link")); err != nil {
t.Fatal(err)
}
client := &recordingUploadClient{}
var skipped []string
summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", uploadPathOptions{Skipped: func(localPath, _ string) {
skipped = append(skipped, filepath.Base(localPath))
}})
if err != nil {
t.Fatal(err)
}
if summary.Files != 2 || summary.Directories != 3 || summary.Skipped != 2 || summary.Bytes != 9 {
t.Fatalf("summary = %+v", summary)
}
if want := []string{"/backup/mac", "/backup/mac/empty-dir", "/backup/mac/nested"}; !reflect.DeepEqual(client.directories, want) {
t.Fatalf("directories = %q, want %q", client.directories, want)
}
if !bytes.Equal(client.files["/backup/mac/top.txt"], []byte("top")) {
t.Fatalf("top.txt = %q", client.files["/backup/mac/top.txt"])
}
if !bytes.Equal(client.files["/backup/mac/nested/file.bin"], []byte("nested")) {
t.Fatalf("nested/file.bin = %q", client.files["/backup/mac/nested/file.bin"])
}
if want := []string{"top-link", "zero.txt"}; !reflect.DeepEqual(skipped, want) {
t.Fatalf("skipped = %q, want %q", skipped, want)
}
}
func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) {
localPath := filepath.Join(t.TempDir(), "source.txt")
if err := os.WriteFile(localPath, []byte("content"), 0o600); err != nil {
t.Fatal(err)
}
client := &recordingUploadClient{}
summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", uploadPathOptions{})
if err != nil {
t.Fatal(err)
}
if summary.Files != 1 || summary.Directories != 0 || summary.Bytes != 7 {
t.Fatalf("summary = %+v", summary)
}
if !bytes.Equal(client.files["/backup/renamed.txt"], []byte("content")) {
t.Fatalf("uploaded files = %v", client.files)
}
}
func TestUploadLocalDirectoryHonorsExcludePatterns(t *testing.T) {
root := t.TempDir()
files := map[string]string{
"keep.txt": "keep",
"scratch.tmp": "temporary",
filepath.Join(".Trash", "old"): "trash",
filepath.Join("Library", "keep"): "library",
filepath.Join("Library", "Caches", "cache.db"): "cache",
}
for name, content := range files {
localPath := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(localPath), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(localPath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
client := &recordingUploadClient{}
summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{
Excludes: []string{".Trash", "Library/Caches/**", "*.tmp"},
})
if err != nil {
t.Fatal(err)
}
if summary.Files != 2 || summary.Directories != 2 || summary.Skipped != 3 {
t.Fatalf("summary = %+v", summary)
}
if want := []string{"/backup/mac", "/backup/mac/Library"}; !reflect.DeepEqual(client.directories, want) {
t.Fatalf("directories = %q, want %q", client.directories, want)
}
if _, ok := client.files["/backup/mac/keep.txt"]; !ok {
t.Fatalf("keep.txt was not uploaded: %v", client.files)
}
if _, ok := client.files["/backup/mac/Library/keep"]; !ok {
t.Fatalf("Library/keep was not uploaded: %v", client.files)
}
}
func TestUploadLocalDirectoryContinuesAfterReadError(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "blocked"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "good.txt"), []byte("good"), 0o600); err != nil {
t.Fatal(err)
}
entries, err := os.ReadDir(root)
if err != nil {
t.Fatal(err)
}
entryByName := make(map[string]os.DirEntry, len(entries))
for _, entry := range entries {
entryByName[entry.Name()] = entry
}
rootInfo, err := os.Stat(root)
if err != nil {
t.Fatal(err)
}
walk := func(_ string, visit fs.WalkDirFunc) error {
if err := visit(root, fs.FileInfoToDirEntry(rootInfo), nil); err != nil {
return err
}
blockedPath := filepath.Join(root, "blocked")
if err := visit(blockedPath, entryByName["blocked"], fs.ErrPermission); err != nil && err != fs.SkipDir {
return err
}
return visit(filepath.Join(root, "good.txt"), entryByName["good.txt"], nil)
}
client := &recordingUploadClient{}
var skipped []string
summary, err := uploadLocalDirectory(context.Background(), client, root, "/backup/mac", uploadPathOptions{Skipped: func(localPath, reason string) {
skipped = append(skipped, filepath.Base(localPath)+":"+reason)
}}, walk)
if err != nil {
t.Fatal(err)
}
if summary.Files != 1 || summary.Skipped != 1 {
t.Fatalf("summary = %+v", summary)
}
if len(skipped) != 1 || !strings.Contains(skipped[0], "blocked:无法读取: permission denied") {
t.Fatalf("skipped = %q", skipped)
}
if !bytes.Equal(client.files["/backup/mac/good.txt"], []byte("good")) {
t.Fatalf("good.txt was not uploaded: %v", client.files)
}
}
func TestUploadLocalDirectoryContinuesAfterUploadFailure(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "a-fails.txt"), []byte("failed"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "b-succeeds.txt"), []byte("success"), 0o600); err != nil {
t.Fatal(err)
}
client := &recordingUploadClient{failures: map[string]error{
"/backup/mac/a-fails.txt": errors.New("gateway timeout"),
}}
var failed []string
summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{Failed: func(localPath, remotePath string, err error) {
failed = append(failed, filepath.Base(localPath)+":"+remotePath+":"+err.Error())
}})
if err == nil || !strings.Contains(err.Error(), "1 files failed to upload") {
t.Fatalf("error = %v", err)
}
if summary.Files != 1 || summary.Failed != 1 || summary.Bytes != 7 {
t.Fatalf("summary = %+v", summary)
}
if len(failed) != 1 || !strings.Contains(failed[0], "a-fails.txt:/backup/mac/a-fails.txt:") || !strings.Contains(failed[0], "gateway timeout") {
t.Fatalf("failed = %q", failed)
}
if !bytes.Equal(client.files["/backup/mac/b-succeeds.txt"], []byte("success")) {
t.Fatalf("b-succeeds.txt was not uploaded: %v", client.files)
}
}
func TestUploadExcludePatternMatching(t *testing.T) {
patterns, err := normalizeUploadExcludes([]string{".Trash/", "Library/Caches/**", "**/*.tmp"})
if err != nil {
t.Fatal(err)
}
for name, want := range map[string]bool{
".Trash": true,
"nested/.Trash": true,
"Library/Caches": true,
"Library/Caches/data/file": true,
"root.tmp": true,
"nested/file.tmp": true,
"Library/Application/file": false,
"nested/file.tmp.keep": false,
} {
if got := matchesUploadExclude(name, patterns); got != want {
t.Errorf("matchesUploadExclude(%q) = %v, want %v", name, got, want)
}
}
if _, err := normalizeUploadExcludes([]string{"bad["}); err == nil {
t.Fatal("invalid exclude pattern was accepted")
}
}
func TestUploadProgressPath(t *testing.T) {
if got := uploadProgressPath("/backup/mac/", "/backup/mac/Library/file.txt"); got != "Library/file.txt" {
t.Fatalf("progress path = %q", got)
}
if got := uploadProgressPath("/backup/file.txt", "/backup/file.txt"); got != "file.txt" {
t.Fatalf("single-file progress path = %q", got)
}
}

View File

@@ -33,6 +33,7 @@ type Client struct {
root string root string
uploadParts int uploadParts int
partSize int64 partSize int64
uploadRetryDelay func(int) time.Duration
downloadMu sync.Mutex downloadMu sync.Mutex
downloadURL map[int64]cachedDownloadURL downloadURL map[int64]cachedDownloadURL
@@ -75,6 +76,7 @@ func New(cfg *config.Config, opts ...Option) *Client {
apiBase: defaultAPIBase, oauthBase: defaultOAuthBase, uploadBase: defaultUploadBase, apiBase: defaultAPIBase, oauthBase: defaultOAuthBase, uploadBase: defaultUploadBase,
userAgent: cfg.UserAgent, root: cfg.Root, userAgent: cfg.UserAgent, root: cfg.Root,
uploadParts: cfg.UploadParts, partSize: cfg.PartSize, uploadParts: cfg.UploadParts, partSize: cfg.PartSize,
uploadRetryDelay: defaultUploadRetryDelay,
downloadURL: make(map[int64]cachedDownloadURL), downloadURL: make(map[int64]cachedDownloadURL),
clientID: cfg.ClientID, secret: cfg.ClientSecret, clientID: cfg.ClientID, secret: cfg.ClientSecret,
accessToken: cfg.AccessToken, refresh: cfg.RefreshToken, expiresAt: cfg.ExpiresAt, accessToken: cfg.AccessToken, refresh: cfg.RefreshToken, expiresAt: cfg.ExpiresAt,

View File

@@ -9,7 +9,10 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
"path"
"reflect"
"strconv" "strconv"
"strings"
"sync" "sync"
"testing" "testing"
"time" "time"
@@ -203,6 +206,109 @@ func TestUploadMultipartFlow(t *testing.T) {
} }
} }
func TestUploadPartRetriesGatewayTimeout(t *testing.T) {
file, err := os.CreateTemp(t.TempDir(), "retry-upload-*")
if err != nil {
t.Fatal(err)
}
defer file.Close()
content := []byte("retry payload")
if _, err := file.Write(content); err != nil {
t.Fatal(err)
}
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/rest/2.0/pcs/superfile2" {
http.NotFound(w, r)
return
}
attempts++
_, _ = io.Copy(io.Discard, r.Body)
if attempts < 4 {
http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
return
}
fmt.Fprint(w, `{"md5":"ok"}`)
}))
defer server.Close()
client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL))
client.uploadRetryDelay = func(int) time.Duration { return 0 }
if err := client.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-1", 0, 0, int64(len(content))); err != nil {
t.Fatal(err)
}
if attempts != 4 {
t.Fatalf("attempts = %d, want 4", attempts)
}
failureAttempts := 0
failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
failureAttempts++
_, _ = io.Copy(io.Discard, r.Body)
http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
}))
defer failureServer.Close()
failureClient := New(testConfig("token"), WithEndpoints(failureServer.URL, failureServer.URL, failureServer.URL))
failureClient.uploadRetryDelay = func(int) time.Duration { return 0 }
err = failureClient.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-2", 0, 0, int64(len(content)))
if err == nil || !strings.Contains(err.Error(), "after 6 attempts") {
t.Fatalf("error = %v", err)
}
if failureAttempts != maxUploadAttempts {
t.Fatalf("attempts = %d, want %d", failureAttempts, maxUploadAttempts)
}
}
func TestMkdirAllCreatesMissingRemoteDirectories(t *testing.T) {
directories := map[string]bool{"/apps/bdrclone": true}
var created []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/rest/2.0/xpan/file" {
http.NotFound(w, r)
return
}
switch r.URL.Query().Get("method") {
case "list":
parent := r.URL.Query().Get("dir")
entries := make([]File, 0)
for directory := range directories {
if directory != parent && path.Dir(directory) == parent {
entries = append(entries, File{Path: directory, ServerFilename: path.Base(directory), IsDir: 1})
}
}
_ = json.NewEncoder(w).Encode(map[string]any{"errno": 0, "list": entries})
case "create":
if err := r.ParseForm(); err != nil {
t.Error(err)
return
}
directory := r.Form.Get("path")
if r.Form.Get("isdir") != "1" {
t.Errorf("isdir = %q", r.Form.Get("isdir"))
}
directories[directory] = true
created = append(created, directory)
_ = json.NewEncoder(w).Encode(File{Path: directory, ServerFilename: path.Base(directory), IsDir: 1})
default:
http.Error(w, "unexpected method", http.StatusBadRequest)
}
}))
defer server.Close()
client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL))
if err := client.MkdirAll(context.Background(), "/backup/mac/photos"); err != nil {
t.Fatal(err)
}
if err := client.MkdirAll(context.Background(), "/backup/mac/photos"); err != nil {
t.Fatal(err)
}
want := []string{"/apps/bdrclone/backup", "/apps/bdrclone/backup/mac", "/apps/bdrclone/backup/mac/photos"}
if !reflect.DeepEqual(created, want) {
t.Fatalf("created = %q, want %q", created, want)
}
}
func testConfig(token string) *config.Config { func testConfig(token string) *config.Config {
return &config.Config{ return &config.Config{
ClientID: "client", ClientSecret: "secret", AccessToken: token, ClientID: "client", ClientSecret: "secret", AccessToken: token,

View File

@@ -66,6 +66,38 @@ func (c *Client) Mkdir(ctx context.Context, name string) (File, error) {
return result, nil return result, nil
} }
func (c *Client) MkdirAll(ctx context.Context, name string) error {
target := path.Clean("/" + strings.TrimPrefix(name, "/"))
if target == "/" {
return nil
}
current := "/"
for _, component := range strings.Split(strings.TrimPrefix(target, "/"), "/") {
current = path.Join(current, component)
if err := c.EnsureDirectory(ctx, current); err != nil {
return err
}
}
return nil
}
func (c *Client) EnsureDirectory(ctx context.Context, name string) error {
entry, err := c.Stat(ctx, name)
if err == nil {
if !entry.IsDirectory() {
return fmt.Errorf("remote path exists and is not a directory: %s", name)
}
return nil
}
if !errors.Is(err, ErrNotFound) {
return fmt.Errorf("check remote directory %s: %w", name, err)
}
if _, err := c.Mkdir(ctx, name); err != nil {
return fmt.Errorf("create remote directory %s: %w", name, err)
}
return nil
}
func (c *Client) Delete(ctx context.Context, name string) error { func (c *Client) Delete(ctx context.Context, name string) error {
remote := c.RemotePath(name) remote := c.RemotePath(name)
if remote == c.root { if remote == c.root {

View File

@@ -25,6 +25,7 @@ const (
vipPartSize = int64(16 << 20) vipPartSize = int64(16 << 20)
svipPartSize = int64(32 << 20) svipPartSize = int64(32 << 20)
maxPartCount = 2048 maxPartCount = 2048
maxUploadAttempts = 6
) )
type UploadProgress func(uploaded, total int64) type UploadProgress func(uploaded, total int64)
@@ -247,7 +248,7 @@ func (c *Client) uploadPartsParallel(ctx context.Context, file *os.File, remote,
func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error { func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error {
var lastErr error var lastErr error
for attempt := 0; attempt < 3; attempt++ { for attempt := 0; attempt < maxUploadAttempts; attempt++ {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return err return err
} }
@@ -256,13 +257,18 @@ func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote,
if lastErr == nil { if lastErr == nil {
return nil return nil
} }
if attempt < 2 { if attempt < maxUploadAttempts-1 {
if err := sleepContext(ctx, time.Duration(1<<attempt)*time.Second); err != nil { if err := sleepContext(ctx, c.uploadRetryDelay(attempt)); err != nil {
return err return err
} }
} }
} }
return lastErr return fmt.Errorf("upload failed after %d attempts: %w", maxUploadAttempts, lastErr)
}
func defaultUploadRetryDelay(attempt int) time.Duration {
delay := time.Second * time.Duration(1<<attempt)
return min(delay, 30*time.Second)
} }
func (c *Client) uploadPart(ctx context.Context, section *io.SectionReader, filename, remote, uploadID string, part int) error { func (c *Client) uploadPart(ctx context.Context, section *io.SectionReader, filename, remote, uploadID string, part int) error {