Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0b4c2abf65 | |||
| c602698ce9 | |||
| 977fcefc2f | |||
| c1758cf29c | |||
| c2eecdebbb | |||
| f6111c9715 | |||
| 2de606c27d | |||
| 8cde8109e9 | |||
| a2ecfcf550 | |||
| 46c25768e4 | |||
| f23f1a4e5a |
143
.gitea/workflows/build.yml
Normal file
143
.gitea/workflows/build.yml
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
name: Build
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- "**"
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
pull_request:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
publish_version:
|
||||||
|
description: Version to publish to the Gitea package registry
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Test and build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check out source
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
cache: false
|
||||||
|
|
||||||
|
- name: Test
|
||||||
|
run: go test -race ./...
|
||||||
|
|
||||||
|
- name: Vet
|
||||||
|
run: go vet ./...
|
||||||
|
|
||||||
|
- name: Build release archives
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
BUILD_REF: ${{ github.ref }}
|
||||||
|
BUILD_REF_NAME: ${{ github.ref_name }}
|
||||||
|
BUILD_SHA: ${{ github.sha }}
|
||||||
|
PUBLISH_VERSION: ${{ inputs.publish_version }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
package_version="snapshot-${BUILD_SHA:0:8}"
|
||||||
|
version_flags=""
|
||||||
|
if [[ "$BUILD_REF" == refs/tags/v* ]]; then
|
||||||
|
package_version="$BUILD_REF_NAME"
|
||||||
|
elif [[ -n "$PUBLISH_VERSION" ]]; then
|
||||||
|
package_version="$PUBLISH_VERSION"
|
||||||
|
fi
|
||||||
|
if [[ "$package_version" != snapshot-* ]]; then
|
||||||
|
if [[ ! "$package_version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]]; then
|
||||||
|
echo "Invalid release version: $package_version" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
version_flags="-X main.version=$package_version"
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p dist
|
||||||
|
for target in \
|
||||||
|
linux/amd64 linux/arm64 \
|
||||||
|
darwin/amd64 darwin/arm64 \
|
||||||
|
windows/amd64 windows/arm64
|
||||||
|
do
|
||||||
|
target_os="${target%/*}"
|
||||||
|
target_arch="${target#*/}"
|
||||||
|
archive="bdrclone-${package_version}-${target_os}-${target_arch}"
|
||||||
|
executable="bdrclone"
|
||||||
|
if [[ "$target_os" == "windows" ]]; then
|
||||||
|
executable="bdrclone.exe"
|
||||||
|
fi
|
||||||
|
|
||||||
|
package_dir="dist/$archive"
|
||||||
|
mkdir -p "$package_dir"
|
||||||
|
CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" \
|
||||||
|
go build -trimpath -ldflags "-s -w $version_flags" \
|
||||||
|
-o "$package_dir/$executable" ./cmd/bdrclone
|
||||||
|
cp README.md "$package_dir/"
|
||||||
|
tar -C dist -czf "dist/$archive.tar.gz" "$archive"
|
||||||
|
rm -rf "$package_dir"
|
||||||
|
done
|
||||||
|
|
||||||
|
cd dist
|
||||||
|
sha256sum ./*.tar.gz > SHA256SUMS
|
||||||
|
|
||||||
|
- name: Publish Gitea package
|
||||||
|
if: startsWith(github.ref, 'refs/tags/v') || inputs.publish_version != ''
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
||||||
|
GITEA_SERVER_URL: ${{ github.server_url }}
|
||||||
|
GITEA_OWNER: ${{ github.repository_owner }}
|
||||||
|
GITEA_REPOSITORY: ${{ github.repository }}
|
||||||
|
PACKAGE_INTERNAL_URL: ${{ vars.PACKAGE_INTERNAL_URL }}
|
||||||
|
BUILD_REF_NAME: ${{ github.ref_name }}
|
||||||
|
PUBLISH_VERSION: ${{ inputs.publish_version }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
package_version="${PUBLISH_VERSION:-$BUILD_REF_NAME}"
|
||||||
|
repository_name="${GITEA_REPOSITORY#*/}"
|
||||||
|
package_server="${PACKAGE_INTERNAL_URL:-$GITEA_SERVER_URL}"
|
||||||
|
gitea_host="${GITEA_SERVER_URL#*://}"
|
||||||
|
gitea_host="${gitea_host%%/*}"
|
||||||
|
package_url="$package_server/api/packages/$GITEA_OWNER/generic/bdrclone/$package_version"
|
||||||
|
|
||||||
|
for package_file in dist/*.tar.gz dist/SHA256SUMS; do
|
||||||
|
curl --fail-with-body --silent --show-error \
|
||||||
|
--retry 5 --retry-all-errors --retry-delay 3 \
|
||||||
|
--connect-timeout 20 --max-time 300 \
|
||||||
|
--request PUT \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
--header "Host: $gitea_host" \
|
||||||
|
--upload-file "$package_file" \
|
||||||
|
"$package_url/$(basename "$package_file")"
|
||||||
|
done
|
||||||
|
|
||||||
|
curl --fail-with-body --silent --show-error \
|
||||||
|
--retry 5 --retry-all-errors --retry-delay 3 \
|
||||||
|
--connect-timeout 20 --max-time 300 \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
--header "Host: $gitea_host" \
|
||||||
|
"$package_server/api/v1/packages/$GITEA_OWNER/generic/bdrclone/-/link/$repository_name"
|
||||||
|
|
||||||
|
- name: Upload temporary build artifacts
|
||||||
|
if: ${{ !startsWith(github.ref, 'refs/tags/v') && inputs.publish_version == '' }}
|
||||||
|
continue-on-error: true
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: bdrclone-build-${{ github.run_number }}
|
||||||
|
path: |
|
||||||
|
dist/*.tar.gz
|
||||||
|
dist/SHA256SUMS
|
||||||
|
if-no-files-found: error
|
||||||
|
retention-days: 30
|
||||||
10
README.md
10
README.md
@@ -86,6 +86,13 @@ OOB 模式使用 `redirect_uri=oob`,无需在本机监听端口。
|
|||||||
./bdrclone stat /文档/report.pdf
|
./bdrclone stat /文档/report.pdf
|
||||||
./bdrclone download /文档/report.pdf ./report.pdf
|
./bdrclone download /文档/report.pdf ./report.pdf
|
||||||
./bdrclone upload ./photo.jpg /备份/photo.jpg
|
./bdrclone upload ./photo.jpg /备份/photo.jpg
|
||||||
|
# 递归上传目录内容,保留相对目录结构
|
||||||
|
./bdrclone upload ~/ /备份/mac/
|
||||||
|
# 排除受保护目录和缓存;--exclude 可重复,支持 **
|
||||||
|
./bdrclone upload ~/ /备份/mac/ \
|
||||||
|
--exclude .Trash \
|
||||||
|
--exclude 'Library/Caches/**' \
|
||||||
|
--exclude '.cache/codex-runtimes/**'
|
||||||
./bdrclone mkdir /备份/新目录
|
./bdrclone mkdir /备份/新目录
|
||||||
./bdrclone mv /备份/a.txt /备份/b.txt
|
./bdrclone mv /备份/a.txt /备份/b.txt
|
||||||
./bdrclone cp /备份/b.txt /副本/b.txt
|
./bdrclone cp /备份/b.txt /副本/b.txt
|
||||||
@@ -121,6 +128,9 @@ curl http://127.0.0.1:8080/文档/report.pdf -o report.pdf
|
|||||||
## 百度 API 限制
|
## 百度 API 限制
|
||||||
|
|
||||||
- 百度网盘官方接口不允许创建空文件;挂载下的空文件在 flush 时会失败。
|
- 百度网盘官方接口不允许创建空文件;挂载下的空文件在 flush 时会失败。
|
||||||
|
- 目录上传会创建远端空目录,但会跳过无法读取的路径、符号链接、特殊文件及百度接口无法表示的空文件,并逐项提示。
|
||||||
|
- `upload --exclude` 使用相对于源目录的 glob;无 `/` 的规则匹配任意层级的文件名,`**` 可跨目录匹配。
|
||||||
|
- 分片上传遇到临时网络或网关错误会退避重试 6 次;目录中单个文件最终失败时会记录错误并继续,结束时返回失败状态。
|
||||||
- 大文件下载必须使用 `User-Agent: pan.baidu.com`,客户端已统一设置。
|
- 大文件下载必须使用 `User-Agent: pan.baidu.com`,客户端已统一设置。
|
||||||
- 分片大小由会员等级决定:普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB;百度限制分片数,
|
- 分片大小由会员等级决定:普通用户 4 MiB、会员 16 MiB、超级会员 32 MiB;百度限制分片数,
|
||||||
因此不同等级的单文件上限不同。
|
因此不同等级的单文件上限不同。
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "v1.0.0"
|
var version = "v1.0.1"
|
||||||
|
|
||||||
type application struct {
|
type application struct {
|
||||||
configPath string
|
configPath string
|
||||||
@@ -268,19 +268,48 @@ func (a *application) manageCommand(use, short string, fn func(context.Context,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *application) uploadCommand() *cobra.Command {
|
func (a *application) uploadCommand() *cobra.Command {
|
||||||
return &cobra.Command{Use: "upload <本地文件> <远端文件>", Args: cobra.ExactArgs(2), Short: "分片上传本地文件", RunE: func(cmd *cobra.Command, args []string) error {
|
var excludes []string
|
||||||
|
cmd := &cobra.Command{Use: "upload <本地路径> <远端路径>", Args: cobra.ExactArgs(2), Short: "上传文件或递归上传目录", RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
client, _, err := a.client()
|
client, _, err := a.client()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = client.UploadFile(cmd.Context(), args[0], args[1], func(done, total int64) {
|
stderr := cmd.ErrOrStderr()
|
||||||
fmt.Fprintf(os.Stderr, "\r上传 %d/%d bytes (%d%%)", done, total, done*100/total)
|
lineOpen := false
|
||||||
})
|
summary, err := uploadLocalPath(cmd.Context(), client, args[0], args[1], uploadPathOptions{Excludes: excludes, Progress: func(_, remotePath string, done, total int64) {
|
||||||
if err == nil {
|
displayPath := uploadProgressPath(args[1], remotePath)
|
||||||
fmt.Fprintln(os.Stderr)
|
percent := int64(0)
|
||||||
|
if total > 0 {
|
||||||
|
percent = done * 100 / total
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "\r上传 %s: %d/%d bytes (%d%%)", displayPath, done, total, percent)
|
||||||
|
lineOpen = done < total
|
||||||
|
if !lineOpen {
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
|
}
|
||||||
|
}, Skipped: func(localPath, reason string) {
|
||||||
|
if lineOpen {
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
|
lineOpen = false
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "跳过 %s: %s\n", localPath, reason)
|
||||||
|
}, Failed: func(_, _ string, uploadErr error) {
|
||||||
|
if lineOpen {
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
|
lineOpen = false
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "失败: %v\n", uploadErr)
|
||||||
|
}})
|
||||||
|
if lineOpen {
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
|
}
|
||||||
|
if summary.Directories > 0 || summary.Files > 0 || summary.Skipped > 0 || summary.Failed > 0 {
|
||||||
|
fmt.Fprintf(stderr, "完成:成功上传 %d 个文件(%s),处理 %d 个目录,跳过 %d 项,失败 %d 个文件\n", summary.Files, formatBytes(summary.Bytes), summary.Directories, summary.Skipped, summary.Failed)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}}
|
}}
|
||||||
|
cmd.Flags().StringArrayVar(&excludes, "exclude", nil, "排除源目录中的 glob(可重复,支持 **)")
|
||||||
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *application) downloadCommand() *cobra.Command {
|
func (a *application) downloadCommand() *cobra.Command {
|
||||||
|
|||||||
270
cmd/bdrclone/upload_path.go
Normal file
270
cmd/bdrclone/upload_path.go
Normal file
@@ -0,0 +1,270 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
pathpkg "path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
|
||||||
|
)
|
||||||
|
|
||||||
|
type uploadPathClient interface {
|
||||||
|
MkdirAll(context.Context, string) error
|
||||||
|
EnsureDirectory(context.Context, string) error
|
||||||
|
UploadFile(context.Context, string, string, baidu.UploadProgress) (baidu.File, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type uploadPathSummary struct {
|
||||||
|
Files int
|
||||||
|
Directories int
|
||||||
|
Skipped int
|
||||||
|
Failed int
|
||||||
|
Bytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type uploadPathProgress func(localPath, remotePath string, uploaded, total int64)
|
||||||
|
type uploadPathSkipped func(localPath, reason string)
|
||||||
|
type uploadPathFailed func(localPath, remotePath string, err error)
|
||||||
|
|
||||||
|
type uploadPathOptions struct {
|
||||||
|
Excludes []string
|
||||||
|
Progress uploadPathProgress
|
||||||
|
Skipped uploadPathSkipped
|
||||||
|
Failed uploadPathFailed
|
||||||
|
}
|
||||||
|
|
||||||
|
type uploadWalkDir func(string, fs.WalkDirFunc) error
|
||||||
|
|
||||||
|
func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, options uploadPathOptions) (uploadPathSummary, error) {
|
||||||
|
excludes, err := normalizeUploadExcludes(options.Excludes)
|
||||||
|
if err != nil {
|
||||||
|
return uploadPathSummary{}, err
|
||||||
|
}
|
||||||
|
info, err := os.Stat(localSource)
|
||||||
|
if err != nil {
|
||||||
|
return uploadPathSummary{}, fmt.Errorf("stat local source: %w", err)
|
||||||
|
}
|
||||||
|
remoteDestination = cleanUploadRemotePath(remoteDestination)
|
||||||
|
if !info.IsDir() {
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return uploadPathSummary{}, fmt.Errorf("unsupported local file type: %s", localSource)
|
||||||
|
}
|
||||||
|
var summary uploadPathSummary
|
||||||
|
if matchesUploadExclude(filepath.Base(localSource), excludes) {
|
||||||
|
reportUploadSkip(&summary, options, localSource, "已被 --exclude 规则排除")
|
||||||
|
return summary, nil
|
||||||
|
}
|
||||||
|
err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, options.Progress, &summary)
|
||||||
|
return summary, err
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := filepath.Abs(localSource)
|
||||||
|
if err != nil {
|
||||||
|
return uploadPathSummary{}, fmt.Errorf("resolve local source: %w", err)
|
||||||
|
}
|
||||||
|
root, err = filepath.EvalSymlinks(root)
|
||||||
|
if err != nil {
|
||||||
|
return uploadPathSummary{}, fmt.Errorf("resolve local source links: %w", err)
|
||||||
|
}
|
||||||
|
options.Excludes = excludes
|
||||||
|
return uploadLocalDirectory(ctx, client, root, remoteDestination, options, filepath.WalkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, remoteDestination string, options uploadPathOptions, walk uploadWalkDir) (uploadPathSummary, error) {
|
||||||
|
if err := client.MkdirAll(ctx, remoteDestination); err != nil {
|
||||||
|
return uploadPathSummary{}, err
|
||||||
|
}
|
||||||
|
summary := uploadPathSummary{Directories: 1}
|
||||||
|
var firstUploadErr error
|
||||||
|
err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reportUploadSkip(&summary, options, localPath, "无法读取: "+walkErr.Error())
|
||||||
|
if entry != nil && entry.IsDir() {
|
||||||
|
return fs.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if localPath == root {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(root, localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("make local path relative: %w", err)
|
||||||
|
}
|
||||||
|
relative = filepath.ToSlash(relative)
|
||||||
|
if matchesUploadExclude(relative, options.Excludes) {
|
||||||
|
reportUploadSkip(&summary, options, localPath, "已被 --exclude 规则排除")
|
||||||
|
if entry.IsDir() {
|
||||||
|
return fs.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
remotePath := pathpkg.Join(remoteDestination, relative)
|
||||||
|
if entry.IsDir() {
|
||||||
|
if err := client.EnsureDirectory(ctx, remotePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
summary.Directories++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if entry.Type()&os.ModeSymlink != 0 {
|
||||||
|
reportUploadSkip(&summary, options, localPath, "符号链接不会被跟随")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
reportUploadSkip(&summary, options, localPath, "无法读取文件信息: "+err.Error())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
reportUploadSkip(&summary, options, localPath, "不支持的本地文件类型 "+info.Mode().Type().String())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if info.Size() == 0 {
|
||||||
|
reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if firstUploadErr == nil {
|
||||||
|
firstUploadErr = err
|
||||||
|
}
|
||||||
|
reportUploadFailure(&summary, options, localPath, remotePath, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return summary, err
|
||||||
|
}
|
||||||
|
if summary.Failed > 0 {
|
||||||
|
return summary, fmt.Errorf("%d files failed to upload; first failure: %w", summary.Failed, firstUploadErr)
|
||||||
|
}
|
||||||
|
return summary, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, localPath, reason string) {
|
||||||
|
summary.Skipped++
|
||||||
|
if options.Skipped != nil {
|
||||||
|
options.Skipped(localPath, reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportUploadFailure(summary *uploadPathSummary, options uploadPathOptions, localPath, remotePath string, err error) {
|
||||||
|
summary.Failed++
|
||||||
|
if options.Failed != nil {
|
||||||
|
options.Failed(localPath, remotePath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error {
|
||||||
|
completeReported := false
|
||||||
|
if progress != nil && info.Size() > 0 {
|
||||||
|
progress(localPath, remotePath, 0, info.Size())
|
||||||
|
}
|
||||||
|
_, err := client.UploadFile(ctx, localPath, remotePath, func(uploaded, total int64) {
|
||||||
|
completeReported = uploaded >= total
|
||||||
|
if progress != nil {
|
||||||
|
progress(localPath, remotePath, uploaded, total)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("upload %s to %s: %w", localPath, remotePath, err)
|
||||||
|
}
|
||||||
|
if progress != nil && info.Size() > 0 && !completeReported {
|
||||||
|
progress(localPath, remotePath, info.Size(), info.Size())
|
||||||
|
}
|
||||||
|
summary.Files++
|
||||||
|
summary.Bytes += info.Size()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanUploadRemotePath(name string) string {
|
||||||
|
return pathpkg.Clean("/" + strings.TrimPrefix(name, "/"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadProgressPath(remoteDestination, remotePath string) string {
|
||||||
|
root := cleanUploadRemotePath(remoteDestination)
|
||||||
|
if relative := strings.TrimPrefix(remotePath, root+"/"); relative != remotePath {
|
||||||
|
return relative
|
||||||
|
}
|
||||||
|
return pathpkg.Base(remotePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeUploadExcludes(patterns []string) ([]string, error) {
|
||||||
|
result := make([]string, 0, len(patterns))
|
||||||
|
for _, pattern := range patterns {
|
||||||
|
pattern = filepath.ToSlash(pattern)
|
||||||
|
pattern = strings.TrimPrefix(pattern, "./")
|
||||||
|
pattern = strings.Trim(pattern, "/")
|
||||||
|
if pattern == "" {
|
||||||
|
return nil, errors.New("exclude pattern cannot be empty")
|
||||||
|
}
|
||||||
|
for _, component := range strings.Split(pattern, "/") {
|
||||||
|
if component == "**" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := pathpkg.Match(component, ""); err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid exclude pattern %q: %w", pattern, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = append(result, pattern)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchesUploadExclude(relative string, patterns []string) bool {
|
||||||
|
relative = strings.Trim(filepath.ToSlash(relative), "/")
|
||||||
|
for _, pattern := range patterns {
|
||||||
|
if !strings.Contains(pattern, "/") {
|
||||||
|
matched, _ := pathpkg.Match(pattern, pathpkg.Base(relative))
|
||||||
|
if matched {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if matchUploadPathSegments(strings.Split(pattern, "/"), strings.Split(relative, "/")) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchUploadPathSegments(pattern, name []string) bool {
|
||||||
|
type state struct{ pattern, name int }
|
||||||
|
memo := make(map[state]bool)
|
||||||
|
seen := make(map[state]bool)
|
||||||
|
var match func(int, int) bool
|
||||||
|
match = func(patternIndex, nameIndex int) bool {
|
||||||
|
key := state{patternIndex, nameIndex}
|
||||||
|
if seen[key] {
|
||||||
|
return memo[key]
|
||||||
|
}
|
||||||
|
seen[key] = true
|
||||||
|
var result bool
|
||||||
|
switch {
|
||||||
|
case patternIndex == len(pattern):
|
||||||
|
result = nameIndex == len(name)
|
||||||
|
case pattern[patternIndex] == "**":
|
||||||
|
result = match(patternIndex+1, nameIndex) || (nameIndex < len(name) && match(patternIndex, nameIndex+1))
|
||||||
|
case nameIndex < len(name):
|
||||||
|
segmentMatches, _ := pathpkg.Match(pattern[patternIndex], name[nameIndex])
|
||||||
|
result = segmentMatches && match(patternIndex+1, nameIndex+1)
|
||||||
|
}
|
||||||
|
memo[key] = result
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
return match(0, 0)
|
||||||
|
}
|
||||||
265
cmd/bdrclone/upload_path_test.go
Normal file
265
cmd/bdrclone/upload_path_test.go
Normal file
@@ -0,0 +1,265 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingUploadClient struct {
|
||||||
|
directories []string
|
||||||
|
files map[string][]byte
|
||||||
|
failures map[string]error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *recordingUploadClient) MkdirAll(_ context.Context, remotePath string) error {
|
||||||
|
c.directories = append(c.directories, remotePath)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *recordingUploadClient) EnsureDirectory(_ context.Context, remotePath string) error {
|
||||||
|
c.directories = append(c.directories, remotePath)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *recordingUploadClient) UploadFile(_ context.Context, localPath, remotePath string, progress baidu.UploadProgress) (baidu.File, error) {
|
||||||
|
content, err := os.ReadFile(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return baidu.File{}, err
|
||||||
|
}
|
||||||
|
if c.files == nil {
|
||||||
|
c.files = make(map[string][]byte)
|
||||||
|
}
|
||||||
|
if err := c.failures[remotePath]; err != nil {
|
||||||
|
return baidu.File{}, err
|
||||||
|
}
|
||||||
|
c.files[remotePath] = content
|
||||||
|
if progress != nil {
|
||||||
|
progress(int64(len(content)), int64(len(content)))
|
||||||
|
}
|
||||||
|
return baidu.File{Path: remotePath, Size: int64(len(content))}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadLocalDirectoryPreservesRelativePaths(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "top.txt"), []byte("top"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(root, "empty-dir"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(root, "nested"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "nested", "file.bin"), []byte("nested"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "zero.txt"), nil, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink("top.txt", filepath.Join(root, "top-link")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &recordingUploadClient{}
|
||||||
|
var skipped []string
|
||||||
|
summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac/", uploadPathOptions{Skipped: func(localPath, _ string) {
|
||||||
|
skipped = append(skipped, filepath.Base(localPath))
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if summary.Files != 2 || summary.Directories != 3 || summary.Skipped != 2 || summary.Bytes != 9 {
|
||||||
|
t.Fatalf("summary = %+v", summary)
|
||||||
|
}
|
||||||
|
if want := []string{"/backup/mac", "/backup/mac/empty-dir", "/backup/mac/nested"}; !reflect.DeepEqual(client.directories, want) {
|
||||||
|
t.Fatalf("directories = %q, want %q", client.directories, want)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(client.files["/backup/mac/top.txt"], []byte("top")) {
|
||||||
|
t.Fatalf("top.txt = %q", client.files["/backup/mac/top.txt"])
|
||||||
|
}
|
||||||
|
if !bytes.Equal(client.files["/backup/mac/nested/file.bin"], []byte("nested")) {
|
||||||
|
t.Fatalf("nested/file.bin = %q", client.files["/backup/mac/nested/file.bin"])
|
||||||
|
}
|
||||||
|
if want := []string{"top-link", "zero.txt"}; !reflect.DeepEqual(skipped, want) {
|
||||||
|
t.Fatalf("skipped = %q, want %q", skipped, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadLocalFileKeepsDestinationFilename(t *testing.T) {
|
||||||
|
localPath := filepath.Join(t.TempDir(), "source.txt")
|
||||||
|
if err := os.WriteFile(localPath, []byte("content"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client := &recordingUploadClient{}
|
||||||
|
summary, err := uploadLocalPath(context.Background(), client, localPath, "backup/renamed.txt", uploadPathOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if summary.Files != 1 || summary.Directories != 0 || summary.Bytes != 7 {
|
||||||
|
t.Fatalf("summary = %+v", summary)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(client.files["/backup/renamed.txt"], []byte("content")) {
|
||||||
|
t.Fatalf("uploaded files = %v", client.files)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadLocalDirectoryHonorsExcludePatterns(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"keep.txt": "keep",
|
||||||
|
"scratch.tmp": "temporary",
|
||||||
|
filepath.Join(".Trash", "old"): "trash",
|
||||||
|
filepath.Join("Library", "keep"): "library",
|
||||||
|
filepath.Join("Library", "Caches", "cache.db"): "cache",
|
||||||
|
}
|
||||||
|
for name, content := range files {
|
||||||
|
localPath := filepath.Join(root, name)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(localPath, []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &recordingUploadClient{}
|
||||||
|
summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{
|
||||||
|
Excludes: []string{".Trash", "Library/Caches/**", "*.tmp"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if summary.Files != 2 || summary.Directories != 2 || summary.Skipped != 3 {
|
||||||
|
t.Fatalf("summary = %+v", summary)
|
||||||
|
}
|
||||||
|
if want := []string{"/backup/mac", "/backup/mac/Library"}; !reflect.DeepEqual(client.directories, want) {
|
||||||
|
t.Fatalf("directories = %q, want %q", client.directories, want)
|
||||||
|
}
|
||||||
|
if _, ok := client.files["/backup/mac/keep.txt"]; !ok {
|
||||||
|
t.Fatalf("keep.txt was not uploaded: %v", client.files)
|
||||||
|
}
|
||||||
|
if _, ok := client.files["/backup/mac/Library/keep"]; !ok {
|
||||||
|
t.Fatalf("Library/keep was not uploaded: %v", client.files)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadLocalDirectoryContinuesAfterReadError(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(root, "blocked"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "good.txt"), []byte("good"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entryByName := make(map[string]os.DirEntry, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
entryByName[entry.Name()] = entry
|
||||||
|
}
|
||||||
|
rootInfo, err := os.Stat(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
walk := func(_ string, visit fs.WalkDirFunc) error {
|
||||||
|
if err := visit(root, fs.FileInfoToDirEntry(rootInfo), nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
blockedPath := filepath.Join(root, "blocked")
|
||||||
|
if err := visit(blockedPath, entryByName["blocked"], fs.ErrPermission); err != nil && err != fs.SkipDir {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return visit(filepath.Join(root, "good.txt"), entryByName["good.txt"], nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &recordingUploadClient{}
|
||||||
|
var skipped []string
|
||||||
|
summary, err := uploadLocalDirectory(context.Background(), client, root, "/backup/mac", uploadPathOptions{Skipped: func(localPath, reason string) {
|
||||||
|
skipped = append(skipped, filepath.Base(localPath)+":"+reason)
|
||||||
|
}}, walk)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if summary.Files != 1 || summary.Skipped != 1 {
|
||||||
|
t.Fatalf("summary = %+v", summary)
|
||||||
|
}
|
||||||
|
if len(skipped) != 1 || !strings.Contains(skipped[0], "blocked:无法读取: permission denied") {
|
||||||
|
t.Fatalf("skipped = %q", skipped)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(client.files["/backup/mac/good.txt"], []byte("good")) {
|
||||||
|
t.Fatalf("good.txt was not uploaded: %v", client.files)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadLocalDirectoryContinuesAfterUploadFailure(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "a-fails.txt"), []byte("failed"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "b-succeeds.txt"), []byte("success"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client := &recordingUploadClient{failures: map[string]error{
|
||||||
|
"/backup/mac/a-fails.txt": errors.New("gateway timeout"),
|
||||||
|
}}
|
||||||
|
var failed []string
|
||||||
|
summary, err := uploadLocalPath(context.Background(), client, root, "/backup/mac", uploadPathOptions{Failed: func(localPath, remotePath string, err error) {
|
||||||
|
failed = append(failed, filepath.Base(localPath)+":"+remotePath+":"+err.Error())
|
||||||
|
}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "1 files failed to upload") {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
if summary.Files != 1 || summary.Failed != 1 || summary.Bytes != 7 {
|
||||||
|
t.Fatalf("summary = %+v", summary)
|
||||||
|
}
|
||||||
|
if len(failed) != 1 || !strings.Contains(failed[0], "a-fails.txt:/backup/mac/a-fails.txt:") || !strings.Contains(failed[0], "gateway timeout") {
|
||||||
|
t.Fatalf("failed = %q", failed)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(client.files["/backup/mac/b-succeeds.txt"], []byte("success")) {
|
||||||
|
t.Fatalf("b-succeeds.txt was not uploaded: %v", client.files)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadExcludePatternMatching(t *testing.T) {
|
||||||
|
patterns, err := normalizeUploadExcludes([]string{".Trash/", "Library/Caches/**", "**/*.tmp"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, want := range map[string]bool{
|
||||||
|
".Trash": true,
|
||||||
|
"nested/.Trash": true,
|
||||||
|
"Library/Caches": true,
|
||||||
|
"Library/Caches/data/file": true,
|
||||||
|
"root.tmp": true,
|
||||||
|
"nested/file.tmp": true,
|
||||||
|
"Library/Application/file": false,
|
||||||
|
"nested/file.tmp.keep": false,
|
||||||
|
} {
|
||||||
|
if got := matchesUploadExclude(name, patterns); got != want {
|
||||||
|
t.Errorf("matchesUploadExclude(%q) = %v, want %v", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := normalizeUploadExcludes([]string{"bad["}); err == nil {
|
||||||
|
t.Fatal("invalid exclude pattern was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadProgressPath(t *testing.T) {
|
||||||
|
if got := uploadProgressPath("/backup/mac/", "/backup/mac/Library/file.txt"); got != "Library/file.txt" {
|
||||||
|
t.Fatalf("progress path = %q", got)
|
||||||
|
}
|
||||||
|
if got := uploadProgressPath("/backup/file.txt", "/backup/file.txt"); got != "file.txt" {
|
||||||
|
t.Fatalf("single-file progress path = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,7 @@ type Client struct {
|
|||||||
root string
|
root string
|
||||||
uploadParts int
|
uploadParts int
|
||||||
partSize int64
|
partSize int64
|
||||||
|
uploadRetryDelay func(int) time.Duration
|
||||||
downloadMu sync.Mutex
|
downloadMu sync.Mutex
|
||||||
downloadURL map[int64]cachedDownloadURL
|
downloadURL map[int64]cachedDownloadURL
|
||||||
|
|
||||||
@@ -75,6 +76,7 @@ func New(cfg *config.Config, opts ...Option) *Client {
|
|||||||
apiBase: defaultAPIBase, oauthBase: defaultOAuthBase, uploadBase: defaultUploadBase,
|
apiBase: defaultAPIBase, oauthBase: defaultOAuthBase, uploadBase: defaultUploadBase,
|
||||||
userAgent: cfg.UserAgent, root: cfg.Root,
|
userAgent: cfg.UserAgent, root: cfg.Root,
|
||||||
uploadParts: cfg.UploadParts, partSize: cfg.PartSize,
|
uploadParts: cfg.UploadParts, partSize: cfg.PartSize,
|
||||||
|
uploadRetryDelay: defaultUploadRetryDelay,
|
||||||
downloadURL: make(map[int64]cachedDownloadURL),
|
downloadURL: make(map[int64]cachedDownloadURL),
|
||||||
clientID: cfg.ClientID, secret: cfg.ClientSecret,
|
clientID: cfg.ClientID, secret: cfg.ClientSecret,
|
||||||
accessToken: cfg.AccessToken, refresh: cfg.RefreshToken, expiresAt: cfg.ExpiresAt,
|
accessToken: cfg.AccessToken, refresh: cfg.RefreshToken, expiresAt: cfg.ExpiresAt,
|
||||||
|
|||||||
@@ -9,7 +9,10 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
|
"reflect"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -203,6 +206,109 @@ func TestUploadMultipartFlow(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUploadPartRetriesGatewayTimeout(t *testing.T) {
|
||||||
|
file, err := os.CreateTemp(t.TempDir(), "retry-upload-*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
content := []byte("retry payload")
|
||||||
|
if _, err := file.Write(content); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
attempts := 0
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/rest/2.0/pcs/superfile2" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
attempts++
|
||||||
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
if attempts < 4 {
|
||||||
|
http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprint(w, `{"md5":"ok"}`)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL))
|
||||||
|
client.uploadRetryDelay = func(int) time.Duration { return 0 }
|
||||||
|
if err := client.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-1", 0, 0, int64(len(content))); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if attempts != 4 {
|
||||||
|
t.Fatalf("attempts = %d, want 4", attempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
failureAttempts := 0
|
||||||
|
failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
failureAttempts++
|
||||||
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
|
||||||
|
}))
|
||||||
|
defer failureServer.Close()
|
||||||
|
failureClient := New(testConfig("token"), WithEndpoints(failureServer.URL, failureServer.URL, failureServer.URL))
|
||||||
|
failureClient.uploadRetryDelay = func(int) time.Duration { return 0 }
|
||||||
|
err = failureClient.uploadPartWithRetry(context.Background(), file, "/apps/bdrclone/retry.bin", "upload-2", 0, 0, int64(len(content)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "after 6 attempts") {
|
||||||
|
t.Fatalf("error = %v", err)
|
||||||
|
}
|
||||||
|
if failureAttempts != maxUploadAttempts {
|
||||||
|
t.Fatalf("attempts = %d, want %d", failureAttempts, maxUploadAttempts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMkdirAllCreatesMissingRemoteDirectories(t *testing.T) {
|
||||||
|
directories := map[string]bool{"/apps/bdrclone": true}
|
||||||
|
var created []string
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/rest/2.0/xpan/file" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch r.URL.Query().Get("method") {
|
||||||
|
case "list":
|
||||||
|
parent := r.URL.Query().Get("dir")
|
||||||
|
entries := make([]File, 0)
|
||||||
|
for directory := range directories {
|
||||||
|
if directory != parent && path.Dir(directory) == parent {
|
||||||
|
entries = append(entries, File{Path: directory, ServerFilename: path.Base(directory), IsDir: 1})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{"errno": 0, "list": entries})
|
||||||
|
case "create":
|
||||||
|
if err := r.ParseForm(); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
directory := r.Form.Get("path")
|
||||||
|
if r.Form.Get("isdir") != "1" {
|
||||||
|
t.Errorf("isdir = %q", r.Form.Get("isdir"))
|
||||||
|
}
|
||||||
|
directories[directory] = true
|
||||||
|
created = append(created, directory)
|
||||||
|
_ = json.NewEncoder(w).Encode(File{Path: directory, ServerFilename: path.Base(directory), IsDir: 1})
|
||||||
|
default:
|
||||||
|
http.Error(w, "unexpected method", http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
client := New(testConfig("token"), WithEndpoints(server.URL, server.URL, server.URL))
|
||||||
|
if err := client.MkdirAll(context.Background(), "/backup/mac/photos"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := client.MkdirAll(context.Background(), "/backup/mac/photos"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{"/apps/bdrclone/backup", "/apps/bdrclone/backup/mac", "/apps/bdrclone/backup/mac/photos"}
|
||||||
|
if !reflect.DeepEqual(created, want) {
|
||||||
|
t.Fatalf("created = %q, want %q", created, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func testConfig(token string) *config.Config {
|
func testConfig(token string) *config.Config {
|
||||||
return &config.Config{
|
return &config.Config{
|
||||||
ClientID: "client", ClientSecret: "secret", AccessToken: token,
|
ClientID: "client", ClientSecret: "secret", AccessToken: token,
|
||||||
|
|||||||
@@ -66,6 +66,38 @@ func (c *Client) Mkdir(ctx context.Context, name string) (File, error) {
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Client) MkdirAll(ctx context.Context, name string) error {
|
||||||
|
target := path.Clean("/" + strings.TrimPrefix(name, "/"))
|
||||||
|
if target == "/" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
current := "/"
|
||||||
|
for _, component := range strings.Split(strings.TrimPrefix(target, "/"), "/") {
|
||||||
|
current = path.Join(current, component)
|
||||||
|
if err := c.EnsureDirectory(ctx, current); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) EnsureDirectory(ctx context.Context, name string) error {
|
||||||
|
entry, err := c.Stat(ctx, name)
|
||||||
|
if err == nil {
|
||||||
|
if !entry.IsDirectory() {
|
||||||
|
return fmt.Errorf("remote path exists and is not a directory: %s", name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrNotFound) {
|
||||||
|
return fmt.Errorf("check remote directory %s: %w", name, err)
|
||||||
|
}
|
||||||
|
if _, err := c.Mkdir(ctx, name); err != nil {
|
||||||
|
return fmt.Errorf("create remote directory %s: %w", name, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Client) Delete(ctx context.Context, name string) error {
|
func (c *Client) Delete(ctx context.Context, name string) error {
|
||||||
remote := c.RemotePath(name)
|
remote := c.RemotePath(name)
|
||||||
if remote == c.root {
|
if remote == c.root {
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ const (
|
|||||||
vipPartSize = int64(16 << 20)
|
vipPartSize = int64(16 << 20)
|
||||||
svipPartSize = int64(32 << 20)
|
svipPartSize = int64(32 << 20)
|
||||||
maxPartCount = 2048
|
maxPartCount = 2048
|
||||||
|
maxUploadAttempts = 6
|
||||||
)
|
)
|
||||||
|
|
||||||
type UploadProgress func(uploaded, total int64)
|
type UploadProgress func(uploaded, total int64)
|
||||||
@@ -247,7 +248,7 @@ func (c *Client) uploadPartsParallel(ctx context.Context, file *os.File, remote,
|
|||||||
|
|
||||||
func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error {
|
func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote, uploadID string, part int, offset, size int64) error {
|
||||||
var lastErr error
|
var lastErr error
|
||||||
for attempt := 0; attempt < 3; attempt++ {
|
for attempt := 0; attempt < maxUploadAttempts; attempt++ {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -256,13 +257,18 @@ func (c *Client) uploadPartWithRetry(ctx context.Context, file *os.File, remote,
|
|||||||
if lastErr == nil {
|
if lastErr == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if attempt < 2 {
|
if attempt < maxUploadAttempts-1 {
|
||||||
if err := sleepContext(ctx, time.Duration(1<<attempt)*time.Second); err != nil {
|
if err := sleepContext(ctx, c.uploadRetryDelay(attempt)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return lastErr
|
return fmt.Errorf("upload failed after %d attempts: %w", maxUploadAttempts, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultUploadRetryDelay(attempt int) time.Duration {
|
||||||
|
delay := time.Second * time.Duration(1<<attempt)
|
||||||
|
return min(delay, 30*time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) uploadPart(ctx context.Context, section *io.SectionReader, filename, remote, uploadID string, part int) error {
|
func (c *Client) uploadPart(ctx context.Context, section *io.SectionReader, filename, remote, uploadID string, part int) error {
|
||||||
|
|||||||
Reference in New Issue
Block a user