bdrclone/cmd/bdrclone/upload_path.go

package main

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	pathpkg "path"
	"path/filepath"
	"strings"

	"gitea.dddbg.com/youbin/bdrclone/internal/baidu"
)

type uploadPathClient interface {
	MkdirAll(context.Context, string) error
	EnsureDirectory(context.Context, string) error
	UploadFile(context.Context, string, string, baidu.UploadProgress) (baidu.File, error)
}

type uploadPathSummary struct {
	Files       int
	Directories int
	Skipped     int
	Failed      int
	Bytes       int64
}

type uploadPathProgress func(localPath, remotePath string, uploaded, total int64)
type uploadPathSkipped func(localPath, reason string)
type uploadPathFailed func(localPath, remotePath string, err error)

type uploadPathOptions struct {
	Excludes []string
	Progress uploadPathProgress
	Skipped  uploadPathSkipped
	Failed   uploadPathFailed
}

type uploadWalkDir func(string, fs.WalkDirFunc) error

func uploadLocalPath(ctx context.Context, client uploadPathClient, localSource, remoteDestination string, options uploadPathOptions) (uploadPathSummary, error) {
	excludes, err := normalizeUploadExcludes(options.Excludes)
	if err != nil {
		return uploadPathSummary{}, err
	}
	info, err := os.Stat(localSource)
	if err != nil {
		return uploadPathSummary{}, fmt.Errorf("stat local source: %w", err)
	}
	remoteDestination = cleanUploadRemotePath(remoteDestination)
	if !info.IsDir() {
		if !info.Mode().IsRegular() {
			return uploadPathSummary{}, fmt.Errorf("unsupported local file type: %s", localSource)
		}
		var summary uploadPathSummary
		if matchesUploadExclude(filepath.Base(localSource), excludes) {
			reportUploadSkip(&summary, options, localSource, "已被 --exclude 规则排除")
			return summary, nil
		}
		err := uploadRegularFile(ctx, client, localSource, remoteDestination, info, options.Progress, &summary)
		return summary, err
	}

	root, err := filepath.Abs(localSource)
	if err != nil {
		return uploadPathSummary{}, fmt.Errorf("resolve local source: %w", err)
	}
	root, err = filepath.EvalSymlinks(root)
	if err != nil {
		return uploadPathSummary{}, fmt.Errorf("resolve local source links: %w", err)
	}
	options.Excludes = excludes
	return uploadLocalDirectory(ctx, client, root, remoteDestination, options, filepath.WalkDir)
}

func uploadLocalDirectory(ctx context.Context, client uploadPathClient, root, remoteDestination string, options uploadPathOptions, walk uploadWalkDir) (uploadPathSummary, error) {
	if err := client.MkdirAll(ctx, remoteDestination); err != nil {
		return uploadPathSummary{}, err
	}
	summary := uploadPathSummary{Directories: 1}
	var firstUploadErr error
	err := walk(root, func(localPath string, entry fs.DirEntry, walkErr error) error {
		if walkErr != nil {
			if err := ctx.Err(); err != nil {
				return err
			}
			reportUploadSkip(&summary, options, localPath, "无法读取: "+walkErr.Error())
			if entry != nil && entry.IsDir() {
				return fs.SkipDir
			}
			return nil
		}
		if err := ctx.Err(); err != nil {
			return err
		}
		if localPath == root {
			return nil
		}
		relative, err := filepath.Rel(root, localPath)
		if err != nil {
			return fmt.Errorf("make local path relative: %w", err)
		}
		relative = filepath.ToSlash(relative)
		if matchesUploadExclude(relative, options.Excludes) {
			reportUploadSkip(&summary, options, localPath, "已被 --exclude 规则排除")
			if entry.IsDir() {
				return fs.SkipDir
			}
			return nil
		}
		remotePath := pathpkg.Join(remoteDestination, relative)
		if entry.IsDir() {
			if err := client.EnsureDirectory(ctx, remotePath); err != nil {
				return err
			}
			summary.Directories++
			return nil
		}
		if entry.Type()&os.ModeSymlink != 0 {
			reportUploadSkip(&summary, options, localPath, "符号链接不会被跟随")
			return nil
		}
		info, err := entry.Info()
		if err != nil {
			reportUploadSkip(&summary, options, localPath, "无法读取文件信息: "+err.Error())
			return nil
		}
		if !info.Mode().IsRegular() {
			reportUploadSkip(&summary, options, localPath, "不支持的本地文件类型 "+info.Mode().Type().String())
			return nil
		}
		if info.Size() == 0 {
			reportUploadSkip(&summary, options, localPath, "百度开放 API 不支持空文件")
			return nil
		}
		if err := uploadRegularFile(ctx, client, localPath, remotePath, info, options.Progress, &summary); err != nil {
			if ctx.Err() != nil {
				return ctx.Err()
			}
			if firstUploadErr == nil {
				firstUploadErr = err
			}
			reportUploadFailure(&summary, options, localPath, remotePath, err)
		}
		return nil
	})
	if err != nil {
		return summary, err
	}
	if summary.Failed > 0 {
		return summary, fmt.Errorf("%d files failed to upload; first failure: %w", summary.Failed, firstUploadErr)
	}
	return summary, nil
}

func reportUploadSkip(summary *uploadPathSummary, options uploadPathOptions, localPath, reason string) {
	summary.Skipped++
	if options.Skipped != nil {
		options.Skipped(localPath, reason)
	}
}

func reportUploadFailure(summary *uploadPathSummary, options uploadPathOptions, localPath, remotePath string, err error) {
	summary.Failed++
	if options.Failed != nil {
		options.Failed(localPath, remotePath, err)
	}
}

func uploadRegularFile(ctx context.Context, client uploadPathClient, localPath, remotePath string, info fs.FileInfo, progress uploadPathProgress, summary *uploadPathSummary) error {
	completeReported := false
	if progress != nil && info.Size() > 0 {
		progress(localPath, remotePath, 0, info.Size())
	}
	_, err := client.UploadFile(ctx, localPath, remotePath, func(uploaded, total int64) {
		completeReported = uploaded >= total
		if progress != nil {
			progress(localPath, remotePath, uploaded, total)
		}
	})
	if err != nil {
		return fmt.Errorf("upload %s to %s: %w", localPath, remotePath, err)
	}
	if progress != nil && info.Size() > 0 && !completeReported {
		progress(localPath, remotePath, info.Size(), info.Size())
	}
	summary.Files++
	summary.Bytes += info.Size()
	return nil
}

func cleanUploadRemotePath(name string) string {
	return pathpkg.Clean("/" + strings.TrimPrefix(name, "/"))
}

func uploadProgressPath(remoteDestination, remotePath string) string {
	root := cleanUploadRemotePath(remoteDestination)
	if relative := strings.TrimPrefix(remotePath, root+"/"); relative != remotePath {
		return relative
	}
	return pathpkg.Base(remotePath)
}

func normalizeUploadExcludes(patterns []string) ([]string, error) {
	result := make([]string, 0, len(patterns))
	for _, pattern := range patterns {
		pattern = filepath.ToSlash(pattern)
		pattern = strings.TrimPrefix(pattern, "./")
		pattern = strings.Trim(pattern, "/")
		if pattern == "" {
			return nil, errors.New("exclude pattern cannot be empty")
		}
		for _, component := range strings.Split(pattern, "/") {
			if component == "**" {
				continue
			}
			if _, err := pathpkg.Match(component, ""); err != nil {
				return nil, fmt.Errorf("invalid exclude pattern %q: %w", pattern, err)
			}
		}
		result = append(result, pattern)
	}
	return result, nil
}

func matchesUploadExclude(relative string, patterns []string) bool {
	relative = strings.Trim(filepath.ToSlash(relative), "/")
	for _, pattern := range patterns {
		if !strings.Contains(pattern, "/") {
			matched, _ := pathpkg.Match(pattern, pathpkg.Base(relative))
			if matched {
				return true
			}
			continue
		}
		if matchUploadPathSegments(strings.Split(pattern, "/"), strings.Split(relative, "/")) {
			return true
		}
	}
	return false
}

func matchUploadPathSegments(pattern, name []string) bool {
	type state struct{ pattern, name int }
	memo := make(map[state]bool)
	seen := make(map[state]bool)
	var match func(int, int) bool
	match = func(patternIndex, nameIndex int) bool {
		key := state{patternIndex, nameIndex}
		if seen[key] {
			return memo[key]
		}
		seen[key] = true
		var result bool
		switch {
		case patternIndex == len(pattern):
			result = nameIndex == len(name)
		case pattern[patternIndex] == "**":
			result = match(patternIndex+1, nameIndex) || (nameIndex < len(name) && match(patternIndex, nameIndex+1))
		case nameIndex < len(name):
			segmentMatches, _ := pathpkg.Match(pattern[patternIndex], name[nameIndex])
			result = segmentMatches && match(patternIndex+1, nameIndex+1)
		}
		memo[key] = result
		return result
	}
	return match(0, 0)
}