dukai/robotfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add file copy and rename

Browse Source
This commit is contained in:
dukai 2025-05-22 18:28:36 +08:00
parent d83ad12683
commit f02b72e766
6 changed files with 87 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Makefile
View File

@ -30,6 +30,11 @@ else
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME) ./
endif
.PHONY: build-linux
build-linux:
@echo "Building for Linux..."
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME) ./
.PHONY: clean
clean:
rm -f $(BINARY_NAME)
@ -38,7 +43,8 @@ clean:
.PHONY: help
help:
@echo "Available targets:"
@echo " all - Generate protobuf files and build for current platform (default)"
@echo " proto - Generate protobuf files"
@echo " build - Build for current platform"
@echo " clean - Remove built binaries and generated protobuf files"
@echo " all - Generate protobuf files and build for current platform (default)"
@echo " proto - Generate protobuf files"
@echo " build - Build for current platform"
@echo " build-linux - Build for linux platform"
@echo " clean - Remove built binaries and generated protobuf files"

11
api_file.go
View File

@ -148,5 +148,16 @@ func (s *Service) HandleCopyFile(
req *httpserver.Request,
resp *httpserver.Response,
) *httpserver.Response {
params := req.Binded.(*CopyParams)
isDir := params.IsDir
srcPath := utils.FullPath(utils.NormalizePath(params.SrcPath))
dstPath := utils.FullPath(utils.NormalizePath(params.DstPath))
err := s.FileSystemManager.CopyFile(req.Context(), srcPath, dstPath, isDir)
if err != nil {
logger.Error("copy %s to %s: %v", srcPath, dstPath, err)
return resp.InternalServerError("copy file failed, " + err.Error())
}
return resp.NoContent()
}

2
engine/engine.go
View File

@ -29,7 +29,7 @@ func (e *Engine) Start() error {
if err := meta.Initialize(utils.GetViper(), "redis."); err != nil {
return fmt.Errorf("failed to initialize meta store: %v", err)
}
filesystem := NewFileSystemManager(meta, object)
filesystem := NewFileSystemManager(meta, object, utils.GetViper().GetString("robotfs.data_path"))
e.FileSystemManager = filesystem
return nil

55
engine/filesystem_manager.go
View File

@ -15,12 +15,14 @@ type FileSystemManager struct {
sync.RWMutex
meta store.MetaStore
storage *StorageManager
root string
}
func NewFileSystemManager(meta store.MetaStore, storage *StorageManager) *FileSystemManager {
func NewFileSystemManager(meta store.MetaStore, storage *StorageManager, root string) *FileSystemManager {
return &FileSystemManager{
meta: meta,
storage: storage,
root: root,
}
}
@ -229,12 +231,15 @@ func (f *FileSystemManager) RenameFile(ctx context.Context, srcPath, dstPath uti
}
}
newS3Key := dstPath.ToS3Key()
if err := f.storage.CopyObject(srcEntry.S3Key, newS3Key); err != nil {
return fmt.Errorf("rename file failed: %v", err)
if isDir {
return fmt.Errorf("rename not file")
}
newEntry := utils.NewFileEntry(dstPath, newS3Key, srcEntry.Size, srcEntry.ContentType, srcEntry.Etag, srcEntry.VersionID)
if err := utils.Move(f.root, string(srcPath), string(dstPath)); err != nil {
return err
}
newEntry := utils.NewFileEntry(dstPath, dstPath.ToS3Key(), srcEntry.Size, srcEntry.ContentType, srcEntry.Etag, srcEntry.VersionID)
if err := f.meta.InsertEntry(ctx, newEntry); err != nil {
return fmt.Errorf("insert new entry failed: %v", err)
@ -244,8 +249,44 @@ func (f *FileSystemManager) RenameFile(ctx context.Context, srcPath, dstPath uti
return fmt.Errorf("delete source entry failed: %v", err)
}
if err := f.storage.DeleteObject(srcEntry.S3Key); err != nil {
return fmt.Errorf("delete old file failed: %v", err)
return nil
}
func (f *FileSystemManager) CopyFile(ctx context.Context, srcPath, dstPath utils.FullPath, isDir bool) error {
f.Lock()
defer f.Unlock()
if string(srcPath) == "/" || string(dstPath) == "/" {
return fmt.Errorf("cannot copy root")
}
srcEntry, err := f.FindEntry(ctx, srcPath)
if err != nil {
return fmt.Errorf("find src file failed: %v", err)
}
if dstEntry, _ := f.FindEntry(ctx, dstPath); dstEntry != nil {
return fmt.Errorf("dst file %s already exists", dstPath)
}
parentDir, _ := dstPath.DirAndName()
if parentDir != "/" {
if parentEntry, _ := f.FindEntry(ctx, utils.FullPath(parentDir)); parentEntry == nil {
return fmt.Errorf("parent directory %s does not exist", parentDir)
}
}
if isDir {
return fmt.Errorf("copy not file")
}
if err := utils.Clone(f.root, string(srcPath), string(dstPath)); err != nil {
return err
}
newEntry := utils.NewFileEntry(dstPath, dstPath.ToS3Key(), srcEntry.Size, srcEntry.ContentType, srcEntry.Etag, srcEntry.VersionID)
if err := f.meta.InsertEntry(ctx, newEntry); err != nil {
return fmt.Errorf("insert new entry failed: %v", err)
}
return nil

28
engine/storage_manager.go
View File

@ -55,7 +55,12 @@ func NewStorageManager(s3Config *S3Config) (*StorageManager, error) {
s3Client := s3.New(awsSession)
uploader := s3manager.NewUploader(awsSession)
uploader := s3manager.NewUploader(awsSession, func(u *s3manager.Uploader) {
u.PartSize = 100 * 1024 * 1024
u.Concurrency = 5
u.LeavePartsOnError = false
u.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(25 * 1024 * 1024)
})
downloader := s3manager.NewDownloader(awsSession)
return &StorageManager{
@ -185,24 +190,3 @@ func (sm *StorageManager) CopyObject(oldS3key, newS3Key string) error {
Key: aws.String(newS3Key),
})
}
func (sm *StorageManager) ListObjects(prefix string) ([]string, error) {
var result []string
input := &s3.ListObjectsV2Input{
Bucket: aws.String(sm.bucketName),
Prefix: aws.String(prefix),
}
err := sm.s3Client.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, obj := range page.Contents {
result = append(result, *obj.Key)
}
return !lastPage
})
if err != nil {
return nil, err
}
return result, nil
}

13
utils/operation.go
View File

@ -5,8 +5,8 @@ import (
"os/exec"
)
func CloneFile(src, dst string) error {
cmd := exec.Command("juicefs", "clone", src, dst)
func Clone(root, src, dst string) error {
cmd := exec.Command("juicefs", "clone", root+src, root+dst)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("clone failed: %v, output: %s", err, string(output))
@ -14,3 +14,12 @@ func CloneFile(src, dst string) error {
return nil
}
func Move(root, src, dst string) error {
cmd := exec.Command("mv", root+src, root+dst)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("move failed: %v, output: %s", err, string(output))
}
return nil
}