diff --git a/Makefile b/Makefile index 95490ae..a6f8081 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,11 @@ else GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME) ./ endif +.PHONY: build-linux +build-linux: + @echo "Building for Linux..." + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME) ./ + .PHONY: clean clean: rm -f $(BINARY_NAME) @@ -38,7 +43,8 @@ clean: .PHONY: help help: @echo "Available targets:" - @echo " all - Generate protobuf files and build for current platform (default)" - @echo " proto - Generate protobuf files" - @echo " build - Build for current platform" - @echo " clean - Remove built binaries and generated protobuf files" + @echo " all - Generate protobuf files and build for current platform (default)" + @echo " proto - Generate protobuf files" + @echo " build - Build for current platform" + @echo " build-linux - Build for linux platform" + @echo " clean - Remove built binaries and generated protobuf files" diff --git a/api_file.go b/api_file.go index 176370b..d800a20 100644 --- a/api_file.go +++ b/api_file.go @@ -148,5 +148,16 @@ func (s *Service) HandleCopyFile( req *httpserver.Request, resp *httpserver.Response, ) *httpserver.Response { + params := req.Binded.(*CopyParams) + isDir := params.IsDir + srcPath := utils.FullPath(utils.NormalizePath(params.SrcPath)) + dstPath := utils.FullPath(utils.NormalizePath(params.DstPath)) + + err := s.FileSystemManager.CopyFile(req.Context(), srcPath, dstPath, isDir) + if err != nil { + logger.Error("copy %s to %s: %v", srcPath, dstPath, err) + return resp.InternalServerError("copy file failed, " + err.Error()) + } + return resp.NoContent() } diff --git a/engine/engine.go b/engine/engine.go index 2f042da..4247ee7 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -29,7 +29,7 @@ func (e *Engine) Start() error { if err := meta.Initialize(utils.GetViper(), "redis."); err != nil { return fmt.Errorf("failed to initialize meta store: %v", err) } - filesystem := NewFileSystemManager(meta, object) + filesystem := NewFileSystemManager(meta, object, utils.GetViper().GetString("robotfs.data_path")) e.FileSystemManager = filesystem return nil diff --git a/engine/filesystem_manager.go b/engine/filesystem_manager.go index 4c2ed2c..0b75163 100644 --- a/engine/filesystem_manager.go +++ b/engine/filesystem_manager.go @@ -15,12 +15,14 @@ type FileSystemManager struct { sync.RWMutex meta store.MetaStore storage *StorageManager + root string } -func NewFileSystemManager(meta store.MetaStore, storage *StorageManager) *FileSystemManager { +func NewFileSystemManager(meta store.MetaStore, storage *StorageManager, root string) *FileSystemManager { return &FileSystemManager{ meta: meta, storage: storage, + root: root, } } @@ -229,12 +231,15 @@ func (f *FileSystemManager) RenameFile(ctx context.Context, srcPath, dstPath uti } } - newS3Key := dstPath.ToS3Key() - if err := f.storage.CopyObject(srcEntry.S3Key, newS3Key); err != nil { - return fmt.Errorf("rename file failed: %v", err) + if isDir { + return fmt.Errorf("rename not file") } - newEntry := utils.NewFileEntry(dstPath, newS3Key, srcEntry.Size, srcEntry.ContentType, srcEntry.Etag, srcEntry.VersionID) + if err := utils.Move(f.root, string(srcPath), string(dstPath)); err != nil { + return err + } + + newEntry := utils.NewFileEntry(dstPath, dstPath.ToS3Key(), srcEntry.Size, srcEntry.ContentType, srcEntry.Etag, srcEntry.VersionID) if err := f.meta.InsertEntry(ctx, newEntry); err != nil { return fmt.Errorf("insert new entry failed: %v", err) @@ -244,8 +249,44 @@ func (f *FileSystemManager) RenameFile(ctx context.Context, srcPath, dstPath uti return fmt.Errorf("delete source entry failed: %v", err) } - if err := f.storage.DeleteObject(srcEntry.S3Key); err != nil { - return fmt.Errorf("delete old file failed: %v", err) + return nil +} + +func (f *FileSystemManager) CopyFile(ctx context.Context, srcPath, dstPath utils.FullPath, isDir bool) error { + f.Lock() + defer f.Unlock() + + if string(srcPath) == "/" || string(dstPath) == "/" { + return fmt.Errorf("cannot copy root") + } + + srcEntry, err := f.FindEntry(ctx, srcPath) + if err != nil { + return fmt.Errorf("find src file failed: %v", err) + } + + if dstEntry, _ := f.FindEntry(ctx, dstPath); dstEntry != nil { + return fmt.Errorf("dst file %s already exists", dstPath) + } + + parentDir, _ := dstPath.DirAndName() + if parentDir != "/" { + if parentEntry, _ := f.FindEntry(ctx, utils.FullPath(parentDir)); parentEntry == nil { + return fmt.Errorf("parent directory %s does not exist", parentDir) + } + } + + if isDir { + return fmt.Errorf("copy not file") + } + + if err := utils.Clone(f.root, string(srcPath), string(dstPath)); err != nil { + return err + } + + newEntry := utils.NewFileEntry(dstPath, dstPath.ToS3Key(), srcEntry.Size, srcEntry.ContentType, srcEntry.Etag, srcEntry.VersionID) + if err := f.meta.InsertEntry(ctx, newEntry); err != nil { + return fmt.Errorf("insert new entry failed: %v", err) } return nil diff --git a/engine/storage_manager.go b/engine/storage_manager.go index 296d7b7..7ceb5a7 100644 --- a/engine/storage_manager.go +++ b/engine/storage_manager.go @@ -55,7 +55,12 @@ func NewStorageManager(s3Config *S3Config) (*StorageManager, error) { s3Client := s3.New(awsSession) - uploader := s3manager.NewUploader(awsSession) + uploader := s3manager.NewUploader(awsSession, func(u *s3manager.Uploader) { + u.PartSize = 100 * 1024 * 1024 + u.Concurrency = 5 + u.LeavePartsOnError = false + u.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(25 * 1024 * 1024) + }) downloader := s3manager.NewDownloader(awsSession) return &StorageManager{ @@ -185,24 +190,3 @@ func (sm *StorageManager) CopyObject(oldS3key, newS3Key string) error { Key: aws.String(newS3Key), }) } - -func (sm *StorageManager) ListObjects(prefix string) ([]string, error) { - var result []string - - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(sm.bucketName), - Prefix: aws.String(prefix), - } - - err := sm.s3Client.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool { - for _, obj := range page.Contents { - result = append(result, *obj.Key) - } - return !lastPage - }) - if err != nil { - return nil, err - } - - return result, nil -} diff --git a/utils/operation.go b/utils/operation.go index 1dc4d4a..3cdf512 100644 --- a/utils/operation.go +++ b/utils/operation.go @@ -5,8 +5,8 @@ import ( "os/exec" ) -func CloneFile(src, dst string) error { - cmd := exec.Command("juicefs", "clone", src, dst) +func Clone(root, src, dst string) error { + cmd := exec.Command("juicefs", "clone", root+src, root+dst) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("clone failed: %v, output: %s", err, string(output)) @@ -14,3 +14,12 @@ func CloneFile(src, dst string) error { return nil } + +func Move(root, src, dst string) error { + cmd := exec.Command("mv", root+src, root+dst) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("move failed: %v, output: %s", err, string(output)) + } + return nil +}