From 24b356a984d782e72835a5a0d114a09325e9322f Mon Sep 17 00:00:00 2001 From: dukai Date: Mon, 19 May 2025 23:22:57 +0800 Subject: [PATCH] update: remove the lock of findEntry --- api_file.go | 13 ++++++++++++- engine/filesystem_manager.go | 30 ++++++++++++++++++++++++------ engine/storage_manager.go | 36 ++++++++---------------------------- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/api_file.go b/api_file.go index 0680a44..ad52ca9 100644 --- a/api_file.go +++ b/api_file.go @@ -57,7 +57,9 @@ func (s *Service) HandleUploadFile( defer file.Close() contentType := fileHeader.Header.Get("Content-Type") - if err := s.Engine.FileSystemManager.CreateFile(req.Context(), utils.FullPath(newPath), file, contentType); err != nil { + fileSize := fileHeader.Size + + if err := s.Engine.FileSystemManager.CreateFile(req.Context(), utils.FullPath(newPath), file, contentType, fileSize); err != nil { logger.Error("create %s: %v", path, err) return resp.InternalServerError("create file failed, " + err.Error()) } @@ -76,6 +78,15 @@ func (s *Service) HandleDownloadFile( req *httpserver.Request, resp *httpserver.Response, ) *httpserver.Response { + path := req.QueryString("path") + newPath := utils.NormalizePath(path) + + _, _, err := s.FileSystemManager.DownloadFile(req.Context(), utils.FullPath(newPath)) + if err != nil { + logger.Error("download %s: %v", path, err) + return resp.InternalServerError("download file failed, " + err.Error()) + } + return resp.NoContent() } diff --git a/engine/filesystem_manager.go b/engine/filesystem_manager.go index f30288e..0f7c475 100644 --- a/engine/filesystem_manager.go +++ b/engine/filesystem_manager.go @@ -46,9 +46,6 @@ var ( ) func (f *FileSystemManager) FindEntry(ctx context.Context, p utils.FullPath) (entry *utils.Entry, err error) { - f.RLock() - defer f.RUnlock() - if p == "/" { return Root, nil } @@ -119,7 +116,7 @@ func (f *FileSystemManager) doListDirectoryEntries(ctx context.Context, p utils. return } -func (f *FileSystemManager) CreateFile(ctx context.Context, path utils.FullPath, reader io.Reader, contentType string) error { +func (f *FileSystemManager) CreateFile(ctx context.Context, path utils.FullPath, reader io.Reader, contentType string, fileSize int64) error { f.Lock() defer f.Unlock() @@ -138,12 +135,12 @@ func (f *FileSystemManager) CreateFile(ctx context.Context, path utils.FullPath, } } - output, size, err := f.storage.UploadFile(path.ToS3Key(), reader, contentType) + output, err := f.storage.UploadFile(path.ToS3Key(), reader, contentType) if err != nil { return fmt.Errorf("upload s3 failed: %v", err) } - entry := utils.NewFileEntry(path, path.ToS3Key(), uint64(size), contentType, *output.ETag, *output.VersionID) + entry := utils.NewFileEntry(path, path.ToS3Key(), uint64(fileSize), contentType, *output.ETag, *output.VersionID) if err := f.meta.InsertEntry(ctx, entry); err != nil { return fmt.Errorf("create file entry failed: %v", err) } @@ -151,6 +148,27 @@ func (f *FileSystemManager) CreateFile(ctx context.Context, path utils.FullPath, return nil } +func (f *FileSystemManager) DownloadFile(ctx context.Context, path utils.FullPath) ([]byte, *utils.Entry, error) { + f.RLock() + defer f.RUnlock() + + entry, err := f.FindEntry(ctx, path) + if err != nil { + return nil, nil, fmt.Errorf("find entry failed: %v", err) + } + + if entry.IsDir { + return nil, nil, fmt.Errorf("cannot download directory") + } + + content, err := f.storage.ReadObject(entry.S3Key) + if err != nil { + return nil, nil, fmt.Errorf("read s3 object failed: %v", err) + } + + return content, entry, nil +} + func (f *FileSystemManager) Shutdown() { if f.meta != nil { f.meta.Shutdown() diff --git a/engine/storage_manager.go b/engine/storage_manager.go index 95df58b..570bf2f 100644 --- a/engine/storage_manager.go +++ b/engine/storage_manager.go @@ -86,7 +86,7 @@ func createS3Session(config *S3Config) (*S3Config, *session.Session, error) { return config, sess, nil } -func (sm *StorageManager) UploadData(s3Key string, data []byte, contentType string) (*s3manager.UploadOutput, int64, error) { +func (sm *StorageManager) UploadData(s3Key string, data []byte, contentType string) (*s3manager.UploadOutput, error) { if contentType == "" { contentType = "application/octet-stream" } @@ -98,48 +98,28 @@ func (sm *StorageManager) UploadData(s3Key string, data []byte, contentType stri ContentType: aws.String(contentType), }) if err != nil { - return nil, 0, err + return nil, err } - size := len(data) - return output, int64(size), nil + + return output, nil } -type countReader struct { - reader io.Reader - size int64 -} - -func newCountReader(r io.Reader) *countReader { - return &countReader{ - reader: r, - size: 0, - } -} - -func (r *countReader) Read(p []byte) (n int, err error) { - n, err = r.reader.Read(p) - r.size += int64(n) - return n, err -} - -func (sm *StorageManager) UploadFile(s3Key string, data io.Reader, contentType string) (*s3manager.UploadOutput, int64, error) { +func (sm *StorageManager) UploadFile(s3Key string, data io.Reader, contentType string) (*s3manager.UploadOutput, error) { if contentType == "" { contentType = "application/octet-stream" } - countReader := newCountReader(data) - output, err := sm.s3Uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(sm.bucketName), Key: aws.String(s3Key), - Body: countReader, + Body: data, ContentType: aws.String(contentType), }) if err != nil { - return nil, 0, err + return nil, err } - return output, countReader.size, nil + return output, nil } func (sm *StorageManager) DownloadFile(S3Key string, localFilePath string) error {