diff --git a/Makefile b/Makefile index 89f9418..eb76ff9 100644 --- a/Makefile +++ b/Makefile @@ -3,19 +3,37 @@ COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") BUILD_TIME := $(shell date "+%Y-%m-%d %H:%M:%S") -LDFLAGS := -X gosvc.setVersionNumber=$(VERSION) \ - -X gosvc.setVersionRelease=$(BRANCH) \ - -X gosvc.setVersionBuildTime=$(BUILD_TIME) \ - -X gosvc.setVersionDescription="RobotFS Service (commit: $(COMMIT))" +LDFLAGS := -X 'gosvc.setVersionNumber=$(VERSION)' \ + -X 'gosvc.setVersionRelease=$(BRANCH)' \ + -X 'gosvc.setVersionBuildTime=$(BUILD_TIME)' \ + -X 'gosvc.setVersionDescription=RobotFS Service (commit: $(COMMIT))' + +UNAME_S := $(shell uname -s) + +BINARY_NAME := robotfs +BINARY_MAC := $(BINARY_NAME)-darwin +BINARY_LINUX := $(BINARY_NAME)-linux + +.PHONY: all +all: build .PHONY: build build: - go build -ldflags "$(LDFLAGS)" -o robotfs main.go - -.PHONY: run -run: build - ./robotfs +ifeq ($(UNAME_S),Darwin) + @echo "Building for MacOS..." + GOOS=darwin GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_MAC) ./ +else + @echo "Building for Linux..." + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_LINUX) ./ +endif .PHONY: clean clean: - rm -f robotfs \ No newline at end of file + rm -f $(BINARY_NAME) $(BINARY_MAC) $(BINARY_LINUX) + +.PHONY: help +help: + @echo "Available targets:" + @echo " all - Build for current platform (default)" + @echo " build - Build for current platform" + @echo " clean - Remove built binaries" \ No newline at end of file diff --git a/api_directory.go b/api_directory.go index 6d80c9a..a7410b1 100644 --- a/api_directory.go +++ b/api_directory.go @@ -1,7 +1,10 @@ package main import ( + "context" + "gosvc/httpserver" + "gosvc/logger" "gosvc/validator" "robotfs/utils" ) @@ -12,8 +15,10 @@ type Entry struct { } type ListResult struct { - Entries []Entry `json:"entries"` - LastFileName string `json:"last_file_name"` + Entries []Entry `json:"entries"` + MoreAvailable bool `json:"more_available"` + IsEmptyFolder bool `json:"is_empty_folder"` + LastFileName string `json:"last_file_name"` } type MkdirParams struct { @@ -35,6 +40,7 @@ func (s *Service) HandleMkdir( err := s.FileSystemManager.MakeDirectory(req.Context(), fullPath) if err != nil { + logger.Error("makeDirectory %s: %s", string(fullPath), err.Error()) return resp.InternalServerError("mkdir failed, " + err.Error()) } @@ -45,18 +51,39 @@ func (s *Service) HandleListDirectory( req *httpserver.Request, resp *httpserver.Response, ) *httpserver.Response { - // path := req.QueryString("path") - // startFileName := req.QueryString("startFileName") - // limit := req.QueryInt("limit") - entries := []Entry{ - {Name: "file1.txt", IsDir: false}, - {Name: "folder1", IsDir: true}, + path := req.QueryString("path") + startFileName := req.QueryString("startFileName") + inclusive := req.QueryBool("inclusive") + limit := req.QueryInt("limit") + + newPath := utils.NormalizePath(path) + rawEntries, moreAvailable, err := s.FileSystemManager.ListDirectoryEntries( + context.Background(), + utils.FullPath(newPath), + startFileName, + inclusive, + limit, + ) + if err != nil { + logger.Error("listDirectory %s %s %d: %s", path, startFileName, limit, err) + return resp.InternalServerError("listDirectory failed, " + err.Error()) } - lastFileName := "file1.txt" result := ListResult{ - Entries: entries, - LastFileName: lastFileName, + MoreAvailable: moreAvailable, + IsEmptyFolder: len(rawEntries) == 0, + } + + if len(rawEntries) > 0 { + entries := make([]Entry, len(rawEntries)) + for i, e := range rawEntries { + entries[i] = Entry{ + Name: e.FullPath.Name(), + IsDir: e.IsDir, + } + } + result.Entries = entries + result.LastFileName = entries[len(entries)-1].Name } return resp.OK(result).JSON() diff --git a/engine/filesystem_manager.go b/engine/filesystem_manager.go index cbe2c34..eb1e5c9 100644 --- a/engine/filesystem_manager.go +++ b/engine/filesystem_manager.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "robotfs/pb" "robotfs/store" "robotfs/utils" ) @@ -43,7 +42,7 @@ var ( } ) -func (f *FileSystemManager) FindEntry(ctx context.Context, p utils.FullPath) (entry *pb.FileEntry, err error) { +func (f *FileSystemManager) FindEntry(ctx context.Context, p utils.FullPath) (entry *utils.Entry, err error) { if p == "/" { return nil, nil } @@ -76,6 +75,39 @@ func (f *FileSystemManager) MakeDirectory(ctx context.Context, path utils.FullPa return f.meta.InsertEntry(ctx, entry) } +func (f *FileSystemManager) ListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64) (entries []*utils.Entry, hasMore bool, err error) { + + _, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, func(entry *utils.Entry) bool { + entries = append(entries, entry) + return true + }) + + hasMore = int64(len(entries)) >= limit+1 + if hasMore { + entries = entries[:limit] + } + + return entries, hasMore, err +} + +func (f *FileSystemManager) StreamListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) { + + lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, eachEntryFunc) + return +} + +func (f *FileSystemManager) doListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) { + lastFileName, err = f.meta.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit, func(entry *utils.Entry) bool { + select { + case <-ctx.Done(): + return false + default: + return eachEntryFunc(entry) + } + }) + return +} + func (f *FileSystemManager) Shutdown() error { if f.meta != nil { f.meta.Shutdown() diff --git a/router.go b/router.go index 9c74dbc..71d6692 100644 --- a/router.go +++ b/router.go @@ -38,12 +38,17 @@ func (s *Service) RegisterRouteRules() { Type: httpserver.QueryTypeString, Required: true, }, + { + Key: "inclusive", + Type: httpserver.QueryTypeBool, + Required: true, + }, { Key: "limit", Type: httpserver.QueryTypeInt, Required: true, Min: 1, - Max: 2048, + Max: 4096, }, }, }, diff --git a/store/meta_store.go b/store/meta_store.go index 7a6c2c2..169ceb8 100644 --- a/store/meta_store.go +++ b/store/meta_store.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "robotfs/pb" "robotfs/utils" ) @@ -23,10 +22,11 @@ type MetaStore interface { InsertEntry(context.Context, *utils.Entry) error UpdateEntry(context.Context, *utils.Entry) (err error) - FindEntry(context.Context, utils.FullPath) (entry *pb.FileEntry, err error) + FindEntry(context.Context, utils.FullPath) (entry *utils.Entry, err error) DeleteEntry(context.Context, utils.FullPath) (err error) DeleteFolderChildren(context.Context, utils.FullPath) (err error) ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) + // Todo: implement this in the future ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) BeginTransaction(ctx context.Context) (context.Context, error) diff --git a/store/redis_lua/universal_redis_store.go b/store/redis_lua/universal_redis_store.go index 2ed2dd7..a5e1670 100644 --- a/store/redis_lua/universal_redis_store.go +++ b/store/redis_lua/universal_redis_store.go @@ -2,20 +2,22 @@ package redis_lua import ( "context" + "errors" "fmt" - "github.com/redis/go-redis/v9" - - "robotfs/pb" "robotfs/store" "robotfs/store/redis_lua/stored_procedure" "robotfs/utils" + + "github.com/redis/go-redis/v9" ) const ( DIR_LIST_MARKER = "\x00" ) +var ErrNotFound = errors.New("no entry is found") + type UniversalRedisLuaStore struct { Client redis.UniversalClient superLargeDirectoryHash map[string]bool @@ -44,6 +46,7 @@ func (store *UniversalRedisLuaStore) RollbackTransaction(ctx context.Context) er } func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *utils.Entry) (err error) { + value, err := entry.Encode() if err != nil { return fmt.Errorf("encoding %s: %v", entry.FullPath, err) @@ -68,24 +71,24 @@ func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *uti return store.InsertEntry(ctx, entry) } -func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath utils.FullPath) (entry *pb.FileEntry, err error) { +func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath utils.FullPath) (entry *utils.Entry, err error) { - // data, err := store.Client.Get(ctx, string(fullpath)).Result() - // if err == redis.Nil { - // return nil, pb.ErrNotFound - // } + data, err := store.Client.Get(ctx, string(fullpath)).Result() + if err == redis.Nil { + return nil, ErrNotFound + } - // if err != nil { - // return nil, fmt.Errorf("get %s : %v", fullpath, err) - // } + if err != nil { + return nil, fmt.Errorf("get %s : %v", fullpath, err) + } - // entry = &pb.FileEntry{ - // FullPath: fullpath.Name(), - // } - // err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data))) - // if err != nil { - // return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) - // } + entry = &utils.Entry{ + FullPath: fullpath, + } + err = entry.Decode([]byte(data)) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } return entry, nil } @@ -121,56 +124,49 @@ func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, f return nil } +// Todo: implement this in the future func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) { - // return lastFileName, engine.ErrUnsupportedListDirectoryPrefixed - return lastFileName, err + return lastFileName, nil } func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) { - // dirListKey := genDirectoryListKey(string(dirPath)) + dirListKey := genDirectoryListKey(string(dirPath)) - // min := "-" - // if startFileName != "" { - // if includeStartFile { - // min = "[" + startFileName - // } else { - // min = "(" + startFileName - // } - // } + min := "-" + if startFileName != "" { + if includeStartFile { + min = "[" + startFileName + } else { + min = "(" + startFileName + } + } - // members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{ - // Min: min, - // Max: "+", - // Offset: 0, - // Count: limit, - // }).Result() - // if err != nil { - // return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) - // } + members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{ + Min: min, + Max: "+", + Offset: 0, + Count: limit, + }).Result() + if err != nil { + return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) + } - // // fetch entry meta - // for _, fileName := range members { - // path := utils.NewFullPath(string(dirPath), fileName) - // entry, err := store.FindEntry(ctx, path) - // lastFileName = fileName - // if err != nil { - - // if err == pb.ErrNotFound { - // continue - // } - // } else { - // if entry.TtlSec > 0 { - // if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { - // store.DeleteEntry(ctx, path) - // continue - // } - // } - // if !eachEntryFunc(entry) { - // break - // } - // } - // } + // fetch entry meta + for _, fileName := range members { + path := utils.NewFullPath(string(dirPath), fileName) + entry, err := store.FindEntry(ctx, path) + lastFileName = fileName + if err != nil { + if err == ErrNotFound { + continue + } + } else { + if !eachEntryFunc(entry) { + break + } + } + } return lastFileName, err } diff --git a/utils/entry.go b/utils/entry.go index f95776e..5f80372 100644 --- a/utils/entry.go +++ b/utils/entry.go @@ -2,9 +2,10 @@ package utils import ( "fmt" - "robotfs/pb" "time" + "robotfs/pb" + "google.golang.org/protobuf/proto" ) diff --git a/utils/file.go b/utils/file.go deleted file mode 100644 index 3c02300..0000000 --- a/utils/file.go +++ /dev/null @@ -1,14 +0,0 @@ -package utils - -import ( - "path/filepath" - "strings" -) - -func NormalizePath(path string) string { - if !strings.HasPrefix(path, "/") { - path = "/" + path - } - - return filepath.Clean(path) -} diff --git a/utils/fullPath.go b/utils/fullPath.go index 69c5d7a..cf56a2b 100644 --- a/utils/fullPath.go +++ b/utils/fullPath.go @@ -70,3 +70,11 @@ func StringSplit(separatedValues string, sep string) []string { } return strings.Split(separatedValues, sep) } + +func NormalizePath(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + return filepath.Clean(path) +}