feat: add list directory

Makefile

@@ -3,19 +3,37 @@ COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
 BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
 BUILD_TIME := $(shell date "+%Y-%m-%d %H:%M:%S")
 
-LDFLAGS := -X gosvc.setVersionNumber=$(VERSION) \
+LDFLAGS := -X 'gosvc.setVersionNumber=$(VERSION)' \
-           -X gosvc.setVersionRelease=$(BRANCH) \
+           -X 'gosvc.setVersionRelease=$(BRANCH)' \
-           -X gosvc.setVersionBuildTime=$(BUILD_TIME) \
+           -X 'gosvc.setVersionBuildTime=$(BUILD_TIME)' \
-           -X gosvc.setVersionDescription="RobotFS Service (commit: $(COMMIT))"
+           -X 'gosvc.setVersionDescription=RobotFS Service (commit: $(COMMIT))'
+
+UNAME_S := $(shell uname -s)
+
+BINARY_NAME := robotfs
+BINARY_MAC := $(BINARY_NAME)-darwin
+BINARY_LINUX := $(BINARY_NAME)-linux
+
+.PHONY: all
+all: build
 
 .PHONY: build
 build:
-	go build -ldflags "$(LDFLAGS)" -o robotfs main.go
+ifeq ($(UNAME_S),Darwin)
-
+	@echo "Building for MacOS..."
-.PHONY: run
+	GOOS=darwin GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_MAC) ./
-run: build
+else
-	./robotfs
+	@echo "Building for Linux..."
+	GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_LINUX) ./
+endif
 
 .PHONY: clean
 clean:
-	rm -f robotfs
+	rm -f $(BINARY_NAME) $(BINARY_MAC) $(BINARY_LINUX)
+
+.PHONY: help
+help:
+	@echo "Available targets:"
+	@echo "  all        - Build for current platform (default)"
+	@echo "  build      - Build for current platform"
+	@echo "  clean      - Remove built binaries"

api_directory.go

@@ -1,7 +1,10 @@
 package main
 
 import (
+	"context"
+
 	"gosvc/httpserver"
+	"gosvc/logger"
 	"gosvc/validator"
 	"robotfs/utils"
 )
@@ -13,6 +16,8 @@ type Entry struct {
 
 type ListResult struct {
 	Entries       []Entry `json:"entries"`
+	MoreAvailable bool    `json:"more_available"`
+	IsEmptyFolder bool    `json:"is_empty_folder"`
 	LastFileName  string  `json:"last_file_name"`
 }
 
@@ -35,6 +40,7 @@ func (s *Service) HandleMkdir(
 
 	err := s.FileSystemManager.MakeDirectory(req.Context(), fullPath)
 	if err != nil {
+		logger.Error("makeDirectory %s:  %s", string(fullPath), err.Error())
 		return resp.InternalServerError("mkdir failed, " + err.Error())
 	}
 
@@ -45,18 +51,39 @@ func (s *Service) HandleListDirectory(
 	req *httpserver.Request,
 	resp *httpserver.Response,
 ) *httpserver.Response {
-	// path := req.QueryString("path")
+	path := req.QueryString("path")
-	// startFileName := req.QueryString("startFileName")
+	startFileName := req.QueryString("startFileName")
-	// limit := req.QueryInt("limit")
+	inclusive := req.QueryBool("inclusive")
-	entries := []Entry{
+	limit := req.QueryInt("limit")
-		{Name: "file1.txt", IsDir: false},
+
-		{Name: "folder1", IsDir: true},
+	newPath := utils.NormalizePath(path)
+	rawEntries, moreAvailable, err := s.FileSystemManager.ListDirectoryEntries(
+		context.Background(),
+		utils.FullPath(newPath),
+		startFileName,
+		inclusive,
+		limit,
+	)
+	if err != nil {
+		logger.Error("listDirectory %s %s %d: %s", path, startFileName, limit, err)
+		return resp.InternalServerError("listDirectory failed, " + err.Error())
 	}
-	lastFileName := "file1.txt"
 
 	result := ListResult{
-		Entries:      entries,
+		MoreAvailable: moreAvailable,
-		LastFileName: lastFileName,
+		IsEmptyFolder: len(rawEntries) == 0,
+	}
+
+	if len(rawEntries) > 0 {
+		entries := make([]Entry, len(rawEntries))
+		for i, e := range rawEntries {
+			entries[i] = Entry{
+				Name:  e.FullPath.Name(),
+				IsDir: e.IsDir,
+			}
+		}
+		result.Entries = entries
+		result.LastFileName = entries[len(entries)-1].Name
 	}
 
 	return resp.OK(result).JSON()

engine/filesystem_manager.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"robotfs/pb"
 	"robotfs/store"
 	"robotfs/utils"
 )
@@ -43,7 +42,7 @@ var (
 	}
 )
 
-func (f *FileSystemManager) FindEntry(ctx context.Context, p utils.FullPath) (entry *pb.FileEntry, err error) {
+func (f *FileSystemManager) FindEntry(ctx context.Context, p utils.FullPath) (entry *utils.Entry, err error) {
 	if p == "/" {
 		return nil, nil
 	}
@@ -76,6 +75,39 @@ func (f *FileSystemManager) MakeDirectory(ctx context.Context, path utils.FullPa
 	return f.meta.InsertEntry(ctx, entry)
 }
 
+func (f *FileSystemManager) ListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64) (entries []*utils.Entry, hasMore bool, err error) {
+
+	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, func(entry *utils.Entry) bool {
+		entries = append(entries, entry)
+		return true
+	})
+
+	hasMore = int64(len(entries)) >= limit+1
+	if hasMore {
+		entries = entries[:limit]
+	}
+
+	return entries, hasMore, err
+}
+
+func (f *FileSystemManager) StreamListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
+
+	lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, eachEntryFunc)
+	return
+}
+
+func (f *FileSystemManager) doListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
+	lastFileName, err = f.meta.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit, func(entry *utils.Entry) bool {
+		select {
+		case <-ctx.Done():
+			return false
+		default:
+			return eachEntryFunc(entry)
+		}
+	})
+	return
+}
+
 func (f *FileSystemManager) Shutdown() error {
 	if f.meta != nil {
 		f.meta.Shutdown()

router.go

@@ -38,12 +38,17 @@ func (s *Service) RegisterRouteRules() {
 								Type:     httpserver.QueryTypeString,
 								Required: true,
 							},
+							{
+								Key:      "inclusive",
+								Type:     httpserver.QueryTypeBool,
+								Required: true,
+							},
 							{
 								Key:      "limit",
 								Type:     httpserver.QueryTypeInt,
 								Required: true,
 								Min:      1,
-								Max:      2048,
+								Max:      4096,
 							},
 						},
 					},

store/meta_store.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 
-	"robotfs/pb"
 	"robotfs/utils"
 )
 
@@ -23,10 +22,11 @@ type MetaStore interface {
 
 	InsertEntry(context.Context, *utils.Entry) error
 	UpdateEntry(context.Context, *utils.Entry) (err error)
-	FindEntry(context.Context, utils.FullPath) (entry *pb.FileEntry, err error)
+	FindEntry(context.Context, utils.FullPath) (entry *utils.Entry, err error)
 	DeleteEntry(context.Context, utils.FullPath) (err error)
 	DeleteFolderChildren(context.Context, utils.FullPath) (err error)
 	ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
+	// Todo: implement this in the future
 	ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
 
 	BeginTransaction(ctx context.Context) (context.Context, error)

store/redis_lua/universal_redis_store.go

@@ -2,20 +2,22 @@ package redis_lua
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
-	"github.com/redis/go-redis/v9"
-
-	"robotfs/pb"
 	"robotfs/store"
 	"robotfs/store/redis_lua/stored_procedure"
 	"robotfs/utils"
+
+	"github.com/redis/go-redis/v9"
 )
 
 const (
 	DIR_LIST_MARKER = "\x00"
 )
 
+var ErrNotFound = errors.New("no entry is found")
+
 type UniversalRedisLuaStore struct {
 	Client                  redis.UniversalClient
 	superLargeDirectoryHash map[string]bool
@@ -44,6 +46,7 @@ func (store *UniversalRedisLuaStore) RollbackTransaction(ctx context.Context) er
 }
 
 func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *utils.Entry) (err error) {
+
 	value, err := entry.Encode()
 	if err != nil {
 		return fmt.Errorf("encoding %s: %v", entry.FullPath, err)
@@ -68,24 +71,24 @@ func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *uti
 	return store.InsertEntry(ctx, entry)
 }
 
-func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath utils.FullPath) (entry *pb.FileEntry, err error) {
+func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath utils.FullPath) (entry *utils.Entry, err error) {
 
-	// data, err := store.Client.Get(ctx, string(fullpath)).Result()
+	data, err := store.Client.Get(ctx, string(fullpath)).Result()
-	// if err == redis.Nil {
+	if err == redis.Nil {
-	// 	return nil, pb.ErrNotFound
+		return nil, ErrNotFound
-	// }
+	}
 
-	// if err != nil {
+	if err != nil {
-	// 	return nil, fmt.Errorf("get %s : %v", fullpath, err)
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
-	// }
+	}
 
-	// entry = &pb.FileEntry{
+	entry = &utils.Entry{
-	// 	FullPath: fullpath.Name(),
+		FullPath: fullpath,
-	// }
+	}
-	// err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
+	err = entry.Decode([]byte(data))
-	// if err != nil {
+	if err != nil {
-	// 	return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
-	// }
+	}
 
 	return entry, nil
 }
@@ -121,56 +124,49 @@ func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, f
 	return nil
 }
 
+// Todo: implement this in the future
 func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
-	// return lastFileName, engine.ErrUnsupportedListDirectoryPrefixed
+	return lastFileName, nil
-	return lastFileName, err
 }
 
 func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
 
-	// dirListKey := genDirectoryListKey(string(dirPath))
+	dirListKey := genDirectoryListKey(string(dirPath))
 
-	// min := "-"
+	min := "-"
-	// if startFileName != "" {
+	if startFileName != "" {
-	// 	if includeStartFile {
+		if includeStartFile {
-	// 		min = "[" + startFileName
+			min = "[" + startFileName
-	// 	} else {
+		} else {
-	// 		min = "(" + startFileName
+			min = "(" + startFileName
-	// 	}
+		}
-	// }
+	}
 
-	// members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
+	members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
-	// 	Min:    min,
+		Min:    min,
-	// 	Max:    "+",
+		Max:    "+",
-	// 	Offset: 0,
+		Offset: 0,
-	// 	Count:  limit,
+		Count:  limit,
-	// }).Result()
+	}).Result()
-	// if err != nil {
+	if err != nil {
-	// 	return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
+		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
-	// }
+	}
 
-	// // fetch entry meta
+	// fetch entry meta
-	// for _, fileName := range members {
+	for _, fileName := range members {
-	// 	path := utils.NewFullPath(string(dirPath), fileName)
+		path := utils.NewFullPath(string(dirPath), fileName)
-	// 	entry, err := store.FindEntry(ctx, path)
+		entry, err := store.FindEntry(ctx, path)
-	// 	lastFileName = fileName
+		lastFileName = fileName
-	// 	if err != nil {
+		if err != nil {
-
+			if err == ErrNotFound {
-	// 		if err == pb.ErrNotFound {
+				continue
-	// 			continue
+			}
-	// 		}
+		} else {
-	// 	} else {
+			if !eachEntryFunc(entry) {
-	// 		if entry.TtlSec > 0 {
+				break
-	// 			if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+			}
-	// 				store.DeleteEntry(ctx, path)
+		}
-	// 				continue
+	}
-	// 			}
-	// 		}
-	// 		if !eachEntryFunc(entry) {
-	// 			break
-	// 		}
-	// 	}
-	// }
 
 	return lastFileName, err
 }

utils/entry.go

@@ -2,9 +2,10 @@ package utils
 
 import (
 	"fmt"
-	"robotfs/pb"
 	"time"
 
+	"robotfs/pb"
+
 	"google.golang.org/protobuf/proto"
 )
 

utils/file.go

@@ -1,14 +0,0 @@
-package utils
-
-import (
-	"path/filepath"
-	"strings"
-)
-
-func NormalizePath(path string) string {
-	if !strings.HasPrefix(path, "/") {
-		path = "/" + path
-	}
-
-	return filepath.Clean(path)
-}

utils/fullPath.go

@@ -70,3 +70,11 @@ func StringSplit(separatedValues string, sep string) []string {
 	}
 	return strings.Split(separatedValues, sep)
 }
+
+func NormalizePath(path string) string {
+	if !strings.HasPrefix(path, "/") {
+		path = "/" + path
+	}
+
+	return filepath.Clean(path)
+}