dukai/robotfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add list directory

Browse Source
This commit is contained in:
dukai 2025-05-18 00:21:56 +08:00
parent deb3abe394
commit 4a45900436
9 changed files with 174 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
Makefile
View File

@ -3,19 +3,37 @@ COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
BUILD_TIME := $(shell date "+%Y-%m-%d %H:%M:%S")
LDFLAGS := -X gosvc.setVersionNumber=$(VERSION) \
-X gosvc.setVersionRelease=$(BRANCH) \
-X gosvc.setVersionBuildTime=$(BUILD_TIME) \
-X gosvc.setVersionDescription="RobotFS Service (commit: $(COMMIT))"
LDFLAGS := -X 'gosvc.setVersionNumber=$(VERSION)' \
-X 'gosvc.setVersionRelease=$(BRANCH)' \
-X 'gosvc.setVersionBuildTime=$(BUILD_TIME)' \
-X 'gosvc.setVersionDescription=RobotFS Service (commit: $(COMMIT))'
UNAME_S := $(shell uname -s)
BINARY_NAME := robotfs
BINARY_MAC := $(BINARY_NAME)-darwin
BINARY_LINUX := $(BINARY_NAME)-linux
.PHONY: all
all: build
.PHONY: build
build:
go build -ldflags "$(LDFLAGS)" -o robotfs main.go
.PHONY: run
run: build
./robotfs
ifeq ($(UNAME_S),Darwin)
@echo "Building for MacOS..."
GOOS=darwin GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_MAC) ./
else
@echo "Building for Linux..."
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_LINUX) ./
endif
.PHONY: clean
clean:
rm -f robotfs
rm -f $(BINARY_NAME) $(BINARY_MAC) $(BINARY_LINUX)
.PHONY: help
help:
@echo "Available targets:"
@echo " all - Build for current platform (default)"
@echo " build - Build for current platform"
@echo " clean - Remove built binaries"

49
api_directory.go
View File

@ -1,7 +1,10 @@
package main
import (
"context"
"gosvc/httpserver"
"gosvc/logger"
"gosvc/validator"
"robotfs/utils"
)
@ -12,8 +15,10 @@ type Entry struct {
}
type ListResult struct {
Entries []Entry `json:"entries"`
LastFileName string `json:"last_file_name"`
Entries []Entry `json:"entries"`
MoreAvailable bool `json:"more_available"`
IsEmptyFolder bool `json:"is_empty_folder"`
LastFileName string `json:"last_file_name"`
}
type MkdirParams struct {
@ -35,6 +40,7 @@ func (s *Service) HandleMkdir(
err := s.FileSystemManager.MakeDirectory(req.Context(), fullPath)
if err != nil {
logger.Error("makeDirectory %s: %s", string(fullPath), err.Error())
return resp.InternalServerError("mkdir failed, " + err.Error())
}
@ -45,18 +51,39 @@ func (s *Service) HandleListDirectory(
req *httpserver.Request,
resp *httpserver.Response,
) *httpserver.Response {
// path := req.QueryString("path")
// startFileName := req.QueryString("startFileName")
// limit := req.QueryInt("limit")
entries := []Entry{
{Name: "file1.txt", IsDir: false},
{Name: "folder1", IsDir: true},
path := req.QueryString("path")
startFileName := req.QueryString("startFileName")
inclusive := req.QueryBool("inclusive")
limit := req.QueryInt("limit")
newPath := utils.NormalizePath(path)
rawEntries, moreAvailable, err := s.FileSystemManager.ListDirectoryEntries(
context.Background(),
utils.FullPath(newPath),
startFileName,
inclusive,
limit,
)
if err != nil {
logger.Error("listDirectory %s %s %d: %s", path, startFileName, limit, err)
return resp.InternalServerError("listDirectory failed, " + err.Error())
}
lastFileName := "file1.txt"
result := ListResult{
Entries: entries,
LastFileName: lastFileName,
MoreAvailable: moreAvailable,
IsEmptyFolder: len(rawEntries) == 0,
}
if len(rawEntries) > 0 {
entries := make([]Entry, len(rawEntries))
for i, e := range rawEntries {
entries[i] = Entry{
Name: e.FullPath.Name(),
IsDir: e.IsDir,
}
}
result.Entries = entries
result.LastFileName = entries[len(entries)-1].Name
}
return resp.OK(result).JSON()

36
engine/filesystem_manager.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"time"
"robotfs/pb"
"robotfs/store"
"robotfs/utils"
)
@ -43,7 +42,7 @@ var (
}
)
func (f *FileSystemManager) FindEntry(ctx context.Context, p utils.FullPath) (entry *pb.FileEntry, err error) {
func (f *FileSystemManager) FindEntry(ctx context.Context, p utils.FullPath) (entry *utils.Entry, err error) {
if p == "/" {
return nil, nil
}
@ -76,6 +75,39 @@ func (f *FileSystemManager) MakeDirectory(ctx context.Context, path utils.FullPa
return f.meta.InsertEntry(ctx, entry)
}
func (f *FileSystemManager) ListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64) (entries []*utils.Entry, hasMore bool, err error) {
_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, func(entry *utils.Entry) bool {
entries = append(entries, entry)
return true
})
hasMore = int64(len(entries)) >= limit+1
if hasMore {
entries = entries[:limit]
}
return entries, hasMore, err
}
func (f *FileSystemManager) StreamListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, eachEntryFunc)
return
}
func (f *FileSystemManager) doListDirectoryEntries(ctx context.Context, p utils.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
lastFileName, err = f.meta.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit, func(entry *utils.Entry) bool {
select {
case <-ctx.Done():
return false
default:
return eachEntryFunc(entry)
}
})
return
}
func (f *FileSystemManager) Shutdown() error {
if f.meta != nil {
f.meta.Shutdown()

7
router.go
View File

@ -38,12 +38,17 @@ func (s *Service) RegisterRouteRules() {
Type: httpserver.QueryTypeString,
Required: true,
},
{
Key: "inclusive",
Type: httpserver.QueryTypeBool,
Required: true,
},
{
Key: "limit",
Type: httpserver.QueryTypeInt,
Required: true,
Min: 1,
Max: 2048,
Max: 4096,
},
},
},

4
store/meta_store.go
View File

@ -4,7 +4,6 @@ import (
"context"
"errors"
"robotfs/pb"
"robotfs/utils"
)
@ -23,10 +22,11 @@ type MetaStore interface {
InsertEntry(context.Context, *utils.Entry) error
UpdateEntry(context.Context, *utils.Entry) (err error)
FindEntry(context.Context, utils.FullPath) (entry *pb.FileEntry, err error)
FindEntry(context.Context, utils.FullPath) (entry *utils.Entry, err error)
DeleteEntry(context.Context, utils.FullPath) (err error)
DeleteFolderChildren(context.Context, utils.FullPath) (err error)
ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
// Todo: implement this in the future
ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
BeginTransaction(ctx context.Context) (context.Context, error)

116
store/redis_lua/universal_redis_store.go
View File

@ -2,20 +2,22 @@ package redis_lua
import (
"context"
"errors"
"fmt"
"github.com/redis/go-redis/v9"
"robotfs/pb"
"robotfs/store"
"robotfs/store/redis_lua/stored_procedure"
"robotfs/utils"
"github.com/redis/go-redis/v9"
)
const (
DIR_LIST_MARKER = "\x00"
)
var ErrNotFound = errors.New("no entry is found")
type UniversalRedisLuaStore struct {
Client redis.UniversalClient
superLargeDirectoryHash map[string]bool
@ -44,6 +46,7 @@ func (store *UniversalRedisLuaStore) RollbackTransaction(ctx context.Context) er
}
func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *utils.Entry) (err error) {
value, err := entry.Encode()
if err != nil {
return fmt.Errorf("encoding %s: %v", entry.FullPath, err)
@ -68,24 +71,24 @@ func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *uti
return store.InsertEntry(ctx, entry)
}
func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath utils.FullPath) (entry *pb.FileEntry, err error) {
func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath utils.FullPath) (entry *utils.Entry, err error) {
// data, err := store.Client.Get(ctx, string(fullpath)).Result()
// if err == redis.Nil {
// return nil, pb.ErrNotFound
// }
data, err := store.Client.Get(ctx, string(fullpath)).Result()
if err == redis.Nil {
return nil, ErrNotFound
}
// if err != nil {
// return nil, fmt.Errorf("get %s : %v", fullpath, err)
// }
if err != nil {
return nil, fmt.Errorf("get %s : %v", fullpath, err)
}
// entry = &pb.FileEntry{
// FullPath: fullpath.Name(),
// }
// err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
// if err != nil {
// return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
// }
entry = &utils.Entry{
FullPath: fullpath,
}
err = entry.Decode([]byte(data))
if err != nil {
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
}
return entry, nil
}
@ -121,56 +124,49 @@ func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, f
return nil
}
// Todo: implement this in the future
func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
// return lastFileName, engine.ErrUnsupportedListDirectoryPrefixed
return lastFileName, err
return lastFileName, nil
}
func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) {
// dirListKey := genDirectoryListKey(string(dirPath))
dirListKey := genDirectoryListKey(string(dirPath))
// min := "-"
// if startFileName != "" {
// if includeStartFile {
// min = "[" + startFileName
// } else {
// min = "(" + startFileName
// }
// }
min := "-"
if startFileName != "" {
if includeStartFile {
min = "[" + startFileName
} else {
min = "(" + startFileName
}
}
// members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
// Min: min,
// Max: "+",
// Offset: 0,
// Count: limit,
// }).Result()
// if err != nil {
// return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
// }
members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
Min: min,
Max: "+",
Offset: 0,
Count: limit,
}).Result()
if err != nil {
return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
}
// // fetch entry meta
// for _, fileName := range members {
// path := utils.NewFullPath(string(dirPath), fileName)
// entry, err := store.FindEntry(ctx, path)
// lastFileName = fileName
// if err != nil {
// if err == pb.ErrNotFound {
// continue
// }
// } else {
// if entry.TtlSec > 0 {
// if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
// store.DeleteEntry(ctx, path)
// continue
// }
// }
// if !eachEntryFunc(entry) {
// break
// }
// }
// }
// fetch entry meta
for _, fileName := range members {
path := utils.NewFullPath(string(dirPath), fileName)
entry, err := store.FindEntry(ctx, path)
lastFileName = fileName
if err != nil {
if err == ErrNotFound {
continue
}
} else {
if !eachEntryFunc(entry) {
break
}
}
}
return lastFileName, err
}

3
utils/entry.go
View File

@ -2,9 +2,10 @@ package utils
import (
"fmt"
"robotfs/pb"
"time"
"robotfs/pb"
"google.golang.org/protobuf/proto"
)

14
utils/file.go
View File

@ -1,14 +0,0 @@
package utils
import (
"path/filepath"
"strings"
)
func NormalizePath(path string) string {
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return filepath.Clean(path)
}

8
utils/fullPath.go
View File

@ -70,3 +70,11 @@ func StringSplit(separatedValues string, sep string) []string {
}
return strings.Split(separatedValues, sep)
}
func NormalizePath(path string) string {
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return filepath.Clean(path)
}