commit 3f53e1d922fbb856707cc05af20532297aa5088a Author: d-robotics Date: Mon May 12 20:23:33 2025 +0800 add: init repo diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..89f9418 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +VERSION := $(shell git describe --tags --abbrev=0 2>/dev/null || echo "dev") +COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") +BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +BUILD_TIME := $(shell date "+%Y-%m-%d %H:%M:%S") + +LDFLAGS := -X gosvc.setVersionNumber=$(VERSION) \ + -X gosvc.setVersionRelease=$(BRANCH) \ + -X gosvc.setVersionBuildTime=$(BUILD_TIME) \ + -X gosvc.setVersionDescription="RobotFS Service (commit: $(COMMIT))" + +.PHONY: build +build: + go build -ldflags "$(LDFLAGS)" -o robotfs main.go + +.PHONY: run +run: build + ./robotfs + +.PHONY: clean +clean: + rm -f robotfs \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/api_other.go b/api_other.go new file mode 100644 index 0000000..d338b0b --- /dev/null +++ b/api_other.go @@ -0,0 +1,12 @@ +package main + +import ( + "gosvc/httpserver" +) + +func (s *Service) API_Ping( + req *httpserver.Request, + resp *httpserver.Response, +) *httpserver.Response { + return resp.OK("pong") +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..1ca3c3c --- /dev/null +++ b/config/config.go @@ -0,0 +1,53 @@ +package config + +import ( + "time" + + "gosvc" + "gosvc/httpserver" + "gosvc/logger" +) + +func SetupServiceConfig() { + gosvc.Config = gosvc.ServiceConfig{ + Name: "robotfs", + Version: gosvc.Config.Version, + Runtime: gosvc.RuntimeConfig{ + RootOnly: false, + }, + Logger: gosvc.LoggerConfig{ + Options: logger.Options{ + TargetDirectory: "/var/log/robotfs", + FilePrefix: "robotfs", + CycleFiles: 16, + MaxFileSize: 64 * 1024 * 1024, + }, + }, + Ticker: gosvc.TickerConfig{ + Enable: false, + TickInterval: 1300 * time.Millisecond, + TickMinInterval: 300 * time.Millisecond, + TickOnStop: true, + }, + Saver: gosvc.SaverConfig{ + Enable: false, + SaveInterval: 60 * time.Second, + SaveMinInterval: 500 * time.Millisecond, + }, + HTTPServer: gosvc.HTTPServerConfig{ + Enable: true, + Options: httpserver.Options{ + Address: "0.0.0.0:6201", + TLSEnabled: false, + CrossOriginEnabled: true, + MaxConcurrentStreams: 10000, + MaxTraces: 30 * 10000, + LogSlowRequest: true, + SlowRequestThreshold: 1000 * time.Millisecond, + }, + }, + } + + gosvc.Config.HTTPServer.MaxRequestsPerSecond = 4000 + gosvc.Config.HTTPServer.MaxPendingRequests = 15000 +} diff --git a/config/security.go b/config/security.go new file mode 100644 index 0000000..078cf78 --- /dev/null +++ b/config/security.go @@ -0,0 +1,29 @@ +package config + +type Config struct { + Robotfs struct { + Address string `mapstructure:"ADDRESS"` + LogPath string `mapstructure:"LOGPATH"` + } `mapstructure:"robotfs"` + + Redis struct { + Address string `mapstructure:"address"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` + } `mapstructure:"redis"` + + S3 struct { + Region string `mapstructure:"region"` + Endpoint string `mapstructure:"endpoint"` + AccessKeyID string `mapstructure:"access_key_id"` + SecretAccessKey string `mapstructure:"secret_access_key"` + BucketName string `mapstructure:"bucket_name"` + UsePathStyle bool `mapstructure:"use_path_style"` + } `mapstructure:"s3"` +} + +func GetConfig() *Config { + config := &Config{} + + return config +} diff --git a/engine/engine.go b/engine/engine.go new file mode 100644 index 0000000..c6239f5 --- /dev/null +++ b/engine/engine.go @@ -0,0 +1,47 @@ +package engine + +import ( + "fmt" + + "robotfs/store/redis_lua" + "robotfs/utils" +) + +type Engine struct { + metadataManager *MetaDataManager + storageManager *StorageManager +} + +func NewEngine() *Engine { + return &Engine{} +} + +func (e *Engine) Start() error { + store := redis_lua.NewRedisLuaStore() + if err := store.Initialize(utils.GetViper(), "redis."); err != nil { + return fmt.Errorf("failed to initialize meta store: %v", err) + } + metadata, err := NewMetaDataManager(store) + if err != nil { + return fmt.Errorf("failed to create metadata manager: %v", err) + } + e.metadataManager = metadata + + s3Config, err := Initialize(utils.GetViper(), "s3.") + if err != nil { + return fmt.Errorf("failed to initialize s3 config: %v", err) + } + storageManager, err := NewStorageManager(s3Config) + if err != nil { + return fmt.Errorf("failed to create storage manager: %v", err) + } + e.storageManager = storageManager + + return nil +} + +func (e *Engine) Stop() { + if e.metadataManager != nil { + e.metadataManager.Shutdown() + } +} diff --git a/engine/metadata_manager.go b/engine/metadata_manager.go new file mode 100644 index 0000000..e4eda18 --- /dev/null +++ b/engine/metadata_manager.go @@ -0,0 +1,28 @@ +package engine + +import ( + "fmt" + + "robotfs/store" +) + +type MetaDataManager struct { + store store.MetaStore +} + +func NewMetaDataManager(store store.MetaStore) (*MetaDataManager, error) { + if store == nil { + return nil, fmt.Errorf("meta store is required") + } + return &MetaDataManager{ + store: store, + }, nil +} + +func (m *MetaDataManager) Shutdown() error { + if m.store != nil { + m.store.Shutdown() + } + + return nil +} diff --git a/engine/storage_manager.go b/engine/storage_manager.go new file mode 100644 index 0000000..570bf2f --- /dev/null +++ b/engine/storage_manager.go @@ -0,0 +1,204 @@ +package engine + +import ( + "bytes" + "fmt" + "io" + "os" + + "robotfs/utils" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type S3Config struct { + Region string + Endpoint string + AccessKeyID string + SecretAccessKey string + BucketName string + UsePathStyle bool +} + +func Initialize(configuration utils.Configuration, prefix string) (*S3Config, error) { + s3Cfg := &S3Config{ + Region: configuration.GetString(prefix + "region"), + Endpoint: configuration.GetString(prefix + "endpoint"), + AccessKeyID: configuration.GetString(prefix + "access_key_id"), + SecretAccessKey: configuration.GetString(prefix + "secret_access_key"), + BucketName: configuration.GetString(prefix + "bucket_name"), + UsePathStyle: configuration.GetBool(prefix + "use_path_style"), + } + + if s3Cfg.BucketName == "" { + return nil, fmt.Errorf("bucket_name is required") + } + + return s3Cfg, nil +} + +type StorageManager struct { + s3Client *s3.S3 + s3Uploader *s3manager.Uploader + s3Downloader *s3manager.Downloader + bucketName string +} + +func NewStorageManager(s3Config *S3Config) (*StorageManager, error) { + s3Config, awsSession, err := createS3Session(s3Config) + if err != nil { + return nil, err + } + + s3Client := s3.New(awsSession) + + uploader := s3manager.NewUploader(awsSession) + downloader := s3manager.NewDownloader(awsSession) + + return &StorageManager{ + s3Client: s3Client, + s3Uploader: uploader, + s3Downloader: downloader, + bucketName: s3Config.BucketName, + }, nil +} + +func createS3Session(config *S3Config) (*S3Config, *session.Session, error) { + awsConfig := &aws.Config{ + Credentials: credentials.NewStaticCredentials(config.AccessKeyID, config.SecretAccessKey, ""), + Region: aws.String(config.Region), + S3ForcePathStyle: aws.Bool(config.UsePathStyle), + } + + if config.Endpoint != "" { + awsConfig.Endpoint = aws.String(config.Endpoint) + } + + sess, err := session.NewSession(awsConfig) + if err != nil { + return config, nil, err + } + + return config, sess, nil +} + +func (sm *StorageManager) UploadData(s3Key string, data []byte, contentType string) (*s3manager.UploadOutput, error) { + if contentType == "" { + contentType = "application/octet-stream" + } + + output, err := sm.s3Uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(sm.bucketName), + Key: aws.String(s3Key), + Body: bytes.NewReader(data), + ContentType: aws.String(contentType), + }) + if err != nil { + return nil, err + } + + return output, nil +} + +func (sm *StorageManager) UploadFile(s3Key string, data io.Reader, contentType string) (*s3manager.UploadOutput, error) { + if contentType == "" { + contentType = "application/octet-stream" + } + + output, err := sm.s3Uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(sm.bucketName), + Key: aws.String(s3Key), + Body: data, + ContentType: aws.String(contentType), + }) + if err != nil { + return nil, err + } + + return output, nil +} + +func (sm *StorageManager) DownloadFile(S3Key string, localFilePath string) error { + file, err := os.Create(localFilePath) + if err != nil { + return err + } + defer file.Close() + + _, err = sm.s3Downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(sm.bucketName), + Key: aws.String(S3Key), + }) + + return err +} + +func (sm *StorageManager) ReadObject(S3Key string) ([]byte, error) { + buf := aws.NewWriteAtBuffer([]byte{}) + + _, err := sm.s3Downloader.Download(buf, &s3.GetObjectInput{ + Bucket: aws.String(sm.bucketName), + Key: aws.String(S3Key), + }) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (sm *StorageManager) DeleteObject(s3Key string) error { + _, err := sm.s3Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(sm.bucketName), + Key: aws.String(s3Key), + }) + if err != nil { + return err + } + + return sm.s3Client.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(sm.bucketName), + Key: aws.String(s3Key), + }) +} + +func (sm *StorageManager) CopyObject(oldS3key, newS3Key string) error { + _, err := sm.s3Client.CopyObject(&s3.CopyObjectInput{ + Bucket: aws.String(sm.bucketName), + CopySource: aws.String(fmt.Sprintf("%s/%s", sm.bucketName, oldS3key)), + Key: aws.String(newS3Key), + }) + if err != nil { + return err + } + + return sm.s3Client.WaitUntilObjectExists(&s3.HeadObjectInput{ + Bucket: aws.String(sm.bucketName), + Key: aws.String(newS3Key), + }) +} + +func (sm *StorageManager) ListObjects(prefix string) ([]string, error) { + var result []string + + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(sm.bucketName), + Prefix: aws.String(prefix), + } + + err := sm.s3Client.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, obj := range page.Contents { + result = append(result, *obj.Key) + } + return !lastPage + }) + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/engine/workspace_manager.go b/engine/workspace_manager.go new file mode 100644 index 0000000..08b37b4 --- /dev/null +++ b/engine/workspace_manager.go @@ -0,0 +1,42 @@ +package engine + +import ( + "context" + "time" +) + +type WorkSpaceManager struct{} + +func NewWorkSpaceManager() *WorkSpaceManager { + return &WorkSpaceManager{} +} + +type StorageSpace struct { + ID string + Name string + OwnerID string + Type string + CreateTime time.Time + UpdateTime time.Time + InviteCode string + RootPath string +} + +func (wsm *WorkSpaceManager) CreateWorkSpace(ctx context.Context, ownerID, name, visibility string) (string, error) { + return "", nil +} + +func (wsm *WorkSpaceManager) GetWorkSpace(ctx context.Context, spaceID string) { +} + +func (wsm *WorkSpaceManager) UpdateWorkSpace(ctx context.Context, spaceID string) (*StorageSpace, error) { + return nil, nil +} + +func (wsm *WorkSpaceManager) GenerateInviteCode(ctx context.Context, spaceID string) (string, error) { + return "", nil +} + +func (wsm *WorkSpaceManager) ImportWorkSpace(ctx context.Context, code string, newOwnerID string) (string, error) { + return "", nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..615d240 --- /dev/null +++ b/go.mod @@ -0,0 +1,41 @@ +module robotfs + +go 1.23.8 + +replace gosvc => ../gosvc + +require ( + github.com/aws/aws-sdk-go v1.55.6 + github.com/redis/go-redis/v9 v9.8.0 + github.com/spf13/viper v1.20.1 + google.golang.org/protobuf v1.36.6 + gosvc v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect + github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-ole/go-ole v1.2.5 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/shirou/gopsutil v3.21.3+incompatible // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.21.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..875651b --- /dev/null +++ b/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "flag" + "fmt" + "gosvc" + "os" + + "robotfs/config" + "robotfs/utils" +) + +const usage = `Usage: robotfs [options] + +Options: + --config string Path to config file (required) + --version Show version information + --help Show this help message + +Example: + robotfs --config=/path/to/config.yaml` + +func printUsage() { + fmt.Println(usage) +} + +func main() { + configFile := flag.String("config", "", "path to config file") + version := flag.Bool("version", false, "show version information") + help := flag.Bool("help", false, "show help message") + flag.Parse() + + if *help { + printUsage() + return + } + + if *version { + gosvc.Config.PrintVersion() + return + } + + if *configFile != "" { + utils.ConfigurationFileDirectory.Set(*configFile) + if !utils.LoadConfiguration("config", true) { + os.Exit(1) + } + } else { + fmt.Println("Error: --config flag is required") + printUsage() + os.Exit(1) + } + + config.SetupServiceConfig() + + svc, err := CreateService() + if err != nil { + fmt.Println("create service failed, " + err.Error()) + return + } + + err = gosvc.Run(svc) + if err != nil { + fmt.Println("run service failed, " + err.Error()) + return + } +} diff --git a/pb/metadata.pb.go b/pb/metadata.pb.go new file mode 100644 index 0000000..cb6e8fe --- /dev/null +++ b/pb/metadata.pb.go @@ -0,0 +1,212 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v5.29.3 +// source: pb/metadata.proto + +package pb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + FullPath string `protobuf:"bytes,1,opt,name=full_path,json=fullPath,proto3" json:"full_path,omitempty"` + IsDir bool `protobuf:"varint,2,opt,name=is_dir,json=isDir,proto3" json:"is_dir,omitempty"` + Size uint64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + CreateTime int64 `protobuf:"varint,4,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"` + S3Key string `protobuf:"bytes,5,opt,name=s3_key,json=s3Key,proto3" json:"s3_key,omitempty"` + ContentType string `protobuf:"bytes,6,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"` + Etag string `protobuf:"bytes,7,opt,name=etag,proto3" json:"etag,omitempty"` + VersionId string `protobuf:"bytes,8,opt,name=version_id,json=versionId,proto3" json:"version_id,omitempty"` + LastModificationTime int64 `protobuf:"varint,9,opt,name=last_modification_time,json=lastModificationTime,proto3" json:"last_modification_time,omitempty"` + Extended map[string][]byte `protobuf:"bytes,10,rep,name=extended,proto3" json:"extended,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileEntry) Reset() { + *x = FileEntry{} + mi := &file_pb_metadata_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileEntry) ProtoMessage() {} + +func (x *FileEntry) ProtoReflect() protoreflect.Message { + mi := &file_pb_metadata_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileEntry.ProtoReflect.Descriptor instead. +func (*FileEntry) Descriptor() ([]byte, []int) { + return file_pb_metadata_proto_rawDescGZIP(), []int{0} +} + +func (x *FileEntry) GetFullPath() string { + if x != nil { + return x.FullPath + } + return "" +} + +func (x *FileEntry) GetIsDir() bool { + if x != nil { + return x.IsDir + } + return false +} + +func (x *FileEntry) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *FileEntry) GetCreateTime() int64 { + if x != nil { + return x.CreateTime + } + return 0 +} + +func (x *FileEntry) GetS3Key() string { + if x != nil { + return x.S3Key + } + return "" +} + +func (x *FileEntry) GetContentType() string { + if x != nil { + return x.ContentType + } + return "" +} + +func (x *FileEntry) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *FileEntry) GetVersionId() string { + if x != nil { + return x.VersionId + } + return "" +} + +func (x *FileEntry) GetLastModificationTime() int64 { + if x != nil { + return x.LastModificationTime + } + return 0 +} + +func (x *FileEntry) GetExtended() map[string][]byte { + if x != nil { + return x.Extended + } + return nil +} + +var File_pb_metadata_proto protoreflect.FileDescriptor + +const file_pb_metadata_proto_rawDesc = "" + + "\n" + + "\x11pb/metadata.proto\x12\arobotfs\"\x92\x03\n" + + "\tFileEntry\x12\x1b\n" + + "\tfull_path\x18\x01 \x01(\tR\bfullPath\x12\x15\n" + + "\x06is_dir\x18\x02 \x01(\bR\x05isDir\x12\x12\n" + + "\x04size\x18\x03 \x01(\x04R\x04size\x12\x1f\n" + + "\vcreate_time\x18\x04 \x01(\x03R\n" + + "createTime\x12\x15\n" + + "\x06s3_key\x18\x05 \x01(\tR\x05s3Key\x12!\n" + + "\fcontent_type\x18\x06 \x01(\tR\vcontentType\x12\x12\n" + + "\x04etag\x18\a \x01(\tR\x04etag\x12\x1d\n" + + "\n" + + "version_id\x18\b \x01(\tR\tversionId\x124\n" + + "\x16last_modification_time\x18\t \x01(\x03R\x14lastModificationTime\x12<\n" + + "\bextended\x18\n" + + " \x03(\v2 .robotfs.FileEntry.ExtendedEntryR\bextended\x1a;\n" + + "\rExtendedEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\fR\x05value:\x028\x01B\fZ\n" + + "robotfs/pbb\x06proto3" + +var ( + file_pb_metadata_proto_rawDescOnce sync.Once + file_pb_metadata_proto_rawDescData []byte +) + +func file_pb_metadata_proto_rawDescGZIP() []byte { + file_pb_metadata_proto_rawDescOnce.Do(func() { + file_pb_metadata_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_pb_metadata_proto_rawDesc), len(file_pb_metadata_proto_rawDesc))) + }) + return file_pb_metadata_proto_rawDescData +} + +var file_pb_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_pb_metadata_proto_goTypes = []any{ + (*FileEntry)(nil), // 0: robotfs.FileEntry + nil, // 1: robotfs.FileEntry.ExtendedEntry +} +var file_pb_metadata_proto_depIdxs = []int32{ + 1, // 0: robotfs.FileEntry.extended:type_name -> robotfs.FileEntry.ExtendedEntry + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_pb_metadata_proto_init() } +func file_pb_metadata_proto_init() { + if File_pb_metadata_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_pb_metadata_proto_rawDesc), len(file_pb_metadata_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_pb_metadata_proto_goTypes, + DependencyIndexes: file_pb_metadata_proto_depIdxs, + MessageInfos: file_pb_metadata_proto_msgTypes, + }.Build() + File_pb_metadata_proto = out.File + file_pb_metadata_proto_goTypes = nil + file_pb_metadata_proto_depIdxs = nil +} diff --git a/pb/metadata.proto b/pb/metadata.proto new file mode 100644 index 0000000..4c5f5a9 --- /dev/null +++ b/pb/metadata.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package robotfs; + +option go_package = "robotfs/pb"; + +message FileEntry { + string full_path = 1; + bool is_dir = 2; + uint64 size = 3; + int64 create_time = 4; + string s3_key = 5; + string content_type = 6; + string etag = 7; + string version_id = 8; + int64 last_modification_time = 9; + map extended = 10; +} diff --git a/router.go b/router.go new file mode 100644 index 0000000..4a689a6 --- /dev/null +++ b/router.go @@ -0,0 +1,19 @@ +package main + +import ( + "net/http" + + "gosvc/httpserver" +) + +func (s *Service) RegisterRouteRules() { + s.HTTPServer.RegisterRouteRules( + []*httpserver.RouteRule{ + { + Path: "/ping", + Method: http.MethodGet, + Handler: s.API_Ping, + }, + }, + ) +} diff --git a/service.go b/service.go new file mode 100644 index 0000000..3bfba40 --- /dev/null +++ b/service.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "gosvc" + + "robotfs/engine" +) + +type Service struct { + gosvc.ServiceBase + + engine *engine.Engine +} + +func CreateService() (*Service, error) { + return &Service{ + engine: engine.NewEngine(), + }, nil +} + +func (s *Service) Start() error { + if err := s.engine.Start(); err != nil { + return fmt.Errorf("failed to start engine: %v", err) + } + + return nil +} + +func (s *Service) Stop() { + s.engine.Stop() +} + +func (s *Service) Tick() { +} diff --git a/store/meta_store.go b/store/meta_store.go new file mode 100644 index 0000000..b57b774 --- /dev/null +++ b/store/meta_store.go @@ -0,0 +1,30 @@ +package store + +import ( + "context" + + "robotfs/pb" + "robotfs/utils" +) + +type ListEachEntryFunc func(entry *pb.FileEntry) bool + +type MetaStore interface { + GetName() string + + Initialize(configuration utils.Configuration, prefix string) error + + InsertEntry(context.Context, *pb.FileEntry) error + UpdateEntry(context.Context, *pb.FileEntry) (err error) + FindEntry(context.Context, utils.FullPath) (entry *pb.FileEntry, err error) + DeleteEntry(context.Context, utils.FullPath) (err error) + DeleteFolderChildren(context.Context, utils.FullPath) (err error) + ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) + ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) + + BeginTransaction(ctx context.Context) (context.Context, error) + CommitTransaction(ctx context.Context) error + RollbackTransaction(ctx context.Context) error + + Shutdown() +} diff --git a/store/redis_lua/redis_store.go b/store/redis_lua/redis_store.go new file mode 100644 index 0000000..2636a5c --- /dev/null +++ b/store/redis_lua/redis_store.go @@ -0,0 +1,36 @@ +package redis_lua + +import ( + "robotfs/utils" + + "github.com/redis/go-redis/v9" +) + +type RedisLuaStore struct { + UniversalRedisLuaStore +} + +func NewRedisLuaStore() *RedisLuaStore { + return &RedisLuaStore{} +} + +func (store *RedisLuaStore) GetName() string { + return "redis_lua" +} + +func (store *RedisLuaStore) Initialize(configuration utils.Configuration, prefix string) (err error) { + return store.initialize( + configuration.GetString(prefix+"address"), + configuration.GetString(prefix+"password"), + configuration.GetInt(prefix+"database"), + ) +} + +func (store *RedisLuaStore) initialize(hostPort string, password string, database int) (err error) { + store.Client = redis.NewClient(&redis.Options{ + Addr: hostPort, + Password: password, + DB: database, + }) + return +} diff --git a/store/redis_lua/stored_procedure/delete_entry.lua b/store/redis_lua/stored_procedure/delete_entry.lua new file mode 100644 index 0000000..9730937 --- /dev/null +++ b/store/redis_lua/stored_procedure/delete_entry.lua @@ -0,0 +1,19 @@ +-- KEYS[1]: full path of entry +local fullpath = KEYS[1] +-- KEYS[2]: full path of entry +local fullpath_list_key = KEYS[2] +-- KEYS[3]: dir of the entry +local dir_list_key = KEYS[3] + +-- ARGV[1]: isSuperLargeDirectory +local isSuperLargeDirectory = ARGV[1] == "1" +-- ARGV[2]: name of the entry +local name = ARGV[2] + +redis.call("DEL", fullpath, fullpath_list_key) + +if not isSuperLargeDirectory and name ~= "" then + redis.call("ZREM", dir_list_key, name) +end + +return 0 diff --git a/store/redis_lua/stored_procedure/delete_folder_children.lua b/store/redis_lua/stored_procedure/delete_folder_children.lua new file mode 100644 index 0000000..18ea74d --- /dev/null +++ b/store/redis_lua/stored_procedure/delete_folder_children.lua @@ -0,0 +1,15 @@ +-- KEYS[1]: full path of entry +local fullpath = KEYS[1] + +if fullpath ~= "" and string.sub(fullpath, -1) == "/" then + fullpath = string.sub(fullpath, 0, -2) +end + +local files = redis.call("ZRANGE", fullpath .. "\0", "0", "-1") + +for _, name in ipairs(files) do + local file_path = fullpath .. "/" .. name + redis.call("DEL", file_path, file_path .. "\0") +end + +return 0 diff --git a/store/redis_lua/stored_procedure/init.go b/store/redis_lua/stored_procedure/init.go new file mode 100644 index 0000000..9373cc5 --- /dev/null +++ b/store/redis_lua/stored_procedure/init.go @@ -0,0 +1,24 @@ +package stored_procedure + +import ( + _ "embed" + "github.com/redis/go-redis/v9" +) + +func init() { + InsertEntryScript = redis.NewScript(insertEntry) + DeleteEntryScript = redis.NewScript(deleteEntry) + DeleteFolderChildrenScript = redis.NewScript(deleteFolderChildren) +} + +//go:embed insert_entry.lua +var insertEntry string +var InsertEntryScript *redis.Script + +//go:embed delete_entry.lua +var deleteEntry string +var DeleteEntryScript *redis.Script + +//go:embed delete_folder_children.lua +var deleteFolderChildren string +var DeleteFolderChildrenScript *redis.Script diff --git a/store/redis_lua/stored_procedure/insert_entry.lua b/store/redis_lua/stored_procedure/insert_entry.lua new file mode 100644 index 0000000..a990aff --- /dev/null +++ b/store/redis_lua/stored_procedure/insert_entry.lua @@ -0,0 +1,27 @@ +-- KEYS[1]: full path of entry +local full_path = KEYS[1] +-- KEYS[2]: dir of the entry +local dir_list_key = KEYS[2] + +-- ARGV[1]: content of the entry +local entry = ARGV[1] +-- ARGV[2]: TTL of the entry +local ttlSec = tonumber(ARGV[2]) +-- ARGV[3]: isSuperLargeDirectory +local isSuperLargeDirectory = ARGV[3] == "1" +-- ARGV[4]: zscore of the entry in zset +local zscore = tonumber(ARGV[4]) +-- ARGV[5]: name of the entry +local name = ARGV[5] + +if ttlSec > 0 then + redis.call("SET", full_path, entry, "EX", ttlSec) +else + redis.call("SET", full_path, entry) +end + +if not isSuperLargeDirectory and name ~= "" then + redis.call("ZADD", dir_list_key, "NX", zscore, name) +end + +return 0 diff --git a/store/redis_lua/universal_redis_store.go b/store/redis_lua/universal_redis_store.go new file mode 100644 index 0000000..327c862 --- /dev/null +++ b/store/redis_lua/universal_redis_store.go @@ -0,0 +1,189 @@ +package redis_lua + +import ( + "context" + "fmt" + + "github.com/redis/go-redis/v9" + + "robotfs/pb" + "robotfs/store" + "robotfs/store/redis_lua/stored_procedure" + "robotfs/utils" +) + +const ( + DIR_LIST_MARKER = "\x00" +) + +type UniversalRedisLuaStore struct { + Client redis.UniversalClient + superLargeDirectoryHash map[string]bool +} + +func (store *UniversalRedisLuaStore) isSuperLargeDirectory(dir string) (isSuperLargeDirectory bool) { + _, isSuperLargeDirectory = store.superLargeDirectoryHash[dir] + return +} + +func (store *UniversalRedisLuaStore) loadSuperLargeDirectories(superLargeDirectories []string) { + store.superLargeDirectoryHash = make(map[string]bool) + for _, dir := range superLargeDirectories { + store.superLargeDirectoryHash[dir] = true + } +} + +func (store *UniversalRedisLuaStore) BeginTransaction(ctx context.Context) (context.Context, error) { + return ctx, nil +} +func (store *UniversalRedisLuaStore) CommitTransaction(ctx context.Context) error { + return nil +} +func (store *UniversalRedisLuaStore) RollbackTransaction(ctx context.Context) error { + return nil +} + +func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *pb.FileEntry) (err error) { + + // value, err := entry.EncodeAttributesAndChunks() + // if err != nil { + // return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + // } + + // if len(entry.GetChunks()) > CountEntryChunksForGzip { + // value = util.MaybeGzipData(value) + // } + + // dir, name := entry.FullPath.DirAndName() + + // err = stored_procedure.InsertEntryScript.Run(ctx, store.Client, + // []string{string(entry.FullPath), genDirectoryListKey(dir)}, + // value, entry.TtlSec, + // store.isSuperLargeDirectory(dir), 0, name).Err() + + // if err != nil { + // return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + // } + + return nil +} + +func (store *UniversalRedisLuaStore) UpdateEntry(ctx context.Context, entry *pb.FileEntry) (err error) { + + return store.InsertEntry(ctx, entry) +} + +func (store *UniversalRedisLuaStore) FindEntry(ctx context.Context, fullpath utils.FullPath) (entry *pb.FileEntry, err error) { + + // data, err := store.Client.Get(ctx, string(fullpath)).Result() + // if err == redis.Nil { + // return nil, pb.ErrNotFound + // } + + // if err != nil { + // return nil, fmt.Errorf("get %s : %v", fullpath, err) + // } + + // entry = &pb.FileEntry{ + // FullPath: fullpath.Name(), + // } + // err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data))) + // if err != nil { + // return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + // } + + return entry, nil +} + +func (store *UniversalRedisLuaStore) DeleteEntry(ctx context.Context, fullpath utils.FullPath) (err error) { + + dir, name := fullpath.DirAndName() + + err = stored_procedure.DeleteEntryScript.Run(ctx, store.Client, + []string{string(fullpath), genDirectoryListKey(string(fullpath)), genDirectoryListKey(dir)}, + store.isSuperLargeDirectory(dir), name).Err() + + if err != nil { + return fmt.Errorf("DeleteEntry %s : %v", fullpath, err) + } + + return nil +} + +func (store *UniversalRedisLuaStore) DeleteFolderChildren(ctx context.Context, fullpath utils.FullPath) (err error) { + + if store.isSuperLargeDirectory(string(fullpath)) { + return nil + } + + err = stored_procedure.DeleteFolderChildrenScript.Run(ctx, store.Client, + []string{string(fullpath)}).Err() + + if err != nil { + return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err) + } + + return nil +} + +func (store *UniversalRedisLuaStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) { + // return lastFileName, engine.ErrUnsupportedListDirectoryPrefixed + return lastFileName, err +} + +func (store *UniversalRedisLuaStore) ListDirectoryEntries(ctx context.Context, dirPath utils.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc store.ListEachEntryFunc) (lastFileName string, err error) { + + // dirListKey := genDirectoryListKey(string(dirPath)) + + // min := "-" + // if startFileName != "" { + // if includeStartFile { + // min = "[" + startFileName + // } else { + // min = "(" + startFileName + // } + // } + + // members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{ + // Min: min, + // Max: "+", + // Offset: 0, + // Count: limit, + // }).Result() + // if err != nil { + // return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) + // } + + // // fetch entry meta + // for _, fileName := range members { + // path := utils.NewFullPath(string(dirPath), fileName) + // entry, err := store.FindEntry(ctx, path) + // lastFileName = fileName + // if err != nil { + + // if err == pb.ErrNotFound { + // continue + // } + // } else { + // if entry.TtlSec > 0 { + // if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { + // store.DeleteEntry(ctx, path) + // continue + // } + // } + // if !eachEntryFunc(entry) { + // break + // } + // } + // } + + return lastFileName, err +} + +func genDirectoryListKey(dir string) (dirList string) { + return dir + DIR_LIST_MARKER +} + +func (store *UniversalRedisLuaStore) Shutdown() { + store.Client.Close() +} diff --git a/store/redis_lua/universal_redis_store.go_bak b/store/redis_lua/universal_redis_store.go_bak new file mode 100644 index 0000000..e535a4d --- /dev/null +++ b/store/redis_lua/universal_redis_store.go_bak @@ -0,0 +1,210 @@ +package redis + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +const ( + DIR_LIST_MARKER = "\x00" +) + +type UniversalRedis2Store struct { + Client redis.UniversalClient +} + +func (store *UniversalRedis2Store) BeginTransaction(ctx context.Context) (context.Context, error) { + return ctx, nil +} +func (store *UniversalRedis2Store) CommitTransaction(ctx context.Context) error { + return nil +} +func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) error { + return nil +} + +func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + + if err = store.doInsertEntry(ctx, entry); err != nil { + return err + } + + dir, name := entry.FullPath.DirAndName() + if store.isSuperLargeDirectory(dir) { + return nil + } + + if name != "" { + if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil { + return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) + } + } + + return nil +} + +func (store *UniversalRedis2Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error { + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + if len(entry.GetChunks()) > filer.CountEntryChunksForGzip { + value = util.MaybeGzipData(value) + } + + if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + return nil +} + +func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + + return store.doInsertEntry(ctx, entry) +} + +func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + + data, err := store.Client.Get(ctx, string(fullpath)).Result() + if err == redis.Nil { + return nil, filer_pb.ErrNotFound + } + + if err != nil { + return nil, fmt.Errorf("get %s : %v", fullpath, err) + } + + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data))) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + + _, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result() + if err != nil { + return fmt.Errorf("delete dir list %s : %v", fullpath, err) + } + + _, err = store.Client.Del(ctx, string(fullpath)).Result() + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + dir, name := fullpath.DirAndName() + if store.isSuperLargeDirectory(dir) { + return nil + } + if name != "" { + _, err = store.Client.ZRem(ctx, genDirectoryListKey(dir), name).Result() + if err != nil { + return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err) + } + } + + return nil +} + +func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { + + if store.isSuperLargeDirectory(string(fullpath)) { + return nil + } + + members, err := store.Client.ZRangeByLex(ctx, genDirectoryListKey(string(fullpath)), &redis.ZRangeBy{ + Min: "-", + Max: "+", + }).Result() + if err != nil { + return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err) + } + + for _, fileName := range members { + path := util.NewFullPath(string(fullpath), fileName) + _, err = store.Client.Del(ctx, string(path)).Result() + if err != nil { + return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err) + } + // not efficient, but need to remove if it is a directory + store.Client.Del(ctx, genDirectoryListKey(string(path))) + } + + return nil +} + +func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed +} + +func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + + dirListKey := genDirectoryListKey(string(dirPath)) + + min := "-" + if startFileName != "" { + if includeStartFile { + min = "[" + startFileName + } else { + min = "(" + startFileName + } + } + + members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{ + Min: min, + Max: "+", + Offset: 0, + Count: limit, + }).Result() + if err != nil { + return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) + } + + // fetch entry meta + for _, fileName := range members { + path := util.NewFullPath(string(dirPath), fileName) + entry, err := store.FindEntry(ctx, path) + lastFileName = fileName + if err != nil { + glog.V(0).Infof("list %s : %v", path, err) + if err == filer_pb.ErrNotFound { + continue + } + } else { + if entry.TtlSec > 0 { + if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { + store.Client.Del(ctx, string(path)).Result() + store.Client.ZRem(ctx, dirListKey, fileName).Result() + continue + } + } + if !eachEntryFunc(entry) { + break + } + } + } + + return lastFileName, err +} + +func genDirectoryListKey(dir string) (dirList string) { + return dir + DIR_LIST_MARKER +} + +func (store *UniversalRedis2Store) Shutdown() { + store.Client.Close() +} diff --git a/store/redis_lua/universal_redis_store_kv.go b/store/redis_lua/universal_redis_store_kv.go new file mode 100644 index 0000000..9bb59c2 --- /dev/null +++ b/store/redis_lua/universal_redis_store_kv.go @@ -0,0 +1,41 @@ +package redis_lua + +import ( + "context" + "fmt" + + "github.com/redis/go-redis/v9" +) + +func (store *UniversalRedisLuaStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + _, err = store.Client.Set(ctx, string(key), value, 0).Result() + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *UniversalRedisLuaStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + data, err := store.Client.Get(ctx, string(key)).Result() + + if err == redis.Nil { + return nil, fmt.Errorf("kv: not found") + } + + return []byte(data), err +} + +func (store *UniversalRedisLuaStore) KvDelete(ctx context.Context, key []byte) (err error) { + + _, err = store.Client.Del(ctx, string(key)).Result() + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/utils/config.go b/utils/config.go new file mode 100644 index 0000000..768c9a5 --- /dev/null +++ b/utils/config.go @@ -0,0 +1,115 @@ +package utils + +import ( + "strings" + "sync" + + "gosvc/logger" + + "github.com/spf13/viper" +) + +var ( + ConfigurationFileDirectory DirectoryValueType + loadSecurityConfigOnce sync.Once +) + +type DirectoryValueType string + +func (s *DirectoryValueType) Set(value string) error { + *s = DirectoryValueType(value) + return nil +} +func (s *DirectoryValueType) String() string { + return string(*s) +} + +type Configuration interface { + GetString(key string) string + GetBool(key string) bool + GetInt(key string) int + GetStringSlice(key string) []string + SetDefault(key string, value interface{}) +} + +func LoadSecurityConfiguration() { + loadSecurityConfigOnce.Do(func() { + LoadConfiguration("security", false) + }) +} + +func LoadConfiguration(configFileName string, required bool) (loaded bool) { + viper.SetConfigName(configFileName) // name of config file (without extension) + viper.AddConfigPath(".") // look for config in the working directory + + if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file + if strings.Contains(err.Error(), "Not Found") { + logger.Info("Reading %s: %v", viper.ConfigFileUsed(), err) + } else { + logger.Error("Reading %s: %v", viper.ConfigFileUsed(), err) + } + if required { + logger.Error("Failed to load %s.yaml file from current directory\n"+ + "\nPlease use this command to run the program:\n"+ + " robotfs --config=%s.yaml\n\n\n", + configFileName, configFileName) + } + return false + } + logger.Info("Reading %s.yaml from %s", configFileName, viper.ConfigFileUsed()) + + return true +} + +type ViperProxy struct { + *viper.Viper + sync.Mutex +} + +var ( + vp = &ViperProxy{} +) + +func (vp *ViperProxy) SetDefault(key string, value interface{}) { + vp.Lock() + defer vp.Unlock() + vp.Viper.SetDefault(key, value) +} + +func (vp *ViperProxy) GetString(key string) string { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetString(key) +} + +func (vp *ViperProxy) GetBool(key string) bool { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetBool(key) +} + +func (vp *ViperProxy) GetInt(key string) int { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetInt(key) +} + +func (vp *ViperProxy) GetStringSlice(key string) []string { + vp.Lock() + defer vp.Unlock() + return vp.Viper.GetStringSlice(key) +} + +func GetViper() *ViperProxy { + vp.Lock() + defer vp.Unlock() + + if vp.Viper == nil { + vp.Viper = viper.GetViper() + vp.AutomaticEnv() + vp.SetEnvPrefix("robotfs") + vp.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + } + + return vp +} diff --git a/utils/file.go b/utils/file.go new file mode 100644 index 0000000..2bc7e9a --- /dev/null +++ b/utils/file.go @@ -0,0 +1,14 @@ +package utils + +import ( + "strings" + "path/filepath" +) + +func NormalizePath(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + return filepath.Clean(path) +} diff --git a/utils/fullPath.go b/utils/fullPath.go new file mode 100644 index 0000000..69c5d7a --- /dev/null +++ b/utils/fullPath.go @@ -0,0 +1,72 @@ +package utils + +import ( + "path/filepath" + "strings" +) + +type FullPath string + +func NewFullPath(dir, name string) FullPath { + return FullPath(dir).Child(name) +} + +func (fp FullPath) DirAndName() (string, string) { + dir, name := filepath.Split(string(fp)) + name = strings.ToValidUTF8(name, "?") + if dir == "/" { + return dir, name + } + if len(dir) < 1 { + return "/", "" + } + return dir[:len(dir)-1], name +} + +func (fp FullPath) Name() string { + _, name := filepath.Split(string(fp)) + name = strings.ToValidUTF8(name, "?") + return name +} + +func (fp FullPath) Child(name string) FullPath { + dir := string(fp) + noPrefix := name + if strings.HasPrefix(name, "/") { + noPrefix = name[1:] + } + if strings.HasSuffix(dir, "/") { + return FullPath(dir + noPrefix) + } + return FullPath(dir + "/" + noPrefix) +} + +// split, but skipping the root +func (fp FullPath) Split() []string { + if fp == "" || fp == "/" { + return []string{} + } + return strings.Split(string(fp)[1:], "/") +} + +func Join(names ...string) string { + return filepath.ToSlash(filepath.Join(names...)) +} + +func JoinPath(names ...string) FullPath { + return FullPath(Join(names...)) +} + +func (fp FullPath) IsUnder(other FullPath) bool { + if other == "/" { + return true + } + return strings.HasPrefix(string(fp), string(other)+"/") +} + +func StringSplit(separatedValues string, sep string) []string { + if separatedValues == "" { + return nil + } + return strings.Split(separatedValues, sep) +} diff --git a/utils/s3.go b/utils/s3.go new file mode 100644 index 0000000..5b5077f --- /dev/null +++ b/utils/s3.go @@ -0,0 +1,14 @@ +package utils + +import ( + "strings" +) + +func PathToS3Key(path string) string { + key := strings.TrimPrefix(path, "/") + if key == "" { + key = "root" + } + + return key +}