🔒 Enable scoped internal test slicing and delivery
This commit is contained in:
parent
62889285f5
commit
949daabd02
@ -2,41 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/bootstrap"
|
||||
)
|
||||
|
||||
type apiBootstrapStore interface {
|
||||
RunMigrations(context.Context) error
|
||||
EnsureIndexes(context.Context) error
|
||||
EnsureDefaultOrganization(context.Context) (*domain.OrganizationRecord, error)
|
||||
GetOrganization(context.Context, string) (*domain.OrganizationRecord, error)
|
||||
}
|
||||
|
||||
// verify-existing is only for already initialized databases with separately
|
||||
// verified schema/index compatibility. It never initializes or repairs data.
|
||||
func bootstrapAPI(ctx context.Context, db apiBootstrapStore, mode string) error {
|
||||
switch mode {
|
||||
case "verify-existing":
|
||||
org, err := db.GetOrganization(ctx, domain.DefaultOrganizationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if org == nil {
|
||||
return fmt.Errorf("database is not initialized; verify-existing cannot bootstrap it")
|
||||
}
|
||||
return nil
|
||||
case "", "migrate":
|
||||
if err := db.RunMigrations(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.EnsureIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := db.EnsureDefaultOrganization(ctx)
|
||||
return err
|
||||
default:
|
||||
return fmt.Errorf("invalid DROBOTICFLOW_DATABASE_BOOTSTRAP mode")
|
||||
}
|
||||
func bootstrapAPI(ctx context.Context, db bootstrap.Store, mode string) error {
|
||||
return bootstrap.Run(ctx, db, mode)
|
||||
}
|
||||
|
||||
@ -92,7 +92,12 @@ func main() {
|
||||
|
||||
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
|
||||
if os.Getenv("ANNOTATION_PROCESSOR_MODE") == "processor" {
|
||||
if err := dataStore.EnsureIndexes(ctx); err != nil {
|
||||
checkIndexes := dataStore.EnsureIndexes
|
||||
mode := os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP")
|
||||
if mode != "" && mode != "migrate" {
|
||||
checkIndexes = func(ctx context.Context) error { return bootstrapAPI(ctx, dataStore, mode) }
|
||||
}
|
||||
if err := checkIndexes(ctx); err != nil {
|
||||
fmt.Printf("[annotation-processor] indexes: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/bootstrap"
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/retry"
|
||||
"droboticflow/internal/store"
|
||||
@ -51,18 +52,11 @@ func main() {
|
||||
}()
|
||||
|
||||
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
|
||||
if err := dataStore.RunMigrations(ctx); err != nil {
|
||||
fmt.Printf("[worker] failed to run mongo migrations: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := dataStore.EnsureIndexes(ctx); err != nil {
|
||||
fmt.Printf("[worker] failed to ensure mongo indexes: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if _, err := dataStore.EnsureDefaultOrganization(ctx); err != nil {
|
||||
fmt.Printf("[worker] failed to ensure default organization: %v\n", err)
|
||||
if err := bootstrap.Run(ctx, dataStore, os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP")); err != nil {
|
||||
fmt.Printf("[worker] database bootstrap: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if cfg.OperatorRunnerImage == "" {
|
||||
fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; set it explicitly or configure REGISTRY_HOST and REGISTRY_NAMESPACE for deploy/render scripts")
|
||||
os.Exit(1)
|
||||
|
||||
@ -42,7 +42,7 @@
|
||||
}
|
||||
```
|
||||
|
||||
只读委托必须指定连接的真实所属组织,且不能配置任何 writePrefixes;所有写入、发布和清理请求均拒绝。省略委托字段仍严格检查连接属于请求组织。其他组织不能继承这项委托。凭证只在后台使用,不复制给测试组织或返回浏览器。授权仅覆盖读取测试副本,不能配置客户原始素材目录。
|
||||
仅设置 readConnectionOrganizationId 的只读委托必须指定连接的真实所属组织,且不能配置任何 writePrefixes;所有写入、发布和清理请求均拒绝。省略委托字段仍严格检查连接属于请求组织。其他组织不能继承这项委托。凭证只在后台使用,不复制给测试组织或返回浏览器。授权仅覆盖读取测试副本,不能配置客户原始素材目录。
|
||||
|
||||
此方案不修改 BOS 权限,但原凭证仍有较大权限,隔离依赖应用配置与路径检查,不等同于独立受限凭证。只读委托仅解决素材与帧范围加载,不支持生成切片产物或完整交付。
|
||||
此配置不处理工作流 worker 的最终打包、BOS 交付目录、项目/供应商删除配置,也不隔离计算资源。最终交付存储须另行配置和核验,不得据帧范围通过宣称全流程验收通过。每次新增 run 必须同步核对允许的输出目录。
|
||||
@ -64,3 +64,28 @@ go test ./internal/api -run 'TestAnnotation(TenantWorkspace|BOSVersion|TimelineC
|
||||
API 设置 `DROBOTICFLOW_DATABASE_BOOTSTRAP=verify-existing` 时,启动仅读取默认组织验证数据库已初始化,不运行迁移、索引创建或默认组织初始化。默认组织缺失或读取失败会拒绝启动;未知配置值也拒绝启动。未设置或设为 `migrate` 保留旧行为。该选项不替代 schema/index 兼容性核验,发布前必须确认候选版本没有新增 schema/index 要求。
|
||||
|
||||
本次只读委托只部署 API,并保留现有 processor,以免打断客户在途切片。processor 不具备新的测试组织读写能力,测试切片生成不在本次验收范围。此选项只禁止 API 的启动初始化,不禁止客户正常业务写入或既有后台业务处理。
|
||||
|
||||
|
||||
## 经授权的测试全流程委托
|
||||
|
||||
需要切片、送检和交付时,再显式添加 `writeConnectionOrganizationId`(必须与读取连接所属组织一致)、`writePrefixes` 和 `deliveryPrefixes`。只读配置不会自动获得写权限。
|
||||
|
||||
```json
|
||||
{
|
||||
"example-test-org": {
|
||||
"connectionId": "existing-storage-connection",
|
||||
"readConnectionOrganizationId": "existing-connection-owner",
|
||||
"writeConnectionOrganizationId": "existing-connection-owner",
|
||||
"bucket": "example-workspace-bucket",
|
||||
"readPrefixes": ["internal-test/example/assets"],
|
||||
"writePrefixes": ["system/runs/example-test-run"],
|
||||
"deliveryPrefixes": ["internal-test/example/deliveries"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
输入目录和输出目录不得包含或相互覆盖。切片写入仅允许列出的具体 run 目录,不能授予整个 `system/runs`。最终交付另由 worker 校验测试组织、项目、Capture、获准父 run 的发布子任务关系,以及当前子任务的本地输出路径;最终目录只允许 `deliveryPrefixes`。沿用条件创建和所有权校验,不覆盖已有无关对象。普通工作流存储、连接管理和其他组织不获得跨组织授权。
|
||||
|
||||
产物预览与交付下载按请求组织、连接、bucket、完整目录边界分别检查。API、processor、worker 的映射须一致。worker 也支持 `DROBOTICFLOW_DATABASE_BOOTSTRAP=verify-existing`;processor 在该模式不建索引,只读校验已有数据库。发布前仍需确认索引兼容及在途任务,保留原资源和并发配置。
|
||||
|
||||
此授权依赖应用校验,不是 BOS 账号级隔离;共用节点、队列和存储仍可能竞争资源。只允许少量功能测试,不能据此承诺多人容量或候选代码完全不影响客户。实际验证应记录切片、送检、质检、终审、打包和下载各环节结果,以及客户只读回归。
|
||||
|
||||
@ -2,89 +2,19 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workspacescope"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
// Deployment-owned mappings may explicitly delegate prefix-scoped reads.
|
||||
// They never delegate cross-organization writes or expose credentials to users.
|
||||
type annotationWorkspaceMapping struct {
|
||||
ConnectionID string `json:"connectionId"`
|
||||
ReadConnectionOrganizationID string `json:"readConnectionOrganizationId,omitempty"`
|
||||
Bucket string `json:"bucket"`
|
||||
ReadPrefixes []string `json:"readPrefixes"`
|
||||
WritePrefixes []string `json:"writePrefixes"`
|
||||
}
|
||||
type annotationWorkspaceMapping = workspacescope.Mapping
|
||||
|
||||
func annotationMappingForOrganization(org string) (*annotationWorkspaceMapping, error) {
|
||||
raw := os.Getenv("ANNOTATION_WORKSPACE_MAPPINGS")
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
var mappings map[string]annotationWorkspaceMapping
|
||||
if err := decoder.Decode(&mappings); err != nil {
|
||||
return nil, fmt.Errorf("invalid annotation workspace mappings")
|
||||
}
|
||||
if err := decoder.Decode(new(any)); err != io.EOF || mappings == nil {
|
||||
return nil, fmt.Errorf("invalid annotation workspace mappings")
|
||||
}
|
||||
mapping, ok := mappings[org]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if strings.TrimSpace(org) == "" || strings.TrimSpace(mapping.ConnectionID) == "" || strings.TrimSpace(mapping.Bucket) == "" || len(mapping.ReadPrefixes) == 0 {
|
||||
return nil, fmt.Errorf("annotation organization workspace mapping incomplete")
|
||||
}
|
||||
if mapping.ReadConnectionOrganizationID != "" && (strings.TrimSpace(mapping.ReadConnectionOrganizationID) != mapping.ReadConnectionOrganizationID || mapping.ReadConnectionOrganizationID == org || len(mapping.WritePrefixes) != 0) {
|
||||
return nil, fmt.Errorf("delegated annotation workspace must be read-only with an explicit foreign owner")
|
||||
}
|
||||
for _, prefixes := range [][]string{mapping.ReadPrefixes, mapping.WritePrefixes} {
|
||||
for _, prefix := range prefixes {
|
||||
if prefix == "" || prefix == "." || prefix == ".." || strings.HasPrefix(prefix, "../") || path.IsAbs(prefix) || path.Clean(prefix) != prefix || strings.ContainsAny(prefix, `\*?`) {
|
||||
return nil, fmt.Errorf("invalid annotation workspace prefix")
|
||||
}
|
||||
}
|
||||
}
|
||||
return &mapping, nil
|
||||
}
|
||||
|
||||
func annotationPrefixAllows(key string, prefixes []string) bool {
|
||||
for _, prefix := range prefixes {
|
||||
if key == prefix || strings.HasPrefix(key, prefix+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A scoped path must not redirect through a symlink, including an existing
|
||||
// parent of a not-yet-created output. The configured workspace root is trusted.
|
||||
func annotationScopedPath(root, relative string) error {
|
||||
current := root
|
||||
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, component)
|
||||
info, err := os.Lstat(current)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("annotation scoped path contains symlink")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var annotationMappingForOrganization = workspacescope.ForOrganization
|
||||
var annotationPrefixAllows = workspacescope.PrefixAllows
|
||||
var annotationScopedPath = workspacescope.ScopedPath
|
||||
|
||||
func (s *server) annotationWorkspaceObjectAccess(ctx context.Context, org, filename string, write bool) (*domain.StorageConnectionRecord, string, string, error) {
|
||||
mapping, err := annotationMappingForOrganization(org)
|
||||
@ -110,7 +40,7 @@ func (s *server) annotationWorkspaceObjectAccess(ctx context.Context, org, filen
|
||||
}
|
||||
owner := org
|
||||
if mapping != nil && mapping.ReadConnectionOrganizationID != "" {
|
||||
if write {
|
||||
if write && mapping.WriteConnectionOrganizationID == "" {
|
||||
return nil, "", "", fmt.Errorf("delegated annotation workspace is read-only")
|
||||
}
|
||||
owner = mapping.ReadConnectionOrganizationID
|
||||
@ -126,7 +56,7 @@ func (s *server) annotationWorkspaceObjectAccess(ctx context.Context, org, filen
|
||||
if mapping != nil {
|
||||
allowed := annotationPrefixAllows(key, mapping.WritePrefixes)
|
||||
if !write {
|
||||
allowed = allowed || annotationPrefixAllows(key, mapping.ReadPrefixes)
|
||||
allowed = allowed || annotationPrefixAllows(key, mapping.ReadPrefixes) || annotationPrefixAllows(key, mapping.DeliveryPrefixes)
|
||||
}
|
||||
if !allowed {
|
||||
return nil, "", "", fmt.Errorf("annotation path outside organization workspace scope")
|
||||
|
||||
40
internal/api/delegated_storage_read.go
Normal file
40
internal/api/delegated_storage_read.go
Normal file
@ -0,0 +1,40 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workspacescope"
|
||||
)
|
||||
|
||||
// Credentials stay server-side. This grants one exact scoped read, never
|
||||
// connection enumeration, bucket listing or arbitrary mutation endpoints.
|
||||
func (s *server) delegatedStorageRead(ctx context.Context, org, id, bucket, key string, deliveryOnly bool) (*domain.StorageConnectionRecord, error) {
|
||||
m, err := workspacescope.ForOrganization(org)
|
||||
if err != nil || m == nil {
|
||||
return nil, err
|
||||
}
|
||||
if id == "" {
|
||||
id = m.ConnectionID
|
||||
}
|
||||
prefixes := m.DeliveryPrefixes
|
||||
if !deliveryOnly {
|
||||
prefixes = append(append(append([]string{}, prefixes...), m.ReadPrefixes...), m.WritePrefixes...)
|
||||
}
|
||||
if id != m.ConnectionID || bucket != m.Bucket || !workspacescope.PrefixAllows(key, prefixes) {
|
||||
return nil, fmt.Errorf("object outside delegated storage scope")
|
||||
}
|
||||
c, err := s.store.GetStorageConnection(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owner := org
|
||||
if m.ReadConnectionOrganizationID != "" {
|
||||
owner = m.ReadConnectionOrganizationID
|
||||
}
|
||||
if c == nil || c.OrganizationID != owner || c.Provider != domain.StorageProviderBOS {
|
||||
return nil, fmt.Errorf("delegated storage owner mismatch")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
40
internal/api/delegated_storage_read_test.go
Normal file
40
internal/api/delegated_storage_read_test.go
Normal file
@ -0,0 +1,40 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDelegatedStorageReadAndWriteArePrefixBound(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", `{"test":{"connectionId":"c","bucket":"b","readConnectionOrganizationId":"customer","writeConnectionOrganizationId":"customer","readPrefixes":["internal-test/assets"],"writePrefixes":["system/runs/test-run"],"deliveryPrefixes":["internal-test/delivery"]}}`)
|
||||
db := store.NewMemoryStore()
|
||||
ctx := t.Context()
|
||||
db.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "c", OrganizationID: "customer", Provider: domain.StorageProviderBOS})
|
||||
s := &server{store: db, workspaceMountPath: t.TempDir()}
|
||||
for _, key := range []string{"internal-test/assets/video.mp4", "system/runs/test-run/clip.mp4", "internal-test/delivery/manifest.json"} {
|
||||
if c, e := s.delegatedStorageRead(ctx, "test", "c", "b", key, false); e != nil || c == nil {
|
||||
t.Fatal(key, e)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"customer/video.mp4", "system/runs/test-run-other/clip.mp4", "internal-test/delivery/../assets/video.mp4"} {
|
||||
if _, e := s.delegatedStorageRead(ctx, "test", "c", "b", key, false); e == nil {
|
||||
t.Fatal("foreign read accepted", key)
|
||||
}
|
||||
}
|
||||
if _, e := s.delegatedStorageRead(ctx, "test", "c", "b", "internal-test/assets/video.mp4", true); e == nil {
|
||||
t.Fatal("input treated as delivery")
|
||||
}
|
||||
if _, e := s.delegatedStorageRead(ctx, "test", "c", "other", "internal-test/delivery/x", true); e == nil {
|
||||
t.Fatal("wrong bucket accepted")
|
||||
}
|
||||
for _, key := range []string{"internal-test/assets/video.mp4", "internal-test/delivery/manifest.json", "system/runs/customer/file"} {
|
||||
if _, _, _, e := s.annotationWorkspaceObject(ctx, "test", filepath.Join(s.workspaceMountPath, key)); e == nil {
|
||||
t.Fatal("annotation wrote outside run", key)
|
||||
}
|
||||
}
|
||||
if _, _, _, e := s.annotationWorkspaceObject(ctx, "test", filepath.Join(s.workspaceMountPath, "system/runs/test-run/clip.mp4")); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
@ -108,9 +108,12 @@ func (s *server) handleDeliveryDownloadProjects(w http.ResponseWriter, r *http.R
|
||||
renderJSON(w, 200, map[string]any{"items": result})
|
||||
}
|
||||
|
||||
func (s *server) downloadBrowser(ctx context.Context, org, id, bucket string) (objectStorageBrowser, func(), error) {
|
||||
func (s *server) downloadBrowser(ctx context.Context, org, id, bucket, prefix string) (objectStorageBrowser, func(), error) {
|
||||
c, e := s.store.GetStorageConnection(ctx, id)
|
||||
if e != nil || c == nil || c.OrganizationID != org || c.Provider != domain.StorageProviderBOS {
|
||||
if e == nil && c != nil && c.OrganizationID != org {
|
||||
c, e = s.delegatedStorageRead(ctx, org, id, bucket, prefix, true)
|
||||
}
|
||||
if e != nil || c == nil || c.Provider != domain.StorageProviderBOS {
|
||||
return nil, nil, fmt.Errorf("交付存储不存在或不可访问")
|
||||
}
|
||||
if _, e = resolveObjectStorageBucket(c, bucket); e != nil {
|
||||
@ -352,7 +355,7 @@ func (s *server) handleDeliveryDownloadManifest(w http.ResponseWriter, r *http.R
|
||||
renderValidationError(w, validationIssue{Field: "directory", Message: "a non-system delivery directory is required"})
|
||||
return
|
||||
}
|
||||
b, cleanup, e := s.downloadBrowser(ctx, s.requestOrganizationID(r), connectionID, u.Host)
|
||||
b, cleanup, e := s.downloadBrowser(ctx, s.requestOrganizationID(r), connectionID, u.Host, prefix)
|
||||
if cleanup != nil {
|
||||
defer cleanup()
|
||||
}
|
||||
@ -373,11 +376,15 @@ func (s *server) handleDeliveryDownloadManifest(w http.ResponseWriter, r *http.R
|
||||
files := []deliveryDownloadFile{}
|
||||
browsers := map[string]objectStorageBrowser{}
|
||||
for _, t := range targets {
|
||||
if _, scopeErr := s.delegatedStorageRead(ctx, s.requestOrganizationID(r), t.connectionID, t.bucket, t.prefix, true); scopeErr != nil {
|
||||
renderJSON(w, 404, map[string]string{"error": "交付目录超出授权范围"})
|
||||
return
|
||||
}
|
||||
k := t.connectionID + "/" + t.bucket
|
||||
b := browsers[k]
|
||||
if b == nil {
|
||||
var cleanup func()
|
||||
b, cleanup, err = s.downloadBrowser(ctx, s.requestOrganizationID(r), t.connectionID, t.bucket)
|
||||
b, cleanup, err = s.downloadBrowser(ctx, s.requestOrganizationID(r), t.connectionID, t.bucket, t.prefix)
|
||||
if cleanup != nil {
|
||||
defer cleanup()
|
||||
}
|
||||
|
||||
@ -2272,6 +2272,16 @@ func (server *server) handleObjectStorageObjectRestore(writer http.ResponseWrite
|
||||
func (server *server) handleObjectStorageContent(writer http.ResponseWriter, request *http.Request) {
|
||||
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
||||
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
||||
if err != nil && (request.Method == http.MethodGet || request.Method == http.MethodHead) {
|
||||
// Only this content endpoint can use the scoped fallback. All storage
|
||||
// administration/mutation routes keep their tenant-owned connection gate.
|
||||
scoped, scopeErr := server.delegatedStorageRead(request.Context(), server.requestOrganizationID(request), connectionID, request.URL.Query().Get("bucket"), request.URL.Query().Get("key"), false)
|
||||
if scopeErr == nil && scoped != nil {
|
||||
connection = server.storageConnectionWithResolvedEndpoint(scoped)
|
||||
browser, cleanup, err = storageConnectionBrowserResolver(request.Context(), connection)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
|
||||
@ -7450,6 +7450,11 @@ func (server *server) resolveArtifactStorageConnection(
|
||||
if run == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if objectBucket, key, ok := artifactBOSLocation(artifact); ok {
|
||||
if connection, err := server.delegatedStorageRead(ctx, run.OrganizationID, "", objectBucket, key, false); err != nil || connection != nil {
|
||||
return connection, err
|
||||
}
|
||||
}
|
||||
if workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID); err != nil {
|
||||
return nil, err
|
||||
} else if workflow != nil {
|
||||
|
||||
42
internal/bootstrap/bootstrap.go
Normal file
42
internal/bootstrap/bootstrap.go
Normal file
@ -0,0 +1,42 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
RunMigrations(context.Context) error
|
||||
EnsureIndexes(context.Context) error
|
||||
EnsureDefaultOrganization(context.Context) (*domain.OrganizationRecord, error)
|
||||
GetOrganization(context.Context, string) (*domain.OrganizationRecord, error)
|
||||
}
|
||||
|
||||
// verify-existing is only for already initialized databases with separately
|
||||
// verified schema/index compatibility. It never initializes or repairs data.
|
||||
func Run(ctx context.Context, db Store, mode string) error {
|
||||
switch mode {
|
||||
case "verify-existing":
|
||||
org, err := db.GetOrganization(ctx, domain.DefaultOrganizationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if org == nil {
|
||||
return fmt.Errorf("database is not initialized; verify-existing cannot bootstrap it")
|
||||
}
|
||||
return nil
|
||||
case "", "migrate":
|
||||
if err := db.RunMigrations(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.EnsureIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := db.EnsureDefaultOrganization(ctx)
|
||||
return err
|
||||
default:
|
||||
return fmt.Errorf("invalid DROBOTICFLOW_DATABASE_BOOTSTRAP mode")
|
||||
}
|
||||
}
|
||||
@ -106,7 +106,7 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.
|
||||
return artifact, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
|
||||
}
|
||||
}
|
||||
connection, err := runtime.artifactStorageConnection(ctx, target.ConnectionID, run.OrganizationID)
|
||||
connection, err := runtime.captureDeliveryConnection(ctx, run, record, artifact)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
|
||||
67
internal/worker/delegated_delivery.go
Normal file
67
internal/worker/delegated_delivery.go
Normal file
@ -0,0 +1,67 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workspacescope"
|
||||
)
|
||||
|
||||
// Only final publication may borrow a connection. General workflow artifact
|
||||
// storage remains tenant-owned. The accepted source run and output directory
|
||||
// must both be explicitly approved by the deployment.
|
||||
func (r *Runtime) captureDeliveryConnection(ctx context.Context, run *domain.RunRecord, record *domain.CaptureRecord, artifact domain.ExecutionArtifact) (*domain.StorageConnectionRecord, error) {
|
||||
m, err := workspacescope.ForOrganization(run.OrganizationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m == nil || m.WriteConnectionOrganizationID == "" {
|
||||
return r.artifactStorageConnection(ctx, record.Delivery.ConnectionID, run.OrganizationID)
|
||||
}
|
||||
target := record.Delivery
|
||||
if record.OrganizationID != run.OrganizationID || record.ProjectID != run.ProjectID ||
|
||||
target.ConnectionID != m.ConnectionID || target.Bucket != m.Bucket || !workspacescope.PrefixAllows(target.Prefix, m.DeliveryPrefixes) {
|
||||
return nil, fmt.Errorf("delivery outside delegated organization scope")
|
||||
}
|
||||
rootID := run.ID
|
||||
if run.Trigger == domain.RunTriggerHumanRelease {
|
||||
parent, err := r.store.GetRun(ctx, run.ParentRunID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parent == nil || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != record.ID {
|
||||
return nil, fmt.Errorf("delegated delivery source lineage mismatch")
|
||||
}
|
||||
rootID = parent.ID
|
||||
}
|
||||
if !workspacescope.PrefixAllows("system/runs/"+rootID, m.WritePrefixes) {
|
||||
return nil, fmt.Errorf("delivery source run not approved")
|
||||
}
|
||||
for _, id := range []string{run.ID, rootID} {
|
||||
if id == "" || filepath.Base(id) != id || strings.ContainsAny(id, "\\/") || id == "." || id == ".." {
|
||||
return nil, fmt.Errorf("invalid delegated run ID")
|
||||
}
|
||||
}
|
||||
relative, err := filepath.Rel(r.workspaceRoot, artifact.Path)
|
||||
if err != nil || !workspacescope.PrefixAllows(filepath.ToSlash(relative), []string{"system/runs/" + run.ID}) {
|
||||
return nil, fmt.Errorf("delivery source outside current run workspace")
|
||||
}
|
||||
if err = workspacescope.ScopedPath(r.workspaceRoot, relative); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := r.store.GetStorageConnection(ctx, m.ConnectionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil || c.Provider != domain.StorageProviderBOS || c.OrganizationID != m.WriteConnectionOrganizationID {
|
||||
return nil, fmt.Errorf("delegated delivery connection owner mismatch")
|
||||
}
|
||||
copy := *c
|
||||
if copy.Endpoint == "" {
|
||||
copy.Endpoint = r.globalBOSEndpoint
|
||||
}
|
||||
return ©, nil
|
||||
}
|
||||
59
internal/worker/delegated_delivery_test.go
Normal file
59
internal/worker/delegated_delivery_test.go
Normal file
@ -0,0 +1,59 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDelegatedDeliveryRequiresApprovedLineageAndOutput(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", `{"test":{"connectionId":"c","bucket":"b","readConnectionOrganizationId":"customer","writeConnectionOrganizationId":"customer","readPrefixes":["internal-test/assets"],"writePrefixes":["system/runs/parent"],"deliveryPrefixes":["internal-test/delivery"]}}`)
|
||||
db := store.NewMemoryStore()
|
||||
ctx := t.Context()
|
||||
db.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "c", OrganizationID: "customer", Provider: domain.StorageProviderBOS})
|
||||
parent := domain.RunRecord{ID: "parent", OrganizationID: "test", ProjectID: "p", WorkflowID: "w", TaskID: "capture"}
|
||||
db.UpsertRun(ctx, parent)
|
||||
run := domain.RunRecord{ID: "child", OrganizationID: "test", ProjectID: "p", WorkflowID: "w", ParentRunID: "parent", Trigger: domain.RunTriggerHumanRelease}
|
||||
record := domain.CaptureRecord{ID: "capture", OrganizationID: "test", ProjectID: "p", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: "internal-test/delivery/capture"}}
|
||||
root := t.TempDir()
|
||||
r := &Runtime{store: db, workspaceRoot: root}
|
||||
a := domain.ExecutionArtifact{Path: filepath.Join(root, "system/runs/child/output")}
|
||||
if _, e := r.captureDeliveryConnection(ctx, &run, &record, a); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, prefix := range []string{"customer/delivery", "internal-test/delivery-other", "internal-test/assets"} {
|
||||
copy := record
|
||||
target := *record.Delivery
|
||||
target.Prefix = prefix
|
||||
copy.Delivery = &target
|
||||
if _, e := r.captureDeliveryConnection(ctx, &run, ©, a); e == nil {
|
||||
t.Fatal("foreign destination accepted", prefix)
|
||||
}
|
||||
}
|
||||
badRun := run
|
||||
badRun.ParentRunID = "unapproved"
|
||||
if _, e := r.captureDeliveryConnection(ctx, &badRun, &record, a); e == nil {
|
||||
t.Fatal("unapproved parent accepted")
|
||||
}
|
||||
for _, p := range []string{"system/runs/parent/output", "system/runs/customer/output", "internal-test/assets"} {
|
||||
bad := a
|
||||
bad.Path = filepath.Join(root, p)
|
||||
if _, e := r.captureDeliveryConnection(ctx, &run, &record, bad); e == nil {
|
||||
t.Fatal("wrong source accepted", p)
|
||||
}
|
||||
}
|
||||
parent.TaskID = "other-capture"
|
||||
db.UpsertRun(ctx, parent)
|
||||
if _, e := r.captureDeliveryConnection(ctx, &run, &record, a); e == nil {
|
||||
t.Fatal("wrong capture lineage accepted")
|
||||
}
|
||||
parent.TaskID = "capture"
|
||||
db.UpsertRun(ctx, parent)
|
||||
os.MkdirAll(filepath.Join(root, "system/runs/child"), 0700)
|
||||
os.Symlink(t.TempDir(), a.Path)
|
||||
if _, e := r.captureDeliveryConnection(ctx, &run, &record, a); e == nil {
|
||||
t.Fatal("symlink accepted")
|
||||
}
|
||||
}
|
||||
107
internal/workspacescope/scope.go
Normal file
107
internal/workspacescope/scope.go
Normal file
@ -0,0 +1,107 @@
|
||||
package workspacescope
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Deployment-owned mappings may explicitly delegate prefix-scoped reads.
|
||||
// Writes require a second explicit owner grant and disjoint output directories.
|
||||
type Mapping struct {
|
||||
ConnectionID string `json:"connectionId"`
|
||||
ReadConnectionOrganizationID string `json:"readConnectionOrganizationId,omitempty"`
|
||||
Bucket string `json:"bucket"`
|
||||
ReadPrefixes []string `json:"readPrefixes"`
|
||||
WriteConnectionOrganizationID string `json:"writeConnectionOrganizationId,omitempty"`
|
||||
DeliveryPrefixes []string `json:"deliveryPrefixes,omitempty"`
|
||||
WritePrefixes []string `json:"writePrefixes"`
|
||||
}
|
||||
|
||||
func ForOrganization(org string) (*Mapping, error) {
|
||||
raw := os.Getenv("ANNOTATION_WORKSPACE_MAPPINGS")
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(raw))
|
||||
decoder.DisallowUnknownFields()
|
||||
var mappings map[string]Mapping
|
||||
if err := decoder.Decode(&mappings); err != nil {
|
||||
return nil, fmt.Errorf("invalid annotation workspace mappings")
|
||||
}
|
||||
if err := decoder.Decode(new(any)); err != io.EOF || mappings == nil {
|
||||
return nil, fmt.Errorf("invalid annotation workspace mappings")
|
||||
}
|
||||
mapping, ok := mappings[org]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
if strings.TrimSpace(org) == "" || strings.TrimSpace(mapping.ConnectionID) == "" || strings.TrimSpace(mapping.Bucket) == "" || len(mapping.ReadPrefixes) == 0 {
|
||||
return nil, fmt.Errorf("annotation organization workspace mapping incomplete")
|
||||
}
|
||||
if mapping.ReadConnectionOrganizationID != "" && (strings.TrimSpace(mapping.ReadConnectionOrganizationID) != mapping.ReadConnectionOrganizationID || mapping.ReadConnectionOrganizationID == org) {
|
||||
return nil, fmt.Errorf("delegated workspace requires an explicit foreign owner")
|
||||
}
|
||||
if mapping.WriteConnectionOrganizationID != "" && (mapping.WriteConnectionOrganizationID != mapping.ReadConnectionOrganizationID || len(mapping.WritePrefixes) == 0) {
|
||||
return nil, fmt.Errorf("write delegation must explicitly match read owner and output prefixes")
|
||||
}
|
||||
if mapping.ReadConnectionOrganizationID != "" && mapping.WriteConnectionOrganizationID == "" && (len(mapping.WritePrefixes) != 0 || len(mapping.DeliveryPrefixes) != 0) {
|
||||
return nil, fmt.Errorf("read-only delegation cannot grant outputs")
|
||||
}
|
||||
if len(mapping.DeliveryPrefixes) != 0 && mapping.WriteConnectionOrganizationID == "" {
|
||||
return nil, fmt.Errorf("delivery requires explicit write delegation")
|
||||
}
|
||||
if mapping.WriteConnectionOrganizationID != "" {
|
||||
for _, input := range mapping.ReadPrefixes {
|
||||
for _, output := range append(append([]string{}, mapping.WritePrefixes...), mapping.DeliveryPrefixes...) {
|
||||
if PrefixAllows(input, []string{output}) || PrefixAllows(output, []string{input}) {
|
||||
return nil, fmt.Errorf("delegated input and output directories must be disjoint")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, prefixes := range [][]string{mapping.ReadPrefixes, mapping.WritePrefixes, mapping.DeliveryPrefixes} {
|
||||
for _, prefix := range prefixes {
|
||||
if prefix == "" || prefix == "." || prefix == ".." || strings.HasPrefix(prefix, "../") || path.IsAbs(prefix) || path.Clean(prefix) != prefix || strings.ContainsAny(prefix, `\*?`) {
|
||||
return nil, fmt.Errorf("invalid annotation workspace prefix")
|
||||
}
|
||||
}
|
||||
}
|
||||
return &mapping, nil
|
||||
}
|
||||
|
||||
func PrefixAllows(key string, prefixes []string) bool {
|
||||
if key == "" || key == "." || key == ".." || path.IsAbs(key) || path.Clean(key) != key || strings.HasPrefix(key, "../") || strings.ContainsAny(key, `\*?`) {
|
||||
return false
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
if key == prefix || strings.HasPrefix(key, prefix+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A scoped path must not redirect through a symlink, including an existing
|
||||
// parent of a not-yet-created output. The configured workspace root is trusted.
|
||||
func ScopedPath(root, relative string) error {
|
||||
current := root
|
||||
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, component)
|
||||
info, err := os.Lstat(current)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("annotation scoped path contains symlink")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
28
internal/workspacescope/scope_test.go
Normal file
28
internal/workspacescope/scope_test.go
Normal file
@ -0,0 +1,28 @@
|
||||
package workspacescope
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDelegatedOutputsRequireExplicitDisjointGrant(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
`{"o":{"connectionId":"c","bucket":"b","readConnectionOrganizationId":"owner","readPrefixes":["input"],"writePrefixes":["out"]}}`,
|
||||
`{"o":{"connectionId":"c","bucket":"b","readConnectionOrganizationId":"owner","writeConnectionOrganizationId":"other","readPrefixes":["input"],"writePrefixes":["out"]}}`,
|
||||
`{"o":{"connectionId":"c","bucket":"b","readConnectionOrganizationId":"owner","writeConnectionOrganizationId":"owner","readPrefixes":["input"],"writePrefixes":["input/out"]}}`,
|
||||
} {
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", raw)
|
||||
if _, err := ForOrganization("o"); err == nil {
|
||||
t.Fatal("unsafe grant accepted")
|
||||
}
|
||||
}
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", `{"o":{"connectionId":"c","bucket":"b","readConnectionOrganizationId":"owner","writeConnectionOrganizationId":"owner","readPrefixes":["input"],"writePrefixes":["out"],"deliveryPrefixes":["delivery"]}}`)
|
||||
if m, e := ForOrganization("o"); e != nil || m == nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, k := range []string{"out-other/x", "out/../input/x", "out//x", "/out/x", "out\\x"} {
|
||||
if PrefixAllows(k, []string{"out"}) {
|
||||
t.Fatal(k)
|
||||
}
|
||||
}
|
||||
if !PrefixAllows("out/clip.mp4", []string{"out"}) {
|
||||
t.Fatal("valid output rejected")
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user