droboticflow/internal/api/delivery_index_batches.go

511 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package api
// Batch publication is an explicit admin operation. GET is a read-only preview;
// POST requires the exact preview ID and only conditionally creates new objects.
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"path"
"sort"
"strings"
"time"
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/sourceidentity"
"droboticflow/internal/workspacescope"
)
type indexBatchRequest struct {
ProjectID string `json:"projectId"`
PackageIDs []string `json:"packageIds"`
SnapshotID string `json:"snapshotId"`
}
type conditionalIndexWriter interface {
PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
}
type batchPlan struct {
snapshot deliveryindex.Snapshot
browser objectStorageBrowser
target deliveryDownloadTarget
cleanup func()
}
func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPlan, error) {
if q.ProjectID == "" || len(q.PackageIDs) == 0 || len(q.PackageIDs) > 100 {
return nil, fmt.Errorf("选择 1–100 个正式验收交付包")
}
project, e := s.store.GetProject(r.Context(), q.ProjectID)
if e != nil || project == nil || project.OrganizationID != s.requestOrganizationID(r) {
return nil, fmt.Errorf("项目不存在或不可访问")
}
packages, e := s.deliveryDownloadPackages(r, q.ProjectID)
if e != nil {
return nil, e
}
selected := []domain.DeliveryPackageRecord{}
wanted := map[string]bool{}
for _, id := range q.PackageIDs {
if id == "" || wanted[id] {
return nil, fmt.Errorf("交付包选择重复或为空")
}
wanted[id] = true
}
for _, pkg := range packages {
if wanted[pkg.ID] {
selected = append(selected, pkg)
}
}
if len(selected) != len(wanted) {
return nil, fmt.Errorf("仅可选择本项目已正式验收且有最终交付物的包")
}
targets, e := s.packageDownloadTargets(r, q.ProjectID, selected)
if e != nil {
return nil, e
}
if len(targets) != len(selected) {
return nil, fmt.Errorf("每个交付包必须有唯一且已完成的最终目标")
}
t := targets[0]
root := path.Dir(path.Dir(t.prefix))
if !strings.HasSuffix(path.Base(root), "__"+q.ProjectID) || !domain.ValidDeliveryPrefix(root) {
return nil, fmt.Errorf("交付路径缺少冻结项目层")
}
c, e := s.store.GetStorageConnection(r.Context(), t.connectionID)
if e != nil || c == nil {
return nil, fmt.Errorf("批次交付连接不存在")
}
b, cleanup, e := s.downloadBrowser(r.Context(), s.requestOrganizationID(r), t.connectionID, t.bucket, t.prefix)
if e != nil {
return nil, e
}
success := false
defer func() {
if !success && cleanup != nil {
cleanup()
}
}()
sources := []deliveryindex.Source{}
var sourceBytes int64
sourceRuns := []string{}
for _, target := range targets {
if target.connectionID != t.connectionID || target.bucket != t.bucket || path.Dir(path.Dir(target.prefix)) != root {
return nil, fmt.Errorf("选择的包不在同一冻结项目交付根")
}
files, e := readDownloadManifest(r.Context(), b, target)
if e != nil {
return nil, e
}
if len(files) != 2 && len(files) != 3 {
return nil, fmt.Errorf("原生 v3 part 包必须只含 part.json 与独立审核历史")
}
var partBytes, historyBytes []byte
for _, f := range files {
if f.key != target.prefix+"/part.json" && f.key != target.prefix+"/review-history.json" {
continue
}
if f.SizeBytes > 8<<20 {
return nil, fmt.Errorf("单份 JSON 超过 8 MiB")
}
raw, _, e := b.ReadObjectPreview(r.Context(), f.bucket, f.key, 8<<20+1)
if e != nil || int64(len(raw)) != f.SizeBytes || deliveryindex.Hash(raw) != f.SHA256 {
return nil, fmt.Errorf("交付 JSON 与完成清单校验不一致")
}
sourceBytes += int64(len(raw))
if sourceBytes > 64<<20 {
return nil, fmt.Errorf("选择的源 JSON 合计超过 64 MiB,请缩小批次")
}
if strings.HasSuffix(f.key, "/part.json") {
partBytes = raw
} else {
historyBytes = raw
}
}
if len(partBytes) == 0 {
return nil, fmt.Errorf("选择的包尚未由原生 v3 算子交付;旧格式必须另行重交,不会覆盖旧文件")
}
var part struct {
Schema string `json:"schemaVersion"`
Capture struct {
ID string `json:"captureId"`
Prefix string `json:"prefix"`
} `json:"capture"`
OriginalSource struct {
Provider string `json:"provider"`
Bucket string `json:"bucket"`
Prefix string `json:"prefix"`
PartPrefix string `json:"partPrefix"`
Files map[string]struct {
Path string `json:"path"`
} `json:"files"`
} `json:"originalSource"`
PartID string `json:"partId"`
Lineage struct {
RunID string `json:"sourceRunId"`
} `json:"lineage"`
}
if json.Unmarshal(partBytes, &part) != nil || target.prefix != root+"/"+part.Capture.ID+"/"+part.PartID {
return nil, fmt.Errorf("part 身份与冻结交付目标不一致")
}
switch part.Schema {
case deliveryindex.ClassificationPartSchema:
if len(files) != 2 || len(historyBytes) != 0 {
return nil, fmt.Errorf("分类交付包中包含额外文件")
}
historyBytes, part.Lineage.RunID, e = s.classificationPackageAudit(r, selected, target, partBytes)
if e != nil {
return nil, e
}
case deliveryindex.PartSchema:
if len(files) != 3 || len(historyBytes) == 0 {
return nil, fmt.Errorf("旧 v3 交付缺少审核历史")
}
default:
return nil, fmt.Errorf("不支持的交付格式;历史交付不会被覆盖")
}
record, e := s.store.GetCaptureRecordByRunID(r.Context(), part.Lineage.RunID)
if e != nil || record == nil || record.OrganizationID != project.OrganizationID || record.ProjectID != project.ID || record.Prefix != part.Capture.Prefix+"/"+part.PartID || record.Delivery == nil || record.Delivery.Prefix != target.prefix || record.Delivery.ConnectionID != t.connectionID || record.Delivery.Bucket != t.bucket {
return nil, fmt.Errorf("part 来源与采集记录不一致")
}
if part.Schema == deliveryindex.ClassificationPartSchema && (part.OriginalSource.Provider != "BOS" || part.OriginalSource.Bucket != record.Bucket || part.OriginalSource.Prefix != part.Capture.Prefix || part.OriginalSource.PartPrefix != record.Prefix) {
return nil, fmt.Errorf("分类 JSON 原始存储来源与采集记录不一致,停止发布")
}
sourceRuns = append(sourceRuns, part.Lineage.RunID)
sourceBrowser, closeSource, e := s.deliveryReadBrowser(r.Context(), project.OrganizationID, record.ConnectionID, record.Bucket, part.Capture.Prefix, false)
if e != nil {
return nil, e
}
expected, meta, e := indexBatchSourceEvidence(r.Context(), sourceBrowser, record.Bucket, part.Capture.Prefix, part.Capture.ID)
if e == nil && part.Schema == deliveryindex.ClassificationPartSchema {
originalRun, runErr := s.store.GetRun(r.Context(), part.Lineage.RunID)
if runErr != nil {
e = runErr
} else {
var selection *sourceidentity.Selection
selection, e = sourceidentity.FromRun(originalRun, record)
if e == nil {
paths := map[string]string{}
for role, file := range part.OriginalSource.Files {
paths[role] = file.Path
}
e = selection.MatchFiles(part.OriginalSource.Provider, part.OriginalSource.Bucket, part.OriginalSource.Prefix, part.OriginalSource.PartPrefix, paths)
}
if e == nil {
e = selection.Recheck(r.Context(), sourceBrowser.GetObjectSummary)
}
}
if e != nil {
e = fmt.Errorf("原始数据身份核验失败,停止发布:%w", e)
}
}
if closeSource != nil {
closeSource()
}
if e != nil {
return nil, e
}
sources = append(sources, deliveryindex.Source{Part: partBytes, History: historyBytes, ExpectedParts: expected, MetadataSHA256: deliveryindex.Hash(meta), MetadataBytes: int64(len(meta))})
}
snapshot, e := deliveryindex.Build(project.ID, project.Name, root, sources)
if e != nil {
return nil, e
}
if e := indexBatchWriteScope(project.OrganizationID, c, t.bucket, snapshot.Root, sourceRuns); e != nil {
return nil, e
}
validationFiles := make([]deliveryDownloadFile, 0, len(snapshot.Files))
for name, body := range snapshot.Files {
if len(body) > 8<<20 {
return nil, fmt.Errorf("规范化后的单份 JSON 超过 8 MiB,请缩小交付内容")
}
validationFiles = append(validationFiles, deliveryDownloadFile{RelativePath: name, SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)})
}
if _, e := finalizeDownloadManifest(validationFiles); e != nil {
return nil, e
}
// Share the existing publisher's overlap rules; destination is server-derived.
captures, e := s.store.ListCaptureRecords(r.Context(), project.OrganizationID, "", 0)
if e != nil {
return nil, e
}
for _, record := range captures {
if record.Delivery != nil && record.Delivery.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Delivery.Prefix, snapshot.Root) {
return nil, fmt.Errorf("批次目录与已有采集交付目标重叠")
}
if record.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Prefix, snapshot.Root) {
return nil, fmt.Errorf("批次目录与来源采集重叠")
}
}
listeners, e := s.store.ListEnabledCaptureAutomations(r.Context(), project.OrganizationID)
if e != nil {
return nil, e
}
for _, workflow := range listeners {
if c := workflow.CaptureAutomation; c != nil && c.Bucket == t.bucket && domain.StoragePrefixesOverlap(c.Prefix, snapshot.Root) {
return nil, fmt.Errorf("批次目录与已启用接入监听目录重叠")
}
}
t.prefix = snapshot.Root
t.relative = path.Base(root) + "/index-batches/" + snapshot.ID
success = true
return &batchPlan{snapshot, b, t, cleanup}, nil
}
// Borrowed connections require the deployment's existing explicit final-output
// grant and approved source runs. Read delegation never becomes write access.
func indexBatchWriteScope(org string, c *domain.StorageConnectionRecord, bucket, prefix string, sourceRuns []string) error {
m, err := workspacescope.ForOrganization(org)
if err != nil {
return err
}
if c.OrganizationID == org && (m == nil || m.ConnectionID != c.ID) {
return nil
}
if m == nil || m.ConnectionID != c.ID || m.Bucket != bucket || !workspacescope.PrefixAllows(prefix, m.DeliveryPrefixes) {
return fmt.Errorf("批次目录超出已授权交付前缀")
}
if c.OrganizationID != org {
if m.WriteConnectionOrganizationID != c.OrganizationID {
return fmt.Errorf("委派连接没有明确交付写授权")
}
for _, runID := range sourceRuns {
if !workspacescope.PrefixAllows("system/runs/"+runID, m.WritePrefixes) {
return fmt.Errorf("批次来源运行不在已批准委派范围")
}
}
}
return nil
}
func indexBatchSourceEvidence(ctx context.Context, b objectStorageBrowser, bucket, prefix, captureID string) ([]string, []byte, error) {
paged, ok := b.(pagedObjectStorageBrowser)
if !ok {
return nil, nil, fmt.Errorf("原始存储不支持完整分页列举")
}
expected := map[string]bool{}
marker := ""
markers := map[string]bool{}
count := 0
for {
page, e := paged.ListObjectsPage(ctx, bucket, prefix+"/", marker, 1000)
if e != nil {
return nil, nil, fmt.Errorf("列举原始 part 失败")
}
for _, item := range page.Items {
count++
if count > 50000 {
return nil, nil, fmt.Errorf("原始采集目录超过 50000 对象")
}
if !strings.HasPrefix(item.Key, prefix+"/") {
return nil, nil, fmt.Errorf("原始列举返回越界对象")
}
first := strings.SplitN(strings.TrimPrefix(item.Key, prefix+"/"), "/", 2)[0]
if isIndexPartName(first) {
expected[first] = true
}
}
if !page.HasMore {
break
}
if page.NextMarker == "" || page.NextMarker == marker || markers[page.NextMarker] {
return nil, nil, fmt.Errorf("原始存储分页无进展")
}
marker = page.NextMarker
markers[marker] = true
}
meta, _, e := b.ReadObjectPreview(ctx, bucket, prefix+"/meta_"+captureID+".json", 8<<20+1)
if e != nil || len(meta) == 0 || len(meta) > 8<<20 || !json.Valid(meta) {
return nil, nil, fmt.Errorf("原始 meta JSON 不存在、不可读或过大")
}
parts := []string{}
for p := range expected {
parts = append(parts, p)
}
sort.Strings(parts)
if len(parts) == 0 {
return nil, nil, fmt.Errorf("原始采集没有 part 清单")
}
return parts, meta, nil
}
func isIndexPartName(s string) bool {
if !strings.HasPrefix(s, "part") || len(s) < 8 {
return false
}
for _, c := range s[4:] {
if c < '0' || c > '9' {
return false
}
}
return true
}
// A failed/ambiguous conditional PUT is successful only when a subsequent read
// verifies the exact bytes. No overwrite, deletion or rollback of old batches.
func createIndexObject(ctx context.Context, b objectStorageBrowser, writer conditionalIndexWriter, bucket, key string, body []byte) error {
_, err := writer.PutObjectConditional(ctx, bucket, key, body, "application/json", "", "*", map[string]string{"sha256": deliveryindex.Hash(body)})
got, _, e := b.ReadObjectPreview(ctx, bucket, key, int64(len(body))+1)
if e == nil && bytes.Equal(got, body) {
return nil
}
if err != nil {
return fmt.Errorf("批次对象条件创建失败或存在冲突")
}
return fmt.Errorf("批次对象写入后校验失败")
}
func publishIndexBatch(ctx context.Context, plan *batchPlan) error {
writer, ok := plan.browser.(conditionalIndexWriter)
if !ok {
return fmt.Errorf("存储不支持条件创建,拒绝发布")
}
owner, _ := json.Marshal(map[string]string{"owner": "index-batch:" + plan.snapshot.ID})
// Empty-prefix proof before taking ownership. Existing marker must match;
// read failures never count as absence: a successful complete listing proves it.
markerKey := plan.target.prefix + "/.delivery-owner.json"
existing, _, err := plan.browser.ReadObjectPreview(ctx, plan.target.bucket, markerKey, 4096)
if err == nil {
if !bytes.Equal(existing, owner) {
return fmt.Errorf("批次归属冲突")
}
} else {
paged, ok := plan.browser.(pagedObjectStorageBrowser)
if !ok {
return fmt.Errorf("存储无法校验空目录")
}
page, e := paged.ListObjectsPage(ctx, plan.target.bucket, plan.target.prefix+"/", "", 1)
if e != nil || len(page.Items) > 0 || page.HasMore {
return fmt.Errorf("批次目录非空或无法确认,拒绝接管")
}
}
if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, markerKey, owner); e != nil {
return e
}
names := []string{}
for name := range plan.snapshot.Files {
if !validDeliveryDownloadPath(name) {
return fmt.Errorf("不安全批次路径")
}
if name != "delivery_index.json" {
names = append(names, name)
}
}
sort.Strings(names)
// Capture manifests follow their part/history files; total index is last.
sort.SliceStable(names, func(i, j int) bool {
a := strings.HasSuffix(names[i], "/annotation_manifest.json")
b := strings.HasSuffix(names[j], "/annotation_manifest.json")
return !a && b
})
for _, name := range append(names, "delivery_index.json") {
if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, plan.target.prefix+"/"+name, plan.snapshot.Files[name]); e != nil {
return e
}
}
return nil
}
func (s *server) handleDeliveryIndexBatch(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Referrer-Policy", "no-referrer")
principal, ok := currentPrincipal(r)
if !ok || principal == nil || principal.User.Role != domain.UserRoleAdmin {
renderJSON(w, 403, map[string]string{"error": "交付批次生成仅限组织管理员"})
return
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
defer cancel()
r = r.WithContext(ctx)
q := indexBatchRequest{ProjectID: r.URL.Query().Get("projectId"), PackageIDs: r.URL.Query()["packageId"]}
if r.Method == http.MethodPost {
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
decoder.DisallowUnknownFields()
if decoder.Decode(&q) != nil || q.SnapshotID == "" {
renderJSON(w, 400, map[string]string{"error": "发布必须提交预览的 snapshotId、projectId 和 packageIds"})
return
}
}
plan, e := s.planIndexBatch(r, q)
if e != nil {
renderJSON(w, 409, map[string]string{"error": e.Error()})
return
}
if plan.cleanup != nil {
defer plan.cleanup()
}
if r.Method == http.MethodPost {
if q.SnapshotID != plan.snapshot.ID {
renderJSON(w, 409, map[string]string{"error": "交付证据已变化,请重新预览", "code": "index_snapshot_changed"})
return
}
if e := publishIndexBatch(ctx, plan); e != nil {
renderJSON(w, 409, map[string]string{"error": e.Error()})
return
}
}
renderJSON(w, 200, map[string]any{"snapshotId": plan.snapshot.ID, "rootKey": plan.snapshot.Root, "index": plan.snapshot.Index, "published": r.Method == http.MethodPost})
}
// Immutable batches participate in the existing directory-download protocol,
// including per-file hash checks, resume snapshots and refreshed signed URLs.
func readIndexBatchDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliveryDownloadTarget) ([]deliveryDownloadFile, error) {
id := path.Base(t.prefix)
digest, digestErr := hex.DecodeString(id)
if digestErr != nil || len(digest) != 32 || path.Base(path.Dir(t.prefix)) != "index-batches" {
return nil, fmt.Errorf("交付清单不可读或过大")
}
marker, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/.delivery-owner.json", 4096)
var owner struct {
Owner string `json:"owner"`
}
if e != nil || json.Unmarshal(marker, &owner) != nil || owner.Owner != "index-batch:"+id {
return nil, fmt.Errorf("批次归属标记无效")
}
raw, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/delivery_index.json", deliveryDownloadMaxManifestBytes+1)
if e != nil || len(raw) > deliveryDownloadMaxManifestBytes {
return nil, fmt.Errorf("批次尚未完成或清单过大")
}
var index struct {
Schema int `json:"schemaVersion"`
Kind string `json:"kind"`
ID string `json:"snapshotId"`
Count int `json:"fileCount"`
Bytes int64 `json:"totalBytes"`
Project struct {
Root string `json:"rootKey"`
} `json:"project"`
Entries []deliveryindex.Entry `json:"entries"`
}
if json.Unmarshal(raw, &index) != nil || (index.Schema != 3 && index.Schema != 4) || index.Kind != "part-index" || index.ID != id || index.Project.Root != t.prefix || index.Count != len(index.Entries) || index.Count < 1 || index.Count > 400 {
return nil, fmt.Errorf("批次总索引身份或成员数量无效")
}
files := []deliveryDownloadFile{}
var size int64
for _, entry := range index.Entries {
sha, shaErr := hex.DecodeString(entry.SHA256)
if (index.Schema == 4 && !validClassificationBatchPath(entry.Path)) || !validDeliveryDownloadPath(entry.Path) || entry.Path == "delivery_index.json" || strings.HasPrefix(entry.Path, ".") || entry.Key != t.prefix+"/"+entry.Path || entry.SizeBytes < 0 || entry.SizeBytes > 8<<20 || shaErr != nil || len(sha) != 32 {
return nil, fmt.Errorf("批次成员校验信息无效")
}
size += entry.SizeBytes
files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/" + entry.Path, SizeBytes: entry.SizeBytes, SHA256: entry.SHA256, key: entry.Key, bucket: t.bucket, connectionID: t.connectionID})
}
if size != index.Bytes || size > 64<<20 {
return nil, fmt.Errorf("批次大小校验不一致")
}
files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/delivery_index.json", SizeBytes: int64(len(raw)), SHA256: deliveryindex.Hash(raw), key: t.prefix + "/delivery_index.json", bucket: t.bucket, connectionID: t.connectionID})
return files, nil
}
func validClassificationBatchPath(value string) bool {
parts := strings.Split(value, "/")
if len(parts) != 2 {
return false
}
if parts[1] == "annotation_manifest.json" {
return true
}
name := strings.TrimSuffix(parts[1], ".json")
return strings.HasSuffix(parts[1], ".json") && len(name) >= 8 && strings.HasPrefix(name, "part") && strings.Trim(name[4:], "0123456789") == ""
}