forked from longtao.wu/droboticflow
465 lines
18 KiB
Go
465 lines
18 KiB
Go
// Package deliveryindex builds deterministic immutable delivery snapshots. It
|
||
// performs no storage operations; callers must verify the accepted source bytes.
|
||
package deliveryindex
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"math"
|
||
"path"
|
||
"regexp"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
const PartSchema = "droboticflow.delivery.part-index/v3"
|
||
const ClassificationPartSchema = "droboticflow.delivery.part-index/v4"
|
||
|
||
var partName = regexp.MustCompile(`^part[0-9]{4,}$`)
|
||
|
||
type Source struct {
|
||
Part json.RawMessage `json:"part"`
|
||
History json.RawMessage `json:"history"`
|
||
ExpectedParts []string `json:"expectedParts"`
|
||
MetadataSHA256 string `json:"metadataSha256"`
|
||
MetadataBytes int64 `json:"metadataBytes"`
|
||
}
|
||
type Snapshot struct {
|
||
ID string `json:"snapshotId"`
|
||
Root string `json:"rootKey"`
|
||
Files map[string][]byte `json:"-"`
|
||
Index json.RawMessage `json:"index"`
|
||
}
|
||
type Entry struct {
|
||
Path string `json:"path"`
|
||
Key string `json:"key"`
|
||
SizeBytes int64 `json:"sizeBytes"`
|
||
SHA256 string `json:"sha256"`
|
||
}
|
||
type Summary struct {
|
||
CaptureCount int `json:"captureCount"`
|
||
PartCount int `json:"partCount"`
|
||
ExpectedPartCount int `json:"expectedPartCount"`
|
||
UndeliveredParts []string `json:"undeliveredParts,omitempty"`
|
||
Completion string `json:"deliveryCompletion"`
|
||
KeptCount int `json:"keptSliceCount"`
|
||
DiscardedCount int `json:"discardedSliceCount"`
|
||
KeptDuration float64 `json:"keptDurationSeconds"`
|
||
DiscardedDuration float64 `json:"discardedDurationSeconds"`
|
||
Acceptance map[string]string `json:"acceptance,omitempty"`
|
||
}
|
||
type part struct {
|
||
Schema string `json:"schemaVersion"`
|
||
Capture struct {
|
||
ID string `json:"captureId"`
|
||
Prefix string `json:"prefix"`
|
||
} `json:"capture"`
|
||
PartID string `json:"partId"`
|
||
Revision int `json:"revision"`
|
||
SourceMetadata struct {
|
||
Path string `json:"path"`
|
||
} `json:"sourceMetadata"`
|
||
Slices []slice `json:"slices"`
|
||
Discarded []slice `json:"discardedSlices"`
|
||
Classification *struct {
|
||
KeptCount int `json:"keptSliceCount"`
|
||
DiscardedCount int `json:"discardedSliceCount"`
|
||
KeptDuration float64 `json:"keptDurationSeconds"`
|
||
DiscardedDuration float64 `json:"discardedDurationSeconds"`
|
||
} `json:"deliveryClassification"`
|
||
Acceptance struct {
|
||
Final struct {
|
||
Confirmed bool `json:"confirmed"`
|
||
AcceptedAt string `json:"acceptedAt"`
|
||
} `json:"finalAcceptance"`
|
||
Reviews json.RawMessage `json:"qualityReviews"`
|
||
} `json:"acceptance"`
|
||
}
|
||
type slice struct {
|
||
ID string `json:"sliceId"`
|
||
Split struct {
|
||
Start float64 `json:"startSecond"`
|
||
End float64 `json:"endSecond"`
|
||
Semantics string `json:"semantics"`
|
||
} `json:"taskSplit"`
|
||
Classification struct {
|
||
Disposition string `json:"deliveryDisposition"`
|
||
DescriptionStatus string `json:"descriptionStatus"`
|
||
Discard struct {
|
||
Reason string `json:"reason"`
|
||
} `json:"discardEvidence"`
|
||
} `json:"classification"`
|
||
Segments []struct {
|
||
Description string `json:"description"`
|
||
Invalid bool `json:"isInvalid"`
|
||
Kind string `json:"contentKind"`
|
||
Status *string `json:"status"`
|
||
} `json:"segments"`
|
||
}
|
||
|
||
func Hash(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||
func encode(v any) []byte {
|
||
b, e := json.MarshalIndent(v, "", " ")
|
||
if e != nil {
|
||
panic(e)
|
||
}
|
||
return append(b, '\n')
|
||
}
|
||
func safeName(s string) bool {
|
||
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\:\x00") && !strings.HasPrefix(s, ".")
|
||
}
|
||
func addSummary(a *Summary, b Summary) {
|
||
a.PartCount += b.PartCount
|
||
a.ExpectedPartCount += b.ExpectedPartCount
|
||
a.KeptCount += b.KeptCount
|
||
a.DiscardedCount += b.DiscardedCount
|
||
a.KeptDuration += b.KeptDuration
|
||
a.DiscardedDuration += b.DiscardedDuration
|
||
if b.Acceptance == nil {
|
||
return
|
||
}
|
||
if a.Acceptance == nil {
|
||
a.Acceptance = map[string]string{"decision": "approved"}
|
||
}
|
||
before, _ := time.Parse(time.RFC3339Nano, a.Acceptance["acceptedAt"])
|
||
after, _ := time.Parse(time.RFC3339Nano, b.Acceptance["acceptedAt"])
|
||
if after.After(before) {
|
||
a.Acceptance["acceptedAt"] = b.Acceptance["acceptedAt"]
|
||
}
|
||
}
|
||
func checkPart(p part) (Summary, error) {
|
||
s := Summary{PartCount: 1, Acceptance: map[string]string{"decision": "approved", "acceptedAt": p.Acceptance.Final.AcceptedAt}}
|
||
if (p.Schema != PartSchema && p.Schema != ClassificationPartSchema) || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 {
|
||
return s, fmt.Errorf("part must pin an accepted native v3 revision without embedded review history")
|
||
}
|
||
if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil && p.Schema != ClassificationPartSchema {
|
||
return s, fmt.Errorf("missing acceptance timestamp")
|
||
}
|
||
if p.SourceMetadata.Path != "meta_"+p.Capture.ID+".json" || path.Base(p.Capture.Prefix) != p.Capture.ID || path.Clean(p.Capture.Prefix) != p.Capture.Prefix || strings.HasPrefix(p.Capture.Prefix, "/") || strings.Contains(p.Capture.Prefix, "../") {
|
||
return s, fmt.Errorf("capture/source metadata identity mismatch")
|
||
}
|
||
seen := map[string]bool{}
|
||
var intervals [][2]float64
|
||
for group, items := range [][]slice{p.Slices, p.Discarded} {
|
||
for _, v := range items {
|
||
if v.ID == "" || seen[v.ID] || v.Split.Semantics != "half-open" || math.IsNaN(v.Split.Start) || v.Split.Start < 0 || v.Split.End <= v.Split.Start {
|
||
return s, fmt.Errorf("invalid slice identity or interval")
|
||
}
|
||
seen[v.ID] = true
|
||
intervals = append(intervals, [2]float64{v.Split.Start, v.Split.End})
|
||
duration := v.Split.End - v.Split.Start
|
||
if group == 1 {
|
||
if v.Classification.Disposition != "discard" || strings.TrimSpace(v.Classification.Discard.Reason) == "" || len(v.Segments) > 0 {
|
||
return s, fmt.Errorf("invalid discarded evidence")
|
||
}
|
||
s.DiscardedCount++
|
||
s.DiscardedDuration += duration
|
||
continue
|
||
}
|
||
if v.Classification.Disposition != "keep" {
|
||
return s, fmt.Errorf("invalid kept disposition")
|
||
}
|
||
s.KeptCount++
|
||
s.KeptDuration += duration
|
||
described, invalid, blank := false, false, false
|
||
for _, a := range v.Segments {
|
||
kind := "gap_placeholder"
|
||
if a.Invalid {
|
||
kind = "invalid_marked"
|
||
invalid = true
|
||
} else if strings.TrimSpace(a.Description) != "" {
|
||
kind = "described"
|
||
if p.Schema == ClassificationPartSchema {
|
||
kind = "annotated"
|
||
}
|
||
described = true
|
||
} else {
|
||
blank = true
|
||
}
|
||
if a.Kind != kind || a.Status != nil {
|
||
return s, fmt.Errorf("action content classification mismatch")
|
||
}
|
||
}
|
||
status := "empty_confirmed"
|
||
if invalid {
|
||
status = "invalid_only"
|
||
}
|
||
if invalid && blank {
|
||
status = "invalid_and_blank"
|
||
}
|
||
if described {
|
||
status = "described"
|
||
}
|
||
if v.Classification.DescriptionStatus != status {
|
||
return s, fmt.Errorf("descriptionStatus mismatch")
|
||
}
|
||
}
|
||
}
|
||
if s.KeptCount == 0 && (p.Schema != ClassificationPartSchema || s.DiscardedCount == 0) {
|
||
return s, fmt.Errorf("no deliverable slices")
|
||
}
|
||
if math.IsInf(s.KeptDuration, 0) || math.IsInf(s.DiscardedDuration, 0) {
|
||
return s, fmt.Errorf("slice duration overflow")
|
||
}
|
||
if c := p.Classification; c == nil || c.KeptCount != s.KeptCount || c.DiscardedCount != s.DiscardedCount || math.Abs(c.KeptDuration-s.KeptDuration) > 0.000001 || math.Abs(c.DiscardedDuration-s.DiscardedDuration) > 0.000001 {
|
||
return s, fmt.Errorf("part summary disagrees with its selected slices")
|
||
}
|
||
sort.Slice(intervals, func(i, j int) bool { return intervals[i][0] < intervals[j][0] })
|
||
for i := 1; i < len(intervals); i++ {
|
||
if math.Abs(intervals[i][0]-intervals[i-1][1]) > 0.000001 {
|
||
return s, fmt.Errorf("slice coverage gap or overlap")
|
||
}
|
||
}
|
||
if p.Schema == ClassificationPartSchema {
|
||
s.Acceptance = nil
|
||
}
|
||
return s, nil
|
||
}
|
||
|
||
// Build uses a hash of canonical source documents, source metadata and expected
|
||
// part inventory as its identity. Signed URLs and build time never affect it.
|
||
func Build(projectID, projectName, projectRoot string, sources []Source) (Snapshot, error) {
|
||
out := Snapshot{Files: map[string][]byte{}}
|
||
if projectID == "" || projectRoot == "." || path.Clean(projectRoot) != projectRoot || strings.HasPrefix(projectRoot, "/") || strings.Contains(projectRoot, "../") || len(sources) == 0 || len(sources) > 100 {
|
||
return out, fmt.Errorf("select 1–100 accepted part packages")
|
||
}
|
||
var sourceBytes int
|
||
for _, source := range sources {
|
||
sourceBytes += len(source.Part) + len(source.History)
|
||
if sourceBytes > 64<<20 {
|
||
return out, fmt.Errorf("source JSON exceeds 64 MiB")
|
||
}
|
||
}
|
||
schema := ""
|
||
for _, src := range sources {
|
||
var p part
|
||
if err := json.Unmarshal(src.Part, &p); err != nil {
|
||
return out, err
|
||
}
|
||
if schema != "" && schema != p.Schema {
|
||
return out, fmt.Errorf("cannot mix part schema versions in one batch")
|
||
}
|
||
schema = p.Schema
|
||
if p.Schema == ClassificationPartSchema {
|
||
if err := validateConsumerPart(src.Part); err != nil {
|
||
return out, err
|
||
}
|
||
if _, err := classificationAudit(src); err != nil {
|
||
return out, err
|
||
}
|
||
}
|
||
}
|
||
// Normalize JSON before hashing to ignore transport whitespace/map ordering.
|
||
for i := range sources {
|
||
var d, h any
|
||
if json.Unmarshal(sources[i].Part, &d) != nil || json.Unmarshal(sources[i].History, &h) != nil {
|
||
return out, fmt.Errorf("invalid source JSON")
|
||
}
|
||
sources[i].Part = encode(d)
|
||
sources[i].History = encode(h)
|
||
sources[i].ExpectedParts = append([]string(nil), sources[i].ExpectedParts...)
|
||
sort.Strings(sources[i].ExpectedParts)
|
||
}
|
||
sort.Slice(sources, func(i, j int) bool { return string(sources[i].Part) < string(sources[j].Part) })
|
||
identitySources := append([]Source(nil), sources...)
|
||
if schema == ClassificationPartSchema {
|
||
for i := range identitySources {
|
||
identitySources[i].History = nil
|
||
}
|
||
}
|
||
identity := encode([]any{"droboticflow.delivery.batch/v1", projectID, projectName, projectRoot, identitySources})
|
||
version := 3
|
||
if schema == ClassificationPartSchema {
|
||
version = 4
|
||
}
|
||
out.ID = Hash(identity)
|
||
out.Root = projectRoot + "/index-batches/" + out.ID
|
||
type group struct {
|
||
capture map[string]string
|
||
expected []string
|
||
metadata Source
|
||
parts []map[string]any
|
||
summary Summary
|
||
entries []Entry
|
||
}
|
||
groups := map[string]*group{}
|
||
seen := map[string]bool{}
|
||
var totalBytes int
|
||
add := func(name string, body []byte) Entry {
|
||
out.Files[name] = body
|
||
totalBytes += len(body)
|
||
return Entry{name, out.Root + "/" + name, int64(len(body)), Hash(body)}
|
||
}
|
||
for _, src := range sources {
|
||
var p part
|
||
if e := json.Unmarshal(src.Part, &p); e != nil {
|
||
return out, e
|
||
}
|
||
if p.Schema == ClassificationPartSchema {
|
||
// The original bytes were checked before canonicalization above.
|
||
var audit classificationHistory
|
||
if json.Unmarshal(src.History, &audit) != nil {
|
||
return out, fmt.Errorf("invalid internal audit")
|
||
}
|
||
p.Acceptance.Final.Confirmed = audit.Acceptance.Final.Confirmed
|
||
p.Acceptance.Final.AcceptedAt = audit.Acceptance.Final.AcceptedAt
|
||
}
|
||
s, e := checkPart(p)
|
||
if e != nil {
|
||
return out, e
|
||
}
|
||
name := p.Capture.ID + "/" + p.PartID
|
||
if seen[strings.ToLower(name)] {
|
||
return out, fmt.Errorf("duplicate capture/part, select exactly one revision")
|
||
}
|
||
seen[strings.ToLower(name)] = true
|
||
if len(src.ExpectedParts) == 0 || len(src.ExpectedParts) > 10000 || src.MetadataBytes <= 0 || len(src.MetadataSHA256) != 64 {
|
||
return out, fmt.Errorf("missing original source inventory/metadata evidence")
|
||
}
|
||
expectedSeen := map[string]bool{}
|
||
found := false
|
||
for _, n := range src.ExpectedParts {
|
||
if !partName.MatchString(n) || expectedSeen[n] {
|
||
return out, fmt.Errorf("invalid expected part inventory")
|
||
}
|
||
expectedSeen[n] = true
|
||
found = found || n == p.PartID
|
||
}
|
||
if !found {
|
||
return out, fmt.Errorf("delivered part missing from original inventory")
|
||
}
|
||
var history struct {
|
||
Schema string `json:"schemaVersion"`
|
||
CaptureID string `json:"captureId"`
|
||
PartID string `json:"partId"`
|
||
Revision int `json:"revision"`
|
||
Reviews []json.RawMessage `json:"qualityReviews"`
|
||
}
|
||
if p.Schema != ClassificationPartSchema && (json.Unmarshal(src.History, &history) != nil || history.Schema != "droboticflow.delivery.review-history/v1" || history.CaptureID != p.Capture.ID || history.PartID != p.PartID || history.Revision != p.Revision || history.Reviews == nil) {
|
||
return out, fmt.Errorf("review history revision mismatch")
|
||
}
|
||
g := groups[p.Capture.ID]
|
||
if g == nil {
|
||
g = &group{capture: map[string]string{"captureId": p.Capture.ID, "prefix": p.Capture.Prefix}, expected: src.ExpectedParts, metadata: src}
|
||
groups[p.Capture.ID] = g
|
||
} else if g.capture["prefix"] != p.Capture.Prefix || strings.Join(g.expected, ",") != strings.Join(src.ExpectedParts, ",") || g.metadata.MetadataSHA256 != src.MetadataSHA256 {
|
||
return out, fmt.Errorf("inconsistent capture source evidence")
|
||
}
|
||
var doc map[string]any
|
||
json.Unmarshal(src.Part, &doc)
|
||
doc["delivery"] = map[string]string{"kind": "part_index", "fileName": p.PartID + ".json", "artifactName": "delivery.part-index"}
|
||
g.entries = append(g.entries, add(name+".json", encode(doc)))
|
||
reference := map[string]any{"part": p.PartID, "revision": p.Revision, "path": p.PartID + ".json"}
|
||
if p.Schema != ClassificationPartSchema {
|
||
doc["reviewHistory"] = map[string]string{"path": p.PartID + ".review-history.json", "pathBase": "capture_delivery"}
|
||
totalBytes -= len(out.Files[name+".json"])
|
||
g.entries[len(g.entries)-1] = add(name+".json", encode(doc))
|
||
g.entries = append(g.entries, add(name+".review-history.json", src.History))
|
||
reference["reviewHistoryPath"] = p.PartID + ".review-history.json"
|
||
}
|
||
g.parts = append(g.parts, reference)
|
||
addSummary(&g.summary, s)
|
||
if math.IsInf(g.summary.KeptDuration, 0) || math.IsInf(g.summary.DiscardedDuration, 0) {
|
||
return out, fmt.Errorf("capture duration overflow")
|
||
}
|
||
}
|
||
names := make([]string, 0, len(groups))
|
||
for n := range groups {
|
||
names = append(names, n)
|
||
}
|
||
sort.Strings(names)
|
||
allEntries := []Entry{}
|
||
captures := []map[string]any{}
|
||
total := Summary{Completion: "full_run", CaptureCount: len(groups)}
|
||
for _, n := range names {
|
||
g := groups[n]
|
||
sort.Slice(g.parts, func(i, j int) bool { return g.parts[i]["part"].(string) < g.parts[j]["part"].(string) })
|
||
g.summary.CaptureCount = 1
|
||
g.summary.ExpectedPartCount = len(g.expected)
|
||
g.summary.Completion = "full_run"
|
||
g.summary.UndeliveredParts = []string{}
|
||
for _, pn := range g.expected {
|
||
if !seen[strings.ToLower(n+"/"+pn)] {
|
||
g.summary.UndeliveredParts = append(g.summary.UndeliveredParts, pn)
|
||
}
|
||
}
|
||
if len(g.summary.UndeliveredParts) > 0 {
|
||
g.summary.Completion = "partial_run"
|
||
total.Completion = "partial_run"
|
||
}
|
||
sort.Slice(g.entries, func(i, j int) bool { return g.entries[i].Path < g.entries[j].Path })
|
||
local := append([]Entry(nil), g.entries...)
|
||
var size int64
|
||
for i := range local {
|
||
local[i].Path = strings.TrimPrefix(local[i].Path, n+"/")
|
||
size += local[i].SizeBytes
|
||
}
|
||
metadata := map[string]any{"path": "meta_" + n + ".json", "pathBase": "original_capture", "sha256": g.metadata.MetadataSHA256, "sizeBytes": g.metadata.MetadataBytes}
|
||
manifest := map[string]any{"schemaVersion": version, "kind": "part-index", "capture": g.capture, "pathBase": "capture_delivery", "sourceMetadata": metadata, "summary": g.summary, "parts": g.parts, "entries": local, "fileCount": len(local), "totalBytes": size}
|
||
allEntries = append(allEntries, g.entries...)
|
||
allEntries = append(allEntries, add(n+"/annotation_manifest.json", encode(manifest)))
|
||
captures = append(captures, map[string]any{"captureId": n, "prefix": g.capture["prefix"], "manifestPath": n + "/annotation_manifest.json", "summary": g.summary})
|
||
addSummary(&total, g.summary)
|
||
if math.IsInf(total.KeptDuration, 0) || math.IsInf(total.DiscardedDuration, 0) {
|
||
return out, fmt.Errorf("batch duration overflow")
|
||
}
|
||
}
|
||
if totalBytes > 64<<20 {
|
||
return out, fmt.Errorf("batch exceeds 64 MiB")
|
||
}
|
||
sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Path < allEntries[j].Path })
|
||
var size int64
|
||
for _, v := range allEntries {
|
||
size += v.SizeBytes
|
||
}
|
||
out.Index = encode(map[string]any{"schemaVersion": version, "kind": "part-index", "snapshotId": out.ID, "pathBase": "batch_project", "project": map[string]string{"projectId": projectID, "deliveryName": projectName, "rootKey": out.Root}, "summary": total, "captures": captures, "entries": allEntries, "fileCount": len(allEntries), "totalBytes": size})
|
||
out.Files["delivery_index.json"] = out.Index
|
||
return out, nil
|
||
}
|
||
|
||
// classificationHistory is internal acceptance proof, excluded from v4 batch
|
||
// identity and customer files. The digest binds it to the original native bytes.
|
||
type classificationHistory struct {
|
||
Schema string `json:"schemaVersion"`
|
||
CaptureID string `json:"captureId"`
|
||
PartID string `json:"partId"`
|
||
Revision int `json:"revision"`
|
||
PartSHA string `json:"partSha256"`
|
||
Acceptance struct {
|
||
Final struct {
|
||
Confirmed bool `json:"confirmed"`
|
||
AcceptedAt string `json:"acceptedAt"`
|
||
Sampled *bool `json:"sampled"`
|
||
SamplingDecision string `json:"samplingDecision"`
|
||
} `json:"finalAcceptance"`
|
||
} `json:"acceptance"`
|
||
}
|
||
|
||
func classificationAudit(src Source) (classificationHistory, error) {
|
||
var audit classificationHistory
|
||
var p part
|
||
var doc map[string]json.RawMessage
|
||
if json.Unmarshal(src.Part, &p) != nil || json.Unmarshal(src.Part, &doc) != nil || json.Unmarshal(src.History, &audit) != nil {
|
||
return audit, fmt.Errorf("invalid classification/audit JSON")
|
||
}
|
||
for _, field := range []string{"acceptance", "lineage", "reviewHistory"} {
|
||
if _, ok := doc[field]; ok {
|
||
return audit, fmt.Errorf("customer classification contains internal trace")
|
||
}
|
||
}
|
||
if audit.Schema != "droboticflow.delivery.review-history/v2" || audit.CaptureID != p.Capture.ID || audit.PartID != p.PartID || audit.Revision != p.Revision || audit.PartSHA != Hash(src.Part) || !audit.Acceptance.Final.Confirmed {
|
||
return audit, fmt.Errorf("internal audit does not pin the accepted classification")
|
||
}
|
||
if _, err := time.Parse(time.RFC3339Nano, audit.Acceptance.Final.AcceptedAt); err != nil {
|
||
final := audit.Acceptance.Final
|
||
exempt := final.AcceptedAt == "" && final.Sampled != nil && !*final.Sampled && final.SamplingDecision == "not_selected"
|
||
if !exempt {
|
||
return audit, fmt.Errorf("missing internal acceptance timestamp or explicit sampling exemption")
|
||
}
|
||
}
|
||
return audit, nil
|
||
}
|