465 lines
18 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// Package deliveryindex builds deterministic immutable delivery snapshots. It
// performs no storage operations; callers must verify the accepted source bytes.
package deliveryindex
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"math"
"path"
"regexp"
"sort"
"strings"
"time"
)
const PartSchema = "droboticflow.delivery.part-index/v3"
const ClassificationPartSchema = "droboticflow.delivery.part-index/v4"
var partName = regexp.MustCompile(`^part[0-9]{4,}$`)
type Source struct {
Part json.RawMessage `json:"part"`
History json.RawMessage `json:"history"`
ExpectedParts []string `json:"expectedParts"`
MetadataSHA256 string `json:"metadataSha256"`
MetadataBytes int64 `json:"metadataBytes"`
}
type Snapshot struct {
ID string `json:"snapshotId"`
Root string `json:"rootKey"`
Files map[string][]byte `json:"-"`
Index json.RawMessage `json:"index"`
}
type Entry struct {
Path string `json:"path"`
Key string `json:"key"`
SizeBytes int64 `json:"sizeBytes"`
SHA256 string `json:"sha256"`
}
type Summary struct {
CaptureCount int `json:"captureCount"`
PartCount int `json:"partCount"`
ExpectedPartCount int `json:"expectedPartCount"`
UndeliveredParts []string `json:"undeliveredParts,omitempty"`
Completion string `json:"deliveryCompletion"`
KeptCount int `json:"keptSliceCount"`
DiscardedCount int `json:"discardedSliceCount"`
KeptDuration float64 `json:"keptDurationSeconds"`
DiscardedDuration float64 `json:"discardedDurationSeconds"`
Acceptance map[string]string `json:"acceptance,omitempty"`
}
type part struct {
Schema string `json:"schemaVersion"`
Capture struct {
ID string `json:"captureId"`
Prefix string `json:"prefix"`
} `json:"capture"`
PartID string `json:"partId"`
Revision int `json:"revision"`
SourceMetadata struct {
Path string `json:"path"`
} `json:"sourceMetadata"`
Slices []slice `json:"slices"`
Discarded []slice `json:"discardedSlices"`
Classification *struct {
KeptCount int `json:"keptSliceCount"`
DiscardedCount int `json:"discardedSliceCount"`
KeptDuration float64 `json:"keptDurationSeconds"`
DiscardedDuration float64 `json:"discardedDurationSeconds"`
} `json:"deliveryClassification"`
Acceptance struct {
Final struct {
Confirmed bool `json:"confirmed"`
AcceptedAt string `json:"acceptedAt"`
} `json:"finalAcceptance"`
Reviews json.RawMessage `json:"qualityReviews"`
} `json:"acceptance"`
}
type slice struct {
ID string `json:"sliceId"`
Split struct {
Start float64 `json:"startSecond"`
End float64 `json:"endSecond"`
Semantics string `json:"semantics"`
} `json:"taskSplit"`
Classification struct {
Disposition string `json:"deliveryDisposition"`
DescriptionStatus string `json:"descriptionStatus"`
Discard struct {
Reason string `json:"reason"`
} `json:"discardEvidence"`
} `json:"classification"`
Segments []struct {
Description string `json:"description"`
Invalid bool `json:"isInvalid"`
Kind string `json:"contentKind"`
Status *string `json:"status"`
} `json:"segments"`
}
func Hash(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
func encode(v any) []byte {
b, e := json.MarshalIndent(v, "", " ")
if e != nil {
panic(e)
}
return append(b, '\n')
}
func safeName(s string) bool {
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\:\x00") && !strings.HasPrefix(s, ".")
}
func addSummary(a *Summary, b Summary) {
a.PartCount += b.PartCount
a.ExpectedPartCount += b.ExpectedPartCount
a.KeptCount += b.KeptCount
a.DiscardedCount += b.DiscardedCount
a.KeptDuration += b.KeptDuration
a.DiscardedDuration += b.DiscardedDuration
if b.Acceptance == nil {
return
}
if a.Acceptance == nil {
a.Acceptance = map[string]string{"decision": "approved"}
}
before, _ := time.Parse(time.RFC3339Nano, a.Acceptance["acceptedAt"])
after, _ := time.Parse(time.RFC3339Nano, b.Acceptance["acceptedAt"])
if after.After(before) {
a.Acceptance["acceptedAt"] = b.Acceptance["acceptedAt"]
}
}
func checkPart(p part) (Summary, error) {
s := Summary{PartCount: 1, Acceptance: map[string]string{"decision": "approved", "acceptedAt": p.Acceptance.Final.AcceptedAt}}
if (p.Schema != PartSchema && p.Schema != ClassificationPartSchema) || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 {
return s, fmt.Errorf("part must pin an accepted native v3 revision without embedded review history")
}
if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil && p.Schema != ClassificationPartSchema {
return s, fmt.Errorf("missing acceptance timestamp")
}
if p.SourceMetadata.Path != "meta_"+p.Capture.ID+".json" || path.Base(p.Capture.Prefix) != p.Capture.ID || path.Clean(p.Capture.Prefix) != p.Capture.Prefix || strings.HasPrefix(p.Capture.Prefix, "/") || strings.Contains(p.Capture.Prefix, "../") {
return s, fmt.Errorf("capture/source metadata identity mismatch")
}
seen := map[string]bool{}
var intervals [][2]float64
for group, items := range [][]slice{p.Slices, p.Discarded} {
for _, v := range items {
if v.ID == "" || seen[v.ID] || v.Split.Semantics != "half-open" || math.IsNaN(v.Split.Start) || v.Split.Start < 0 || v.Split.End <= v.Split.Start {
return s, fmt.Errorf("invalid slice identity or interval")
}
seen[v.ID] = true
intervals = append(intervals, [2]float64{v.Split.Start, v.Split.End})
duration := v.Split.End - v.Split.Start
if group == 1 {
if v.Classification.Disposition != "discard" || strings.TrimSpace(v.Classification.Discard.Reason) == "" || len(v.Segments) > 0 {
return s, fmt.Errorf("invalid discarded evidence")
}
s.DiscardedCount++
s.DiscardedDuration += duration
continue
}
if v.Classification.Disposition != "keep" {
return s, fmt.Errorf("invalid kept disposition")
}
s.KeptCount++
s.KeptDuration += duration
described, invalid, blank := false, false, false
for _, a := range v.Segments {
kind := "gap_placeholder"
if a.Invalid {
kind = "invalid_marked"
invalid = true
} else if strings.TrimSpace(a.Description) != "" {
kind = "described"
if p.Schema == ClassificationPartSchema {
kind = "annotated"
}
described = true
} else {
blank = true
}
if a.Kind != kind || a.Status != nil {
return s, fmt.Errorf("action content classification mismatch")
}
}
status := "empty_confirmed"
if invalid {
status = "invalid_only"
}
if invalid && blank {
status = "invalid_and_blank"
}
if described {
status = "described"
}
if v.Classification.DescriptionStatus != status {
return s, fmt.Errorf("descriptionStatus mismatch")
}
}
}
if s.KeptCount == 0 && (p.Schema != ClassificationPartSchema || s.DiscardedCount == 0) {
return s, fmt.Errorf("no deliverable slices")
}
if math.IsInf(s.KeptDuration, 0) || math.IsInf(s.DiscardedDuration, 0) {
return s, fmt.Errorf("slice duration overflow")
}
if c := p.Classification; c == nil || c.KeptCount != s.KeptCount || c.DiscardedCount != s.DiscardedCount || math.Abs(c.KeptDuration-s.KeptDuration) > 0.000001 || math.Abs(c.DiscardedDuration-s.DiscardedDuration) > 0.000001 {
return s, fmt.Errorf("part summary disagrees with its selected slices")
}
sort.Slice(intervals, func(i, j int) bool { return intervals[i][0] < intervals[j][0] })
for i := 1; i < len(intervals); i++ {
if math.Abs(intervals[i][0]-intervals[i-1][1]) > 0.000001 {
return s, fmt.Errorf("slice coverage gap or overlap")
}
}
if p.Schema == ClassificationPartSchema {
s.Acceptance = nil
}
return s, nil
}
// Build uses a hash of canonical source documents, source metadata and expected
// part inventory as its identity. Signed URLs and build time never affect it.
func Build(projectID, projectName, projectRoot string, sources []Source) (Snapshot, error) {
out := Snapshot{Files: map[string][]byte{}}
if projectID == "" || projectRoot == "." || path.Clean(projectRoot) != projectRoot || strings.HasPrefix(projectRoot, "/") || strings.Contains(projectRoot, "../") || len(sources) == 0 || len(sources) > 100 {
return out, fmt.Errorf("select 1–100 accepted part packages")
}
var sourceBytes int
for _, source := range sources {
sourceBytes += len(source.Part) + len(source.History)
if sourceBytes > 64<<20 {
return out, fmt.Errorf("source JSON exceeds 64 MiB")
}
}
schema := ""
for _, src := range sources {
var p part
if err := json.Unmarshal(src.Part, &p); err != nil {
return out, err
}
if schema != "" && schema != p.Schema {
return out, fmt.Errorf("cannot mix part schema versions in one batch")
}
schema = p.Schema
if p.Schema == ClassificationPartSchema {
if err := validateConsumerPart(src.Part); err != nil {
return out, err
}
if _, err := classificationAudit(src); err != nil {
return out, err
}
}
}
// Normalize JSON before hashing to ignore transport whitespace/map ordering.
for i := range sources {
var d, h any
if json.Unmarshal(sources[i].Part, &d) != nil || json.Unmarshal(sources[i].History, &h) != nil {
return out, fmt.Errorf("invalid source JSON")
}
sources[i].Part = encode(d)
sources[i].History = encode(h)
sources[i].ExpectedParts = append([]string(nil), sources[i].ExpectedParts...)
sort.Strings(sources[i].ExpectedParts)
}
sort.Slice(sources, func(i, j int) bool { return string(sources[i].Part) < string(sources[j].Part) })
identitySources := append([]Source(nil), sources...)
if schema == ClassificationPartSchema {
for i := range identitySources {
identitySources[i].History = nil
}
}
identity := encode([]any{"droboticflow.delivery.batch/v1", projectID, projectName, projectRoot, identitySources})
version := 3
if schema == ClassificationPartSchema {
version = 4
}
out.ID = Hash(identity)
out.Root = projectRoot + "/index-batches/" + out.ID
type group struct {
capture map[string]string
expected []string
metadata Source
parts []map[string]any
summary Summary
entries []Entry
}
groups := map[string]*group{}
seen := map[string]bool{}
var totalBytes int
add := func(name string, body []byte) Entry {
out.Files[name] = body
totalBytes += len(body)
return Entry{name, out.Root + "/" + name, int64(len(body)), Hash(body)}
}
for _, src := range sources {
var p part
if e := json.Unmarshal(src.Part, &p); e != nil {
return out, e
}
if p.Schema == ClassificationPartSchema {
// The original bytes were checked before canonicalization above.
var audit classificationHistory
if json.Unmarshal(src.History, &audit) != nil {
return out, fmt.Errorf("invalid internal audit")
}
p.Acceptance.Final.Confirmed = audit.Acceptance.Final.Confirmed
p.Acceptance.Final.AcceptedAt = audit.Acceptance.Final.AcceptedAt
}
s, e := checkPart(p)
if e != nil {
return out, e
}
name := p.Capture.ID + "/" + p.PartID
if seen[strings.ToLower(name)] {
return out, fmt.Errorf("duplicate capture/part, select exactly one revision")
}
seen[strings.ToLower(name)] = true
if len(src.ExpectedParts) == 0 || len(src.ExpectedParts) > 10000 || src.MetadataBytes <= 0 || len(src.MetadataSHA256) != 64 {
return out, fmt.Errorf("missing original source inventory/metadata evidence")
}
expectedSeen := map[string]bool{}
found := false
for _, n := range src.ExpectedParts {
if !partName.MatchString(n) || expectedSeen[n] {
return out, fmt.Errorf("invalid expected part inventory")
}
expectedSeen[n] = true
found = found || n == p.PartID
}
if !found {
return out, fmt.Errorf("delivered part missing from original inventory")
}
var history struct {
Schema string `json:"schemaVersion"`
CaptureID string `json:"captureId"`
PartID string `json:"partId"`
Revision int `json:"revision"`
Reviews []json.RawMessage `json:"qualityReviews"`
}
if p.Schema != ClassificationPartSchema && (json.Unmarshal(src.History, &history) != nil || history.Schema != "droboticflow.delivery.review-history/v1" || history.CaptureID != p.Capture.ID || history.PartID != p.PartID || history.Revision != p.Revision || history.Reviews == nil) {
return out, fmt.Errorf("review history revision mismatch")
}
g := groups[p.Capture.ID]
if g == nil {
g = &group{capture: map[string]string{"captureId": p.Capture.ID, "prefix": p.Capture.Prefix}, expected: src.ExpectedParts, metadata: src}
groups[p.Capture.ID] = g
} else if g.capture["prefix"] != p.Capture.Prefix || strings.Join(g.expected, ",") != strings.Join(src.ExpectedParts, ",") || g.metadata.MetadataSHA256 != src.MetadataSHA256 {
return out, fmt.Errorf("inconsistent capture source evidence")
}
var doc map[string]any
json.Unmarshal(src.Part, &doc)
doc["delivery"] = map[string]string{"kind": "part_index", "fileName": p.PartID + ".json", "artifactName": "delivery.part-index"}
g.entries = append(g.entries, add(name+".json", encode(doc)))
reference := map[string]any{"part": p.PartID, "revision": p.Revision, "path": p.PartID + ".json"}
if p.Schema != ClassificationPartSchema {
doc["reviewHistory"] = map[string]string{"path": p.PartID + ".review-history.json", "pathBase": "capture_delivery"}
totalBytes -= len(out.Files[name+".json"])
g.entries[len(g.entries)-1] = add(name+".json", encode(doc))
g.entries = append(g.entries, add(name+".review-history.json", src.History))
reference["reviewHistoryPath"] = p.PartID + ".review-history.json"
}
g.parts = append(g.parts, reference)
addSummary(&g.summary, s)
if math.IsInf(g.summary.KeptDuration, 0) || math.IsInf(g.summary.DiscardedDuration, 0) {
return out, fmt.Errorf("capture duration overflow")
}
}
names := make([]string, 0, len(groups))
for n := range groups {
names = append(names, n)
}
sort.Strings(names)
allEntries := []Entry{}
captures := []map[string]any{}
total := Summary{Completion: "full_run", CaptureCount: len(groups)}
for _, n := range names {
g := groups[n]
sort.Slice(g.parts, func(i, j int) bool { return g.parts[i]["part"].(string) < g.parts[j]["part"].(string) })
g.summary.CaptureCount = 1
g.summary.ExpectedPartCount = len(g.expected)
g.summary.Completion = "full_run"
g.summary.UndeliveredParts = []string{}
for _, pn := range g.expected {
if !seen[strings.ToLower(n+"/"+pn)] {
g.summary.UndeliveredParts = append(g.summary.UndeliveredParts, pn)
}
}
if len(g.summary.UndeliveredParts) > 0 {
g.summary.Completion = "partial_run"
total.Completion = "partial_run"
}
sort.Slice(g.entries, func(i, j int) bool { return g.entries[i].Path < g.entries[j].Path })
local := append([]Entry(nil), g.entries...)
var size int64
for i := range local {
local[i].Path = strings.TrimPrefix(local[i].Path, n+"/")
size += local[i].SizeBytes
}
metadata := map[string]any{"path": "meta_" + n + ".json", "pathBase": "original_capture", "sha256": g.metadata.MetadataSHA256, "sizeBytes": g.metadata.MetadataBytes}
manifest := map[string]any{"schemaVersion": version, "kind": "part-index", "capture": g.capture, "pathBase": "capture_delivery", "sourceMetadata": metadata, "summary": g.summary, "parts": g.parts, "entries": local, "fileCount": len(local), "totalBytes": size}
allEntries = append(allEntries, g.entries...)
allEntries = append(allEntries, add(n+"/annotation_manifest.json", encode(manifest)))
captures = append(captures, map[string]any{"captureId": n, "prefix": g.capture["prefix"], "manifestPath": n + "/annotation_manifest.json", "summary": g.summary})
addSummary(&total, g.summary)
if math.IsInf(total.KeptDuration, 0) || math.IsInf(total.DiscardedDuration, 0) {
return out, fmt.Errorf("batch duration overflow")
}
}
if totalBytes > 64<<20 {
return out, fmt.Errorf("batch exceeds 64 MiB")
}
sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Path < allEntries[j].Path })
var size int64
for _, v := range allEntries {
size += v.SizeBytes
}
out.Index = encode(map[string]any{"schemaVersion": version, "kind": "part-index", "snapshotId": out.ID, "pathBase": "batch_project", "project": map[string]string{"projectId": projectID, "deliveryName": projectName, "rootKey": out.Root}, "summary": total, "captures": captures, "entries": allEntries, "fileCount": len(allEntries), "totalBytes": size})
out.Files["delivery_index.json"] = out.Index
return out, nil
}
// classificationHistory is internal acceptance proof, excluded from v4 batch
// identity and customer files. The digest binds it to the original native bytes.
type classificationHistory struct {
Schema string `json:"schemaVersion"`
CaptureID string `json:"captureId"`
PartID string `json:"partId"`
Revision int `json:"revision"`
PartSHA string `json:"partSha256"`
Acceptance struct {
Final struct {
Confirmed bool `json:"confirmed"`
AcceptedAt string `json:"acceptedAt"`
Sampled *bool `json:"sampled"`
SamplingDecision string `json:"samplingDecision"`
} `json:"finalAcceptance"`
} `json:"acceptance"`
}
func classificationAudit(src Source) (classificationHistory, error) {
var audit classificationHistory
var p part
var doc map[string]json.RawMessage
if json.Unmarshal(src.Part, &p) != nil || json.Unmarshal(src.Part, &doc) != nil || json.Unmarshal(src.History, &audit) != nil {
return audit, fmt.Errorf("invalid classification/audit JSON")
}
for _, field := range []string{"acceptance", "lineage", "reviewHistory"} {
if _, ok := doc[field]; ok {
return audit, fmt.Errorf("customer classification contains internal trace")
}
}
if audit.Schema != "droboticflow.delivery.review-history/v2" || audit.CaptureID != p.Capture.ID || audit.PartID != p.PartID || audit.Revision != p.Revision || audit.PartSHA != Hash(src.Part) || !audit.Acceptance.Final.Confirmed {
return audit, fmt.Errorf("internal audit does not pin the accepted classification")
}
if _, err := time.Parse(time.RFC3339Nano, audit.Acceptance.Final.AcceptedAt); err != nil {
final := audit.Acceptance.Final
exempt := final.AcceptedAt == "" && final.Sampled != nil && !*final.Sampled && final.SamplingDecision == "not_selected"
if !exempt {
return audit, fmt.Errorf("missing internal acceptance timestamp or explicit sampling exemption")
}
}
return audit, nil
}