data-gen-bpu/export/trajectory.py

664 lines
34 KiB
Python

"""Lossless Pi event materialization and source-based validation.
The Pi 0.73 event adapter treats agent_end as a segment mirror, never a session
replacement. Message identity is a source location/lifecycle, not content. The
validator reads the frozen event bytes again and audits every raw observation.
Raw bytes stay private; exported lineage contains only locations and hashes.
"""
from __future__ import annotations
import copy
import hashlib
import json
import re
from collections import Counter
from dataclasses import dataclass
from pathlib import Path
from export.redact import redact_preserving
RULES_VERSION = "sft-preserve-v1"
# Pin the implemented spec, not caller-supplied release metadata.
RULES_SPEC_SHA256 = "651d8126ec049198dc14b4de455b14ff5bc654e8a0fa9df3df4e31418e330ada"
FILES = ("rollout.json", "agent/events.jsonl", "agent/system.md", "agent/tools.json", "agent/prompt.md")
TERMINAL_STATES = frozenset({"DONE", "FAILED", "FAIL", "TIMEOUT", "CANCELLED", "ENV_ERROR",
"BUDGET_EXHAUSTED", "INTERRUPTED", "COMPLETED", "ERROR", "DEADLINE_EXCEEDED",
"INFRA_ERROR", "AGENT_ERROR", "LOST"})
OPERATIONS = frozenset({"session", "agent_start", "turn_start", "auto_retry_start", "auto_retry_end",
"tool_execution_start", "tool_execution_update", "tool_execution_end"})
SUPPORTED_UPDATES = frozenset(f"{kind}_{suffix}" for kind in ("text", "thinking", "toolcall")
for suffix in ("start", "delta", "end"))
def sha(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()
def payload_sha(value) -> str:
return sha(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode())
@dataclass
class FrozenSource:
files: dict[str, bytes]
manifest: dict
errors: list[str]
def freeze(rd: Path) -> FrozenSource:
files, manifest, errors = {}, {}, []
for name in FILES:
path = rd / name
try:
before = path.stat()
data = path.read_bytes()
after = path.stat()
except FileNotFoundError:
continue
files[name] = data
manifest[name] = {"sha256": sha(data), "bytes": len(data), "watermark": len(data)}
if (before.st_ino, before.st_size, before.st_mtime_ns) != (after.st_ino, after.st_size, after.st_mtime_ns):
errors.append("SOURCE_CHANGED")
# Recheck the entire input set, not just each file in isolation.
for name, data in files.items():
try:
if (rd / name).read_bytes() != data:
errors.append("SOURCE_CHANGED")
except FileNotFoundError:
errors.append("SOURCE_CHANGED")
return FrozenSource(files, manifest, errors)
def source_unchanged(rd: Path, source: FrozenSource) -> bool:
"""Check the entire known input set and EOF again immediately before output."""
for name in FILES:
path = rd / name
try:
digest, size = hashlib.sha256(), 0
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1 << 20), b""):
digest.update(chunk); size += len(chunk)
if source.manifest.get(name) != {"sha256": digest.hexdigest(), "bytes": size, "watermark": size}:
return False
except FileNotFoundError:
if name in source.manifest:
return False
return True
def _json(source, name, default):
try:
return json.loads(source.files[name])
except (KeyError, ValueError, UnicodeDecodeError):
return default
def _events(source):
offset = 0
digest = source.manifest.get("agent/events.jsonl", {}).get("sha256", "missing")
for line in source.files.get("agent/events.jsonl", b"").splitlines(keepends=True):
location = f"{digest}:{offset}"
offset += len(line)
try:
event = json.loads(line)
if not isinstance(event, dict):
raise ValueError("not an event object")
yield location, event
except (ValueError, UnicodeDecodeError):
yield location, {"type": "!parse_error"}
def _observation(location, pointer, payload):
return {"observation_id": f"{location}:{pointer}", "payload_sha256": payload_sha(payload)}
def _calls(message):
content = message.get("content")
calls = [block for block in content if isinstance(block, dict) and block.get("type") == "toolCall"] if isinstance(content, list) else []
return calls + list(message.get("tool_calls") or [])
def _empty_content(message):
if "content" not in message:
return False
content = message["content"]
if content is None or content == "" or content == []:
return True
return isinstance(content, list) and all(
isinstance(block, dict) and block.get("type") in ("text", "output_text")
and block.get("text") == "" and set(block) <= {"type", "text"} for block in content)
def _identity_shape(message):
"""Shape is only a segment-position check; payload equality is not identity."""
return (message.get("role"), message.get("toolCallId"), tuple(c.get("id") for c in _calls(message)))
def inventory(source: FrozenSource) -> dict:
errors = list(source.errors)
metadata = _json(source, "rollout.json", {})
if metadata.get("runner", "pi") != "pi" or metadata.get("runner_version") not in (None, "legacy-pi", "0.73.1", "pi-sdk-v1"):
errors.append("UNSUPPORTED_SOURCE")
if metadata.get("state") not in TERMINAL_STATES:
errors.append("SOURCE_NOT_TERMINAL")
messages, operations, segment, turn = [], [], [], []
pending = None
ended = False
sdk_headers = []
session_ids = set()
context_started = False
def add(payload, observation, *, streamed=False, event_type="message_end"):
if not isinstance(payload, dict) or not isinstance(payload.get("role"), str):
errors.append("UNSUPPORTED_SOURCE")
return None
item = {"message_id": observation["observation_id"], "payload": copy.deepcopy(payload),
"observations": [observation], "event_type": event_type, "streamed": streamed}
messages.append(item)
return item
def alias(item, payload, observation):
if item["payload"] != payload:
errors.append("IDENTITY_PAYLOAD_CONFLICT")
item["observations"].append(observation)
def mirror(payloads, location, pointer, candidates, allow_context=False):
"""One-to-one segment slots establish identity; no content search/dedup."""
if not isinstance(payloads, list):
errors.append("UNSUPPORTED_SOURCE")
return
cursor, ordered = 0, []
for index, payload in enumerate(payloads):
observation = _observation(location, f"{pointer}/{index}", payload)
if cursor < len(candidates) and _identity_shape(payload) == _identity_shape(candidates[cursor]["payload"]):
item = candidates[cursor]
cursor += 1
alias(item, payload, observation)
elif allow_context and payload.get("role") in ("user", "custom", "system"):
item = add(payload, observation, event_type="context_mirror")
# A complete segment mirror gives the new context its position.
else:
errors.append("IDENTITY_AMBIGUOUS")
item = add(payload, observation, event_type="unresolved_mirror")
if item:
ordered.append(item)
if cursor != len(candidates):
errors.append("OBSERVATION_UNACCOUNTED")
if allow_context:
# Replace only this segment's slots, retaining every prior segment.
identities = {id(item) for item in ordered + candidates}
positions = [i for i, item in enumerate(messages) if id(item) in identities]
start = min(positions, default=len(messages))
messages[:] = [item for item in messages if id(item) not in identities]
messages[start:start] = ordered
candidates[:] = ordered
for location, event in _events(source):
kind = event.get("type")
if kind in ("agent_start", "turn_start", "message_start", "message_update", "message_end", "turn_end", "agent_end"):
context_started = True
if kind == "!parse_error":
errors.append("PARSE_ERROR")
elif kind in OPERATIONS:
operations.append({"type": kind, "observation_id": location, "payload_sha256": payload_sha(event)})
if kind == "session":
if event.get("id"):
session_ids.add(event["id"])
if len(session_ids) > 1:
errors.append("SESSION_IDENTITY_CONFLICT")
header = event.get("bench")
if isinstance(header, dict) and header.get("runner_version") == "pi-sdk-v1":
if not sdk_headers and context_started:
errors.append("SDK_CONTEXT_HEADER_LATE")
if sdk_headers and not ended:
errors.append("SDK_RESUME_INCOMPLETE_SEGMENT")
sdk_headers.append(event)
elif event.get("version") != 3:
errors.append("UNSUPPORTED_SOURCE")
elif kind == "agent_start":
if pending:
errors.append("INCOMPLETE_GENERATION")
segment, turn, pending, ended = [], [], None, False
elif kind == "turn_start":
turn = []
elif kind == "tool_execution_start" and pending:
pending["streamed"] = True # an observed side effect forbids empty-fragment deletion
elif kind == "message_start":
if pending:
errors.append("INCOMPLETE_GENERATION")
start = event.get("message")
pending = {"location": location, "message": start,
"streamed": not isinstance(start, dict) or not _empty_content(start)}
elif kind == "message_update":
update = event.get("assistantMessageEvent", {})
if not pending or update.get("type") not in SUPPORTED_UPDATES:
errors.append("SOURCE_EVENT_UNKNOWN")
elif update.get("delta") not in (None, "", {}, []):
pending["streamed"] = True
elif kind == "message_end":
payload = event.get("message")
observation = _observation(location, "/message", payload)
if pending and isinstance(payload, dict) and isinstance(pending["message"], dict):
if pending["message"].get("role") != payload.get("role"):
errors.append("CONTEXT_ORDER_UNKNOWN")
item = add(payload, observation, streamed=bool(pending and pending["streamed"]))
if item:
if pending:
item["lifecycle_start"] = pending["location"]
segment.append(item)
turn.append(item)
pending, ended = None, False
elif kind == "turn_end":
candidates = [item for item in turn if item["payload"].get("role") in ("assistant", "toolResult")]
if not candidates:
# Legacy fixture/event streams without turn_start: the current
# turn is the suffix starting at the last assistant message.
candidates = [item for item in segment if item["payload"].get("role") in ("assistant", "toolResult")]
payloads = [event.get("message")] + list(event.get("toolResults") or [])
if not all(isinstance(m, dict) for m in payloads):
errors.append("UNSUPPORTED_SOURCE")
continue
# Use actual Pi JSON pointers, not the synthetic combined list.
for i, payload in enumerate(payloads):
pointer = "/message" if i == 0 else f"/toolResults/{i - 1}"
observation = _observation(location, pointer, payload)
if i < len(candidates) and _identity_shape(payload) == _identity_shape(candidates[i]["payload"]):
alias(candidates[i], payload, observation)
else:
errors.append("IDENTITY_AMBIGUOUS")
add(payload, observation, event_type="unresolved_mirror")
if len(payloads) != len(candidates):
errors.append("OBSERVATION_UNACCOUNTED")
elif kind == "agent_end":
payloads = event.get("messages")
if not segment and isinstance(payloads, list):
# An isolated end container proves its own order but provides no
# lifecycle proof. Keep it for analysis; never invent aliases.
errors.append("MISSING_MESSAGE_LIFECYCLES")
if isinstance(payloads, list) and all(isinstance(m, dict) for m in payloads):
mirror(payloads, location, "/messages", segment, allow_context=True)
else:
errors.append("UNSUPPORTED_SOURCE")
ended = True
else:
errors.append("SOURCE_EVENT_UNKNOWN")
operations.append({"type": "unknown", "type_sha256": payload_sha(kind),
"observation_id": location, "payload_sha256": payload_sha(event), "unknown": True})
if pending:
errors.append("INCOMPLETE_GENERATION")
if not ended:
errors.append("INCOMPLETE_TRAJECTORY")
# Only saved runner context is admitted. The initial prompt is not injected
# by text equality; real Pi streams include the initial user message.
if "agent/system.md" in source.files:
try:
payload = {"role": "system", "content": source.files["agent/system.md"].decode("utf-8")}
observation = _observation(source.manifest["agent/system.md"]["sha256"] + ":0", "/system", payload)
item = add(payload, observation, event_type="system_snapshot")
messages.insert(0, messages.pop())
except UnicodeDecodeError:
errors.append("PARSE_ERROR")
else:
errors.append("CONTEXT_SNAPSHOT_MISSING")
tools = _json(source, "agent/tools.json", None)
if not isinstance(tools, list) or not tools:
tools = []
errors.append("CONTEXT_SNAPSHOT_MISSING")
context = {"status": "UNKNOWN", "source": "unverified-snapshot"}
if metadata.get("runner_version") == "legacy-pi":
context["source"] = "legacy-append-only"
if sdk_headers and metadata.get("runner_version") != "pi-sdk-v1":
errors.append("SOURCE_CONTEXT_CONFLICT")
if metadata.get("runner_version") == "pi-sdk-v1":
if not sdk_headers:
errors.append("SDK_CONTEXT_HEADER_MISSING")
else:
first_header = sdk_headers[0]
header = first_header["bench"]
expected_system = source.files.get("agent/system.md", b"").decode("utf-8", errors="replace")
if (first_header.get("version") != 3 or not isinstance(first_header.get("id"), str) or not first_header["id"]
or any(other != first_header for other in sdk_headers[1:])
or not isinstance(header.get("execution_id"), str) or not header["execution_id"]
or not isinstance(header.get("system_prompt"), str) or not header["system_prompt"]
or header["system_prompt"] != expected_system or header.get("tools") != tools
or metadata.get("execution_id") != header["execution_id"]):
errors.append("SDK_CONTEXT_SNAPSHOT_MISMATCH")
elif not {"SDK_CONTEXT_HEADER_LATE", "SDK_RESUME_INCOMPLETE_SEGMENT"} & set(errors):
context = {"status": "COMPLETE", "source": "pi-sdk-session-header"}
if context["status"] != "COMPLETE":
errors.append("REQUIRED_CONTEXT_UNKNOWN")
if not any(item["payload"].get("role") == "user" for item in messages):
errors.append("INITIAL_CONTEXT_MISSING")
excluded, retained = [], []
for item in messages:
msg = item["payload"]
if msg.get("role") not in ("system", "user", "custom", "assistant", "toolResult"):
errors.append("UNSUPPORTED_SOURCE")
empty_error = (item["event_type"] == "message_end" and msg.get("role") == "assistant"
and msg.get("stopReason") == "error" and _empty_content(msg)
and not _calls(msg) and not any(k in msg for k in ("toolCallId", "tool_call_id", "function_call", "tool_calls"))
and not item["streamed"])
if empty_error:
excluded.append({**item, "reason": "EMPTY_ERROR_FRAGMENT"})
else:
retained.append(item)
if msg.get("role") == "assistant" and msg.get("stopReason") in ("error", "aborted"):
errors.append("ERROR_CONTENT_RETAINED")
return {"messages": retained, "excluded": excluded, "operations": operations,
"errors": sorted(set(errors)), "tools": tools, "agent_end": ended, "context": context}
def closure(messages):
calls, results = {}, Counter()
missing_id, reused, orphan, early = 0, 0, 0, 0
for position, message in enumerate(messages):
for call in _calls(message):
ident = call.get("id")
if not isinstance(ident, str) or not ident:
missing_id += 1
elif ident in calls:
reused += 1
else:
calls[ident] = position
if message.get("role") in ("toolResult", "tool"):
ident = message.get("toolCallId", message.get("tool_call_id"))
results[ident] += 1
if ident not in calls:
orphan += 1
early += 1
metrics = {"missing": sum(results[key] == 0 for key in calls), "orphan": orphan,
"duplicate_final": sum(max(0, n - 1) for n in results.values()),
"call_id_conflict": reused + missing_id, "result_before_call": early}
return {"pass": not any(metrics.values()), **metrics}
def _view(items, tools):
# Pi-native blocks preserve thinking, tool args/results and unknown fields.
return redact_preserving({"messages": [item["payload"] for item in items], "tools": tools})
def reconstruct(source: FrozenSource) -> dict:
scanned = inventory(source)
view, redaction = _view(scanned["messages"], scanned["tools"])
lineage = [{k: item[k] for k in ("message_id", "observations", "event_type")} for item in scanned["messages"]]
excluded = [{k: item[k] for k in ("message_id", "observations", "event_type", "reason")} for item in scanned["excluded"]]
result = {**view, "format": "pi-native-messages/1", "rules_version": RULES_VERSION,
"source_ids": [item["message_id"] for item in scanned["messages"]], "lineage": lineage,
"excluded": excluded, "operations": scanned["operations"], "errors": scanned["errors"],
"agent_end": scanned["agent_end"], "context": scanned["context"],
"redaction": redaction, "source_manifest": source.manifest}
result["annotations"] = wait_annotations(result["messages"], result["source_ids"])
return result
def wait_annotations(messages, source_ids):
"""Advisory-only parser for the recorded stageclient v1 grammar.
Unknown output stays UNKNOWN. We compare all non-heartbeat text, so the same
phase with newly printed evidence cannot be mistaken for an empty poll.
No current status file is read and no conversation message is rewritten.
"""
pending, previous, annotations = {}, {}, []
command_re = re.compile(r"(?:^|\s)bench stage (?:wait|status) ([A-Za-z0-9_-]+)(?=\s|$)")
heartbeat_re = re.compile(r"^\[\d{2}:\d{2}:\d{2}\] ([A-Za-z0-9_-]+): (QUEUED|RUNNING)(?:(等 ([^\n]+) 池槽位))? 已 \d+s$")
result_re = re.compile(r"^result: (OK|FAIL|CANCELLED|SKIPPED) exit_code: (\S+) error_class: (\S+) [0-9.]+s$")
for message, source_id in zip(messages, source_ids):
for call in _calls(message):
args = call.get("arguments") or {}
matches = command_re.findall(args.get("command", "")) if isinstance(args, dict) else []
if len(matches) == 1:
pending[call.get("id")] = matches[0]
ident = message.get("toolCallId", message.get("tool_call_id"))
if message.get("role") not in ("toolResult", "tool") or ident not in pending:
continue
handle = pending[ident]
content = message.get("content")
if isinstance(content, str):
text = content
elif isinstance(content, list) and all(isinstance(b, dict) and b.get("type") == "text" for b in content):
text = "\n".join(b.get("text", "") for b in content)
else:
text = ""
states, evidence, unknown = [], [], not bool(text)
waiter = "TIMEOUT" if re.search(r"(?:^|\n)Command timed out after \d+ seconds(?:\n|$)", text) else None
for line in text.splitlines():
heartbeat = heartbeat_re.fullmatch(line)
result = result_re.fullmatch(line)
if heartbeat and heartbeat[1] == handle:
states.append({"state": heartbeat[2], "stage_state": heartbeat[2], "stage_result": None})
if heartbeat[3]:
evidence.append({"pool": heartbeat[3]})
elif result:
states.append({"state": result[1] if result[1] != "SKIPPED" else "UNKNOWN",
"stage_state": "DONE", "stage_result": result[1]})
evidence.append({"exit_code": result[2], "error_class": result[3]})
elif line.startswith(("stdout: ", "stderr: ", "meta: ", "measurements: ")):
evidence.append(line)
elif line:
unknown = True
evidence.append(line)
digest = payload_sha(evidence)
if not states:
states = [{"state": "UNKNOWN", "stage_state": None, "stage_result": None}]
for state in states:
prior = previous.get(handle)
if unknown or state["state"] == "UNKNOWN":
tag = "WAIT_UNKNOWN"
elif prior and prior["state"] in ("OK", "FAIL", "CANCELLED") and state["state"] != prior["state"]:
tag = "STATE_CONFLICT"
elif not prior:
tag = "FIRST_OBSERVATION"
elif prior["state"] != state["state"]:
tag = "STATE_CHANGED"
elif prior["evidence_sha256"] != digest:
tag = "SAME_STATE_NEW_EVIDENCE"
else:
tag = "HEARTBEAT_REPEAT"
annotation = {"source_id": source_id, "handle": handle, "call_id": ident,
**state, "waiter_outcome": waiter, "evidence_sha256": digest, "tag": tag,
"mutate_sft_messages": False}
annotations.append(annotation)
previous[handle] = annotation
return annotations
def truncation_annotations(removed_fields, later_messages, reference_candidates=None):
"""Classify declared historical loss using explicit structured references.
Natural language cannot prove the absence of a dependency. Unsupported or
implicit references therefore remain UNKNOWN, never permission to delete.
`removed_fields` come from a source/export diff, not from a text classifier.
"""
results = []
candidates = removed_fields if reference_candidates is None else reference_candidates
key_counts = Counter((field.get("field_key"), field.get("sha256")) for field in candidates)
pointer_counts = Counter(field.get("pointer") for field in candidates)
for removed in removed_fields:
source_id, pointer = removed.get("source_message_id"), removed.get("pointer")
matches, paired, paths, literal = [], [], [], False
for message in later_messages:
if message.get("role") != "assistant":
continue
for ref in message.get("evidence_refs", []):
if (ref.get("source_message_id"), ref.get("pointer")) == (source_id, pointer):
matches.append(ref)
content = message.get("content")
texts = [content] if isinstance(content, str) else [b.get("text", "") for b in content or [] if isinstance(b, dict)]
objects = [call.get("arguments") for call in _calls(message)]
for text in texts:
if pointer and re.search(r'(?<![A-Za-z0-9_/])' + re.escape(pointer) + r'(?![A-Za-z0-9_/])', text):
paths.append(pointer)
if removed.get("_value") is not None and str(removed["_value"]) in text:
literal = True
try:
objects.append(json.loads(text))
except (ValueError, TypeError):
pass
def inspect(value):
if isinstance(value, dict):
for key, child in value.items():
if key == removed.get("field_key") and payload_sha(child) == removed.get("sha256"):
paired.append(key)
inspect(child)
elif isinstance(value, list):
for child in value:
inspect(child)
for obj in objects:
inspect(obj)
unique_pair = paired and key_counts[(removed.get("field_key"), removed.get("sha256"))] == 1
unique_path = paths and pointer_counts[pointer] == 1
ambiguous = (paired or paths) and not (unique_pair or unique_path)
tag = ("REFERENCED_CONTENT_REMOVED" if matches or unique_pair or unique_path else
"REFERENCE_AMBIGUOUS" if ambiguous else "LITERAL_MATCH_UNRESOLVED" if literal else "REFERENCE_UNKNOWN")
results.append({**{k: v for k, v in removed.items() if not k.startswith("_")},
"tag": tag,
"mutate_sft_messages": False})
return results
def _json_leaf_spans(text):
"""JSON Pointer and exact character span for each scalar in valid JSON."""
decoder = json.JSONDecoder()
leaves = []
def skip(index):
while index < len(text) and text[index].isspace():
index += 1
return index
def walk(index, pointer="", key=None):
index = skip(index)
value, end = decoder.raw_decode(text, index)
if isinstance(value, dict):
cursor = skip(index + 1)
for name in value:
parsed, cursor = decoder.raw_decode(text, cursor)
if parsed != name:
raise ValueError("duplicate or inconsistent JSON keys")
cursor = skip(cursor)
cursor = walk(cursor + 1, pointer + "/" + name.replace("~", "~0").replace("/", "~1"), name)
cursor = skip(cursor)
if text[cursor] == ",":
cursor = skip(cursor + 1)
elif isinstance(value, list):
cursor = skip(index + 1)
for item in range(len(value)):
cursor = skip(walk(cursor, pointer + f"/{item}", key))
if text[cursor] == ",":
cursor = skip(cursor + 1)
else:
leaves.append({"pointer": pointer, "field_key": key, "_value": value,
"sha256": payload_sha(value), "_start": index, "_end": end})
return end
if skip(walk(0)) != len(text):
raise ValueError("trailing JSON content")
return leaves
def legacy_truncation_audit(messages, source_ids, legacy_messages):
"""Compare the old cleaner's actual output to lossless source messages.
Alignment uses unique tool-call identity. R7's declared marker defines the
missing range; unknown shapes stay advisory UNKNOWN. This only annotates.
"""
old = {}
for message in legacy_messages:
if message.get("role") == "tool":
old.setdefault(message.get("tool_call_id"), []).append(message.get("content"))
annotations, losses = [], []
for position, (message, source_id) in enumerate(zip(messages, source_ids)):
if message.get("role") != "toolResult":
continue
prior = old.get(message.get("toolCallId"), [])
content = message.get("content")
if not isinstance(content, list) or not all(isinstance(b, dict) and b.get("type") == "text" for b in content):
continue
full = "\n".join(b.get("text", "") for b in content)
if len(prior) != 1 or not isinstance(prior[0], str) or prior[0] == full:
continue
marker = re.search(r"\n…\[truncated (\d+) chars\]…\n", prior[0])
if not marker:
annotations.append({"source_id": source_id, "tag": "REFERENCE_UNKNOWN", "mutate_sft_messages": False})
continue
start, stop = marker.start(), len(full) - len(prior[0][marker.end():])
if full[:start] != prior[0][:start] or full[stop:] != prior[0][marker.end():] or stop - start != int(marker[1]):
annotations.append({"source_id": source_id, "tag": "REFERENCE_UNKNOWN", "mutate_sft_messages": False})
continue
removed = []
try:
for field in _json_leaf_spans(full):
if field["_start"] < stop and field["_end"] > start:
removed.append({**field, "source_message_id": source_id,
"span": [len(full[:max(start, field['_start'])].encode()), len(full[:min(stop, field['_end'])].encode())]})
except (ValueError, IndexError):
pass
if not removed:
removed = [{"source_message_id": source_id, "pointer": "/content/text", "sha256": sha(full[start:stop].encode()),
"span": [len(full[:start].encode()), len(full[:stop].encode())]}]
losses.append((position, source_id, removed))
candidates = [field for _, _, fields in losses for field in fields]
for position, source_id, removed in losses:
for annotation in truncation_annotations(removed, messages[position + 1:], candidates):
annotations.append({**annotation, "source_id": source_id, "kind": "historical_truncation",
"comparison_source": "legacy-clean-replay", "span_unit": "utf8-bytes"})
return annotations
def validate(source: FrozenSource, output: dict) -> dict:
"""Re-read source bytes, not the materializer's denominator or clean stats.
Besides rebuilding expectations, independently enumerate all raw message
observations. A shared parser omission therefore cannot make coverage pass.
"""
scanned = inventory(source)
expected = [item["message_id"] for item in scanned["messages"]]
actual = output.get("source_ids", [])
wanted_observations = []
direct_order = []
for location, event in _events(source):
if event.get("type") == "message_end":
ident = f"{location}:/message"
wanted_observations.append(ident)
direct_order.append(ident)
elif event.get("type") == "agent_end" and isinstance(event.get("messages"), list):
wanted_observations += [f"{location}:/messages/{i}" for i in range(len(event["messages"]))]
elif event.get("type") == "turn_end":
wanted_observations += [f"{location}:/message"]
wanted_observations += [f"{location}:/toolResults/{i}" for i in range(len(event.get("toolResults") or []))]
if "agent/system.md" in source.files:
wanted_observations.append(f"{source.manifest['agent/system.md']['sha256']}:0:/system")
supplied = output.get("lineage", []) + output.get("excluded", [])
observed = [obs["observation_id"] for item in supplied for obs in item.get("observations", [])]
expected_view, redaction = _view(scanned["messages"], scanned["tools"])
expected_lineage = [{k: item[k] for k in ("message_id", "observations", "event_type")} for item in scanned["messages"]]
expected_excluded = [{k: item[k] for k in ("message_id", "observations", "event_type", "reason")} for item in scanned["excluded"]]
counts = Counter(actual)
covered = sum(counts[ident] == 1 for ident in expected)
native_ids = {item["message_id"] for item in scanned["messages"] + scanned["excluded"]}
valid_direct = [ident for ident in direct_order if ident in set(expected)]
source_consistent = all(name in source.files and info["sha256"] == sha(source.files[name])
and info["bytes"] == len(source.files[name]) for name, info in source.manifest.items())
observation_counts = Counter(observed)
checks = {
"OBSERVATION_ACCOUNTING": {"pass": Counter(wanted_observations) == observation_counts,
"expected": len(wanted_observations), "actual": len(observed)},
"VALID_MESSAGE_COVERAGE": {"pass": covered == len(expected) > 0 and not (set(actual) - set(expected))
and len(actual) == len(expected), "numerator": covered, "denominator": len(expected)},
"PAYLOAD_COVERAGE": {"pass": output.get("messages") == expected_view["messages"] and output.get("tools") == expected_view["tools"]},
"IDENTITY_DEDUP": {"pass": output.get("lineage") == expected_lineage and output.get("excluded") == expected_excluded
and all(ident in native_ids for ident in direct_order)
and not ({"IDENTITY_AMBIGUOUS", "IDENTITY_PAYLOAD_CONFLICT"} & set(scanned["errors"]))},
"ORDER_AND_CAUSALITY": {"pass": actual == expected and [ident for ident in actual if ident in set(valid_direct)] == valid_direct},
"CALL_TOOL_CLOSURE": closure(output.get("messages", [])),
"REQUIRED_CONTEXT": {"pass": scanned["context"]["status"] == "COMPLETE"
and output.get("context") == scanned["context"], **scanned["context"]},
"SOURCE_INTEGRITY": {"pass": source_consistent and not scanned["errors"]
and output.get("source_manifest") == source.manifest, "errors": scanned["errors"]},
"ERROR_EXCLUSION": {"pass": output.get("excluded") == expected_excluded
and not ({"ERROR_CONTENT_RETAINED", "INCOMPLETE_GENERATION"} & set(scanned["errors"]))},
"PRIVACY": {"pass": output.get("redaction") == redaction and not redaction["protected_sensitive_fields"]},
"USABLE_ASSISTANT": {"pass": any(m.get("role") == "assistant" and (not _empty_content(m) or _calls(m))
for m in output.get("messages", []))},
}
return {"pass": all(check["pass"] for check in checks.values()), "rules_version": RULES_VERSION, "checks": checks}