664 lines
34 KiB
Python
664 lines
34 KiB
Python
"""Lossless Pi event materialization and source-based validation.
|
|
|
|
The Pi 0.73 event adapter treats agent_end as a segment mirror, never a session
|
|
replacement. Message identity is a source location/lifecycle, not content. The
|
|
validator reads the frozen event bytes again and audits every raw observation.
|
|
Raw bytes stay private; exported lineage contains only locations and hashes.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
from export.redact import redact_preserving
|
|
|
|
RULES_VERSION = "sft-preserve-v1"
|
|
# Pin the implemented spec, not caller-supplied release metadata.
|
|
RULES_SPEC_SHA256 = "651d8126ec049198dc14b4de455b14ff5bc654e8a0fa9df3df4e31418e330ada"
|
|
FILES = ("rollout.json", "agent/events.jsonl", "agent/system.md", "agent/tools.json", "agent/prompt.md")
|
|
TERMINAL_STATES = frozenset({"DONE", "FAILED", "FAIL", "TIMEOUT", "CANCELLED", "ENV_ERROR",
|
|
"BUDGET_EXHAUSTED", "INTERRUPTED", "COMPLETED", "ERROR", "DEADLINE_EXCEEDED",
|
|
"INFRA_ERROR", "AGENT_ERROR", "LOST"})
|
|
OPERATIONS = frozenset({"session", "agent_start", "turn_start", "auto_retry_start", "auto_retry_end",
|
|
"tool_execution_start", "tool_execution_update", "tool_execution_end"})
|
|
SUPPORTED_UPDATES = frozenset(f"{kind}_{suffix}" for kind in ("text", "thinking", "toolcall")
|
|
for suffix in ("start", "delta", "end"))
|
|
|
|
|
|
def sha(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def payload_sha(value) -> str:
|
|
return sha(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode())
|
|
|
|
|
|
@dataclass
|
|
class FrozenSource:
|
|
files: dict[str, bytes]
|
|
manifest: dict
|
|
errors: list[str]
|
|
|
|
|
|
def freeze(rd: Path) -> FrozenSource:
|
|
files, manifest, errors = {}, {}, []
|
|
for name in FILES:
|
|
path = rd / name
|
|
try:
|
|
before = path.stat()
|
|
data = path.read_bytes()
|
|
after = path.stat()
|
|
except FileNotFoundError:
|
|
continue
|
|
files[name] = data
|
|
manifest[name] = {"sha256": sha(data), "bytes": len(data), "watermark": len(data)}
|
|
if (before.st_ino, before.st_size, before.st_mtime_ns) != (after.st_ino, after.st_size, after.st_mtime_ns):
|
|
errors.append("SOURCE_CHANGED")
|
|
# Recheck the entire input set, not just each file in isolation.
|
|
for name, data in files.items():
|
|
try:
|
|
if (rd / name).read_bytes() != data:
|
|
errors.append("SOURCE_CHANGED")
|
|
except FileNotFoundError:
|
|
errors.append("SOURCE_CHANGED")
|
|
return FrozenSource(files, manifest, errors)
|
|
|
|
|
|
def source_unchanged(rd: Path, source: FrozenSource) -> bool:
|
|
"""Check the entire known input set and EOF again immediately before output."""
|
|
for name in FILES:
|
|
path = rd / name
|
|
try:
|
|
digest, size = hashlib.sha256(), 0
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
|
digest.update(chunk); size += len(chunk)
|
|
if source.manifest.get(name) != {"sha256": digest.hexdigest(), "bytes": size, "watermark": size}:
|
|
return False
|
|
except FileNotFoundError:
|
|
if name in source.manifest:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _json(source, name, default):
|
|
try:
|
|
return json.loads(source.files[name])
|
|
except (KeyError, ValueError, UnicodeDecodeError):
|
|
return default
|
|
|
|
|
|
def _events(source):
|
|
offset = 0
|
|
digest = source.manifest.get("agent/events.jsonl", {}).get("sha256", "missing")
|
|
for line in source.files.get("agent/events.jsonl", b"").splitlines(keepends=True):
|
|
location = f"{digest}:{offset}"
|
|
offset += len(line)
|
|
try:
|
|
event = json.loads(line)
|
|
if not isinstance(event, dict):
|
|
raise ValueError("not an event object")
|
|
yield location, event
|
|
except (ValueError, UnicodeDecodeError):
|
|
yield location, {"type": "!parse_error"}
|
|
|
|
|
|
def _observation(location, pointer, payload):
|
|
return {"observation_id": f"{location}:{pointer}", "payload_sha256": payload_sha(payload)}
|
|
|
|
|
|
def _calls(message):
|
|
content = message.get("content")
|
|
calls = [block for block in content if isinstance(block, dict) and block.get("type") == "toolCall"] if isinstance(content, list) else []
|
|
return calls + list(message.get("tool_calls") or [])
|
|
|
|
|
|
def _empty_content(message):
|
|
if "content" not in message:
|
|
return False
|
|
content = message["content"]
|
|
if content is None or content == "" or content == []:
|
|
return True
|
|
return isinstance(content, list) and all(
|
|
isinstance(block, dict) and block.get("type") in ("text", "output_text")
|
|
and block.get("text") == "" and set(block) <= {"type", "text"} for block in content)
|
|
|
|
|
|
def _identity_shape(message):
|
|
"""Shape is only a segment-position check; payload equality is not identity."""
|
|
return (message.get("role"), message.get("toolCallId"), tuple(c.get("id") for c in _calls(message)))
|
|
|
|
|
|
def inventory(source: FrozenSource) -> dict:
|
|
errors = list(source.errors)
|
|
metadata = _json(source, "rollout.json", {})
|
|
if metadata.get("runner", "pi") != "pi" or metadata.get("runner_version") not in (None, "legacy-pi", "0.73.1", "pi-sdk-v1"):
|
|
errors.append("UNSUPPORTED_SOURCE")
|
|
if metadata.get("state") not in TERMINAL_STATES:
|
|
errors.append("SOURCE_NOT_TERMINAL")
|
|
messages, operations, segment, turn = [], [], [], []
|
|
pending = None
|
|
ended = False
|
|
sdk_headers = []
|
|
session_ids = set()
|
|
context_started = False
|
|
|
|
def add(payload, observation, *, streamed=False, event_type="message_end"):
|
|
if not isinstance(payload, dict) or not isinstance(payload.get("role"), str):
|
|
errors.append("UNSUPPORTED_SOURCE")
|
|
return None
|
|
item = {"message_id": observation["observation_id"], "payload": copy.deepcopy(payload),
|
|
"observations": [observation], "event_type": event_type, "streamed": streamed}
|
|
messages.append(item)
|
|
return item
|
|
|
|
def alias(item, payload, observation):
|
|
if item["payload"] != payload:
|
|
errors.append("IDENTITY_PAYLOAD_CONFLICT")
|
|
item["observations"].append(observation)
|
|
|
|
def mirror(payloads, location, pointer, candidates, allow_context=False):
|
|
"""One-to-one segment slots establish identity; no content search/dedup."""
|
|
if not isinstance(payloads, list):
|
|
errors.append("UNSUPPORTED_SOURCE")
|
|
return
|
|
cursor, ordered = 0, []
|
|
for index, payload in enumerate(payloads):
|
|
observation = _observation(location, f"{pointer}/{index}", payload)
|
|
if cursor < len(candidates) and _identity_shape(payload) == _identity_shape(candidates[cursor]["payload"]):
|
|
item = candidates[cursor]
|
|
cursor += 1
|
|
alias(item, payload, observation)
|
|
elif allow_context and payload.get("role") in ("user", "custom", "system"):
|
|
item = add(payload, observation, event_type="context_mirror")
|
|
# A complete segment mirror gives the new context its position.
|
|
else:
|
|
errors.append("IDENTITY_AMBIGUOUS")
|
|
item = add(payload, observation, event_type="unresolved_mirror")
|
|
if item:
|
|
ordered.append(item)
|
|
if cursor != len(candidates):
|
|
errors.append("OBSERVATION_UNACCOUNTED")
|
|
if allow_context:
|
|
# Replace only this segment's slots, retaining every prior segment.
|
|
identities = {id(item) for item in ordered + candidates}
|
|
positions = [i for i, item in enumerate(messages) if id(item) in identities]
|
|
start = min(positions, default=len(messages))
|
|
messages[:] = [item for item in messages if id(item) not in identities]
|
|
messages[start:start] = ordered
|
|
candidates[:] = ordered
|
|
|
|
for location, event in _events(source):
|
|
kind = event.get("type")
|
|
if kind in ("agent_start", "turn_start", "message_start", "message_update", "message_end", "turn_end", "agent_end"):
|
|
context_started = True
|
|
if kind == "!parse_error":
|
|
errors.append("PARSE_ERROR")
|
|
elif kind in OPERATIONS:
|
|
operations.append({"type": kind, "observation_id": location, "payload_sha256": payload_sha(event)})
|
|
if kind == "session":
|
|
if event.get("id"):
|
|
session_ids.add(event["id"])
|
|
if len(session_ids) > 1:
|
|
errors.append("SESSION_IDENTITY_CONFLICT")
|
|
header = event.get("bench")
|
|
if isinstance(header, dict) and header.get("runner_version") == "pi-sdk-v1":
|
|
if not sdk_headers and context_started:
|
|
errors.append("SDK_CONTEXT_HEADER_LATE")
|
|
if sdk_headers and not ended:
|
|
errors.append("SDK_RESUME_INCOMPLETE_SEGMENT")
|
|
sdk_headers.append(event)
|
|
elif event.get("version") != 3:
|
|
errors.append("UNSUPPORTED_SOURCE")
|
|
elif kind == "agent_start":
|
|
if pending:
|
|
errors.append("INCOMPLETE_GENERATION")
|
|
segment, turn, pending, ended = [], [], None, False
|
|
elif kind == "turn_start":
|
|
turn = []
|
|
elif kind == "tool_execution_start" and pending:
|
|
pending["streamed"] = True # an observed side effect forbids empty-fragment deletion
|
|
elif kind == "message_start":
|
|
if pending:
|
|
errors.append("INCOMPLETE_GENERATION")
|
|
start = event.get("message")
|
|
pending = {"location": location, "message": start,
|
|
"streamed": not isinstance(start, dict) or not _empty_content(start)}
|
|
elif kind == "message_update":
|
|
update = event.get("assistantMessageEvent", {})
|
|
if not pending or update.get("type") not in SUPPORTED_UPDATES:
|
|
errors.append("SOURCE_EVENT_UNKNOWN")
|
|
elif update.get("delta") not in (None, "", {}, []):
|
|
pending["streamed"] = True
|
|
elif kind == "message_end":
|
|
payload = event.get("message")
|
|
observation = _observation(location, "/message", payload)
|
|
if pending and isinstance(payload, dict) and isinstance(pending["message"], dict):
|
|
if pending["message"].get("role") != payload.get("role"):
|
|
errors.append("CONTEXT_ORDER_UNKNOWN")
|
|
item = add(payload, observation, streamed=bool(pending and pending["streamed"]))
|
|
if item:
|
|
if pending:
|
|
item["lifecycle_start"] = pending["location"]
|
|
segment.append(item)
|
|
turn.append(item)
|
|
pending, ended = None, False
|
|
elif kind == "turn_end":
|
|
candidates = [item for item in turn if item["payload"].get("role") in ("assistant", "toolResult")]
|
|
if not candidates:
|
|
# Legacy fixture/event streams without turn_start: the current
|
|
# turn is the suffix starting at the last assistant message.
|
|
candidates = [item for item in segment if item["payload"].get("role") in ("assistant", "toolResult")]
|
|
payloads = [event.get("message")] + list(event.get("toolResults") or [])
|
|
if not all(isinstance(m, dict) for m in payloads):
|
|
errors.append("UNSUPPORTED_SOURCE")
|
|
continue
|
|
# Use actual Pi JSON pointers, not the synthetic combined list.
|
|
for i, payload in enumerate(payloads):
|
|
pointer = "/message" if i == 0 else f"/toolResults/{i - 1}"
|
|
observation = _observation(location, pointer, payload)
|
|
if i < len(candidates) and _identity_shape(payload) == _identity_shape(candidates[i]["payload"]):
|
|
alias(candidates[i], payload, observation)
|
|
else:
|
|
errors.append("IDENTITY_AMBIGUOUS")
|
|
add(payload, observation, event_type="unresolved_mirror")
|
|
if len(payloads) != len(candidates):
|
|
errors.append("OBSERVATION_UNACCOUNTED")
|
|
elif kind == "agent_end":
|
|
payloads = event.get("messages")
|
|
if not segment and isinstance(payloads, list):
|
|
# An isolated end container proves its own order but provides no
|
|
# lifecycle proof. Keep it for analysis; never invent aliases.
|
|
errors.append("MISSING_MESSAGE_LIFECYCLES")
|
|
if isinstance(payloads, list) and all(isinstance(m, dict) for m in payloads):
|
|
mirror(payloads, location, "/messages", segment, allow_context=True)
|
|
else:
|
|
errors.append("UNSUPPORTED_SOURCE")
|
|
ended = True
|
|
else:
|
|
errors.append("SOURCE_EVENT_UNKNOWN")
|
|
operations.append({"type": "unknown", "type_sha256": payload_sha(kind),
|
|
"observation_id": location, "payload_sha256": payload_sha(event), "unknown": True})
|
|
if pending:
|
|
errors.append("INCOMPLETE_GENERATION")
|
|
if not ended:
|
|
errors.append("INCOMPLETE_TRAJECTORY")
|
|
|
|
# Only saved runner context is admitted. The initial prompt is not injected
|
|
# by text equality; real Pi streams include the initial user message.
|
|
if "agent/system.md" in source.files:
|
|
try:
|
|
payload = {"role": "system", "content": source.files["agent/system.md"].decode("utf-8")}
|
|
observation = _observation(source.manifest["agent/system.md"]["sha256"] + ":0", "/system", payload)
|
|
item = add(payload, observation, event_type="system_snapshot")
|
|
messages.insert(0, messages.pop())
|
|
except UnicodeDecodeError:
|
|
errors.append("PARSE_ERROR")
|
|
else:
|
|
errors.append("CONTEXT_SNAPSHOT_MISSING")
|
|
tools = _json(source, "agent/tools.json", None)
|
|
if not isinstance(tools, list) or not tools:
|
|
tools = []
|
|
errors.append("CONTEXT_SNAPSHOT_MISSING")
|
|
context = {"status": "UNKNOWN", "source": "unverified-snapshot"}
|
|
if metadata.get("runner_version") == "legacy-pi":
|
|
context["source"] = "legacy-append-only"
|
|
if sdk_headers and metadata.get("runner_version") != "pi-sdk-v1":
|
|
errors.append("SOURCE_CONTEXT_CONFLICT")
|
|
if metadata.get("runner_version") == "pi-sdk-v1":
|
|
if not sdk_headers:
|
|
errors.append("SDK_CONTEXT_HEADER_MISSING")
|
|
else:
|
|
first_header = sdk_headers[0]
|
|
header = first_header["bench"]
|
|
expected_system = source.files.get("agent/system.md", b"").decode("utf-8", errors="replace")
|
|
if (first_header.get("version") != 3 or not isinstance(first_header.get("id"), str) or not first_header["id"]
|
|
or any(other != first_header for other in sdk_headers[1:])
|
|
or not isinstance(header.get("execution_id"), str) or not header["execution_id"]
|
|
or not isinstance(header.get("system_prompt"), str) or not header["system_prompt"]
|
|
or header["system_prompt"] != expected_system or header.get("tools") != tools
|
|
or metadata.get("execution_id") != header["execution_id"]):
|
|
errors.append("SDK_CONTEXT_SNAPSHOT_MISMATCH")
|
|
elif not {"SDK_CONTEXT_HEADER_LATE", "SDK_RESUME_INCOMPLETE_SEGMENT"} & set(errors):
|
|
context = {"status": "COMPLETE", "source": "pi-sdk-session-header"}
|
|
if context["status"] != "COMPLETE":
|
|
errors.append("REQUIRED_CONTEXT_UNKNOWN")
|
|
if not any(item["payload"].get("role") == "user" for item in messages):
|
|
errors.append("INITIAL_CONTEXT_MISSING")
|
|
|
|
excluded, retained = [], []
|
|
for item in messages:
|
|
msg = item["payload"]
|
|
if msg.get("role") not in ("system", "user", "custom", "assistant", "toolResult"):
|
|
errors.append("UNSUPPORTED_SOURCE")
|
|
empty_error = (item["event_type"] == "message_end" and msg.get("role") == "assistant"
|
|
and msg.get("stopReason") == "error" and _empty_content(msg)
|
|
and not _calls(msg) and not any(k in msg for k in ("toolCallId", "tool_call_id", "function_call", "tool_calls"))
|
|
and not item["streamed"])
|
|
if empty_error:
|
|
excluded.append({**item, "reason": "EMPTY_ERROR_FRAGMENT"})
|
|
else:
|
|
retained.append(item)
|
|
if msg.get("role") == "assistant" and msg.get("stopReason") in ("error", "aborted"):
|
|
errors.append("ERROR_CONTENT_RETAINED")
|
|
return {"messages": retained, "excluded": excluded, "operations": operations,
|
|
"errors": sorted(set(errors)), "tools": tools, "agent_end": ended, "context": context}
|
|
|
|
|
|
def closure(messages):
|
|
calls, results = {}, Counter()
|
|
missing_id, reused, orphan, early = 0, 0, 0, 0
|
|
for position, message in enumerate(messages):
|
|
for call in _calls(message):
|
|
ident = call.get("id")
|
|
if not isinstance(ident, str) or not ident:
|
|
missing_id += 1
|
|
elif ident in calls:
|
|
reused += 1
|
|
else:
|
|
calls[ident] = position
|
|
if message.get("role") in ("toolResult", "tool"):
|
|
ident = message.get("toolCallId", message.get("tool_call_id"))
|
|
results[ident] += 1
|
|
if ident not in calls:
|
|
orphan += 1
|
|
early += 1
|
|
metrics = {"missing": sum(results[key] == 0 for key in calls), "orphan": orphan,
|
|
"duplicate_final": sum(max(0, n - 1) for n in results.values()),
|
|
"call_id_conflict": reused + missing_id, "result_before_call": early}
|
|
return {"pass": not any(metrics.values()), **metrics}
|
|
|
|
|
|
def _view(items, tools):
|
|
# Pi-native blocks preserve thinking, tool args/results and unknown fields.
|
|
return redact_preserving({"messages": [item["payload"] for item in items], "tools": tools})
|
|
|
|
|
|
def reconstruct(source: FrozenSource) -> dict:
|
|
scanned = inventory(source)
|
|
view, redaction = _view(scanned["messages"], scanned["tools"])
|
|
lineage = [{k: item[k] for k in ("message_id", "observations", "event_type")} for item in scanned["messages"]]
|
|
excluded = [{k: item[k] for k in ("message_id", "observations", "event_type", "reason")} for item in scanned["excluded"]]
|
|
result = {**view, "format": "pi-native-messages/1", "rules_version": RULES_VERSION,
|
|
"source_ids": [item["message_id"] for item in scanned["messages"]], "lineage": lineage,
|
|
"excluded": excluded, "operations": scanned["operations"], "errors": scanned["errors"],
|
|
"agent_end": scanned["agent_end"], "context": scanned["context"],
|
|
"redaction": redaction, "source_manifest": source.manifest}
|
|
result["annotations"] = wait_annotations(result["messages"], result["source_ids"])
|
|
return result
|
|
|
|
|
|
def wait_annotations(messages, source_ids):
|
|
"""Advisory-only parser for the recorded stageclient v1 grammar.
|
|
|
|
Unknown output stays UNKNOWN. We compare all non-heartbeat text, so the same
|
|
phase with newly printed evidence cannot be mistaken for an empty poll.
|
|
No current status file is read and no conversation message is rewritten.
|
|
"""
|
|
pending, previous, annotations = {}, {}, []
|
|
command_re = re.compile(r"(?:^|\s)bench stage (?:wait|status) ([A-Za-z0-9_-]+)(?=\s|$)")
|
|
heartbeat_re = re.compile(r"^\[\d{2}:\d{2}:\d{2}\] ([A-Za-z0-9_-]+): (QUEUED|RUNNING)(?:(等 ([^\n]+) 池槽位))? 已 \d+s$")
|
|
result_re = re.compile(r"^result: (OK|FAIL|CANCELLED|SKIPPED) exit_code: (\S+) error_class: (\S+) [0-9.]+s$")
|
|
for message, source_id in zip(messages, source_ids):
|
|
for call in _calls(message):
|
|
args = call.get("arguments") or {}
|
|
matches = command_re.findall(args.get("command", "")) if isinstance(args, dict) else []
|
|
if len(matches) == 1:
|
|
pending[call.get("id")] = matches[0]
|
|
ident = message.get("toolCallId", message.get("tool_call_id"))
|
|
if message.get("role") not in ("toolResult", "tool") or ident not in pending:
|
|
continue
|
|
handle = pending[ident]
|
|
content = message.get("content")
|
|
if isinstance(content, str):
|
|
text = content
|
|
elif isinstance(content, list) and all(isinstance(b, dict) and b.get("type") == "text" for b in content):
|
|
text = "\n".join(b.get("text", "") for b in content)
|
|
else:
|
|
text = ""
|
|
states, evidence, unknown = [], [], not bool(text)
|
|
waiter = "TIMEOUT" if re.search(r"(?:^|\n)Command timed out after \d+ seconds(?:\n|$)", text) else None
|
|
for line in text.splitlines():
|
|
heartbeat = heartbeat_re.fullmatch(line)
|
|
result = result_re.fullmatch(line)
|
|
if heartbeat and heartbeat[1] == handle:
|
|
states.append({"state": heartbeat[2], "stage_state": heartbeat[2], "stage_result": None})
|
|
if heartbeat[3]:
|
|
evidence.append({"pool": heartbeat[3]})
|
|
elif result:
|
|
states.append({"state": result[1] if result[1] != "SKIPPED" else "UNKNOWN",
|
|
"stage_state": "DONE", "stage_result": result[1]})
|
|
evidence.append({"exit_code": result[2], "error_class": result[3]})
|
|
elif line.startswith(("stdout: ", "stderr: ", "meta: ", "measurements: ")):
|
|
evidence.append(line)
|
|
elif line:
|
|
unknown = True
|
|
evidence.append(line)
|
|
digest = payload_sha(evidence)
|
|
if not states:
|
|
states = [{"state": "UNKNOWN", "stage_state": None, "stage_result": None}]
|
|
for state in states:
|
|
prior = previous.get(handle)
|
|
if unknown or state["state"] == "UNKNOWN":
|
|
tag = "WAIT_UNKNOWN"
|
|
elif prior and prior["state"] in ("OK", "FAIL", "CANCELLED") and state["state"] != prior["state"]:
|
|
tag = "STATE_CONFLICT"
|
|
elif not prior:
|
|
tag = "FIRST_OBSERVATION"
|
|
elif prior["state"] != state["state"]:
|
|
tag = "STATE_CHANGED"
|
|
elif prior["evidence_sha256"] != digest:
|
|
tag = "SAME_STATE_NEW_EVIDENCE"
|
|
else:
|
|
tag = "HEARTBEAT_REPEAT"
|
|
annotation = {"source_id": source_id, "handle": handle, "call_id": ident,
|
|
**state, "waiter_outcome": waiter, "evidence_sha256": digest, "tag": tag,
|
|
"mutate_sft_messages": False}
|
|
annotations.append(annotation)
|
|
previous[handle] = annotation
|
|
return annotations
|
|
|
|
|
|
def truncation_annotations(removed_fields, later_messages, reference_candidates=None):
|
|
"""Classify declared historical loss using explicit structured references.
|
|
|
|
Natural language cannot prove the absence of a dependency. Unsupported or
|
|
implicit references therefore remain UNKNOWN, never permission to delete.
|
|
`removed_fields` come from a source/export diff, not from a text classifier.
|
|
"""
|
|
results = []
|
|
candidates = removed_fields if reference_candidates is None else reference_candidates
|
|
key_counts = Counter((field.get("field_key"), field.get("sha256")) for field in candidates)
|
|
pointer_counts = Counter(field.get("pointer") for field in candidates)
|
|
for removed in removed_fields:
|
|
source_id, pointer = removed.get("source_message_id"), removed.get("pointer")
|
|
matches, paired, paths, literal = [], [], [], False
|
|
for message in later_messages:
|
|
if message.get("role") != "assistant":
|
|
continue
|
|
for ref in message.get("evidence_refs", []):
|
|
if (ref.get("source_message_id"), ref.get("pointer")) == (source_id, pointer):
|
|
matches.append(ref)
|
|
content = message.get("content")
|
|
texts = [content] if isinstance(content, str) else [b.get("text", "") for b in content or [] if isinstance(b, dict)]
|
|
objects = [call.get("arguments") for call in _calls(message)]
|
|
for text in texts:
|
|
if pointer and re.search(r'(?<![A-Za-z0-9_/])' + re.escape(pointer) + r'(?![A-Za-z0-9_/])', text):
|
|
paths.append(pointer)
|
|
if removed.get("_value") is not None and str(removed["_value"]) in text:
|
|
literal = True
|
|
try:
|
|
objects.append(json.loads(text))
|
|
except (ValueError, TypeError):
|
|
pass
|
|
def inspect(value):
|
|
if isinstance(value, dict):
|
|
for key, child in value.items():
|
|
if key == removed.get("field_key") and payload_sha(child) == removed.get("sha256"):
|
|
paired.append(key)
|
|
inspect(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
inspect(child)
|
|
for obj in objects:
|
|
inspect(obj)
|
|
unique_pair = paired and key_counts[(removed.get("field_key"), removed.get("sha256"))] == 1
|
|
unique_path = paths and pointer_counts[pointer] == 1
|
|
ambiguous = (paired or paths) and not (unique_pair or unique_path)
|
|
tag = ("REFERENCED_CONTENT_REMOVED" if matches or unique_pair or unique_path else
|
|
"REFERENCE_AMBIGUOUS" if ambiguous else "LITERAL_MATCH_UNRESOLVED" if literal else "REFERENCE_UNKNOWN")
|
|
results.append({**{k: v for k, v in removed.items() if not k.startswith("_")},
|
|
"tag": tag,
|
|
"mutate_sft_messages": False})
|
|
return results
|
|
|
|
|
|
def _json_leaf_spans(text):
|
|
"""JSON Pointer and exact character span for each scalar in valid JSON."""
|
|
decoder = json.JSONDecoder()
|
|
leaves = []
|
|
def skip(index):
|
|
while index < len(text) and text[index].isspace():
|
|
index += 1
|
|
return index
|
|
def walk(index, pointer="", key=None):
|
|
index = skip(index)
|
|
value, end = decoder.raw_decode(text, index)
|
|
if isinstance(value, dict):
|
|
cursor = skip(index + 1)
|
|
for name in value:
|
|
parsed, cursor = decoder.raw_decode(text, cursor)
|
|
if parsed != name:
|
|
raise ValueError("duplicate or inconsistent JSON keys")
|
|
cursor = skip(cursor)
|
|
cursor = walk(cursor + 1, pointer + "/" + name.replace("~", "~0").replace("/", "~1"), name)
|
|
cursor = skip(cursor)
|
|
if text[cursor] == ",":
|
|
cursor = skip(cursor + 1)
|
|
elif isinstance(value, list):
|
|
cursor = skip(index + 1)
|
|
for item in range(len(value)):
|
|
cursor = skip(walk(cursor, pointer + f"/{item}", key))
|
|
if text[cursor] == ",":
|
|
cursor = skip(cursor + 1)
|
|
else:
|
|
leaves.append({"pointer": pointer, "field_key": key, "_value": value,
|
|
"sha256": payload_sha(value), "_start": index, "_end": end})
|
|
return end
|
|
if skip(walk(0)) != len(text):
|
|
raise ValueError("trailing JSON content")
|
|
return leaves
|
|
|
|
|
|
def legacy_truncation_audit(messages, source_ids, legacy_messages):
|
|
"""Compare the old cleaner's actual output to lossless source messages.
|
|
|
|
Alignment uses unique tool-call identity. R7's declared marker defines the
|
|
missing range; unknown shapes stay advisory UNKNOWN. This only annotates.
|
|
"""
|
|
old = {}
|
|
for message in legacy_messages:
|
|
if message.get("role") == "tool":
|
|
old.setdefault(message.get("tool_call_id"), []).append(message.get("content"))
|
|
annotations, losses = [], []
|
|
for position, (message, source_id) in enumerate(zip(messages, source_ids)):
|
|
if message.get("role") != "toolResult":
|
|
continue
|
|
prior = old.get(message.get("toolCallId"), [])
|
|
content = message.get("content")
|
|
if not isinstance(content, list) or not all(isinstance(b, dict) and b.get("type") == "text" for b in content):
|
|
continue
|
|
full = "\n".join(b.get("text", "") for b in content)
|
|
if len(prior) != 1 or not isinstance(prior[0], str) or prior[0] == full:
|
|
continue
|
|
marker = re.search(r"\n…\[truncated (\d+) chars\]…\n", prior[0])
|
|
if not marker:
|
|
annotations.append({"source_id": source_id, "tag": "REFERENCE_UNKNOWN", "mutate_sft_messages": False})
|
|
continue
|
|
start, stop = marker.start(), len(full) - len(prior[0][marker.end():])
|
|
if full[:start] != prior[0][:start] or full[stop:] != prior[0][marker.end():] or stop - start != int(marker[1]):
|
|
annotations.append({"source_id": source_id, "tag": "REFERENCE_UNKNOWN", "mutate_sft_messages": False})
|
|
continue
|
|
removed = []
|
|
try:
|
|
for field in _json_leaf_spans(full):
|
|
if field["_start"] < stop and field["_end"] > start:
|
|
removed.append({**field, "source_message_id": source_id,
|
|
"span": [len(full[:max(start, field['_start'])].encode()), len(full[:min(stop, field['_end'])].encode())]})
|
|
except (ValueError, IndexError):
|
|
pass
|
|
if not removed:
|
|
removed = [{"source_message_id": source_id, "pointer": "/content/text", "sha256": sha(full[start:stop].encode()),
|
|
"span": [len(full[:start].encode()), len(full[:stop].encode())]}]
|
|
losses.append((position, source_id, removed))
|
|
candidates = [field for _, _, fields in losses for field in fields]
|
|
for position, source_id, removed in losses:
|
|
for annotation in truncation_annotations(removed, messages[position + 1:], candidates):
|
|
annotations.append({**annotation, "source_id": source_id, "kind": "historical_truncation",
|
|
"comparison_source": "legacy-clean-replay", "span_unit": "utf8-bytes"})
|
|
return annotations
|
|
|
|
|
|
def validate(source: FrozenSource, output: dict) -> dict:
|
|
"""Re-read source bytes, not the materializer's denominator or clean stats.
|
|
|
|
Besides rebuilding expectations, independently enumerate all raw message
|
|
observations. A shared parser omission therefore cannot make coverage pass.
|
|
"""
|
|
scanned = inventory(source)
|
|
expected = [item["message_id"] for item in scanned["messages"]]
|
|
actual = output.get("source_ids", [])
|
|
wanted_observations = []
|
|
direct_order = []
|
|
for location, event in _events(source):
|
|
if event.get("type") == "message_end":
|
|
ident = f"{location}:/message"
|
|
wanted_observations.append(ident)
|
|
direct_order.append(ident)
|
|
elif event.get("type") == "agent_end" and isinstance(event.get("messages"), list):
|
|
wanted_observations += [f"{location}:/messages/{i}" for i in range(len(event["messages"]))]
|
|
elif event.get("type") == "turn_end":
|
|
wanted_observations += [f"{location}:/message"]
|
|
wanted_observations += [f"{location}:/toolResults/{i}" for i in range(len(event.get("toolResults") or []))]
|
|
if "agent/system.md" in source.files:
|
|
wanted_observations.append(f"{source.manifest['agent/system.md']['sha256']}:0:/system")
|
|
supplied = output.get("lineage", []) + output.get("excluded", [])
|
|
observed = [obs["observation_id"] for item in supplied for obs in item.get("observations", [])]
|
|
expected_view, redaction = _view(scanned["messages"], scanned["tools"])
|
|
expected_lineage = [{k: item[k] for k in ("message_id", "observations", "event_type")} for item in scanned["messages"]]
|
|
expected_excluded = [{k: item[k] for k in ("message_id", "observations", "event_type", "reason")} for item in scanned["excluded"]]
|
|
counts = Counter(actual)
|
|
covered = sum(counts[ident] == 1 for ident in expected)
|
|
native_ids = {item["message_id"] for item in scanned["messages"] + scanned["excluded"]}
|
|
valid_direct = [ident for ident in direct_order if ident in set(expected)]
|
|
source_consistent = all(name in source.files and info["sha256"] == sha(source.files[name])
|
|
and info["bytes"] == len(source.files[name]) for name, info in source.manifest.items())
|
|
observation_counts = Counter(observed)
|
|
checks = {
|
|
"OBSERVATION_ACCOUNTING": {"pass": Counter(wanted_observations) == observation_counts,
|
|
"expected": len(wanted_observations), "actual": len(observed)},
|
|
"VALID_MESSAGE_COVERAGE": {"pass": covered == len(expected) > 0 and not (set(actual) - set(expected))
|
|
and len(actual) == len(expected), "numerator": covered, "denominator": len(expected)},
|
|
"PAYLOAD_COVERAGE": {"pass": output.get("messages") == expected_view["messages"] and output.get("tools") == expected_view["tools"]},
|
|
"IDENTITY_DEDUP": {"pass": output.get("lineage") == expected_lineage and output.get("excluded") == expected_excluded
|
|
and all(ident in native_ids for ident in direct_order)
|
|
and not ({"IDENTITY_AMBIGUOUS", "IDENTITY_PAYLOAD_CONFLICT"} & set(scanned["errors"]))},
|
|
"ORDER_AND_CAUSALITY": {"pass": actual == expected and [ident for ident in actual if ident in set(valid_direct)] == valid_direct},
|
|
"CALL_TOOL_CLOSURE": closure(output.get("messages", [])),
|
|
"REQUIRED_CONTEXT": {"pass": scanned["context"]["status"] == "COMPLETE"
|
|
and output.get("context") == scanned["context"], **scanned["context"]},
|
|
"SOURCE_INTEGRITY": {"pass": source_consistent and not scanned["errors"]
|
|
and output.get("source_manifest") == source.manifest, "errors": scanned["errors"]},
|
|
"ERROR_EXCLUSION": {"pass": output.get("excluded") == expected_excluded
|
|
and not ({"ERROR_CONTENT_RETAINED", "INCOMPLETE_GENERATION"} & set(scanned["errors"]))},
|
|
"PRIVACY": {"pass": output.get("redaction") == redaction and not redaction["protected_sensitive_fields"]},
|
|
"USABLE_ASSISTANT": {"pass": any(m.get("role") == "assistant" and (not _empty_content(m) or _calls(m))
|
|
for m in output.get("messages", []))},
|
|
}
|
|
return {"pass": all(check["pass"] for check in checks.values()), "rules_version": RULES_VERSION, "checks": checks}
|