✨ Export inherited classification tags and original source coordinates

This commit is contained in:
d-robotics 2026-10-21 12:42:45 +08:00
parent 1469e1d9e1
commit e66c77261d
21 changed files with 1275 additions and 10 deletions

View File

@ -0,0 +1,21 @@
Implement approved local v4 contract only. Freeze source ref metadata in revision; require complete customer v4 original source fields. Derive tags from own dictionary or parent frozen dictionary without overwriting selection provenance. Explicit timestamps + local/part seconds, half-open time bases and original PTS resolution. Consumer roundtrip uses only exported JSON + synthetic original source. Legacy formats remain unchanged. Commit local only, no deploy.
Status: resolved
Acceptance:
- Effective tags and action/slice origin are explicit; independent selections and invalid flags remain intact.
- part → slices → segments hierarchy and discard-reason-only semantics remain intact.
- Frozen original BOS references survive metadata-only reuse; no connection credentials or workspace paths enter customer output.
- Action local/part times and hardware PTS boundaries are complete and continuous.
- Missing tags, source, timestamps, source files or conflicting identities reject v4 package/batch publication.
- Independent reader resolves every synthetic original camera frame from JSON alone in physical and source-index modes.
- Immutable retry/recovery, separate private audit, legacy formats and local Mongo publication remain verified.
Compatibility: v4 default stays off; no production changes, retrospective tag guessing, old delivery replacement, downstream recutting or source writes.
Verification:
- 73 relevant Python tests passed, including physical/source-index independent real-media consumption, legacy v1/v2/v3 and 60-second materialization.
- Native Python v4 output passed Go batch/API publication, failure recovery and download tests with an isolated local Mongo 27018 database.
- Go full baseline gate: the same 70 named known failures, no new failures and no baseline edits.
- Operator repository generation check and git diff whitespace check passed.
- Local evidence: `local-data/verification/delivery-consumer-20261021/`; final customer samples under `final-samples/{physical,source-index}/customer/part0001/part.json`.

View File

@ -6,6 +6,10 @@
**最终分类结果**:最终验收通过的固定版本中,视频各区间的基础分类、动作细分、显式无效及舍弃判断。未写动作描述不撤销基础分类;审核批注和返工过程不属于分类结果。
**动作有效标签**:动作独立选择描述字典词条时,使用当次选择冻结的标签;未独立选择时,继承所属切片冻结的分类标签。是否填写自由描述不决定继承,无效判断与标签分别保留。
**原始区间定位**:以原始采集及 part 的文件身份和硬件时间戳定位人工确认区间;切片内相对时间与原始 part 相对时间是不同坐标,不能混用。下游按定位和分类处理原始数据,不依赖平台内部审核文件。
## 标注对象
**Capture(源采集)**:一次源采集所关联的双目视频、时间戳与传感器数据,是本流程组织人工处理和审核的对象。

View File

@ -5,6 +5,7 @@ import hashlib
import bisect
import csv
import json
import math
import pathlib
import re
import shutil
@ -845,7 +846,174 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
return "empty_confirmed"
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None):
def _v4_tags(selection: Any, kind: str) -> list[str]:
if not isinstance(selection, dict) or selection.get("kind") != kind or not str(selection.get("id") or "").strip() or type(selection.get("version")) is not int or selection["version"] < 1 or not str(selection.get("name") or "").strip():
raise RuntimeError(f"v4 missing frozen {kind} dictionary; confirm historical classification before delivery")
tags = selection.get("tags")
if not isinstance(tags, list) or not tags or any(not isinstance(tag, str) or not tag.strip() or tag != tag.strip() for tag in tags) or len(set(tags)) != len(tags):
raise RuntimeError(f"v4 missing frozen {kind} tags; confirm classification before delivery")
return list(tags)
def _v4_original_sources(manifest, capture_id, part_id, capture_prefix):
source = manifest.get("originalSource")
if not isinstance(source, dict) or source.get("provider") != "BOS" or not str(source.get("bucket") or "").strip():
raise RuntimeError("v4 requires frozen original BOS source selection; workspace paths cannot identify original data")
prefix = str(source.get("prefix") or "")
part_prefix = str(source.get("partPrefix") or "")
if prefix != capture_prefix or part_prefix != prefix + "/" + part_id or source.get("captureUnit") != capture_id + "/" + part_id:
raise RuntimeError("v4 original source identity disagrees with accepted capture/part")
if not prefix or str(pathlib.PurePosixPath(prefix)) != prefix or pathlib.PurePosixPath(prefix).is_absolute() or ".." in pathlib.PurePosixPath(prefix).parts or any(c in prefix for c in ("\\", "\x00")):
raise RuntimeError("unsafe v4 original source prefix")
keys = source.get("objectKeys")
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) for key in keys) or len(set(keys)) != len(keys):
raise RuntimeError("v4 requires an unambiguous original object selection")
files = {}
cap, part = re.escape(capture_id), re.escape(part_id)
patterns = {
"leftVideo": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
"rightVideo": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
"leftPts": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?_pts\.csv",
"rightPts": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?_pts\.csv",
"imu": rf"{part}/{cap}_imu_{part}\.csv",
"cameraCalibration": rf"{cap}_calibration_camera\.yaml",
"imuCalibration": rf"{cap}_calibration_imu\.yaml",
"metadata": rf"meta_{cap}\.json",
}
for key in keys:
if not key.startswith(prefix + "/") or str(pathlib.PurePosixPath(key)) != key or ".." in pathlib.PurePosixPath(key).parts or any(c in key for c in ("\\", "\x00")):
raise RuntimeError("unsafe v4 original source object")
relative = key[len(prefix) + 1:]
if "/" in relative and not relative.startswith(part_id + "/"):
raise RuntimeError("v4 original selection contains another part")
for role, pattern in patterns.items():
if re.fullmatch(pattern, relative, re.IGNORECASE):
if role in files: raise RuntimeError(f"ambiguous v4 original {role} selection")
files[role] = {"path": relative}
missing = sorted(set(patterns) - files.keys())
if missing: raise RuntimeError("v4 missing original files: " + ", ".join(missing))
return {"provider": "BOS", "bucket": source["bucket"], "prefix": prefix, "partPrefix": part_prefix,
"snapshotFingerprint": str(source.get("snapshotFingerprint") or ""), "files": files}
def _v4_number(value, label):
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
raise RuntimeError(f"v4 missing or invalid {label}")
return float(value)
def _v4_timestamp(value, label):
if type(value) is not int or value <= 0: raise RuntimeError(f"v4 missing or invalid {label}")
return value
def _v4_consumer_payload(payload, manifest, source_root):
"""Project immutable accepted facts into a self-contained consumer contract.
No dictionary lookup, source media reads, transcoding or BOS writes here.
Hardware PTS are authoritative; media seconds are explicitly named and never
used to manufacture missing hardware timestamps.
"""
if not isinstance(manifest, dict): raise RuntimeError("v4 requires accepted source manifest")
original = _v4_original_sources(manifest, payload["capture"]["captureId"], payload["partId"], payload["capture"]["prefix"])
payload["originalSource"] = original
payload["consumerContract"] = "droboticflow.classification-consumer/v1"
payload["timeBases"] = {"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video",
"segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts",
"interval": "half-open", "frameRanges": "zero-based-half-open"}
payload["pathBases"] = {"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
# Workspace sources are private transport references, not the original files.
payload["sources"] = {kind: original["files"][kind] for kind in ("leftVideo", "rightVideo", "leftPts", "rightPts", "imu")}
payload["sourceMetadata"]["path"] = original["files"]["metadata"]["path"]
for item in payload["calibration"]:
# Bind by actual calibration kind, not input ordering.
kind = "imuCalibration" if "imu" in item["name"].lower() else "cameraCalibration" if "camera" in item["name"].lower() else ""
if not kind: raise RuntimeError("v4 unknown calibration kind")
item["path"] = original["files"][kind]["path"]
rows = {str(row["sliceId"]): row for row in manifest["items"]}
if len(rows) != len(manifest["items"]): raise RuntimeError("v4 duplicate slice identities")
for item in payload["slices"] + payload["discardedSlices"]:
split = item["taskSplit"]
start, end = (_v4_number(split[k], k) for k in ("startSecond", "endSecond"))
first, last = (_v4_timestamp(split[k], k) for k in ("startTimestampUs", "endTimestampUs"))
if start < 0 or end <= start or last <= first: raise RuntimeError("v4 invalid accepted slice range")
if item["classification"]["deliveryDisposition"] == "discard":
item["sliceName"] = ""
evidence = item["classification"]["discardEvidence"]
selected = evidence.get("dictionary")
if selected is not None:
if (not isinstance(selected, dict) or selected.get("kind") != "discard_reason" or
not selected.get("id") or not selected.get("name") or
type(selected.get("version")) is not int or selected["version"] < 1):
raise RuntimeError("v4 invalid frozen discard dictionary")
for alias, field in (("reasonLabelId", "id"), ("reasonName", "name"), ("reasonVersion", "version")):
if evidence.get(alias) not in (None, "", 0, selected[field]):
raise RuntimeError("v4 discard reason disagrees with frozen dictionary")
evidence[alias] = selected[field]
continue
parent_tags = _v4_tags(item["classification"].get("dictionary"), "slice")
item["classification"]["tags"] = parent_tags
row = rows[item["sliceId"]]
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
seconds, stamps = sync.get("boundarySeconds", []), sync.get("boundaryTimestampUs", [])
if seconds or stamps:
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
raise RuntimeError("v4 invalid hardware boundary mapping")
seconds = [_v4_number(value, "boundary") for value in seconds]
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
stamps[0] != first or stamps[-1] != last or
any(a >= b for a, b in zip(seconds, seconds[1:])) or
any(a >= b for a, b in zip(stamps, stamps[1:]))):
raise RuntimeError("v4 invalid hardware boundary mapping")
def timestamp(second):
if abs(second) < 1e-7: return first
if abs(second - (end-start)) < 1e-7: return last
hits = [stamps[i] for i, value in enumerate(seconds) if abs(_v4_number(value,"boundary") - second) < 1e-7]
if len(hits) != 1: raise RuntimeError("v4 action boundary has no original PTS mapping")
return _v4_timestamp(hits[0], "boundary timestamp")
actions = []
for action in item["segments"]:
a = _v4_number(action.get("clipStartMs"), "action clipStartMs") / 1000
b = _v4_number(action.get("clipEndMs"), "action clipEndMs") / 1000
if a < -1e-7 or b <= a or b > end-start+1e-7: raise RuntimeError("v4 action escapes its parent slice")
ta, tb = (_v4_timestamp(action.get(k), k) for k in ("startTimestampUs", "endTimestampUs"))
if not first <= ta < tb <= last: raise RuntimeError("v4 action timestamps escape its parent slice")
if seconds and (timestamp(a) != ta or timestamp(b) != tb): raise RuntimeError("v4 action seconds disagree with original PTS")
action.update(startSecond=a, endSecond=b, partStartSecond=start+a, partEndSecond=start+b)
own = action.get("dictionary")
action["tags"] = _v4_tags(own, "action") if own is not None else list(parent_tags)
action["tagSource"] = "action" if own is not None else "slice"
actions.append(action)
actions.sort(key=lambda a: a["startSecond"])
complete, previous = [], 0.0
def gap(a,b):
ta,tb=timestamp(a),timestamp(b)
return {"id":f"{item['sliceId']}:gap:{ta}:{tb}", "description":"", "isInvalid":False,
"contentKind":"gap_placeholder", "startSecond":a,"endSecond":b,
"clipStartMs":a*1000,"clipEndMs":b*1000,"partStartSecond":start+a,"partEndSecond":start+b,
"startTimestampUs":ta,"endTimestampUs":tb,"tags":list(parent_tags),"tagSource":"slice"}
for action in actions:
if action["startSecond"] < previous-1e-7: raise RuntimeError("v4 overlapping actions")
if action["startSecond"] > previous+1e-7: complete.append(gap(previous,action["startSecond"]))
complete.append(action);previous=action["endSecond"]
if previous < end-start-1e-7: complete.append(gap(previous,end-start))
if not complete: raise RuntimeError("v4 kept slice has no classified action coverage")
previous_stamp = first
for action in complete:
if action["startTimestampUs"] != previous_stamp or action["endTimestampUs"] <= previous_stamp:
raise RuntimeError("v4 action PTS coverage is discontinuous")
previous_stamp = action["endTimestampUs"]
if previous_stamp != last: raise RuntimeError("v4 action PTS coverage is incomplete")
item["segments"] = complete
counts = {kind:sum(a["contentKind"]==kind for a in complete) for kind in ("annotated","gap_placeholder","invalid_marked")}
item["classification"]["contentCounts"] = counts
item["classification"]["descriptionStatus"] = _description_status(complete)
if not counts["annotated"] and counts["invalid_marked"] and counts["gap_placeholder"]: item["classification"]["descriptionStatus"]="invalid_and_blank"
item["synchronization"]["rightBoundarySeconds"] = sync.get("rightBoundarySeconds", [])
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None, revision_manifest=None, source_root=None):
"""Use the established immutable publisher for versioned customer JSON.
V3 retains its historical sibling review file; v4 stores that audit in a
@ -929,6 +1097,8 @@ def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payl
for key in ("source", "confidence", "phrases", "labelId"):
action.pop(key, None)
if action.get("dictionary") is None: action.pop("dictionary", None)
audit["materializedSources"] = payload.get("sources", {})
_v4_consumer_payload(payload, revision_manifest, source_root)
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8")
audit["partSha256"] = hashlib.sha256(body).hexdigest()
if len(json.dumps(audit, ensure_ascii=False, indent=2).encode("utf8")) > 8 * 1024 * 1024:
@ -1782,7 +1952,7 @@ def _run_accepted_slice_directory(
item["sourceFrames"] = frames[item["sliceId"]]
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
if _bool(params, "DELIVERY_JSON_V4", False):
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload)
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload, revision_manifest, source_root)
if _bool(params, "DELIVERY_JSON_V3", False):
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload)
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)

View File

@ -51,3 +51,11 @@ status: accepted
独立字典 tags 由管理员维护,与 ID/版本/名称一起在确认时冻结;category 与 aliases 不当作 tags。未写动作描述不等于没有动作,也不撤销切片已有分类。历史没有标签快照时不反查最新字典拼造历史。全舍弃也属于人工分类结果,因此 v4 仍交付该 part 的分类 JSON;这显式替代 v4 路径的旧 no_deliverables 规则,旧媒体目录和 v1–v3 路径继续保持原行为。
采用独立 v4 协议与默认关闭的开关,避免同一 v3 schema 悄悄改变字段、枚举或文件集合。保持已约定的 capture/part、slices/taskSplit/segments 层级及不可变批次身份,动作有描述使用约定的 annotated。保留旧版本读取与发布恢复,不原地修补历史完成目录。单独写内部追溯会增加一份内部产物,但避免客户为读取分类被迫接触人员、批注和平台流程字段。
## 2026-10-21 补充:动作继承及原始数据消费
用户确认切片与动作是父子分类。动作独立选择词条时使用该快照的标签,否则继承父切片冻结标签;不按描述是否为空决定,不伪造动作词条。v4 显式输出有效 tags 和 action/slice 来源,保留独立无效判断;舍弃仅保留原因分类及定位,不带旧动作或审核批注。缺标签不反查最新词条猜补。
下游需要仅凭总 JSON 和原始数据定位、切分、按类归组。新 v4 使用任务原始输入的 BOS 选择冻结八类源文件及 bucket/prefix,主文件不再依赖平台 workspace。动作同时明确切片内秒数、原始 part 秒数与原始硬件时间戳;用原始左右 PTS 各自解析硬件区间,避免用左视频秒数误切另一相机。输入快照指纹不是单个 BOS 对象内容版本的保证。实际切割和分类打包仍由下游执行。
缺少可靠来源、分类标签或时间证据时,选择拒绝新 v4 交付,而非“尽量输出”后让客户承受丢分类或错切的风险。发布同时验证消费契约、来源归属和文件存在。已交付结果不原地改写;历史只按证据补齐并产生新批次。这个要求适用于尚未发布的 v4,不改变 v1–v3;旧本地 v4 候选须重新生成。实现和本地消费验证见 `docs/delivery-classification-v4.zh-CN.md`。

View File

@ -8,12 +8,32 @@
- `sliceName` 是已确认的基础分类名称;`classification.dictionary` 是可获得的字典快照。动作描述为空也保留这个分类,不代表无动作。
- 字典快照固定 `id/kind/version/code/name/tags`。`tags` 是独立分类标签,与 `category`、别名和描述原义不混用。新选择由服务端从项目字典校验,客户不能伪造标签;历史缺失快照不按当前词条补填。
- 每个动作显式导出有效 `tags` 和 `tagSource`。独立选择动作词条用该快照标签,来源为 `action`;没有独立选择则继承本切片冻结标签,来源为 `slice`,不伪造动作 `dictionary`。自由填写文字也沿用这个判定;有文字不代表选择了独立词条,没有文字不代表没有动作。独立词条缺 tags 时拒绝交付,不偷偷退回父标签。
- 动作 `contentKind` 为 `annotated`(有描述)、`invalid_marked`(明确无效)、`gap_placeholder`(保留的留空区间)。`descriptionStatus` 描述文字覆盖情况,不能用于推断整个切片是否有动作。
- `isInvalid` 与标签独立;无效动作可保留继承标签,但仍必须按无效判断排除有效动作处理。历史动作未覆盖的留空范围保留为继承父标签的 `gap_placeholder`,不能把该段视频漏掉。
- `discardedSlices[].classification.discardEvidence` 保留业务舍弃原因、原因词条/版本/标签和补充说明。全舍弃 part 也输出总分类 JSON。审核退回原因不放在这里。
- `sourceMetadata.path` 与标定路径相对原采集目录。源索引 `sources` 的 workspace 路径基准显式标记。内部审核文件不在客户目录内,也不从主 JSON 引用。
- 舍弃原因有冻结字典时,旧兼容字段 `reasonLabelId/reasonVersion/reasonName` 从同一快照补齐,不导出空名称或版本 0;已有非空值若与快照冲突即拒绝交付。手填原因仍按原有规则保留,不猜造字典选择。
- `originalSource` 冻结 BOS bucket、原采集 prefix、partPrefix、输入快照指纹和所选原始文件路径。来自任务创建时的输入引用,不查询当前目录猜原文件,不输出连接 ID、凭据或签名 URL。八类文件为左右视频、左右 PTS、IMU、相机标定、IMU 标定和采集 meta。`sources`、`sourceMetadata.path` 与标定路径统一相对原采集目录,`pathBases` 显式写为 `original_capture`;原 workspace 路径只留内部追溯。
内部 `droboticflow.delivery.review-history/v2` JSON 按 capture/part/revision 关联,以 `partSha256` 固定原生主文件字节;包含最终验收、抽检、审核人/批注/轮次/确认项、已有正式退回基线、处理记录和阶段事件。该 artifact 留在内部 Run 存储,不进入最终交付列表、不可变客户批次或客户下载。分类选择的服务端快照另外保存在当前任务的恢复记录中,使字典改名、更新或停用后的撤销仍可恢复原版本,客户端不能伪造历史标签。旧记录没有的追溯不制造补写。追溯超出限制时拒绝交付,不静默截断。
最终分类取通过验收的固定物化修订;沿用既有“未抽中”验收策略的任务,内部保留明确豁免,不能伪造人工通过日期。返工正式修改分类后必须重新送检、验收;查看或处理意见本身不修改分类。相同名称而字典版本/标签不同的切片不能自动合并成一个分类。
时间按两级明确表达,全部区间采用左闭右开:
| 字段 | 坐标 |
| --- | --- |
| `taskSplit.startSecond/endSecond` | 原始 part 左相机视频秒数 |
| `segments[].startSecond/endSecond`、`clipStartMs/clipEndMs` | 本切片左相机内秒数/毫秒数 |
| `segments[].partStartSecond/partEndSecond` | 原始 part 左相机视频秒数 |
| `startTimestampUs/endTimestampUs` | 原始硬件时间戳,非墙钟时间、非从秒数估算 |
`timeBases` 描述以上坐标及帧区间基准。下游从原始左右 PTS 分别定位同一硬件时间窗,不能把左相机秒数直接当右相机秒数。原始输入快照指纹标识当时选择,不冒充每个 BOS 对象的版本号或内容 SHA;原始数据应保留,当前发布时仅以 HEAD 校验所引用文件存在,不证明同名文件从未被外部覆盖。
新 v4 要求完整的冻结分类、非空标签、原始来源和可靠时间边界;任一缺失或动作重叠、时间戳不连续即停止交付。批次发布再核对原始 bucket/part 归属、文件存在性和消费契约,不能只凭主文件的 SHA 放行。元数据修订复用原始引用,不追随后来改变的采集记录。未发布的旧本地 v4 样例须重新生成;正式 v1/v2/v3 及其恢复路径保持兼容。历史缺证据应先核实补齐,已交付文件不覆盖,修正交付另建不可变批次。
平台只输出分类索引,不按标签执行 BOS 数据切割、搬运或集中打包。内部审核文件不在客户目录内,也不从主 JSON 引用。
本地可用 `python -m tests.backend.export_delivery_v4_fixture <全新输出目录>` 生成明确标记的合成样例。目录中 `partNNNN/part.json` 是客户主文件;`internal/partNNNN/review-history.json` 是分离的内部示例,不属于客户包。不要把整个验证输出目录直接交付给客户。
独立消费验证:设置 `FANYU_DELIVERY_CONSUMER_FIXTURE_DIR` 为全新绝对目录,先运行 Go `TestDeliveryV4ConsumerReplicaFixture`,再以同一环境变量运行 `tests/backend/test_delivery_v4_consumer.py`。合成真实双目视频分别经过物理切片、源索引和元数据复用;独立进程仅读取导出 JSON 与原始文件,核对每帧覆盖、非零 part 起点、标签继承、无效与舍弃。独立读取器 `tests/backend/consume_delivery_v4.py` 只生成处理计划,不执行切割或写源数据。

View File

@ -72,6 +72,15 @@ func (s *server) tryAnnotationMetadataRevision(ctx context.Context, task domain.
if e != nil {
return nil, nil
}
if manifest["originalSource"] == nil {
if record, err := s.store.GetCaptureRecordByRunID(ctx, task.RunID); err != nil {
return nil, err
} else if original, err := annotationOriginalSource(run, record); err != nil {
return nil, err
} else if original != nil {
manifest["originalSource"] = original
}
}
entries := listValue(manifest["items"])
if len(entries) != len(slices) {
return nil, nil

View File

@ -0,0 +1,63 @@
package api
import (
"encoding/json"
"fmt"
"path"
"sort"
"strings"
"droboticflow/internal/domain"
)
// Freeze only credential-free facts from the run's original input, never the
// current workflow configuration or a later source-directory listing.
func annotationOriginalSource(run *domain.RunRecord, capture *domain.CaptureRecord) (map[string]any, error) {
if run == nil || capture == nil || len(run.Inputs) == 0 {
return nil, nil
}
if capture.RunID != run.ID || capture.OrganizationID != run.OrganizationID {
return nil, fmt.Errorf("original capture scope mismatch")
}
root := capture.SourcePrefix
if root == "" {
root = capture.Prefix
}
var result map[string]any
for _, input := range run.Inputs {
ref := input.Ref
if ref == nil || ref.Provider != domain.StorageProviderBOS || ref.Kind != domain.ArtifactRefKindDirectory || ref.Bucket != capture.Bucket || ref.Prefix != root {
continue
}
if ref.CaptureUnit != "" && ref.CaptureUnit != capture.CaptureID {
continue
}
if len(ref.ObjectKeys) == 0 {
continue
} // Older non-selected directory inputs need explicit recovery.
if root == "" || path.Clean(root) != root || strings.HasPrefix(root, "/") || strings.Contains(root, "\\") {
return nil, fmt.Errorf("unsafe original capture prefix")
}
keys := append([]string(nil), ref.ObjectKeys...)
sort.Strings(keys)
for i, key := range keys {
if !strings.HasPrefix(key, root+"/") || path.Clean(key) != key || strings.Contains(key, "\\") || i > 0 && key == keys[i-1] {
return nil, fmt.Errorf("unsafe original capture object selection")
}
relative := strings.TrimPrefix(key, root+"/")
if strings.Contains(relative, "/") && !strings.HasPrefix(key, capture.Prefix+"/") {
return nil, fmt.Errorf("original capture selection includes another part")
}
}
candidate := map[string]any{"provider": string(ref.Provider), "bucket": ref.Bucket, "prefix": root, "partPrefix": capture.Prefix, "captureUnit": capture.CaptureID, "objectKeys": keys, "snapshotFingerprint": ref.VersionID}
if result != nil {
a, _ := json.Marshal(result)
b, _ := json.Marshal(candidate)
if string(a) != string(b) {
return nil, fmt.Errorf("ambiguous original capture inputs")
}
}
result = candidate
}
return result, nil
}

View File

@ -0,0 +1,171 @@
package api
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"droboticflow/internal/domain"
)
func consumerSourceRef() *domain.ArtifactReference {
root := "incoming/capture-A"
keys := []string{root + "/meta_capture-A.json", root + "/capture-A_calibration_camera.yaml", root + "/capture-A_calibration_imu.yaml", root + "/part0001/capture-A_imu_part0001.csv"}
for _, side := range []string{"left", "right"} {
keys = append(keys, root+"/part0001/capture-A_camera_"+side+"_part0001.mp4", root+"/part0001/capture-A_camera_"+side+"_part0001_pts.csv")
}
return &domain.ArtifactReference{Provider: domain.StorageProviderBOS, Kind: domain.ArtifactRefKindDirectory, ConnectionID: "PRIVATE_CONNECTION", Bucket: "synthetic-original", Prefix: root, CaptureUnit: "capture-A/part0001", ObjectKeys: keys, VersionID: "frozen-input-version"}
}
func TestAnnotationOriginalSourceUsesFrozenInputSelection(t *testing.T) {
ref := consumerSourceRef()
run := &domain.RunRecord{ID: "run", OrganizationID: "org", Inputs: map[string]domain.ArtifactPayload{"raw": {Ref: ref}}}
capture := &domain.CaptureRecord{RunID: "run", OrganizationID: "org", Bucket: ref.Bucket, Prefix: ref.Prefix + "/part0001", SourcePrefix: ref.Prefix, CaptureID: ref.CaptureUnit, SourceKeys: []string{"later-current-directory-must-not-be-used"}}
source, err := annotationOriginalSource(run, capture)
if err != nil {
t.Fatal(err)
}
if len(source["objectKeys"].([]string)) != 8 {
t.Fatal("frozen source selection lost")
}
if _, ok := source["connectionId"]; ok {
t.Fatal("private connection leaked")
}
ref.ObjectKeys = append(ref.ObjectKeys, ref.Prefix+"/part0002/foreign.mp4")
if _, err = annotationOriginalSource(run, capture); err == nil {
t.Fatal("foreign part accepted")
}
ref.ObjectKeys = ref.ObjectKeys[:len(ref.ObjectKeys)-1]
capture.OrganizationID = "other"
if _, err = annotationOriginalSource(run, capture); err == nil {
t.Fatal("foreign organization accepted")
}
}
// Retain real synthetic source data for the independent Python consumer test.
func TestDeliveryV4ConsumerReplicaFixture(t *testing.T) {
parent := os.Getenv("FANYU_DELIVERY_CONSUMER_FIXTURE_DIR")
if parent == "" {
t.Skip("opt-in retained synthetic fixture")
}
if !filepath.IsAbs(parent) {
t.Fatal("absolute local output required")
}
for _, sourceIndex := range []bool{false, true} {
mode := "physical"
if sourceIndex {
mode = "source-index"
}
t.Run(mode, func(t *testing.T) {
f, temp := annotationProcessorFixture(t)
dir := filepath.Join(parent, mode)
root := filepath.Join(dir, "workspace")
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
if err := copyAnnotationTree(f.ctx, temp, root); err != nil {
t.Fatal(err)
}
run, _ := f.store.GetRun(f.ctx, f.task.RunID)
for i := range run.NodeExecutions {
for j := range run.NodeExecutions[i].InputBindings {
for k := range run.NodeExecutions[i].InputBindings[j].Artifacts {
a := &run.NodeExecutions[i].InputBindings[j].Artifacts[k]
a.Path = filepath.Join(root, filepath.Base(a.Path))
}
}
}
ref := consumerSourceRef()
run.Inputs = map[string]domain.ArtifactPayload{"raw": {Ref: ref}}
if err := f.store.UpsertRun(f.ctx, *run); err != nil {
t.Fatal(err)
}
capture := domain.CaptureRecord{ID: "consumer-capture", RunID: run.ID, OrganizationID: run.OrganizationID, Bucket: ref.Bucket, Prefix: ref.Prefix + "/part0001", SourcePrefix: ref.Prefix, CaptureID: ref.CaptureUnit}
if err := f.store.UpsertCaptureRecord(f.ctx, capture); err != nil {
t.Fatal(err)
}
base := &domain.DictionarySelection{ID: "base", Kind: "slice", Version: 2, Code: "operate", Name: "物品操作", Tags: []string{"商品", "手部"}}
reason := &domain.DictionarySelection{ID: "discard", Kind: "discard_reason", Version: 1, Code: "occluded", Name: "遮挡", Tags: []string{"不可用"}}
action := &domain.DictionarySelection{ID: "pick", Kind: "action", Version: 3, Code: "pick", Name: "拿取", Tags: []string{"抓取"}}
input := combinedAnnotationSubmissionInput{SliceSegments: []domain.SliceSegment{{ID: "first", Name: base.Name, Dictionary: base, Start: 0, End: 1}, {ID: "discard", Start: 1, End: 3, DeliveryDisposition: "discard", DiscardReason: "遮挡", DiscardReasonDictionary: reason}, {ID: "last", Name: base.Name, Dictionary: base, Start: 3, End: 4}}, Segments: []domain.AnnotationSegment{{ID: "blank", Start: 0, End: .5}, {ID: "pick", Start: .5, End: 1, Description: "拿起杯子", Dictionary: action}, {ID: "invalid", Start: 3, End: 3.5, IsInvalid: true}, {ID: "free-text", Start: 3.5, End: 4, Description: "人工补充描述"}}}
s := &server{store: f.store, workspaceMountPath: root, annotationSourceIndex: sourceIndex}
result, err := s.materializeCombinedSliceSubmission(f.ctx, f.task, f.item, input, 1)
if err != nil {
t.Fatal(err)
}
if !sourceIndex {
before, err := os.ReadFile(filepath.Join(result.Artifact.Path, "revision-manifest.json"))
if err != nil {
t.Fatal(err)
}
var first map[string]any
if err = json.Unmarshal(before, &first); err != nil {
t.Fatal(err)
}
// Metadata-only reuse must keep the original source, even if discovery changes.
capture.Bucket = "later-bucket"
if err = f.store.UpsertCaptureRecord(f.ctx, capture); err != nil {
t.Fatal(err)
}
run, _ = f.store.GetRun(f.ctx, f.task.RunID)
result, err = s.materializeCombinedSliceSubmission(f.ctx, f.task, f.item, input, 2)
if err != nil {
t.Fatal(err)
}
if result.Payload["metadataOnly"] != true {
t.Fatal("metadata-only recovery not exercised")
}
after, err := os.ReadFile(filepath.Join(result.Artifact.Path, "revision-manifest.json"))
if err != nil {
t.Fatal(err)
}
var second map[string]any
if err = json.Unmarshal(after, &second); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(first["originalSource"], second["originalSource"]) {
t.Fatal("metadata revision followed mutable source")
}
}
original := filepath.Join(dir, "original")
for _, key := range ref.ObjectKeys {
relative := key[len(ref.Prefix)+1:]
dst := filepath.Join(original, filepath.FromSlash(relative))
if err = os.MkdirAll(filepath.Dir(dst), 0700); err != nil {
t.Fatal(err)
}
var body []byte
switch {
case filepath.Ext(dst) == ".yaml":
body = []byte("synthetic: true\n")
case filepath.Ext(dst) == ".json":
body = []byte("{\"synthetic\":true,\"partId\":\"part0001\"}\n")
default:
name := "imu.csv"
for _, side := range []string{"left", "right"} {
if strings.Contains(relative, "camera_"+side) {
name = side + ".mp4"
if strings.HasSuffix(relative, "_pts.csv") {
name = side + "_pts.csv"
}
}
}
body, err = os.ReadFile(filepath.Join(root, name))
if err != nil {
t.Fatal(err)
}
}
if err = os.WriteFile(dst, body, 0600); err != nil {
t.Fatal(err)
}
}
body, _ := json.MarshalIndent(map[string]any{"artifact": result.Artifact, "original": original, "workspace": root}, "", " ")
if err = os.WriteFile(filepath.Join(dir, "context.json"), body, 0600); err != nil {
t.Fatal(err)
}
})
}
}

View File

@ -126,12 +126,20 @@ func (s *server) materializeSourceIndexRevision(ctx context.Context, t domain.An
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": grid.Seconds[0], "endSecond": grid.Seconds[previous], "durationSeconds": grid.Seconds[previous] - grid.Seconds[0], "startTimestampUs": grid.Timestamps[0], "endTimestampUs": grid.Timestamps[previous], "ptsToMediaScale": cert.Timeline.Scale, "sliceCount": len(slices), "reconstruction": map[string]any{"leftFrameCount": grid.LeftFrames[previous] - grid.LeftFrames[0], "rightFrameCount": grid.RightFrames[previous] - grid.RightFrames[0], "imuRowCount": cert.IMURowsBeforeBoundary[previous] - cert.IMURowsBeforeBoundary[0]}}
selection := map[string]any{"keptSliceCount": kept, "discardedSliceCount": discarded, "keptDurationSeconds": keepDuration, "discardedDurationSeconds": discardDuration}
capture := map[string]any{}
var originalSource map[string]any
if record, e := s.store.GetCaptureRecordByRunID(ctx, t.RunID); e != nil {
return nil, e
} else if record != nil {
originalSource, e = annotationOriginalSource(run, record)
if e != nil {
return nil, e
}
capture = map[string]any{"captureRecordId": record.ID, "captureId": record.CaptureID, "prefix": record.Prefix, "fingerprint": record.Fingerprint, "contractRevision": record.ContractRevision}
}
manifest := map[string]any{"schemaVersion": sourceIndexSchema, "revision": revision, "taskId": t.ID, "humanItemId": h.ID, "runId": t.RunID, "collectionId": cert.CollectionID, "capture": capture, "sourceFingerprint": fingerprint, "sources": cert.Sources, "sourceDurations": map[string]any{"left": cert.LeftDuration, "right": cert.RightDuration}, "status": "materialized", "sliceCount": len(items), "coverage": coverage, "deliverySelection": selection, "items": items}
if originalSource != nil {
manifest["originalSource"] = originalSource
}
manifest["sourceFrameCounts"] = map[string]any{"left": len(cert.LeftPTS), "right": len(cert.RightPTS)}
encoded, e := json.Marshal(manifest)
if e != nil || len(encoded)+1 > maxRunArtifactManifestBytes {

View File

@ -133,6 +133,15 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla
ID string `json:"captureId"`
Prefix string `json:"prefix"`
} `json:"capture"`
OriginalSource struct {
Provider string `json:"provider"`
Bucket string `json:"bucket"`
Prefix string `json:"prefix"`
PartPrefix string `json:"partPrefix"`
Files map[string]struct {
Path string `json:"path"`
} `json:"files"`
} `json:"originalSource"`
PartID string `json:"partId"`
Lineage struct {
RunID string `json:"sourceRunId"`
@ -161,12 +170,33 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla
if e != nil || record == nil || record.OrganizationID != project.OrganizationID || record.ProjectID != project.ID || record.Prefix != part.Capture.Prefix+"/"+part.PartID || record.Delivery == nil || record.Delivery.Prefix != target.prefix || record.Delivery.ConnectionID != t.connectionID || record.Delivery.Bucket != t.bucket {
return nil, fmt.Errorf("part 来源与采集记录不一致")
}
if part.Schema == deliveryindex.ClassificationPartSchema && (part.OriginalSource.Provider != "BOS" || part.OriginalSource.Bucket != record.Bucket || part.OriginalSource.Prefix != part.Capture.Prefix || part.OriginalSource.PartPrefix != record.Prefix) {
return nil, fmt.Errorf("分类 JSON 原始存储来源与采集记录不一致,停止发布")
}
sourceRuns = append(sourceRuns, part.Lineage.RunID)
sourceBrowser, closeSource, e := s.deliveryReadBrowser(r.Context(), project.OrganizationID, record.ConnectionID, record.Bucket, part.Capture.Prefix, false)
if e != nil {
return nil, e
}
expected, meta, e := indexBatchSourceEvidence(r.Context(), sourceBrowser, record.Bucket, part.Capture.Prefix, part.Capture.ID)
if e == nil && part.Schema == deliveryindex.ClassificationPartSchema {
// HEAD only: prove the frozen references still resolve without reading
// video bodies, recutting media or expanding the original selection.
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
rel := part.OriginalSource.Files[role].Path
media := role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu"
if rel == "" || path.Clean(rel) != rel || strings.ContainsAny(rel, "\\\x00") || strings.HasPrefix(rel, "/") || strings.HasPrefix(rel, "../") || rel == ".." || media && !strings.HasPrefix(rel, part.PartID+"/") || !media && path.Base(rel) != rel {
e = fmt.Errorf("分类 JSON 原始文件路径无效:%s", role)
break
}
key := part.Capture.Prefix + "/" + rel
info, err := sourceBrowser.GetObjectSummary(r.Context(), record.Bucket, key)
if err != nil || info.Key != key || info.Kind != "file" || info.SizeBytes <= 0 {
e = fmt.Errorf("分类 JSON 原始文件不存在或不可读:%s", role)
break
}
}
}
if closeSource != nil {
closeSource()
}

View File

@ -25,6 +25,14 @@ type indexTestBrowser struct {
ambiguous bool
}
func (b *indexTestBrowser) GetObjectSummary(_ context.Context, bucket, key string) (domain.ObjectStorageObjectSummary, error) {
body, ok := b.objects[key]
if !ok {
return domain.ObjectStorageObjectSummary{}, os.ErrNotExist
}
return domain.ObjectStorageObjectSummary{Bucket: bucket, Key: key, Kind: "file", SizeBytes: int64(len(body))}, nil
}
func (b *indexTestBrowser) PutObjectConditional(_ context.Context, _ string, key string, body []byte, _ string, match, none string, _ map[string]string) (domain.ObjectStorageObjectSummary, error) {
if match != "" || none != "*" {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("unconditional write")
@ -137,6 +145,7 @@ func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authent
delete(doc, "acceptance")
delete(doc, "lineage")
doc["schemaVersion"] = deliveryindex.ClassificationPartSchema
enrichConsumerFixture(doc)
body, _ = json.Marshal(doc)
history, _ = json.Marshal(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2", "captureId": "capture-A", "partId": part, "revision": 1, "partSha256": deliveryindex.Hash(body), "acceptance": acceptance, "lineage": map[string]any{"sourceRunId": sourceRun}, "qualityReviews": []any{map[string]any{"comment": "PRIVATE_REWORK"}}, "trace": []any{}})
}
@ -151,6 +160,19 @@ func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authent
b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 1, "totalBytes": len(body), "entries": entries})
}
b.objects["incoming/capture-A/"+part+"/left.mp4"] = []byte("synthetic")
if v4 {
var doc struct {
Original struct {
Files map[string]struct {
Path string `json:"path"`
} `json:"files"`
} `json:"originalSource"`
}
must(json.Unmarshal(body, &doc))
for _, file := range doc.Original.Files {
b.objects["incoming/capture-A/"+file.Path] = []byte("synthetic original")
}
}
run := domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: "project-A", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: artifactID, RunID: runID, Name: "delivery.directory.index", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindDirectory, Provider: domain.StorageProviderBOS, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix, ManifestKey: prefix + "/manifest.json"}}}}}
if v4 {
var audit map[string]any
@ -342,3 +364,93 @@ func TestClassificationBatchHTTPPublicationRecoveryAndDownload(t *testing.T) {
t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1")
exerciseIndexBatch(t, store.NewMemoryStore())
}
func TestClassificationBatchRejectsMissingOrChangedOriginalSource(t *testing.T) {
t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1")
t.Setenv("DELIVERY_V4_FIXTURE_DIR", "")
for _, failure := range []string{"bucket", "missing-file"} {
t.Run(failure, func(t *testing.T) {
ds := store.NewMemoryStore()
b, p, _ := seedIndexFixture(t, ds)
if failure == "bucket" {
record, err := ds.GetCaptureRecordByRunID(t.Context(), "source-run-part0001")
if err != nil {
t.Fatal(err)
}
record.Bucket = "changed-bucket"
if err = ds.UpsertCaptureRecord(t.Context(), *record); err != nil {
t.Fatal(err)
}
} else {
delete(b.objects, "incoming/capture-A/part0001/leftVideo")
}
w := indexCall(NewHandler(ds, nil), p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
if w.Code != 409 || len(b.puts) != 0 || !strings.Contains(w.Body.String(), "原始") {
t.Fatalf("incomplete source accepted: %d %s", w.Code, w.Body.String())
}
})
}
}
func enrichConsumerFixture(p map[string]any) {
part := p["partId"].(string)
p["consumerContract"] = "droboticflow.classification-consumer/v1"
p["timeBases"] = map[string]string{"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video", "segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts", "interval": "half-open", "frameRanges": "zero-based-half-open"}
p["pathBases"] = map[string]string{"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
files := map[string]any{}
sources := map[string]any{}
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
name := part + "/" + role
if role == "cameraCalibration" || role == "imuCalibration" {
name = role + ".yaml"
}
if role == "metadata" {
name = "meta_capture-A.json"
}
files[role] = map[string]any{"path": name}
if role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu" {
sources[role] = files[role]
}
}
p["sources"] = sources
p["sourceMetadata"] = files["metadata"]
p["originalSource"] = map[string]any{"provider": "BOS", "bucket": "test-bucket", "prefix": "incoming/capture-A", "partPrefix": "incoming/capture-A/" + part, "files": files}
for _, kind := range []string{"slices", "discardedSlices"} {
for _, raw := range p[kind].([]any) {
item := raw.(map[string]any)
split := item["taskSplit"].(map[string]any)
number := func(v any) float64 {
switch n := v.(type) {
case float64:
return n
case int:
return float64(n)
default:
panic("invalid fixture time")
}
}
start, end := number(split["startSecond"]), number(split["endSecond"])
split["startTimestampUs"] = int64(1e6 + start*1e6)
split["endTimestampUs"] = int64(1e6 + end*1e6)
if kind == "discardedSlices" {
continue
}
class := item["classification"].(map[string]any)
class["dictionary"] = map[string]any{"id": "base", "kind": "slice", "version": 1, "name": "操作", "tags": []string{"操作"}}
class["tags"] = []string{"操作"}
actions := item["segments"].([]any)
for i, raw := range actions {
a := raw.(map[string]any)
low, high := float64(i)*(end-start)/float64(len(actions)), float64(i+1)*(end-start)/float64(len(actions))
a["startSecond"] = low
a["endSecond"] = high
a["partStartSecond"] = start + low
a["partEndSecond"] = start + high
a["startTimestampUs"] = int64(1e6 + (start+low)*1e6)
a["endTimestampUs"] = int64(1e6 + (start+high)*1e6)
a["tags"] = []string{"操作"}
a["tagSource"] = "slice"
}
}
}
}

View File

@ -1005,15 +1005,23 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
}
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": commonMediaStart, "endSecond": commonMediaEnd, "durationSeconds": commonMediaEnd - commonMediaStart, "startTimestampUs": commonStart, "endTimestampUs": commonEnd, "ptsToMediaScale": ptsToMediaScale, "sliceCount": len(manifestItems), "reconstruction": map[string]any{"leftFrameCount": deliveredLeftFrames, "rightFrameCount": deliveredRightFrames, "imuRowCount": deliveredIMURows}}
captureEvidence := map[string]any{}
var originalSource map[string]any
if capture, captureErr := server.store.GetCaptureRecordByRunID(ctx, task.RunID); captureErr != nil {
return nil, captureErr
} else if capture != nil {
originalSource, err = annotationOriginalSource(run, capture)
if err != nil {
return nil, err
}
captureEvidence = map[string]any{
"captureRecordId": capture.ID, "captureId": capture.CaptureID, "prefix": capture.Prefix,
"fingerprint": capture.Fingerprint, "contractRevision": capture.ContractRevision,
}
}
revisionManifest := map[string]any{"schemaVersion": "droboticflow.slice-materialization/v2", "revision": revision, "humanItemId": item.ID, "taskId": task.ID, "runId": task.RunID, "collectionId": collectionID, "capture": captureEvidence, "status": "materialized", "sliceCount": len(manifestItems), "coverage": coverage, "items": manifestItems}
if originalSource != nil {
revisionManifest["originalSource"] = originalSource
}
deliverySelection := map[string]any{"keptSliceCount": keptSliceCount, "discardedSliceCount": discardedSliceCount, "keptDurationSeconds": keptDuration, "discardedDurationSeconds": discardedDuration}
if discardedSliceCount > 0 {
revisionManifest["schemaVersion"] = "droboticflow.slice-materialization/v3"

View File

@ -244,6 +244,9 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh
}
schema = p.Schema
if p.Schema == ClassificationPartSchema {
if err := validateConsumerPart(src.Part); err != nil {
return out, err
}
if _, err := classificationAudit(src); err != nil {
return out, err
}

View File

@ -19,6 +19,7 @@ func classificationFixture(allDiscard bool) Source {
p["discardedSlices"] = []any{map[string]any{"sliceId": "all", "taskSplit": map[string]any{"startSecond": 0, "endSecond": 3, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "discard", "discardEvidence": map[string]any{"reason": "遮挡"}}, "segments": []any{}}}
p["deliveryClassification"] = map[string]any{"keptSliceCount": 0, "discardedSliceCount": 1, "keptDurationSeconds": 0, "discardedDurationSeconds": 3}
}
enrichConsumerFixture(p)
src.Part = encode(p)
src.History = encode(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2", "captureId": "capture-A", "partId": "part0001", "revision": 7, "partSha256": Hash(src.Part), "acceptance": acceptance, "qualityReviews": []any{map[string]string{"comment": "PRIVATE_REWORK"}}, "trace": []any{}})
return src
@ -55,6 +56,40 @@ func TestClassificationBatchRejectsMixedVersions(t *testing.T) {
}
}
func TestClassificationBatchRejectsIncompleteConsumerContract(t *testing.T) {
for _, failure := range []string{"contract", "tags", "tag-source", "part-time", "pts", "original-file"} {
t.Run(failure, func(t *testing.T) {
src := classificationFixture(false)
var doc, audit map[string]any
json.Unmarshal(src.Part, &doc)
slice := doc["slices"].([]any)[0].(map[string]any)
action := slice["segments"].([]any)[0].(map[string]any)
switch failure {
case "contract":
delete(doc, "consumerContract")
case "tags":
delete(action, "tags")
case "tag-source":
action["tagSource"] = "action"
case "part-time":
action["partStartSecond"] = -1
case "pts":
delete(action, "startTimestampUs")
case "original-file":
delete(doc["originalSource"].(map[string]any)["files"].(map[string]any), "leftPts")
}
src.Part = encode(doc)
json.Unmarshal(src.History, &audit)
// Keep acceptance hash valid: rejection must be the consumer guard.
audit["partSha256"] = Hash(src.Part)
src.History = encode(audit)
if _, err := Build("project-A", "项目A", "delivery/项目A__project-A", []Source{src}); err == nil || !strings.Contains(err.Error(), "consumer contract") {
t.Fatalf("incomplete %s accepted: %v", failure, err)
}
})
}
}
func TestClassificationBatchAcceptsExplicitSamplingExemptionWithoutInventingHumanDate(t *testing.T) {
src := classificationFixture(false)
var audit map[string]any
@ -74,3 +109,66 @@ func TestClassificationBatchAcceptsExplicitSamplingExemptionWithoutInventingHuma
t.Fatal("unverified human acceptance accepted")
}
}
func enrichConsumerFixture(p map[string]any) {
part := p["partId"].(string)
p["consumerContract"] = "droboticflow.classification-consumer/v1"
p["timeBases"] = map[string]string{"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video", "segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts", "interval": "half-open", "frameRanges": "zero-based-half-open"}
p["pathBases"] = map[string]string{"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
files := map[string]any{}
sources := map[string]any{}
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
name := part + "/" + role
if role == "cameraCalibration" || role == "imuCalibration" {
name = role + ".yaml"
}
if role == "metadata" {
name = "meta_capture-A.json"
}
files[role] = map[string]any{"path": name}
if role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu" {
sources[role] = files[role]
}
}
p["sources"] = sources
p["sourceMetadata"] = files["metadata"]
p["originalSource"] = map[string]any{"provider": "BOS", "bucket": "test-bucket", "prefix": "incoming/capture-A", "partPrefix": "incoming/capture-A/" + part, "files": files}
for _, kind := range []string{"slices", "discardedSlices"} {
for _, raw := range p[kind].([]any) {
item := raw.(map[string]any)
split := item["taskSplit"].(map[string]any)
number := func(v any) float64 {
switch n := v.(type) {
case float64:
return n
case int:
return float64(n)
default:
panic("invalid fixture time")
}
}
start, end := number(split["startSecond"]), number(split["endSecond"])
split["startTimestampUs"] = int64(1e6 + start*1e6)
split["endTimestampUs"] = int64(1e6 + end*1e6)
if kind == "discardedSlices" {
continue
}
class := item["classification"].(map[string]any)
class["dictionary"] = map[string]any{"id": "base", "kind": "slice", "version": 1, "name": "操作", "tags": []string{"操作"}}
class["tags"] = []string{"操作"}
actions := item["segments"].([]any)
for i, raw := range actions {
a := raw.(map[string]any)
low, high := float64(i)*(end-start)/float64(len(actions)), float64(i+1)*(end-start)/float64(len(actions))
a["startSecond"] = low
a["endSecond"] = high
a["partStartSecond"] = start + low
a["partEndSecond"] = start + high
a["startTimestampUs"] = int64(1e6 + (start+low)*1e6)
a["endTimestampUs"] = int64(1e6 + (start+high)*1e6)
a["tags"] = []string{"操作"}
a["tagSource"] = "slice"
}
}
}
}

View File

@ -0,0 +1,148 @@
package deliveryindex
import (
"encoding/json"
"fmt"
"math"
"path"
"reflect"
"strings"
)
// v4 is unreleased: only complete consumer/v1 projections may enter new batches.
// Keep this guard at publication as well as in the package producer, so older
// local v4 artifacts cannot silently publish incomplete classification JSON.
func validateConsumerPart(body []byte) error {
type sourceFile struct {
Path string `json:"path"`
}
type selection struct {
ID string `json:"id"`
Kind string `json:"kind"`
Version int `json:"version"`
Name string `json:"name"`
Tags []string `json:"tags"`
}
type window struct {
Start *float64 `json:"startSecond"`
End *float64 `json:"endSecond"`
First *int64 `json:"startTimestampUs"`
Last *int64 `json:"endTimestampUs"`
}
type action struct {
window
PartStart *float64 `json:"partStartSecond"`
PartEnd *float64 `json:"partEndSecond"`
Tags []string `json:"tags"`
TagSource string `json:"tagSource"`
Dictionary *selection `json:"dictionary"`
}
type item struct {
Split window `json:"taskSplit"`
Class struct {
Dictionary *selection `json:"dictionary"`
Tags []string `json:"tags"`
} `json:"classification"`
Actions []action `json:"segments"`
}
var p struct {
Contract string `json:"consumerContract"`
Capture struct {
Prefix string `json:"prefix"`
} `json:"capture"`
PartID string `json:"partId"`
Original struct {
Provider string `json:"provider"`
Bucket string `json:"bucket"`
Prefix string `json:"prefix"`
PartPrefix string `json:"partPrefix"`
Files map[string]sourceFile `json:"files"`
} `json:"originalSource"`
Bases map[string]string `json:"timeBases"`
Slices []item `json:"slices"`
Discarded []struct {
Split window `json:"taskSplit"`
} `json:"discardedSlices"`
}
if err := json.Unmarshal(body, &p); err != nil {
return err
}
fail := func() error { return fmt.Errorf("incomplete v4 classification consumer contract") }
var refs struct {
Sources map[string]sourceFile `json:"sources"`
Metadata sourceFile `json:"sourceMetadata"`
Bases map[string]string `json:"pathBases"`
}
if err := json.Unmarshal(body, &refs); err != nil {
return err
}
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
if refs.Sources[role].Path != p.Original.Files[role].Path || !strings.HasPrefix(refs.Sources[role].Path, p.PartID+"/") {
return fail()
}
}
if refs.Metadata.Path != p.Original.Files["metadata"].Path || refs.Bases["sources"] != "original_capture" || refs.Bases["sourceMetadata"] != "original_capture" || refs.Bases["calibration"] != "original_capture" {
return fail()
}
if p.Contract != "droboticflow.classification-consumer/v1" || p.Original.Provider != "BOS" || p.Original.Bucket == "" || p.Original.Prefix == "" || p.Original.Prefix != p.Capture.Prefix || p.Original.PartPrefix != p.Original.Prefix+"/"+p.PartID {
return fail()
}
for key, value := range map[string]string{"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video", "segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts", "interval": "half-open", "frameRanges": "zero-based-half-open"} {
if p.Bases[key] != value {
return fail()
}
}
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
file := p.Original.Files[role].Path
if file == "" || path.Clean(file) != file || strings.HasPrefix(file, "/") || strings.ContainsAny(file, "\\\x00") || file == ".." || strings.HasPrefix(file, "../") {
return fail()
}
}
validWindow := func(w window) bool {
return w.Start != nil && w.End != nil && w.First != nil && w.Last != nil && !math.IsNaN(*w.Start) && !math.IsInf(*w.Start, 0) && !math.IsNaN(*w.End) && !math.IsInf(*w.End, 0) && *w.Start >= 0 && *w.End > *w.Start && *w.First > 0 && *w.Last > *w.First
}
validTags := func(tags []string) bool {
seen := map[string]bool{}
if len(tags) == 0 {
return false
}
for _, tag := range tags {
if strings.TrimSpace(tag) == "" || tag != strings.TrimSpace(tag) || seen[tag] {
return false
}
seen[tag] = true
}
return true
}
for _, slice := range p.Slices {
d := slice.Class.Dictionary
if !validWindow(slice.Split) || d == nil || strings.TrimSpace(d.ID) == "" || strings.TrimSpace(d.Name) == "" || d.Kind != "slice" || d.Version < 1 || !validTags(d.Tags) || !reflect.DeepEqual(d.Tags, slice.Class.Tags) || len(slice.Actions) == 0 {
return fail()
}
previous := 0.0
stamp := *slice.Split.First
for _, a := range slice.Actions {
if !validWindow(a.window) || a.PartStart == nil || a.PartEnd == nil || !validTags(a.Tags) || math.Abs(*a.Start-previous) > 1e-6 || *a.First != stamp || *a.Last > *slice.Split.Last || math.Abs(*a.PartStart-*slice.Split.Start-*a.Start) > 1e-6 || math.Abs(*a.PartEnd-*slice.Split.Start-*a.End) > 1e-6 {
return fail()
}
if a.Dictionary == nil {
if a.TagSource != "slice" || !reflect.DeepEqual(a.Tags, d.Tags) {
return fail()
}
} else if a.TagSource != "action" || a.Dictionary.Kind != "action" || strings.TrimSpace(a.Dictionary.ID) == "" || strings.TrimSpace(a.Dictionary.Name) == "" || a.Dictionary.Version < 1 || !reflect.DeepEqual(a.Tags, a.Dictionary.Tags) {
return fail()
}
previous = *a.End
stamp = *a.Last
}
if math.Abs(previous-(*slice.Split.End-*slice.Split.Start)) > 1e-6 || stamp != *slice.Split.Last {
return fail()
}
}
for _, item := range p.Discarded {
if !validWindow(item.Split) {
return fail()
}
}
return nil
}

View File

@ -5,6 +5,7 @@ import hashlib
import bisect
import csv
import json
import math
import pathlib
import re
import shutil
@ -845,7 +846,174 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
return "empty_confirmed"
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None):
def _v4_tags(selection: Any, kind: str) -> list[str]:
if not isinstance(selection, dict) or selection.get("kind") != kind or not str(selection.get("id") or "").strip() or type(selection.get("version")) is not int or selection["version"] < 1 or not str(selection.get("name") or "").strip():
raise RuntimeError(f"v4 missing frozen {kind} dictionary; confirm historical classification before delivery")
tags = selection.get("tags")
if not isinstance(tags, list) or not tags or any(not isinstance(tag, str) or not tag.strip() or tag != tag.strip() for tag in tags) or len(set(tags)) != len(tags):
raise RuntimeError(f"v4 missing frozen {kind} tags; confirm classification before delivery")
return list(tags)
def _v4_original_sources(manifest, capture_id, part_id, capture_prefix):
source = manifest.get("originalSource")
if not isinstance(source, dict) or source.get("provider") != "BOS" or not str(source.get("bucket") or "").strip():
raise RuntimeError("v4 requires frozen original BOS source selection; workspace paths cannot identify original data")
prefix = str(source.get("prefix") or "")
part_prefix = str(source.get("partPrefix") or "")
if prefix != capture_prefix or part_prefix != prefix + "/" + part_id or source.get("captureUnit") != capture_id + "/" + part_id:
raise RuntimeError("v4 original source identity disagrees with accepted capture/part")
if not prefix or str(pathlib.PurePosixPath(prefix)) != prefix or pathlib.PurePosixPath(prefix).is_absolute() or ".." in pathlib.PurePosixPath(prefix).parts or any(c in prefix for c in ("\\", "\x00")):
raise RuntimeError("unsafe v4 original source prefix")
keys = source.get("objectKeys")
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) for key in keys) or len(set(keys)) != len(keys):
raise RuntimeError("v4 requires an unambiguous original object selection")
files = {}
cap, part = re.escape(capture_id), re.escape(part_id)
patterns = {
"leftVideo": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
"rightVideo": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
"leftPts": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?_pts\.csv",
"rightPts": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?_pts\.csv",
"imu": rf"{part}/{cap}_imu_{part}\.csv",
"cameraCalibration": rf"{cap}_calibration_camera\.yaml",
"imuCalibration": rf"{cap}_calibration_imu\.yaml",
"metadata": rf"meta_{cap}\.json",
}
for key in keys:
if not key.startswith(prefix + "/") or str(pathlib.PurePosixPath(key)) != key or ".." in pathlib.PurePosixPath(key).parts or any(c in key for c in ("\\", "\x00")):
raise RuntimeError("unsafe v4 original source object")
relative = key[len(prefix) + 1:]
if "/" in relative and not relative.startswith(part_id + "/"):
raise RuntimeError("v4 original selection contains another part")
for role, pattern in patterns.items():
if re.fullmatch(pattern, relative, re.IGNORECASE):
if role in files: raise RuntimeError(f"ambiguous v4 original {role} selection")
files[role] = {"path": relative}
missing = sorted(set(patterns) - files.keys())
if missing: raise RuntimeError("v4 missing original files: " + ", ".join(missing))
return {"provider": "BOS", "bucket": source["bucket"], "prefix": prefix, "partPrefix": part_prefix,
"snapshotFingerprint": str(source.get("snapshotFingerprint") or ""), "files": files}
def _v4_number(value, label):
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
raise RuntimeError(f"v4 missing or invalid {label}")
return float(value)
def _v4_timestamp(value, label):
if type(value) is not int or value <= 0: raise RuntimeError(f"v4 missing or invalid {label}")
return value
def _v4_consumer_payload(payload, manifest, source_root):
"""Project immutable accepted facts into a self-contained consumer contract.
No dictionary lookup, source media reads, transcoding or BOS writes here.
Hardware PTS are authoritative; media seconds are explicitly named and never
used to manufacture missing hardware timestamps.
"""
if not isinstance(manifest, dict): raise RuntimeError("v4 requires accepted source manifest")
original = _v4_original_sources(manifest, payload["capture"]["captureId"], payload["partId"], payload["capture"]["prefix"])
payload["originalSource"] = original
payload["consumerContract"] = "droboticflow.classification-consumer/v1"
payload["timeBases"] = {"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video",
"segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts",
"interval": "half-open", "frameRanges": "zero-based-half-open"}
payload["pathBases"] = {"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
# Workspace sources are private transport references, not the original files.
payload["sources"] = {kind: original["files"][kind] for kind in ("leftVideo", "rightVideo", "leftPts", "rightPts", "imu")}
payload["sourceMetadata"]["path"] = original["files"]["metadata"]["path"]
for item in payload["calibration"]:
# Bind by actual calibration kind, not input ordering.
kind = "imuCalibration" if "imu" in item["name"].lower() else "cameraCalibration" if "camera" in item["name"].lower() else ""
if not kind: raise RuntimeError("v4 unknown calibration kind")
item["path"] = original["files"][kind]["path"]
rows = {str(row["sliceId"]): row for row in manifest["items"]}
if len(rows) != len(manifest["items"]): raise RuntimeError("v4 duplicate slice identities")
for item in payload["slices"] + payload["discardedSlices"]:
split = item["taskSplit"]
start, end = (_v4_number(split[k], k) for k in ("startSecond", "endSecond"))
first, last = (_v4_timestamp(split[k], k) for k in ("startTimestampUs", "endTimestampUs"))
if start < 0 or end <= start or last <= first: raise RuntimeError("v4 invalid accepted slice range")
if item["classification"]["deliveryDisposition"] == "discard":
item["sliceName"] = ""
evidence = item["classification"]["discardEvidence"]
selected = evidence.get("dictionary")
if selected is not None:
if (not isinstance(selected, dict) or selected.get("kind") != "discard_reason" or
not selected.get("id") or not selected.get("name") or
type(selected.get("version")) is not int or selected["version"] < 1):
raise RuntimeError("v4 invalid frozen discard dictionary")
for alias, field in (("reasonLabelId", "id"), ("reasonName", "name"), ("reasonVersion", "version")):
if evidence.get(alias) not in (None, "", 0, selected[field]):
raise RuntimeError("v4 discard reason disagrees with frozen dictionary")
evidence[alias] = selected[field]
continue
parent_tags = _v4_tags(item["classification"].get("dictionary"), "slice")
item["classification"]["tags"] = parent_tags
row = rows[item["sliceId"]]
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
seconds, stamps = sync.get("boundarySeconds", []), sync.get("boundaryTimestampUs", [])
if seconds or stamps:
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
raise RuntimeError("v4 invalid hardware boundary mapping")
seconds = [_v4_number(value, "boundary") for value in seconds]
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
stamps[0] != first or stamps[-1] != last or
any(a >= b for a, b in zip(seconds, seconds[1:])) or
any(a >= b for a, b in zip(stamps, stamps[1:]))):
raise RuntimeError("v4 invalid hardware boundary mapping")
def timestamp(second):
if abs(second) < 1e-7: return first
if abs(second - (end-start)) < 1e-7: return last
hits = [stamps[i] for i, value in enumerate(seconds) if abs(_v4_number(value,"boundary") - second) < 1e-7]
if len(hits) != 1: raise RuntimeError("v4 action boundary has no original PTS mapping")
return _v4_timestamp(hits[0], "boundary timestamp")
actions = []
for action in item["segments"]:
a = _v4_number(action.get("clipStartMs"), "action clipStartMs") / 1000
b = _v4_number(action.get("clipEndMs"), "action clipEndMs") / 1000
if a < -1e-7 or b <= a or b > end-start+1e-7: raise RuntimeError("v4 action escapes its parent slice")
ta, tb = (_v4_timestamp(action.get(k), k) for k in ("startTimestampUs", "endTimestampUs"))
if not first <= ta < tb <= last: raise RuntimeError("v4 action timestamps escape its parent slice")
if seconds and (timestamp(a) != ta or timestamp(b) != tb): raise RuntimeError("v4 action seconds disagree with original PTS")
action.update(startSecond=a, endSecond=b, partStartSecond=start+a, partEndSecond=start+b)
own = action.get("dictionary")
action["tags"] = _v4_tags(own, "action") if own is not None else list(parent_tags)
action["tagSource"] = "action" if own is not None else "slice"
actions.append(action)
actions.sort(key=lambda a: a["startSecond"])
complete, previous = [], 0.0
def gap(a,b):
ta,tb=timestamp(a),timestamp(b)
return {"id":f"{item['sliceId']}:gap:{ta}:{tb}", "description":"", "isInvalid":False,
"contentKind":"gap_placeholder", "startSecond":a,"endSecond":b,
"clipStartMs":a*1000,"clipEndMs":b*1000,"partStartSecond":start+a,"partEndSecond":start+b,
"startTimestampUs":ta,"endTimestampUs":tb,"tags":list(parent_tags),"tagSource":"slice"}
for action in actions:
if action["startSecond"] < previous-1e-7: raise RuntimeError("v4 overlapping actions")
if action["startSecond"] > previous+1e-7: complete.append(gap(previous,action["startSecond"]))
complete.append(action);previous=action["endSecond"]
if previous < end-start-1e-7: complete.append(gap(previous,end-start))
if not complete: raise RuntimeError("v4 kept slice has no classified action coverage")
previous_stamp = first
for action in complete:
if action["startTimestampUs"] != previous_stamp or action["endTimestampUs"] <= previous_stamp:
raise RuntimeError("v4 action PTS coverage is discontinuous")
previous_stamp = action["endTimestampUs"]
if previous_stamp != last: raise RuntimeError("v4 action PTS coverage is incomplete")
item["segments"] = complete
counts = {kind:sum(a["contentKind"]==kind for a in complete) for kind in ("annotated","gap_placeholder","invalid_marked")}
item["classification"]["contentCounts"] = counts
item["classification"]["descriptionStatus"] = _description_status(complete)
if not counts["annotated"] and counts["invalid_marked"] and counts["gap_placeholder"]: item["classification"]["descriptionStatus"]="invalid_and_blank"
item["synchronization"]["rightBoundarySeconds"] = sync.get("rightBoundarySeconds", [])
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None, revision_manifest=None, source_root=None):
"""Use the established immutable publisher for versioned customer JSON.
V3 retains its historical sibling review file; v4 stores that audit in a
@ -929,6 +1097,8 @@ def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payl
for key in ("source", "confidence", "phrases", "labelId"):
action.pop(key, None)
if action.get("dictionary") is None: action.pop("dictionary", None)
audit["materializedSources"] = payload.get("sources", {})
_v4_consumer_payload(payload, revision_manifest, source_root)
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8")
audit["partSha256"] = hashlib.sha256(body).hexdigest()
if len(json.dumps(audit, ensure_ascii=False, indent=2).encode("utf8")) > 8 * 1024 * 1024:
@ -1782,7 +1952,7 @@ def _run_accepted_slice_directory(
item["sourceFrames"] = frames[item["sliceId"]]
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
if _bool(params, "DELIVERY_JSON_V4", False):
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload)
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload, revision_manifest, source_root)
if _bool(params, "DELIVERY_JSON_V3", False):
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload)
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)

View File

@ -0,0 +1,68 @@
"""Read-only independent fixture consumer: stdlib, JSON and original files only.
Produces a processing plan; never cuts, copies, uploads or modifies source data.
"""
import argparse
import bisect
import csv
import hashlib
import json
from pathlib import Path
def inspect(part, original):
original=original.resolve()
def file(relative):
p=(original/relative).resolve()
if not p.is_relative_to(original) or not p.is_file():raise ValueError("original file unavailable or outside source root")
return p
assert part["consumerContract"]=="droboticflow.classification-consumer/v1"
assert part["timeBases"]["timestampUs"]=="original_hardware_pts"
assert part["timeBases"]["interval"]=="half-open"
source=part["originalSource"]
assert source["provider"]=="BOS" and source["bucket"]
assert source["prefix"]==part["capture"]["prefix"]
assert source["partPrefix"]==source["prefix"]+"/"+part["partId"]
assert part["pathBases"]["sources"]=="original_capture"
assert part["sources"]=={k:source["files"][k] for k in ("leftVideo","rightVideo","leftPts","rightPts","imu")}
clocks={}
for side in ("left","right"):
file(part["sources"][side+"Video"]["path"])
with file(part["sources"][side+"Pts"]["path"]).open() as f:clocks[side]=[int(row["timestamp_us"]) for row in csv.DictReader(f)]
assert clocks[side] and all(a<b for a,b in zip(clocks[side],clocks[side][1:]))
for c in part["calibration"]:assert hashlib.sha256(file(c["path"]).read_bytes()).hexdigest()==c["sha256"]
file(part["sourceMetadata"]["path"])
with file(part["sources"]["imu"]["path"]).open() as f:imu=[int(r["timestamp_us"]) for r in csv.DictReader(f)]
covered={side:[] for side in clocks};actions=[];discard=[]
def range_for(a,b):return {side:[bisect.bisect_left(pts,a),bisect.bisect_left(pts,b)] for side,pts in clocks.items()}
for item in sorted(part["slices"]+part["discardedSlices"],key=lambda s:s["taskSplit"]["startTimestampUs"]):
parent=item["taskSplit"];a,b=parent["startTimestampUs"],parent["endTimestampUs"]
if item["classification"]["deliveryDisposition"]=="discard":
assert item["segments"]==[]
reason=item["classification"]["discardEvidence"];assert reason["reason"]
frames=range_for(a,b);discard.append({"sliceId":item["sliceId"],"reason":reason,"frames":frames})
for side,(start,end) in frames.items():covered[side]+=list(range(start,end))
continue
previous=a
for action in item["segments"]:
start,end=action["startTimestampUs"],action["endTimestampUs"]
assert start==previous and start<end<=b
previous=end
assert abs(action["partStartSecond"]-parent["startSecond"]-action["startSecond"])<1e-6
assert abs(action["partEndSecond"]-parent["startSecond"]-action["endSecond"])<1e-6
dictionary=action.get("dictionary") if action["tagSource"]=="action" else item["classification"]["dictionary"]
assert dictionary["tags"]==action["tags"] and action["tags"]
frames=range_for(start,end)
for side,(first,last) in frames.items():
assert last>first
covered[side]+=list(range(first,last))
actions.append({"slice":item["sliceId"],"tags":action["tags"],"tagSource":action["tagSource"],"dictionaryId":dictionary["id"],"dictionaryVersion":dictionary["version"],"invalid":action["isInvalid"],"range":[action["partStartSecond"],action["partEndSecond"]],"timestampsUs":[start,end],"frames":frames,"imuRows":bisect.bisect_left(imu,end)-bisect.bisect_left(imu,start)})
assert previous==b
low=part["coverage"]["startTimestampUs"];high=part["coverage"]["endTimestampUs"]
for side,pts in clocks.items():assert covered[side]==[i for i,t in enumerate(pts) if low<=t<high]
return {"passed":True,"actions":actions,"discarded":discard,"coveredFrames":{side:len(frames) for side,frames in covered.items()},"sourceModified":False}
if __name__=="__main__":
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument("part",type=Path);parser.add_argument("original",type=Path)
args=parser.parse_args();print(json.dumps(inspect(json.loads(args.part.read_text()),args.original),ensure_ascii=False))

View File

@ -6,7 +6,7 @@ import shutil
from pathlib import Path
from backend.operators.builtin.package import operator as package
from tests.backend.test_delivery_v4_package import NODE, ACTION, prepare
from tests.backend.test_delivery_v4_package import NODE, ACTION, prepare, original_source
from tests.backend.test_part_index_package import helpers
@ -18,7 +18,7 @@ def export(destination):
inputs=prepare(work,[{"id":"blank","description":"","clipStartMs":0,"clipEndMs":500,"startTimestampUs":1000000,"endTimestampUs":1500000},{"id":"pick","description":"拿起杯子","dictionary":ACTION,"clipStartMs":500,"clipEndMs":1000,"startTimestampUs":1500000,"endTimestampUs":2000000}])
inputs[1]["data"]["capture"].update(captureId="capture-A/"+part,prefix="incoming/capture-A/"+part)
manifest_path=Path(inputs[0]["path"])/"revision-manifest.json"
m=json.loads(manifest_path.read_text());m["runId"]="source-run-"+part
m=json.loads(manifest_path.read_text());m["runId"]="source-run-"+part;m["originalSource"]=original_source(part=part)
if part=="part0002":
m["items"][0]["deliveryDisposition"]="discard"
m["items"][0]["discardEvidence"]=m["items"][1]["discardEvidence"]

View File

@ -0,0 +1,124 @@
"""Consumers must classify and locate every interval without platform workspaces."""
import csv
import hashlib
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
from backend.operators.builtin.package import operator as package
from tests.backend.test_delivery_v4_package import ACTION, BASE, NODE, prepare, read_outputs
from tests.backend.test_part_index_package import helpers
from tests.backend.test_slice_discard_package import accepted_inputs
def test_tags_inherit_by_dictionary_selection_not_description_text(tmp_path):
inputs=prepare(tmp_path,[{"description":""},{"description":"人工补充文字"},{"description":"","dictionary":ACTION},{"description":"无效","isInvalid":True}])
doc,_,_=read_outputs(tmp_path,inputs)
actions=doc["slices"][0]["segments"]
assert [a["tags"] for a in actions]==[BASE["tags"],BASE["tags"],ACTION["tags"],BASE["tags"]]
assert [a["tagSource"] for a in actions]==["slice","slice","action","slice"]
assert actions[-1]["isInvalid"] is True
assert "dictionary" not in actions[0] # Inheritance must not invent an action selection.
assert actions[2]["dictionary"]==ACTION
assert doc["consumerContract"]=="droboticflow.classification-consumer/v1"
assert doc["pathBases"]["sources"]=="original_capture"
assert "workspaceKey" not in json.dumps(doc)
@pytest.mark.parametrize("failure,match",[("parent-tags","slice tags"),("action-tags","action tags"),("source","original BOS"),("timestamps","startTimestampUs"),("overlap","overlapping"),("clock","disagree"),("missing-file","missing original files"),("unsafe-key","unsafe"),("boundary","boundary mapping"),("pts-gap","discontinuous"),("discard-dictionary","discard reason disagrees")])
def test_v4_refuses_incomplete_consumer_instructions(tmp_path,failure,match):
inputs=prepare(tmp_path,[{"description":"one"},{"description":"two","dictionary":dict(ACTION)}])
root=Path(inputs[0]["path"])
manifest_path=root/"revision-manifest.json"
manifest=json.loads(manifest_path.read_text())
sync_path=root/"slices/slice-000001/synchronized-slice.json"
sync=json.loads(sync_path.read_text())
ann_path=root/"slices/slice-000001/annotation.json"
ann=json.loads(ann_path.read_text())
if failure=="parent-tags":sync["dictionary"]["tags"]=[]
elif failure=="action-tags":ann["actions"][1]["dictionary"]["tags"]=[]
elif failure=="source":manifest.pop("originalSource")
elif failure=="missing-file":manifest["originalSource"]["objectKeys"]=[k for k in manifest["originalSource"]["objectKeys"] if "camera_left" not in k]
elif failure=="timestamps":ann["actions"][0].pop("startTimestampUs")
elif failure=="overlap":ann["actions"][1].update(clipStartMs=0,startTimestampUs=1000000)
elif failure=="clock":ann["actions"][0]["endTimestampUs"]-=1
elif failure=="unsafe-key":manifest["originalSource"]["objectKeys"][0]=manifest["originalSource"]["objectKeys"][0].replace("/part0001/", "//part0001/")
elif failure=="boundary":sync["boundaryTimestampUs"][-1]-=1
elif failure=="discard-dictionary":manifest["items"][1]["discardEvidence"]["reasonVersion"]=999
elif failure=="pts-gap":
sync.pop("boundarySeconds");sync.pop("boundaryTimestampUs")
ann["actions"][1]["startTimestampUs"]+=1
sync_path.write_text(json.dumps(sync));ann_path.write_text(json.dumps(ann));manifest_path.write_text(json.dumps(manifest))
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(manifest_path.read_bytes()).hexdigest()
with pytest.raises(RuntimeError,match=match):read_outputs(tmp_path,inputs)
def test_legacy_partial_annotations_keep_inherited_gap_coverage(tmp_path):
inputs=prepare(tmp_path,[{"description":"拿起","dictionary":ACTION,"clipStartMs":250,"clipEndMs":750}])
doc,_,_=read_outputs(tmp_path,inputs)
actions=doc["slices"][0]["segments"]
assert [(a["startSecond"],a["endSecond"],a["tagSource"]) for a in actions]==[(0,.25,"slice"),(.25,.75,"action"),(.75,1,"slice")]
assert [(a["startTimestampUs"],a["endTimestampUs"]) for a in actions]==[(1000000,1250000),(1250000,1750000),(1750000,2000000)]
def test_discard_aliases_use_frozen_dictionary_without_zero_version(tmp_path):
inputs=prepare(tmp_path,[])
manifest_path=Path(inputs[0]["path"])/"revision-manifest.json"
manifest=json.loads(manifest_path.read_text())
reason=manifest["items"][1]["discardEvidence"]
reason.update(reasonLabelId="",reasonName="",reasonVersion=0)
manifest_path.write_text(json.dumps(manifest))
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(manifest_path.read_bytes()).hexdigest()
doc,_,_=read_outputs(tmp_path,inputs)
reason=doc["discardedSlices"][0]["classification"]["discardEvidence"]
assert reason["reasonLabelId"]==reason["dictionary"]["id"]
assert reason["reasonVersion"]==reason["dictionary"]["version"]==4
assert reason["reasonName"]==reason["dictionary"]["name"]
@pytest.mark.parametrize("mode",["physical","source-index"])
def test_real_v4_roundtrip_uses_only_customer_json_and_original_data(tmp_path,monkeypatch,mode):
root=os.environ.get("FANYU_DELIVERY_CONSUMER_FIXTURE_DIR")
if not root:pytest.skip("generate retained local Go source fixture first")
context=json.loads((Path(root)/mode/"context.json").read_text())
artifact=context["artifact"]
inputs=accepted_inputs(artifact)[:2]
inputs[1]["data"].update(trace=[],qualityReviews=[])
inputs[1]["data"]["finalAcceptance"]["acceptedAt"]="2026-10-21T00:00:00Z"
original=Path(context["original"])
for kind in ("camera","imu"):
p=original/f"capture-A_calibration_{kind}.yaml"
inputs.append({"portId":"calibrations","name":p.name,"path":str(p)})
node={**NODE,"hyperparams":NODE["hyperparams"]+[{"key":"SOURCE_INDEX_DELIVERY","value":"true"}]}
with monkeypatch.context() as guard:
def no_subprocess(*args, **kwargs):
raise AssertionError("classification export must not transcode or recut source data")
original_sha = package._sha256_file
def metadata_only(path):
assert Path(path).suffix not in {".mp4", ".mov", ".mkv"}, "classification export rehashed video bodies"
return original_sha(path)
guard.setattr(package.subprocess, "run", no_subprocess)
guard.setattr(package.subprocess, "Popen", no_subprocess)
guard.setattr(package, "_sha256_file", metadata_only)
result=package.run("consumer","package",tmp_path/"out",node,inputs,helpers())
doc=json.loads((Path(result[0]["path"])/"part.json").read_text())
assert not any(word in json.dumps(doc) for word in ("workspaceKey","system/runs/","PRIVATE_CONNECTION"))
# A fresh process can consume copies of only the customer file and original source.
consumer=tmp_path/"downstream";shutil.copytree(original,consumer/"original")
(consumer/"part.json").write_text(json.dumps(doc))
before={p.relative_to(consumer/"original"):hashlib.sha256(p.read_bytes()).hexdigest() for p in (consumer/"original").rglob("*") if p.is_file()}
result=subprocess.run([sys.executable,str(Path(__file__).with_name("consume_delivery_v4.py").resolve()),str(consumer/"part.json"),str(consumer/"original")],cwd=consumer,capture_output=True,text=True,check=True)
assert before=={p.relative_to(consumer/"original"):hashlib.sha256(p.read_bytes()).hexdigest() for p in (consumer/"original").rglob("*") if p.is_file()}
report=json.loads(result.stdout)
assert report["passed"] and report["sourceModified"] is False
assert report["coveredFrames"]=={"left":32,"right":32}
groups=report["actions"]
assert [g["tags"] for g in groups]==[["商品","手部"],["抓取"],["商品","手部"],["商品","手部"]]
assert [g["invalid"] for g in groups]==[False,False,True,False]
assert [g["range"] for g in groups]==[[0,.5],[.5,1],[3,3.5],[3.5,4]]
assert doc["slices"][1]["segments"][0]["startSecond"]==0
assert doc["slices"][1]["segments"][0]["partStartSecond"]==3

View File

@ -19,16 +19,34 @@ BASE = {"id": "slice-word", "kind": "slice", "version": 2, "code": "base", "name
ACTION = {"id": "action-word", "kind": "action", "version": 3, "code": "pick", "name": "拿取", "tags": ["抓取"]}
def original_source(capture="capture-A", part="part0001"):
root=f"incoming/{capture}"
paths=[f"{part}/{capture}_camera_{side}_{part}{suffix}" for side in ("left","right") for suffix in (".mp4","_pts.csv")]
paths += [f"{part}/{capture}_imu_{part}.csv",f"{capture}_calibration_camera.yaml",f"{capture}_calibration_imu.yaml",f"meta_{capture}.json"]
return {"provider":"BOS","bucket":"test-bucket","prefix":root,"partPrefix":root+"/"+part,"captureUnit":capture+"/"+part,"objectKeys":[root+"/"+p for p in paths],"snapshotFingerprint":"synthetic-source-version"}
def prepare(tmp_path, actions):
inputs = inputs_for(tmp_path, actions)
root = Path(inputs[0]["path"])
sync = root / "slices/slice-000001/synchronized-slice.json"
doc = json.loads(sync.read_text())
doc["dictionary"] = BASE
n=max(1,len(actions))
for i,action in enumerate(actions):
action.setdefault("id",f"action-{i}")
action.setdefault("clipStartMs",i*1000/n)
action.setdefault("clipEndMs",(i+1)*1000/n)
action.setdefault("startTimestampUs",1000000+round(action["clipStartMs"]*1000))
action.setdefault("endTimestampUs",1000000+round(action["clipEndMs"]*1000))
(root/"slices/slice-000001/annotation.json").write_text(json.dumps({"actions":actions}))
points=sorted({0.0,1.0,*[a[k]/1000 for a in actions for k in ("clipStartMs","clipEndMs")]})
doc.update(boundarySeconds=points,boundaryTimestampUs=[1000000+round(p*1000000) for p in points],rightBoundarySeconds=points)
sync.write_text(json.dumps(doc))
manifest_path = root / "revision-manifest.json"
m = json.loads(manifest_path.read_text())
m["runId"] = "source-run"
m["originalSource"] = original_source()
m["items"][0]["sliceName"] = BASE["name"]
reason = m["items"][1]["discardEvidence"]
reason.update(reasonLabelId="discard-word", reasonVersion=4, reasonName="遮挡", reasonNote="持续遮挡", at="2026-10-20T00:00:00Z", dictionary={"id":"discard-word","kind":"discard_reason","version":4,"code":"occluded","name":"遮挡","tags":["不可用"]})
@ -86,6 +104,7 @@ def test_v4_all_discard_outputs_classification_and_legacy_stays_unchanged(tmp_pa
inputs = fixture(tmp_path,["discard","discard"])
inputs[1]["data"].update(capture={"captureId":"capture-A/part0001","prefix":"incoming/capture-A/part0001"},trace=[])
inputs[1]["data"]["finalAcceptance"]["acceptedAt"] = "2026-10-20T01:00:00Z"
root=Path(inputs[0]["path"]);p=root/"revision-manifest.json";m=json.loads(p.read_text());m["originalSource"]=original_source();p.write_text(json.dumps(m));inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(p.read_bytes()).hexdigest()
doc, _, _ = read_outputs(tmp_path, inputs)
assert doc["slices"] == [] and len(doc["discardedSlices"]) == 2
assert doc["deliveryClassification"]["discardedSliceCount"] == 2

View File

@ -1,22 +1,28 @@
"""Consume real Go materialization revisions, including metadata-only reuse."""
import hashlib
import json
import shutil
from pathlib import Path
import pytest
from backend.operators.builtin.package import operator as package
from tests.backend.test_part_index_real_materialization import load_context
from tests.backend.test_part_index_package import helpers
from tests.backend.test_delivery_v4_package import NODE
from tests.backend.test_delivery_v4_package import NODE, original_source
@pytest.mark.parametrize("mode", ["keep", "mixed", "discard", "blank"])
def test_v4_real_materialization_retains_final_dictionary_snapshots(tmp_path, mode):
context=load_context()
artifact=context[mode]["artifact"]
root=Path(artifact["path"])
root=tmp_path/"accepted"
shutil.copytree(artifact["path"],root)
artifact={**artifact,"path":str(root)}
mpath=root/"revision-manifest.json"
manifest=json.loads(mpath.read_text())
# Explicit synthetic original reference for this historical, local-only fixture.
manifest["originalSource"]=original_source("real-capture")
mpath.write_text(json.dumps(manifest))
summary={"decision":"approved","processingProfile":"raw_human","capture":{"captureId":"real-capture/part0001","prefix":"incoming/real-capture/part0001"},"trace":[],"qualityReviews":[],"acceptedMaterializations":[{"artifactId":artifact["id"],"revision":manifest["revision"],"manifestSha256":hashlib.sha256(mpath.read_bytes()).hexdigest()}],"finalAcceptance":{"confirmed":True,"acceptedAt":"2026-10-21T00:00:00Z","processingProfile":"raw_human","confirmedByUserId":"internal-acceptor"}}
inputs=[{**artifact,"portId":"acceptedSliceDirectory"},{"portId":"acceptanceSummary","data":summary}]
for kind in ("camera","imu"):
@ -37,7 +43,12 @@ def test_v4_real_materialization_retains_final_dictionary_snapshots(tmp_path, mo
assert snap["version"]==2 and snap["tags"]==["基础分类","商品"]
if mode=="blank": assert item["classification"]["descriptionStatus"]=="empty_confirmed"
for action in item["segments"]:
assert action["dictionary"]["version"]==4 and action["dictionary"]["tags"]==["抓取"]
if "dictionary" in action:
assert action["dictionary"]["version"]==4 and action["dictionary"]["tags"]==["抓取"]
assert action["tags"]==["抓取"] and action["tagSource"]=="action"
else:
assert mode=="blank"
assert action["tags"]==snap["tags"] and action["tagSource"]=="slice"
for item in part["discardedSlices"]:
reason=item["classification"]["discardEvidence"]
assert reason["note"]=="画面被遮挡" and reason["dictionary"]["tags"]==["不可用"]