forked from longtao.wu/droboticflow
✨ Export inherited classification tags and original source coordinates
This commit is contained in:
parent
1469e1d9e1
commit
e66c77261d
21
.scratch/delivery-consumer-contract-20261021/spec.md
Normal file
21
.scratch/delivery-consumer-contract-20261021/spec.md
Normal file
@ -0,0 +1,21 @@
|
||||
Implement approved local v4 contract only. Freeze source ref metadata in revision; require complete customer v4 original source fields. Derive tags from own dictionary or parent frozen dictionary without overwriting selection provenance. Explicit timestamps + local/part seconds, half-open time bases and original PTS resolution. Consumer roundtrip uses only exported JSON + synthetic original source. Legacy formats remain unchanged. Commit local only, no deploy.
|
||||
|
||||
Status: resolved
|
||||
|
||||
Acceptance:
|
||||
- Effective tags and action/slice origin are explicit; independent selections and invalid flags remain intact.
|
||||
- part → slices → segments hierarchy and discard-reason-only semantics remain intact.
|
||||
- Frozen original BOS references survive metadata-only reuse; no connection credentials or workspace paths enter customer output.
|
||||
- Action local/part times and hardware PTS boundaries are complete and continuous.
|
||||
- Missing tags, source, timestamps, source files or conflicting identities reject v4 package/batch publication.
|
||||
- Independent reader resolves every synthetic original camera frame from JSON alone in physical and source-index modes.
|
||||
- Immutable retry/recovery, separate private audit, legacy formats and local Mongo publication remain verified.
|
||||
|
||||
Compatibility: v4 default stays off; no production changes, retrospective tag guessing, old delivery replacement, downstream recutting or source writes.
|
||||
|
||||
Verification:
|
||||
- 73 relevant Python tests passed, including physical/source-index independent real-media consumption, legacy v1/v2/v3 and 60-second materialization.
|
||||
- Native Python v4 output passed Go batch/API publication, failure recovery and download tests with an isolated local Mongo 27018 database.
|
||||
- Go full baseline gate: the same 70 named known failures, no new failures and no baseline edits.
|
||||
- Operator repository generation check and git diff whitespace check passed.
|
||||
- Local evidence: `local-data/verification/delivery-consumer-20261021/`; final customer samples under `final-samples/{physical,source-index}/customer/part0001/part.json`.
|
||||
@ -6,6 +6,10 @@
|
||||
|
||||
**最终分类结果**:最终验收通过的固定版本中,视频各区间的基础分类、动作细分、显式无效及舍弃判断。未写动作描述不撤销基础分类;审核批注和返工过程不属于分类结果。
|
||||
|
||||
**动作有效标签**:动作独立选择描述字典词条时,使用当次选择冻结的标签;未独立选择时,继承所属切片冻结的分类标签。是否填写自由描述不决定继承,无效判断与标签分别保留。
|
||||
|
||||
**原始区间定位**:以原始采集及 part 的文件身份和硬件时间戳定位人工确认区间;切片内相对时间与原始 part 相对时间是不同坐标,不能混用。下游按定位和分类处理原始数据,不依赖平台内部审核文件。
|
||||
|
||||
## 标注对象
|
||||
|
||||
**Capture(源采集)**:一次源采集所关联的双目视频、时间戳与传感器数据,是本流程组织人工处理和审核的对象。
|
||||
|
||||
@ -5,6 +5,7 @@ import hashlib
|
||||
import bisect
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
@ -845,7 +846,174 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
return "empty_confirmed"
|
||||
|
||||
|
||||
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None):
|
||||
def _v4_tags(selection: Any, kind: str) -> list[str]:
|
||||
if not isinstance(selection, dict) or selection.get("kind") != kind or not str(selection.get("id") or "").strip() or type(selection.get("version")) is not int or selection["version"] < 1 or not str(selection.get("name") or "").strip():
|
||||
raise RuntimeError(f"v4 missing frozen {kind} dictionary; confirm historical classification before delivery")
|
||||
tags = selection.get("tags")
|
||||
if not isinstance(tags, list) or not tags or any(not isinstance(tag, str) or not tag.strip() or tag != tag.strip() for tag in tags) or len(set(tags)) != len(tags):
|
||||
raise RuntimeError(f"v4 missing frozen {kind} tags; confirm classification before delivery")
|
||||
return list(tags)
|
||||
|
||||
|
||||
def _v4_original_sources(manifest, capture_id, part_id, capture_prefix):
|
||||
source = manifest.get("originalSource")
|
||||
if not isinstance(source, dict) or source.get("provider") != "BOS" or not str(source.get("bucket") or "").strip():
|
||||
raise RuntimeError("v4 requires frozen original BOS source selection; workspace paths cannot identify original data")
|
||||
prefix = str(source.get("prefix") or "")
|
||||
part_prefix = str(source.get("partPrefix") or "")
|
||||
if prefix != capture_prefix or part_prefix != prefix + "/" + part_id or source.get("captureUnit") != capture_id + "/" + part_id:
|
||||
raise RuntimeError("v4 original source identity disagrees with accepted capture/part")
|
||||
if not prefix or str(pathlib.PurePosixPath(prefix)) != prefix or pathlib.PurePosixPath(prefix).is_absolute() or ".." in pathlib.PurePosixPath(prefix).parts or any(c in prefix for c in ("\\", "\x00")):
|
||||
raise RuntimeError("unsafe v4 original source prefix")
|
||||
keys = source.get("objectKeys")
|
||||
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) for key in keys) or len(set(keys)) != len(keys):
|
||||
raise RuntimeError("v4 requires an unambiguous original object selection")
|
||||
files = {}
|
||||
cap, part = re.escape(capture_id), re.escape(part_id)
|
||||
patterns = {
|
||||
"leftVideo": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
|
||||
"rightVideo": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
|
||||
"leftPts": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?_pts\.csv",
|
||||
"rightPts": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?_pts\.csv",
|
||||
"imu": rf"{part}/{cap}_imu_{part}\.csv",
|
||||
"cameraCalibration": rf"{cap}_calibration_camera\.yaml",
|
||||
"imuCalibration": rf"{cap}_calibration_imu\.yaml",
|
||||
"metadata": rf"meta_{cap}\.json",
|
||||
}
|
||||
for key in keys:
|
||||
if not key.startswith(prefix + "/") or str(pathlib.PurePosixPath(key)) != key or ".." in pathlib.PurePosixPath(key).parts or any(c in key for c in ("\\", "\x00")):
|
||||
raise RuntimeError("unsafe v4 original source object")
|
||||
relative = key[len(prefix) + 1:]
|
||||
if "/" in relative and not relative.startswith(part_id + "/"):
|
||||
raise RuntimeError("v4 original selection contains another part")
|
||||
for role, pattern in patterns.items():
|
||||
if re.fullmatch(pattern, relative, re.IGNORECASE):
|
||||
if role in files: raise RuntimeError(f"ambiguous v4 original {role} selection")
|
||||
files[role] = {"path": relative}
|
||||
missing = sorted(set(patterns) - files.keys())
|
||||
if missing: raise RuntimeError("v4 missing original files: " + ", ".join(missing))
|
||||
return {"provider": "BOS", "bucket": source["bucket"], "prefix": prefix, "partPrefix": part_prefix,
|
||||
"snapshotFingerprint": str(source.get("snapshotFingerprint") or ""), "files": files}
|
||||
|
||||
|
||||
def _v4_number(value, label):
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
||||
raise RuntimeError(f"v4 missing or invalid {label}")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _v4_timestamp(value, label):
|
||||
if type(value) is not int or value <= 0: raise RuntimeError(f"v4 missing or invalid {label}")
|
||||
return value
|
||||
|
||||
|
||||
def _v4_consumer_payload(payload, manifest, source_root):
|
||||
"""Project immutable accepted facts into a self-contained consumer contract.
|
||||
|
||||
No dictionary lookup, source media reads, transcoding or BOS writes here.
|
||||
Hardware PTS are authoritative; media seconds are explicitly named and never
|
||||
used to manufacture missing hardware timestamps.
|
||||
"""
|
||||
if not isinstance(manifest, dict): raise RuntimeError("v4 requires accepted source manifest")
|
||||
original = _v4_original_sources(manifest, payload["capture"]["captureId"], payload["partId"], payload["capture"]["prefix"])
|
||||
payload["originalSource"] = original
|
||||
payload["consumerContract"] = "droboticflow.classification-consumer/v1"
|
||||
payload["timeBases"] = {"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video",
|
||||
"segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts",
|
||||
"interval": "half-open", "frameRanges": "zero-based-half-open"}
|
||||
payload["pathBases"] = {"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
|
||||
# Workspace sources are private transport references, not the original files.
|
||||
payload["sources"] = {kind: original["files"][kind] for kind in ("leftVideo", "rightVideo", "leftPts", "rightPts", "imu")}
|
||||
payload["sourceMetadata"]["path"] = original["files"]["metadata"]["path"]
|
||||
for item in payload["calibration"]:
|
||||
# Bind by actual calibration kind, not input ordering.
|
||||
kind = "imuCalibration" if "imu" in item["name"].lower() else "cameraCalibration" if "camera" in item["name"].lower() else ""
|
||||
if not kind: raise RuntimeError("v4 unknown calibration kind")
|
||||
item["path"] = original["files"][kind]["path"]
|
||||
rows = {str(row["sliceId"]): row for row in manifest["items"]}
|
||||
if len(rows) != len(manifest["items"]): raise RuntimeError("v4 duplicate slice identities")
|
||||
for item in payload["slices"] + payload["discardedSlices"]:
|
||||
split = item["taskSplit"]
|
||||
start, end = (_v4_number(split[k], k) for k in ("startSecond", "endSecond"))
|
||||
first, last = (_v4_timestamp(split[k], k) for k in ("startTimestampUs", "endTimestampUs"))
|
||||
if start < 0 or end <= start or last <= first: raise RuntimeError("v4 invalid accepted slice range")
|
||||
if item["classification"]["deliveryDisposition"] == "discard":
|
||||
item["sliceName"] = ""
|
||||
evidence = item["classification"]["discardEvidence"]
|
||||
selected = evidence.get("dictionary")
|
||||
if selected is not None:
|
||||
if (not isinstance(selected, dict) or selected.get("kind") != "discard_reason" or
|
||||
not selected.get("id") or not selected.get("name") or
|
||||
type(selected.get("version")) is not int or selected["version"] < 1):
|
||||
raise RuntimeError("v4 invalid frozen discard dictionary")
|
||||
for alias, field in (("reasonLabelId", "id"), ("reasonName", "name"), ("reasonVersion", "version")):
|
||||
if evidence.get(alias) not in (None, "", 0, selected[field]):
|
||||
raise RuntimeError("v4 discard reason disagrees with frozen dictionary")
|
||||
evidence[alias] = selected[field]
|
||||
continue
|
||||
parent_tags = _v4_tags(item["classification"].get("dictionary"), "slice")
|
||||
item["classification"]["tags"] = parent_tags
|
||||
row = rows[item["sliceId"]]
|
||||
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
|
||||
seconds, stamps = sync.get("boundarySeconds", []), sync.get("boundaryTimestampUs", [])
|
||||
if seconds or stamps:
|
||||
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
|
||||
raise RuntimeError("v4 invalid hardware boundary mapping")
|
||||
seconds = [_v4_number(value, "boundary") for value in seconds]
|
||||
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
|
||||
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
|
||||
stamps[0] != first or stamps[-1] != last or
|
||||
any(a >= b for a, b in zip(seconds, seconds[1:])) or
|
||||
any(a >= b for a, b in zip(stamps, stamps[1:]))):
|
||||
raise RuntimeError("v4 invalid hardware boundary mapping")
|
||||
def timestamp(second):
|
||||
if abs(second) < 1e-7: return first
|
||||
if abs(second - (end-start)) < 1e-7: return last
|
||||
hits = [stamps[i] for i, value in enumerate(seconds) if abs(_v4_number(value,"boundary") - second) < 1e-7]
|
||||
if len(hits) != 1: raise RuntimeError("v4 action boundary has no original PTS mapping")
|
||||
return _v4_timestamp(hits[0], "boundary timestamp")
|
||||
actions = []
|
||||
for action in item["segments"]:
|
||||
a = _v4_number(action.get("clipStartMs"), "action clipStartMs") / 1000
|
||||
b = _v4_number(action.get("clipEndMs"), "action clipEndMs") / 1000
|
||||
if a < -1e-7 or b <= a or b > end-start+1e-7: raise RuntimeError("v4 action escapes its parent slice")
|
||||
ta, tb = (_v4_timestamp(action.get(k), k) for k in ("startTimestampUs", "endTimestampUs"))
|
||||
if not first <= ta < tb <= last: raise RuntimeError("v4 action timestamps escape its parent slice")
|
||||
if seconds and (timestamp(a) != ta or timestamp(b) != tb): raise RuntimeError("v4 action seconds disagree with original PTS")
|
||||
action.update(startSecond=a, endSecond=b, partStartSecond=start+a, partEndSecond=start+b)
|
||||
own = action.get("dictionary")
|
||||
action["tags"] = _v4_tags(own, "action") if own is not None else list(parent_tags)
|
||||
action["tagSource"] = "action" if own is not None else "slice"
|
||||
actions.append(action)
|
||||
actions.sort(key=lambda a: a["startSecond"])
|
||||
complete, previous = [], 0.0
|
||||
def gap(a,b):
|
||||
ta,tb=timestamp(a),timestamp(b)
|
||||
return {"id":f"{item['sliceId']}:gap:{ta}:{tb}", "description":"", "isInvalid":False,
|
||||
"contentKind":"gap_placeholder", "startSecond":a,"endSecond":b,
|
||||
"clipStartMs":a*1000,"clipEndMs":b*1000,"partStartSecond":start+a,"partEndSecond":start+b,
|
||||
"startTimestampUs":ta,"endTimestampUs":tb,"tags":list(parent_tags),"tagSource":"slice"}
|
||||
for action in actions:
|
||||
if action["startSecond"] < previous-1e-7: raise RuntimeError("v4 overlapping actions")
|
||||
if action["startSecond"] > previous+1e-7: complete.append(gap(previous,action["startSecond"]))
|
||||
complete.append(action);previous=action["endSecond"]
|
||||
if previous < end-start-1e-7: complete.append(gap(previous,end-start))
|
||||
if not complete: raise RuntimeError("v4 kept slice has no classified action coverage")
|
||||
previous_stamp = first
|
||||
for action in complete:
|
||||
if action["startTimestampUs"] != previous_stamp or action["endTimestampUs"] <= previous_stamp:
|
||||
raise RuntimeError("v4 action PTS coverage is discontinuous")
|
||||
previous_stamp = action["endTimestampUs"]
|
||||
if previous_stamp != last: raise RuntimeError("v4 action PTS coverage is incomplete")
|
||||
item["segments"] = complete
|
||||
counts = {kind:sum(a["contentKind"]==kind for a in complete) for kind in ("annotated","gap_placeholder","invalid_marked")}
|
||||
item["classification"]["contentCounts"] = counts
|
||||
item["classification"]["descriptionStatus"] = _description_status(complete)
|
||||
if not counts["annotated"] and counts["invalid_marked"] and counts["gap_placeholder"]: item["classification"]["descriptionStatus"]="invalid_and_blank"
|
||||
item["synchronization"]["rightBoundarySeconds"] = sync.get("rightBoundarySeconds", [])
|
||||
|
||||
|
||||
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None, revision_manifest=None, source_root=None):
|
||||
"""Use the established immutable publisher for versioned customer JSON.
|
||||
|
||||
V3 retains its historical sibling review file; v4 stores that audit in a
|
||||
@ -929,6 +1097,8 @@ def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payl
|
||||
for key in ("source", "confidence", "phrases", "labelId"):
|
||||
action.pop(key, None)
|
||||
if action.get("dictionary") is None: action.pop("dictionary", None)
|
||||
audit["materializedSources"] = payload.get("sources", {})
|
||||
_v4_consumer_payload(payload, revision_manifest, source_root)
|
||||
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8")
|
||||
audit["partSha256"] = hashlib.sha256(body).hexdigest()
|
||||
if len(json.dumps(audit, ensure_ascii=False, indent=2).encode("utf8")) > 8 * 1024 * 1024:
|
||||
@ -1782,7 +1952,7 @@ def _run_accepted_slice_directory(
|
||||
item["sourceFrames"] = frames[item["sliceId"]]
|
||||
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
|
||||
if _bool(params, "DELIVERY_JSON_V4", False):
|
||||
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload)
|
||||
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload, revision_manifest, source_root)
|
||||
if _bool(params, "DELIVERY_JSON_V3", False):
|
||||
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
|
||||
@ -51,3 +51,11 @@ status: accepted
|
||||
独立字典 tags 由管理员维护,与 ID/版本/名称一起在确认时冻结;category 与 aliases 不当作 tags。未写动作描述不等于没有动作,也不撤销切片已有分类。历史没有标签快照时不反查最新字典拼造历史。全舍弃也属于人工分类结果,因此 v4 仍交付该 part 的分类 JSON;这显式替代 v4 路径的旧 no_deliverables 规则,旧媒体目录和 v1–v3 路径继续保持原行为。
|
||||
|
||||
采用独立 v4 协议与默认关闭的开关,避免同一 v3 schema 悄悄改变字段、枚举或文件集合。保持已约定的 capture/part、slices/taskSplit/segments 层级及不可变批次身份,动作有描述使用约定的 annotated。保留旧版本读取与发布恢复,不原地修补历史完成目录。单独写内部追溯会增加一份内部产物,但避免客户为读取分类被迫接触人员、批注和平台流程字段。
|
||||
|
||||
## 2026-10-21 补充:动作继承及原始数据消费
|
||||
|
||||
用户确认切片与动作是父子分类。动作独立选择词条时使用该快照的标签,否则继承父切片冻结标签;不按描述是否为空决定,不伪造动作词条。v4 显式输出有效 tags 和 action/slice 来源,保留独立无效判断;舍弃仅保留原因分类及定位,不带旧动作或审核批注。缺标签不反查最新词条猜补。
|
||||
|
||||
下游需要仅凭总 JSON 和原始数据定位、切分、按类归组。新 v4 使用任务原始输入的 BOS 选择冻结八类源文件及 bucket/prefix,主文件不再依赖平台 workspace。动作同时明确切片内秒数、原始 part 秒数与原始硬件时间戳;用原始左右 PTS 各自解析硬件区间,避免用左视频秒数误切另一相机。输入快照指纹不是单个 BOS 对象内容版本的保证。实际切割和分类打包仍由下游执行。
|
||||
|
||||
缺少可靠来源、分类标签或时间证据时,选择拒绝新 v4 交付,而非“尽量输出”后让客户承受丢分类或错切的风险。发布同时验证消费契约、来源归属和文件存在。已交付结果不原地改写;历史只按证据补齐并产生新批次。这个要求适用于尚未发布的 v4,不改变 v1–v3;旧本地 v4 候选须重新生成。实现和本地消费验证见 `docs/delivery-classification-v4.zh-CN.md`。
|
||||
|
||||
@ -8,12 +8,32 @@
|
||||
|
||||
- `sliceName` 是已确认的基础分类名称;`classification.dictionary` 是可获得的字典快照。动作描述为空也保留这个分类,不代表无动作。
|
||||
- 字典快照固定 `id/kind/version/code/name/tags`。`tags` 是独立分类标签,与 `category`、别名和描述原义不混用。新选择由服务端从项目字典校验,客户不能伪造标签;历史缺失快照不按当前词条补填。
|
||||
- 每个动作显式导出有效 `tags` 和 `tagSource`。独立选择动作词条用该快照标签,来源为 `action`;没有独立选择则继承本切片冻结标签,来源为 `slice`,不伪造动作 `dictionary`。自由填写文字也沿用这个判定;有文字不代表选择了独立词条,没有文字不代表没有动作。独立词条缺 tags 时拒绝交付,不偷偷退回父标签。
|
||||
- 动作 `contentKind` 为 `annotated`(有描述)、`invalid_marked`(明确无效)、`gap_placeholder`(保留的留空区间)。`descriptionStatus` 描述文字覆盖情况,不能用于推断整个切片是否有动作。
|
||||
- `isInvalid` 与标签独立;无效动作可保留继承标签,但仍必须按无效判断排除有效动作处理。历史动作未覆盖的留空范围保留为继承父标签的 `gap_placeholder`,不能把该段视频漏掉。
|
||||
- `discardedSlices[].classification.discardEvidence` 保留业务舍弃原因、原因词条/版本/标签和补充说明。全舍弃 part 也输出总分类 JSON。审核退回原因不放在这里。
|
||||
- `sourceMetadata.path` 与标定路径相对原采集目录。源索引 `sources` 的 workspace 路径基准显式标记。内部审核文件不在客户目录内,也不从主 JSON 引用。
|
||||
- 舍弃原因有冻结字典时,旧兼容字段 `reasonLabelId/reasonVersion/reasonName` 从同一快照补齐,不导出空名称或版本 0;已有非空值若与快照冲突即拒绝交付。手填原因仍按原有规则保留,不猜造字典选择。
|
||||
- `originalSource` 冻结 BOS bucket、原采集 prefix、partPrefix、输入快照指纹和所选原始文件路径。来自任务创建时的输入引用,不查询当前目录猜原文件,不输出连接 ID、凭据或签名 URL。八类文件为左右视频、左右 PTS、IMU、相机标定、IMU 标定和采集 meta。`sources`、`sourceMetadata.path` 与标定路径统一相对原采集目录,`pathBases` 显式写为 `original_capture`;原 workspace 路径只留内部追溯。
|
||||
|
||||
内部 `droboticflow.delivery.review-history/v2` JSON 按 capture/part/revision 关联,以 `partSha256` 固定原生主文件字节;包含最终验收、抽检、审核人/批注/轮次/确认项、已有正式退回基线、处理记录和阶段事件。该 artifact 留在内部 Run 存储,不进入最终交付列表、不可变客户批次或客户下载。分类选择的服务端快照另外保存在当前任务的恢复记录中,使字典改名、更新或停用后的撤销仍可恢复原版本,客户端不能伪造历史标签。旧记录没有的追溯不制造补写。追溯超出限制时拒绝交付,不静默截断。
|
||||
|
||||
最终分类取通过验收的固定物化修订;沿用既有“未抽中”验收策略的任务,内部保留明确豁免,不能伪造人工通过日期。返工正式修改分类后必须重新送检、验收;查看或处理意见本身不修改分类。相同名称而字典版本/标签不同的切片不能自动合并成一个分类。
|
||||
|
||||
时间按两级明确表达,全部区间采用左闭右开:
|
||||
|
||||
| 字段 | 坐标 |
|
||||
| --- | --- |
|
||||
| `taskSplit.startSecond/endSecond` | 原始 part 左相机视频秒数 |
|
||||
| `segments[].startSecond/endSecond`、`clipStartMs/clipEndMs` | 本切片左相机内秒数/毫秒数 |
|
||||
| `segments[].partStartSecond/partEndSecond` | 原始 part 左相机视频秒数 |
|
||||
| `startTimestampUs/endTimestampUs` | 原始硬件时间戳,非墙钟时间、非从秒数估算 |
|
||||
|
||||
`timeBases` 描述以上坐标及帧区间基准。下游从原始左右 PTS 分别定位同一硬件时间窗,不能把左相机秒数直接当右相机秒数。原始输入快照指纹标识当时选择,不冒充每个 BOS 对象的版本号或内容 SHA;原始数据应保留,当前发布时仅以 HEAD 校验所引用文件存在,不证明同名文件从未被外部覆盖。
|
||||
|
||||
新 v4 要求完整的冻结分类、非空标签、原始来源和可靠时间边界;任一缺失或动作重叠、时间戳不连续即停止交付。批次发布再核对原始 bucket/part 归属、文件存在性和消费契约,不能只凭主文件的 SHA 放行。元数据修订复用原始引用,不追随后来改变的采集记录。未发布的旧本地 v4 样例须重新生成;正式 v1/v2/v3 及其恢复路径保持兼容。历史缺证据应先核实补齐,已交付文件不覆盖,修正交付另建不可变批次。
|
||||
|
||||
平台只输出分类索引,不按标签执行 BOS 数据切割、搬运或集中打包。内部审核文件不在客户目录内,也不从主 JSON 引用。
|
||||
|
||||
本地可用 `python -m tests.backend.export_delivery_v4_fixture <全新输出目录>` 生成明确标记的合成样例。目录中 `partNNNN/part.json` 是客户主文件;`internal/partNNNN/review-history.json` 是分离的内部示例,不属于客户包。不要把整个验证输出目录直接交付给客户。
|
||||
|
||||
独立消费验证:设置 `FANYU_DELIVERY_CONSUMER_FIXTURE_DIR` 为全新绝对目录,先运行 Go `TestDeliveryV4ConsumerReplicaFixture`,再以同一环境变量运行 `tests/backend/test_delivery_v4_consumer.py`。合成真实双目视频分别经过物理切片、源索引和元数据复用;独立进程仅读取导出 JSON 与原始文件,核对每帧覆盖、非零 part 起点、标签继承、无效与舍弃。独立读取器 `tests/backend/consume_delivery_v4.py` 只生成处理计划,不执行切割或写源数据。
|
||||
|
||||
@ -72,6 +72,15 @@ func (s *server) tryAnnotationMetadataRevision(ctx context.Context, task domain.
|
||||
if e != nil {
|
||||
return nil, nil
|
||||
}
|
||||
if manifest["originalSource"] == nil {
|
||||
if record, err := s.store.GetCaptureRecordByRunID(ctx, task.RunID); err != nil {
|
||||
return nil, err
|
||||
} else if original, err := annotationOriginalSource(run, record); err != nil {
|
||||
return nil, err
|
||||
} else if original != nil {
|
||||
manifest["originalSource"] = original
|
||||
}
|
||||
}
|
||||
entries := listValue(manifest["items"])
|
||||
if len(entries) != len(slices) {
|
||||
return nil, nil
|
||||
|
||||
63
internal/api/annotation_original_source.go
Normal file
63
internal/api/annotation_original_source.go
Normal file
@ -0,0 +1,63 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
// Freeze only credential-free facts from the run's original input, never the
|
||||
// current workflow configuration or a later source-directory listing.
|
||||
func annotationOriginalSource(run *domain.RunRecord, capture *domain.CaptureRecord) (map[string]any, error) {
|
||||
if run == nil || capture == nil || len(run.Inputs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if capture.RunID != run.ID || capture.OrganizationID != run.OrganizationID {
|
||||
return nil, fmt.Errorf("original capture scope mismatch")
|
||||
}
|
||||
root := capture.SourcePrefix
|
||||
if root == "" {
|
||||
root = capture.Prefix
|
||||
}
|
||||
var result map[string]any
|
||||
for _, input := range run.Inputs {
|
||||
ref := input.Ref
|
||||
if ref == nil || ref.Provider != domain.StorageProviderBOS || ref.Kind != domain.ArtifactRefKindDirectory || ref.Bucket != capture.Bucket || ref.Prefix != root {
|
||||
continue
|
||||
}
|
||||
if ref.CaptureUnit != "" && ref.CaptureUnit != capture.CaptureID {
|
||||
continue
|
||||
}
|
||||
if len(ref.ObjectKeys) == 0 {
|
||||
continue
|
||||
} // Older non-selected directory inputs need explicit recovery.
|
||||
if root == "" || path.Clean(root) != root || strings.HasPrefix(root, "/") || strings.Contains(root, "\\") {
|
||||
return nil, fmt.Errorf("unsafe original capture prefix")
|
||||
}
|
||||
keys := append([]string(nil), ref.ObjectKeys...)
|
||||
sort.Strings(keys)
|
||||
for i, key := range keys {
|
||||
if !strings.HasPrefix(key, root+"/") || path.Clean(key) != key || strings.Contains(key, "\\") || i > 0 && key == keys[i-1] {
|
||||
return nil, fmt.Errorf("unsafe original capture object selection")
|
||||
}
|
||||
relative := strings.TrimPrefix(key, root+"/")
|
||||
if strings.Contains(relative, "/") && !strings.HasPrefix(key, capture.Prefix+"/") {
|
||||
return nil, fmt.Errorf("original capture selection includes another part")
|
||||
}
|
||||
}
|
||||
candidate := map[string]any{"provider": string(ref.Provider), "bucket": ref.Bucket, "prefix": root, "partPrefix": capture.Prefix, "captureUnit": capture.CaptureID, "objectKeys": keys, "snapshotFingerprint": ref.VersionID}
|
||||
if result != nil {
|
||||
a, _ := json.Marshal(result)
|
||||
b, _ := json.Marshal(candidate)
|
||||
if string(a) != string(b) {
|
||||
return nil, fmt.Errorf("ambiguous original capture inputs")
|
||||
}
|
||||
}
|
||||
result = candidate
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
171
internal/api/annotation_original_source_test.go
Normal file
171
internal/api/annotation_original_source_test.go
Normal file
@ -0,0 +1,171 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func consumerSourceRef() *domain.ArtifactReference {
|
||||
root := "incoming/capture-A"
|
||||
keys := []string{root + "/meta_capture-A.json", root + "/capture-A_calibration_camera.yaml", root + "/capture-A_calibration_imu.yaml", root + "/part0001/capture-A_imu_part0001.csv"}
|
||||
for _, side := range []string{"left", "right"} {
|
||||
keys = append(keys, root+"/part0001/capture-A_camera_"+side+"_part0001.mp4", root+"/part0001/capture-A_camera_"+side+"_part0001_pts.csv")
|
||||
}
|
||||
return &domain.ArtifactReference{Provider: domain.StorageProviderBOS, Kind: domain.ArtifactRefKindDirectory, ConnectionID: "PRIVATE_CONNECTION", Bucket: "synthetic-original", Prefix: root, CaptureUnit: "capture-A/part0001", ObjectKeys: keys, VersionID: "frozen-input-version"}
|
||||
}
|
||||
|
||||
func TestAnnotationOriginalSourceUsesFrozenInputSelection(t *testing.T) {
|
||||
ref := consumerSourceRef()
|
||||
run := &domain.RunRecord{ID: "run", OrganizationID: "org", Inputs: map[string]domain.ArtifactPayload{"raw": {Ref: ref}}}
|
||||
capture := &domain.CaptureRecord{RunID: "run", OrganizationID: "org", Bucket: ref.Bucket, Prefix: ref.Prefix + "/part0001", SourcePrefix: ref.Prefix, CaptureID: ref.CaptureUnit, SourceKeys: []string{"later-current-directory-must-not-be-used"}}
|
||||
source, err := annotationOriginalSource(run, capture)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(source["objectKeys"].([]string)) != 8 {
|
||||
t.Fatal("frozen source selection lost")
|
||||
}
|
||||
if _, ok := source["connectionId"]; ok {
|
||||
t.Fatal("private connection leaked")
|
||||
}
|
||||
ref.ObjectKeys = append(ref.ObjectKeys, ref.Prefix+"/part0002/foreign.mp4")
|
||||
if _, err = annotationOriginalSource(run, capture); err == nil {
|
||||
t.Fatal("foreign part accepted")
|
||||
}
|
||||
ref.ObjectKeys = ref.ObjectKeys[:len(ref.ObjectKeys)-1]
|
||||
capture.OrganizationID = "other"
|
||||
if _, err = annotationOriginalSource(run, capture); err == nil {
|
||||
t.Fatal("foreign organization accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// Retain real synthetic source data for the independent Python consumer test.
|
||||
func TestDeliveryV4ConsumerReplicaFixture(t *testing.T) {
|
||||
parent := os.Getenv("FANYU_DELIVERY_CONSUMER_FIXTURE_DIR")
|
||||
if parent == "" {
|
||||
t.Skip("opt-in retained synthetic fixture")
|
||||
}
|
||||
if !filepath.IsAbs(parent) {
|
||||
t.Fatal("absolute local output required")
|
||||
}
|
||||
for _, sourceIndex := range []bool{false, true} {
|
||||
mode := "physical"
|
||||
if sourceIndex {
|
||||
mode = "source-index"
|
||||
}
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
f, temp := annotationProcessorFixture(t)
|
||||
dir := filepath.Join(parent, mode)
|
||||
root := filepath.Join(dir, "workspace")
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := copyAnnotationTree(f.ctx, temp, root); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, _ := f.store.GetRun(f.ctx, f.task.RunID)
|
||||
for i := range run.NodeExecutions {
|
||||
for j := range run.NodeExecutions[i].InputBindings {
|
||||
for k := range run.NodeExecutions[i].InputBindings[j].Artifacts {
|
||||
a := &run.NodeExecutions[i].InputBindings[j].Artifacts[k]
|
||||
a.Path = filepath.Join(root, filepath.Base(a.Path))
|
||||
}
|
||||
}
|
||||
}
|
||||
ref := consumerSourceRef()
|
||||
run.Inputs = map[string]domain.ArtifactPayload{"raw": {Ref: ref}}
|
||||
if err := f.store.UpsertRun(f.ctx, *run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
capture := domain.CaptureRecord{ID: "consumer-capture", RunID: run.ID, OrganizationID: run.OrganizationID, Bucket: ref.Bucket, Prefix: ref.Prefix + "/part0001", SourcePrefix: ref.Prefix, CaptureID: ref.CaptureUnit}
|
||||
if err := f.store.UpsertCaptureRecord(f.ctx, capture); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := &domain.DictionarySelection{ID: "base", Kind: "slice", Version: 2, Code: "operate", Name: "物品操作", Tags: []string{"商品", "手部"}}
|
||||
reason := &domain.DictionarySelection{ID: "discard", Kind: "discard_reason", Version: 1, Code: "occluded", Name: "遮挡", Tags: []string{"不可用"}}
|
||||
action := &domain.DictionarySelection{ID: "pick", Kind: "action", Version: 3, Code: "pick", Name: "拿取", Tags: []string{"抓取"}}
|
||||
input := combinedAnnotationSubmissionInput{SliceSegments: []domain.SliceSegment{{ID: "first", Name: base.Name, Dictionary: base, Start: 0, End: 1}, {ID: "discard", Start: 1, End: 3, DeliveryDisposition: "discard", DiscardReason: "遮挡", DiscardReasonDictionary: reason}, {ID: "last", Name: base.Name, Dictionary: base, Start: 3, End: 4}}, Segments: []domain.AnnotationSegment{{ID: "blank", Start: 0, End: .5}, {ID: "pick", Start: .5, End: 1, Description: "拿起杯子", Dictionary: action}, {ID: "invalid", Start: 3, End: 3.5, IsInvalid: true}, {ID: "free-text", Start: 3.5, End: 4, Description: "人工补充描述"}}}
|
||||
s := &server{store: f.store, workspaceMountPath: root, annotationSourceIndex: sourceIndex}
|
||||
result, err := s.materializeCombinedSliceSubmission(f.ctx, f.task, f.item, input, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sourceIndex {
|
||||
before, err := os.ReadFile(filepath.Join(result.Artifact.Path, "revision-manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var first map[string]any
|
||||
if err = json.Unmarshal(before, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Metadata-only reuse must keep the original source, even if discovery changes.
|
||||
capture.Bucket = "later-bucket"
|
||||
if err = f.store.UpsertCaptureRecord(f.ctx, capture); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, _ = f.store.GetRun(f.ctx, f.task.RunID)
|
||||
result, err = s.materializeCombinedSliceSubmission(f.ctx, f.task, f.item, input, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Payload["metadataOnly"] != true {
|
||||
t.Fatal("metadata-only recovery not exercised")
|
||||
}
|
||||
after, err := os.ReadFile(filepath.Join(result.Artifact.Path, "revision-manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var second map[string]any
|
||||
if err = json.Unmarshal(after, &second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(first["originalSource"], second["originalSource"]) {
|
||||
t.Fatal("metadata revision followed mutable source")
|
||||
}
|
||||
}
|
||||
original := filepath.Join(dir, "original")
|
||||
for _, key := range ref.ObjectKeys {
|
||||
relative := key[len(ref.Prefix)+1:]
|
||||
dst := filepath.Join(original, filepath.FromSlash(relative))
|
||||
if err = os.MkdirAll(filepath.Dir(dst), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var body []byte
|
||||
switch {
|
||||
case filepath.Ext(dst) == ".yaml":
|
||||
body = []byte("synthetic: true\n")
|
||||
case filepath.Ext(dst) == ".json":
|
||||
body = []byte("{\"synthetic\":true,\"partId\":\"part0001\"}\n")
|
||||
default:
|
||||
name := "imu.csv"
|
||||
for _, side := range []string{"left", "right"} {
|
||||
if strings.Contains(relative, "camera_"+side) {
|
||||
name = side + ".mp4"
|
||||
if strings.HasSuffix(relative, "_pts.csv") {
|
||||
name = side + "_pts.csv"
|
||||
}
|
||||
}
|
||||
}
|
||||
body, err = os.ReadFile(filepath.Join(root, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = os.WriteFile(dst, body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
body, _ := json.MarshalIndent(map[string]any{"artifact": result.Artifact, "original": original, "workspace": root}, "", " ")
|
||||
if err = os.WriteFile(filepath.Join(dir, "context.json"), body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -126,12 +126,20 @@ func (s *server) materializeSourceIndexRevision(ctx context.Context, t domain.An
|
||||
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": grid.Seconds[0], "endSecond": grid.Seconds[previous], "durationSeconds": grid.Seconds[previous] - grid.Seconds[0], "startTimestampUs": grid.Timestamps[0], "endTimestampUs": grid.Timestamps[previous], "ptsToMediaScale": cert.Timeline.Scale, "sliceCount": len(slices), "reconstruction": map[string]any{"leftFrameCount": grid.LeftFrames[previous] - grid.LeftFrames[0], "rightFrameCount": grid.RightFrames[previous] - grid.RightFrames[0], "imuRowCount": cert.IMURowsBeforeBoundary[previous] - cert.IMURowsBeforeBoundary[0]}}
|
||||
selection := map[string]any{"keptSliceCount": kept, "discardedSliceCount": discarded, "keptDurationSeconds": keepDuration, "discardedDurationSeconds": discardDuration}
|
||||
capture := map[string]any{}
|
||||
var originalSource map[string]any
|
||||
if record, e := s.store.GetCaptureRecordByRunID(ctx, t.RunID); e != nil {
|
||||
return nil, e
|
||||
} else if record != nil {
|
||||
originalSource, e = annotationOriginalSource(run, record)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
capture = map[string]any{"captureRecordId": record.ID, "captureId": record.CaptureID, "prefix": record.Prefix, "fingerprint": record.Fingerprint, "contractRevision": record.ContractRevision}
|
||||
}
|
||||
manifest := map[string]any{"schemaVersion": sourceIndexSchema, "revision": revision, "taskId": t.ID, "humanItemId": h.ID, "runId": t.RunID, "collectionId": cert.CollectionID, "capture": capture, "sourceFingerprint": fingerprint, "sources": cert.Sources, "sourceDurations": map[string]any{"left": cert.LeftDuration, "right": cert.RightDuration}, "status": "materialized", "sliceCount": len(items), "coverage": coverage, "deliverySelection": selection, "items": items}
|
||||
if originalSource != nil {
|
||||
manifest["originalSource"] = originalSource
|
||||
}
|
||||
manifest["sourceFrameCounts"] = map[string]any{"left": len(cert.LeftPTS), "right": len(cert.RightPTS)}
|
||||
encoded, e := json.Marshal(manifest)
|
||||
if e != nil || len(encoded)+1 > maxRunArtifactManifestBytes {
|
||||
|
||||
@ -133,6 +133,15 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla
|
||||
ID string `json:"captureId"`
|
||||
Prefix string `json:"prefix"`
|
||||
} `json:"capture"`
|
||||
OriginalSource struct {
|
||||
Provider string `json:"provider"`
|
||||
Bucket string `json:"bucket"`
|
||||
Prefix string `json:"prefix"`
|
||||
PartPrefix string `json:"partPrefix"`
|
||||
Files map[string]struct {
|
||||
Path string `json:"path"`
|
||||
} `json:"files"`
|
||||
} `json:"originalSource"`
|
||||
PartID string `json:"partId"`
|
||||
Lineage struct {
|
||||
RunID string `json:"sourceRunId"`
|
||||
@ -161,12 +170,33 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla
|
||||
if e != nil || record == nil || record.OrganizationID != project.OrganizationID || record.ProjectID != project.ID || record.Prefix != part.Capture.Prefix+"/"+part.PartID || record.Delivery == nil || record.Delivery.Prefix != target.prefix || record.Delivery.ConnectionID != t.connectionID || record.Delivery.Bucket != t.bucket {
|
||||
return nil, fmt.Errorf("part 来源与采集记录不一致")
|
||||
}
|
||||
if part.Schema == deliveryindex.ClassificationPartSchema && (part.OriginalSource.Provider != "BOS" || part.OriginalSource.Bucket != record.Bucket || part.OriginalSource.Prefix != part.Capture.Prefix || part.OriginalSource.PartPrefix != record.Prefix) {
|
||||
return nil, fmt.Errorf("分类 JSON 原始存储来源与采集记录不一致,停止发布")
|
||||
}
|
||||
sourceRuns = append(sourceRuns, part.Lineage.RunID)
|
||||
sourceBrowser, closeSource, e := s.deliveryReadBrowser(r.Context(), project.OrganizationID, record.ConnectionID, record.Bucket, part.Capture.Prefix, false)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
expected, meta, e := indexBatchSourceEvidence(r.Context(), sourceBrowser, record.Bucket, part.Capture.Prefix, part.Capture.ID)
|
||||
if e == nil && part.Schema == deliveryindex.ClassificationPartSchema {
|
||||
// HEAD only: prove the frozen references still resolve without reading
|
||||
// video bodies, recutting media or expanding the original selection.
|
||||
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
|
||||
rel := part.OriginalSource.Files[role].Path
|
||||
media := role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu"
|
||||
if rel == "" || path.Clean(rel) != rel || strings.ContainsAny(rel, "\\\x00") || strings.HasPrefix(rel, "/") || strings.HasPrefix(rel, "../") || rel == ".." || media && !strings.HasPrefix(rel, part.PartID+"/") || !media && path.Base(rel) != rel {
|
||||
e = fmt.Errorf("分类 JSON 原始文件路径无效:%s", role)
|
||||
break
|
||||
}
|
||||
key := part.Capture.Prefix + "/" + rel
|
||||
info, err := sourceBrowser.GetObjectSummary(r.Context(), record.Bucket, key)
|
||||
if err != nil || info.Key != key || info.Kind != "file" || info.SizeBytes <= 0 {
|
||||
e = fmt.Errorf("分类 JSON 原始文件不存在或不可读:%s", role)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if closeSource != nil {
|
||||
closeSource()
|
||||
}
|
||||
|
||||
@ -25,6 +25,14 @@ type indexTestBrowser struct {
|
||||
ambiguous bool
|
||||
}
|
||||
|
||||
func (b *indexTestBrowser) GetObjectSummary(_ context.Context, bucket, key string) (domain.ObjectStorageObjectSummary, error) {
|
||||
body, ok := b.objects[key]
|
||||
if !ok {
|
||||
return domain.ObjectStorageObjectSummary{}, os.ErrNotExist
|
||||
}
|
||||
return domain.ObjectStorageObjectSummary{Bucket: bucket, Key: key, Kind: "file", SizeBytes: int64(len(body))}, nil
|
||||
}
|
||||
|
||||
func (b *indexTestBrowser) PutObjectConditional(_ context.Context, _ string, key string, body []byte, _ string, match, none string, _ map[string]string) (domain.ObjectStorageObjectSummary, error) {
|
||||
if match != "" || none != "*" {
|
||||
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("unconditional write")
|
||||
@ -137,6 +145,7 @@ func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authent
|
||||
delete(doc, "acceptance")
|
||||
delete(doc, "lineage")
|
||||
doc["schemaVersion"] = deliveryindex.ClassificationPartSchema
|
||||
enrichConsumerFixture(doc)
|
||||
body, _ = json.Marshal(doc)
|
||||
history, _ = json.Marshal(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2", "captureId": "capture-A", "partId": part, "revision": 1, "partSha256": deliveryindex.Hash(body), "acceptance": acceptance, "lineage": map[string]any{"sourceRunId": sourceRun}, "qualityReviews": []any{map[string]any{"comment": "PRIVATE_REWORK"}}, "trace": []any{}})
|
||||
}
|
||||
@ -151,6 +160,19 @@ func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authent
|
||||
b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 1, "totalBytes": len(body), "entries": entries})
|
||||
}
|
||||
b.objects["incoming/capture-A/"+part+"/left.mp4"] = []byte("synthetic")
|
||||
if v4 {
|
||||
var doc struct {
|
||||
Original struct {
|
||||
Files map[string]struct {
|
||||
Path string `json:"path"`
|
||||
} `json:"files"`
|
||||
} `json:"originalSource"`
|
||||
}
|
||||
must(json.Unmarshal(body, &doc))
|
||||
for _, file := range doc.Original.Files {
|
||||
b.objects["incoming/capture-A/"+file.Path] = []byte("synthetic original")
|
||||
}
|
||||
}
|
||||
run := domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: "project-A", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: artifactID, RunID: runID, Name: "delivery.directory.index", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindDirectory, Provider: domain.StorageProviderBOS, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix, ManifestKey: prefix + "/manifest.json"}}}}}
|
||||
if v4 {
|
||||
var audit map[string]any
|
||||
@ -342,3 +364,93 @@ func TestClassificationBatchHTTPPublicationRecoveryAndDownload(t *testing.T) {
|
||||
t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1")
|
||||
exerciseIndexBatch(t, store.NewMemoryStore())
|
||||
}
|
||||
|
||||
func TestClassificationBatchRejectsMissingOrChangedOriginalSource(t *testing.T) {
|
||||
t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1")
|
||||
t.Setenv("DELIVERY_V4_FIXTURE_DIR", "")
|
||||
for _, failure := range []string{"bucket", "missing-file"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
ds := store.NewMemoryStore()
|
||||
b, p, _ := seedIndexFixture(t, ds)
|
||||
if failure == "bucket" {
|
||||
record, err := ds.GetCaptureRecordByRunID(t.Context(), "source-run-part0001")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record.Bucket = "changed-bucket"
|
||||
if err = ds.UpsertCaptureRecord(t.Context(), *record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
delete(b.objects, "incoming/capture-A/part0001/leftVideo")
|
||||
}
|
||||
w := indexCall(NewHandler(ds, nil), p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
|
||||
if w.Code != 409 || len(b.puts) != 0 || !strings.Contains(w.Body.String(), "原始") {
|
||||
t.Fatalf("incomplete source accepted: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func enrichConsumerFixture(p map[string]any) {
|
||||
part := p["partId"].(string)
|
||||
p["consumerContract"] = "droboticflow.classification-consumer/v1"
|
||||
p["timeBases"] = map[string]string{"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video", "segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts", "interval": "half-open", "frameRanges": "zero-based-half-open"}
|
||||
p["pathBases"] = map[string]string{"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
|
||||
files := map[string]any{}
|
||||
sources := map[string]any{}
|
||||
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
|
||||
name := part + "/" + role
|
||||
if role == "cameraCalibration" || role == "imuCalibration" {
|
||||
name = role + ".yaml"
|
||||
}
|
||||
if role == "metadata" {
|
||||
name = "meta_capture-A.json"
|
||||
}
|
||||
files[role] = map[string]any{"path": name}
|
||||
if role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu" {
|
||||
sources[role] = files[role]
|
||||
}
|
||||
}
|
||||
p["sources"] = sources
|
||||
p["sourceMetadata"] = files["metadata"]
|
||||
p["originalSource"] = map[string]any{"provider": "BOS", "bucket": "test-bucket", "prefix": "incoming/capture-A", "partPrefix": "incoming/capture-A/" + part, "files": files}
|
||||
for _, kind := range []string{"slices", "discardedSlices"} {
|
||||
for _, raw := range p[kind].([]any) {
|
||||
item := raw.(map[string]any)
|
||||
split := item["taskSplit"].(map[string]any)
|
||||
number := func(v any) float64 {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n
|
||||
case int:
|
||||
return float64(n)
|
||||
default:
|
||||
panic("invalid fixture time")
|
||||
}
|
||||
}
|
||||
start, end := number(split["startSecond"]), number(split["endSecond"])
|
||||
split["startTimestampUs"] = int64(1e6 + start*1e6)
|
||||
split["endTimestampUs"] = int64(1e6 + end*1e6)
|
||||
if kind == "discardedSlices" {
|
||||
continue
|
||||
}
|
||||
class := item["classification"].(map[string]any)
|
||||
class["dictionary"] = map[string]any{"id": "base", "kind": "slice", "version": 1, "name": "操作", "tags": []string{"操作"}}
|
||||
class["tags"] = []string{"操作"}
|
||||
actions := item["segments"].([]any)
|
||||
for i, raw := range actions {
|
||||
a := raw.(map[string]any)
|
||||
low, high := float64(i)*(end-start)/float64(len(actions)), float64(i+1)*(end-start)/float64(len(actions))
|
||||
a["startSecond"] = low
|
||||
a["endSecond"] = high
|
||||
a["partStartSecond"] = start + low
|
||||
a["partEndSecond"] = start + high
|
||||
a["startTimestampUs"] = int64(1e6 + (start+low)*1e6)
|
||||
a["endTimestampUs"] = int64(1e6 + (start+high)*1e6)
|
||||
a["tags"] = []string{"操作"}
|
||||
a["tagSource"] = "slice"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1005,15 +1005,23 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta
|
||||
}
|
||||
coverage := map[string]any{"mode": "full_common_range", "contiguous": true, "gapMicroseconds": 0, "overlapMicroseconds": 0, "startSecond": commonMediaStart, "endSecond": commonMediaEnd, "durationSeconds": commonMediaEnd - commonMediaStart, "startTimestampUs": commonStart, "endTimestampUs": commonEnd, "ptsToMediaScale": ptsToMediaScale, "sliceCount": len(manifestItems), "reconstruction": map[string]any{"leftFrameCount": deliveredLeftFrames, "rightFrameCount": deliveredRightFrames, "imuRowCount": deliveredIMURows}}
|
||||
captureEvidence := map[string]any{}
|
||||
var originalSource map[string]any
|
||||
if capture, captureErr := server.store.GetCaptureRecordByRunID(ctx, task.RunID); captureErr != nil {
|
||||
return nil, captureErr
|
||||
} else if capture != nil {
|
||||
originalSource, err = annotationOriginalSource(run, capture)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
captureEvidence = map[string]any{
|
||||
"captureRecordId": capture.ID, "captureId": capture.CaptureID, "prefix": capture.Prefix,
|
||||
"fingerprint": capture.Fingerprint, "contractRevision": capture.ContractRevision,
|
||||
}
|
||||
}
|
||||
revisionManifest := map[string]any{"schemaVersion": "droboticflow.slice-materialization/v2", "revision": revision, "humanItemId": item.ID, "taskId": task.ID, "runId": task.RunID, "collectionId": collectionID, "capture": captureEvidence, "status": "materialized", "sliceCount": len(manifestItems), "coverage": coverage, "items": manifestItems}
|
||||
if originalSource != nil {
|
||||
revisionManifest["originalSource"] = originalSource
|
||||
}
|
||||
deliverySelection := map[string]any{"keptSliceCount": keptSliceCount, "discardedSliceCount": discardedSliceCount, "keptDurationSeconds": keptDuration, "discardedDurationSeconds": discardedDuration}
|
||||
if discardedSliceCount > 0 {
|
||||
revisionManifest["schemaVersion"] = "droboticflow.slice-materialization/v3"
|
||||
|
||||
@ -244,6 +244,9 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh
|
||||
}
|
||||
schema = p.Schema
|
||||
if p.Schema == ClassificationPartSchema {
|
||||
if err := validateConsumerPart(src.Part); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if _, err := classificationAudit(src); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
@ -19,6 +19,7 @@ func classificationFixture(allDiscard bool) Source {
|
||||
p["discardedSlices"] = []any{map[string]any{"sliceId": "all", "taskSplit": map[string]any{"startSecond": 0, "endSecond": 3, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "discard", "discardEvidence": map[string]any{"reason": "遮挡"}}, "segments": []any{}}}
|
||||
p["deliveryClassification"] = map[string]any{"keptSliceCount": 0, "discardedSliceCount": 1, "keptDurationSeconds": 0, "discardedDurationSeconds": 3}
|
||||
}
|
||||
enrichConsumerFixture(p)
|
||||
src.Part = encode(p)
|
||||
src.History = encode(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2", "captureId": "capture-A", "partId": "part0001", "revision": 7, "partSha256": Hash(src.Part), "acceptance": acceptance, "qualityReviews": []any{map[string]string{"comment": "PRIVATE_REWORK"}}, "trace": []any{}})
|
||||
return src
|
||||
@ -55,6 +56,40 @@ func TestClassificationBatchRejectsMixedVersions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassificationBatchRejectsIncompleteConsumerContract(t *testing.T) {
|
||||
for _, failure := range []string{"contract", "tags", "tag-source", "part-time", "pts", "original-file"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
src := classificationFixture(false)
|
||||
var doc, audit map[string]any
|
||||
json.Unmarshal(src.Part, &doc)
|
||||
slice := doc["slices"].([]any)[0].(map[string]any)
|
||||
action := slice["segments"].([]any)[0].(map[string]any)
|
||||
switch failure {
|
||||
case "contract":
|
||||
delete(doc, "consumerContract")
|
||||
case "tags":
|
||||
delete(action, "tags")
|
||||
case "tag-source":
|
||||
action["tagSource"] = "action"
|
||||
case "part-time":
|
||||
action["partStartSecond"] = -1
|
||||
case "pts":
|
||||
delete(action, "startTimestampUs")
|
||||
case "original-file":
|
||||
delete(doc["originalSource"].(map[string]any)["files"].(map[string]any), "leftPts")
|
||||
}
|
||||
src.Part = encode(doc)
|
||||
json.Unmarshal(src.History, &audit)
|
||||
// Keep acceptance hash valid: rejection must be the consumer guard.
|
||||
audit["partSha256"] = Hash(src.Part)
|
||||
src.History = encode(audit)
|
||||
if _, err := Build("project-A", "项目A", "delivery/项目A__project-A", []Source{src}); err == nil || !strings.Contains(err.Error(), "consumer contract") {
|
||||
t.Fatalf("incomplete %s accepted: %v", failure, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassificationBatchAcceptsExplicitSamplingExemptionWithoutInventingHumanDate(t *testing.T) {
|
||||
src := classificationFixture(false)
|
||||
var audit map[string]any
|
||||
@ -74,3 +109,66 @@ func TestClassificationBatchAcceptsExplicitSamplingExemptionWithoutInventingHuma
|
||||
t.Fatal("unverified human acceptance accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func enrichConsumerFixture(p map[string]any) {
|
||||
part := p["partId"].(string)
|
||||
p["consumerContract"] = "droboticflow.classification-consumer/v1"
|
||||
p["timeBases"] = map[string]string{"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video", "segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts", "interval": "half-open", "frameRanges": "zero-based-half-open"}
|
||||
p["pathBases"] = map[string]string{"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
|
||||
files := map[string]any{}
|
||||
sources := map[string]any{}
|
||||
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
|
||||
name := part + "/" + role
|
||||
if role == "cameraCalibration" || role == "imuCalibration" {
|
||||
name = role + ".yaml"
|
||||
}
|
||||
if role == "metadata" {
|
||||
name = "meta_capture-A.json"
|
||||
}
|
||||
files[role] = map[string]any{"path": name}
|
||||
if role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu" {
|
||||
sources[role] = files[role]
|
||||
}
|
||||
}
|
||||
p["sources"] = sources
|
||||
p["sourceMetadata"] = files["metadata"]
|
||||
p["originalSource"] = map[string]any{"provider": "BOS", "bucket": "test-bucket", "prefix": "incoming/capture-A", "partPrefix": "incoming/capture-A/" + part, "files": files}
|
||||
for _, kind := range []string{"slices", "discardedSlices"} {
|
||||
for _, raw := range p[kind].([]any) {
|
||||
item := raw.(map[string]any)
|
||||
split := item["taskSplit"].(map[string]any)
|
||||
number := func(v any) float64 {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n
|
||||
case int:
|
||||
return float64(n)
|
||||
default:
|
||||
panic("invalid fixture time")
|
||||
}
|
||||
}
|
||||
start, end := number(split["startSecond"]), number(split["endSecond"])
|
||||
split["startTimestampUs"] = int64(1e6 + start*1e6)
|
||||
split["endTimestampUs"] = int64(1e6 + end*1e6)
|
||||
if kind == "discardedSlices" {
|
||||
continue
|
||||
}
|
||||
class := item["classification"].(map[string]any)
|
||||
class["dictionary"] = map[string]any{"id": "base", "kind": "slice", "version": 1, "name": "操作", "tags": []string{"操作"}}
|
||||
class["tags"] = []string{"操作"}
|
||||
actions := item["segments"].([]any)
|
||||
for i, raw := range actions {
|
||||
a := raw.(map[string]any)
|
||||
low, high := float64(i)*(end-start)/float64(len(actions)), float64(i+1)*(end-start)/float64(len(actions))
|
||||
a["startSecond"] = low
|
||||
a["endSecond"] = high
|
||||
a["partStartSecond"] = start + low
|
||||
a["partEndSecond"] = start + high
|
||||
a["startTimestampUs"] = int64(1e6 + (start+low)*1e6)
|
||||
a["endTimestampUs"] = int64(1e6 + (start+high)*1e6)
|
||||
a["tags"] = []string{"操作"}
|
||||
a["tagSource"] = "slice"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
148
internal/deliveryindex/consumer.go
Normal file
148
internal/deliveryindex/consumer.go
Normal file
@ -0,0 +1,148 @@
|
||||
package deliveryindex
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"path"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// v4 is unreleased: only complete consumer/v1 projections may enter new batches.
|
||||
// Keep this guard at publication as well as in the package producer, so older
|
||||
// local v4 artifacts cannot silently publish incomplete classification JSON.
|
||||
func validateConsumerPart(body []byte) error {
|
||||
type sourceFile struct {
|
||||
Path string `json:"path"`
|
||||
}
|
||||
type selection struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Version int `json:"version"`
|
||||
Name string `json:"name"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
type window struct {
|
||||
Start *float64 `json:"startSecond"`
|
||||
End *float64 `json:"endSecond"`
|
||||
First *int64 `json:"startTimestampUs"`
|
||||
Last *int64 `json:"endTimestampUs"`
|
||||
}
|
||||
type action struct {
|
||||
window
|
||||
PartStart *float64 `json:"partStartSecond"`
|
||||
PartEnd *float64 `json:"partEndSecond"`
|
||||
Tags []string `json:"tags"`
|
||||
TagSource string `json:"tagSource"`
|
||||
Dictionary *selection `json:"dictionary"`
|
||||
}
|
||||
type item struct {
|
||||
Split window `json:"taskSplit"`
|
||||
Class struct {
|
||||
Dictionary *selection `json:"dictionary"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"classification"`
|
||||
Actions []action `json:"segments"`
|
||||
}
|
||||
var p struct {
|
||||
Contract string `json:"consumerContract"`
|
||||
Capture struct {
|
||||
Prefix string `json:"prefix"`
|
||||
} `json:"capture"`
|
||||
PartID string `json:"partId"`
|
||||
Original struct {
|
||||
Provider string `json:"provider"`
|
||||
Bucket string `json:"bucket"`
|
||||
Prefix string `json:"prefix"`
|
||||
PartPrefix string `json:"partPrefix"`
|
||||
Files map[string]sourceFile `json:"files"`
|
||||
} `json:"originalSource"`
|
||||
Bases map[string]string `json:"timeBases"`
|
||||
Slices []item `json:"slices"`
|
||||
Discarded []struct {
|
||||
Split window `json:"taskSplit"`
|
||||
} `json:"discardedSlices"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &p); err != nil {
|
||||
return err
|
||||
}
|
||||
fail := func() error { return fmt.Errorf("incomplete v4 classification consumer contract") }
|
||||
var refs struct {
|
||||
Sources map[string]sourceFile `json:"sources"`
|
||||
Metadata sourceFile `json:"sourceMetadata"`
|
||||
Bases map[string]string `json:"pathBases"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &refs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu"} {
|
||||
if refs.Sources[role].Path != p.Original.Files[role].Path || !strings.HasPrefix(refs.Sources[role].Path, p.PartID+"/") {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
if refs.Metadata.Path != p.Original.Files["metadata"].Path || refs.Bases["sources"] != "original_capture" || refs.Bases["sourceMetadata"] != "original_capture" || refs.Bases["calibration"] != "original_capture" {
|
||||
return fail()
|
||||
}
|
||||
if p.Contract != "droboticflow.classification-consumer/v1" || p.Original.Provider != "BOS" || p.Original.Bucket == "" || p.Original.Prefix == "" || p.Original.Prefix != p.Capture.Prefix || p.Original.PartPrefix != p.Original.Prefix+"/"+p.PartID {
|
||||
return fail()
|
||||
}
|
||||
for key, value := range map[string]string{"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video", "segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts", "interval": "half-open", "frameRanges": "zero-based-half-open"} {
|
||||
if p.Bases[key] != value {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
|
||||
file := p.Original.Files[role].Path
|
||||
if file == "" || path.Clean(file) != file || strings.HasPrefix(file, "/") || strings.ContainsAny(file, "\\\x00") || file == ".." || strings.HasPrefix(file, "../") {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
validWindow := func(w window) bool {
|
||||
return w.Start != nil && w.End != nil && w.First != nil && w.Last != nil && !math.IsNaN(*w.Start) && !math.IsInf(*w.Start, 0) && !math.IsNaN(*w.End) && !math.IsInf(*w.End, 0) && *w.Start >= 0 && *w.End > *w.Start && *w.First > 0 && *w.Last > *w.First
|
||||
}
|
||||
validTags := func(tags []string) bool {
|
||||
seen := map[string]bool{}
|
||||
if len(tags) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, tag := range tags {
|
||||
if strings.TrimSpace(tag) == "" || tag != strings.TrimSpace(tag) || seen[tag] {
|
||||
return false
|
||||
}
|
||||
seen[tag] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, slice := range p.Slices {
|
||||
d := slice.Class.Dictionary
|
||||
if !validWindow(slice.Split) || d == nil || strings.TrimSpace(d.ID) == "" || strings.TrimSpace(d.Name) == "" || d.Kind != "slice" || d.Version < 1 || !validTags(d.Tags) || !reflect.DeepEqual(d.Tags, slice.Class.Tags) || len(slice.Actions) == 0 {
|
||||
return fail()
|
||||
}
|
||||
previous := 0.0
|
||||
stamp := *slice.Split.First
|
||||
for _, a := range slice.Actions {
|
||||
if !validWindow(a.window) || a.PartStart == nil || a.PartEnd == nil || !validTags(a.Tags) || math.Abs(*a.Start-previous) > 1e-6 || *a.First != stamp || *a.Last > *slice.Split.Last || math.Abs(*a.PartStart-*slice.Split.Start-*a.Start) > 1e-6 || math.Abs(*a.PartEnd-*slice.Split.Start-*a.End) > 1e-6 {
|
||||
return fail()
|
||||
}
|
||||
if a.Dictionary == nil {
|
||||
if a.TagSource != "slice" || !reflect.DeepEqual(a.Tags, d.Tags) {
|
||||
return fail()
|
||||
}
|
||||
} else if a.TagSource != "action" || a.Dictionary.Kind != "action" || strings.TrimSpace(a.Dictionary.ID) == "" || strings.TrimSpace(a.Dictionary.Name) == "" || a.Dictionary.Version < 1 || !reflect.DeepEqual(a.Tags, a.Dictionary.Tags) {
|
||||
return fail()
|
||||
}
|
||||
previous = *a.End
|
||||
stamp = *a.Last
|
||||
}
|
||||
if math.Abs(previous-(*slice.Split.End-*slice.Split.Start)) > 1e-6 || stamp != *slice.Split.Last {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
for _, item := range p.Discarded {
|
||||
if !validWindow(item.Split) {
|
||||
return fail()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -5,6 +5,7 @@ import hashlib
|
||||
import bisect
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
@ -845,7 +846,174 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
return "empty_confirmed"
|
||||
|
||||
|
||||
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None):
|
||||
def _v4_tags(selection: Any, kind: str) -> list[str]:
|
||||
if not isinstance(selection, dict) or selection.get("kind") != kind or not str(selection.get("id") or "").strip() or type(selection.get("version")) is not int or selection["version"] < 1 or not str(selection.get("name") or "").strip():
|
||||
raise RuntimeError(f"v4 missing frozen {kind} dictionary; confirm historical classification before delivery")
|
||||
tags = selection.get("tags")
|
||||
if not isinstance(tags, list) or not tags or any(not isinstance(tag, str) or not tag.strip() or tag != tag.strip() for tag in tags) or len(set(tags)) != len(tags):
|
||||
raise RuntimeError(f"v4 missing frozen {kind} tags; confirm classification before delivery")
|
||||
return list(tags)
|
||||
|
||||
|
||||
def _v4_original_sources(manifest, capture_id, part_id, capture_prefix):
|
||||
source = manifest.get("originalSource")
|
||||
if not isinstance(source, dict) or source.get("provider") != "BOS" or not str(source.get("bucket") or "").strip():
|
||||
raise RuntimeError("v4 requires frozen original BOS source selection; workspace paths cannot identify original data")
|
||||
prefix = str(source.get("prefix") or "")
|
||||
part_prefix = str(source.get("partPrefix") or "")
|
||||
if prefix != capture_prefix or part_prefix != prefix + "/" + part_id or source.get("captureUnit") != capture_id + "/" + part_id:
|
||||
raise RuntimeError("v4 original source identity disagrees with accepted capture/part")
|
||||
if not prefix or str(pathlib.PurePosixPath(prefix)) != prefix or pathlib.PurePosixPath(prefix).is_absolute() or ".." in pathlib.PurePosixPath(prefix).parts or any(c in prefix for c in ("\\", "\x00")):
|
||||
raise RuntimeError("unsafe v4 original source prefix")
|
||||
keys = source.get("objectKeys")
|
||||
if not isinstance(keys, list) or not keys or any(not isinstance(key, str) for key in keys) or len(set(keys)) != len(keys):
|
||||
raise RuntimeError("v4 requires an unambiguous original object selection")
|
||||
files = {}
|
||||
cap, part = re.escape(capture_id), re.escape(part_id)
|
||||
patterns = {
|
||||
"leftVideo": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
|
||||
"rightVideo": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)",
|
||||
"leftPts": rf"{part}/{cap}_camera_left_{part}(?:_[a-z0-9_]+)?_pts\.csv",
|
||||
"rightPts": rf"{part}/{cap}_camera_right_{part}(?:_[a-z0-9_]+)?_pts\.csv",
|
||||
"imu": rf"{part}/{cap}_imu_{part}\.csv",
|
||||
"cameraCalibration": rf"{cap}_calibration_camera\.yaml",
|
||||
"imuCalibration": rf"{cap}_calibration_imu\.yaml",
|
||||
"metadata": rf"meta_{cap}\.json",
|
||||
}
|
||||
for key in keys:
|
||||
if not key.startswith(prefix + "/") or str(pathlib.PurePosixPath(key)) != key or ".." in pathlib.PurePosixPath(key).parts or any(c in key for c in ("\\", "\x00")):
|
||||
raise RuntimeError("unsafe v4 original source object")
|
||||
relative = key[len(prefix) + 1:]
|
||||
if "/" in relative and not relative.startswith(part_id + "/"):
|
||||
raise RuntimeError("v4 original selection contains another part")
|
||||
for role, pattern in patterns.items():
|
||||
if re.fullmatch(pattern, relative, re.IGNORECASE):
|
||||
if role in files: raise RuntimeError(f"ambiguous v4 original {role} selection")
|
||||
files[role] = {"path": relative}
|
||||
missing = sorted(set(patterns) - files.keys())
|
||||
if missing: raise RuntimeError("v4 missing original files: " + ", ".join(missing))
|
||||
return {"provider": "BOS", "bucket": source["bucket"], "prefix": prefix, "partPrefix": part_prefix,
|
||||
"snapshotFingerprint": str(source.get("snapshotFingerprint") or ""), "files": files}
|
||||
|
||||
|
||||
def _v4_number(value, label):
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
||||
raise RuntimeError(f"v4 missing or invalid {label}")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _v4_timestamp(value, label):
|
||||
if type(value) is not int or value <= 0: raise RuntimeError(f"v4 missing or invalid {label}")
|
||||
return value
|
||||
|
||||
|
||||
def _v4_consumer_payload(payload, manifest, source_root):
|
||||
"""Project immutable accepted facts into a self-contained consumer contract.
|
||||
|
||||
No dictionary lookup, source media reads, transcoding or BOS writes here.
|
||||
Hardware PTS are authoritative; media seconds are explicitly named and never
|
||||
used to manufacture missing hardware timestamps.
|
||||
"""
|
||||
if not isinstance(manifest, dict): raise RuntimeError("v4 requires accepted source manifest")
|
||||
original = _v4_original_sources(manifest, payload["capture"]["captureId"], payload["partId"], payload["capture"]["prefix"])
|
||||
payload["originalSource"] = original
|
||||
payload["consumerContract"] = "droboticflow.classification-consumer/v1"
|
||||
payload["timeBases"] = {"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video",
|
||||
"segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts",
|
||||
"interval": "half-open", "frameRanges": "zero-based-half-open"}
|
||||
payload["pathBases"] = {"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
|
||||
# Workspace sources are private transport references, not the original files.
|
||||
payload["sources"] = {kind: original["files"][kind] for kind in ("leftVideo", "rightVideo", "leftPts", "rightPts", "imu")}
|
||||
payload["sourceMetadata"]["path"] = original["files"]["metadata"]["path"]
|
||||
for item in payload["calibration"]:
|
||||
# Bind by actual calibration kind, not input ordering.
|
||||
kind = "imuCalibration" if "imu" in item["name"].lower() else "cameraCalibration" if "camera" in item["name"].lower() else ""
|
||||
if not kind: raise RuntimeError("v4 unknown calibration kind")
|
||||
item["path"] = original["files"][kind]["path"]
|
||||
rows = {str(row["sliceId"]): row for row in manifest["items"]}
|
||||
if len(rows) != len(manifest["items"]): raise RuntimeError("v4 duplicate slice identities")
|
||||
for item in payload["slices"] + payload["discardedSlices"]:
|
||||
split = item["taskSplit"]
|
||||
start, end = (_v4_number(split[k], k) for k in ("startSecond", "endSecond"))
|
||||
first, last = (_v4_timestamp(split[k], k) for k in ("startTimestampUs", "endTimestampUs"))
|
||||
if start < 0 or end <= start or last <= first: raise RuntimeError("v4 invalid accepted slice range")
|
||||
if item["classification"]["deliveryDisposition"] == "discard":
|
||||
item["sliceName"] = ""
|
||||
evidence = item["classification"]["discardEvidence"]
|
||||
selected = evidence.get("dictionary")
|
||||
if selected is not None:
|
||||
if (not isinstance(selected, dict) or selected.get("kind") != "discard_reason" or
|
||||
not selected.get("id") or not selected.get("name") or
|
||||
type(selected.get("version")) is not int or selected["version"] < 1):
|
||||
raise RuntimeError("v4 invalid frozen discard dictionary")
|
||||
for alias, field in (("reasonLabelId", "id"), ("reasonName", "name"), ("reasonVersion", "version")):
|
||||
if evidence.get(alias) not in (None, "", 0, selected[field]):
|
||||
raise RuntimeError("v4 discard reason disagrees with frozen dictionary")
|
||||
evidence[alias] = selected[field]
|
||||
continue
|
||||
parent_tags = _v4_tags(item["classification"].get("dictionary"), "slice")
|
||||
item["classification"]["tags"] = parent_tags
|
||||
row = rows[item["sliceId"]]
|
||||
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
|
||||
seconds, stamps = sync.get("boundarySeconds", []), sync.get("boundaryTimestampUs", [])
|
||||
if seconds or stamps:
|
||||
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
|
||||
raise RuntimeError("v4 invalid hardware boundary mapping")
|
||||
seconds = [_v4_number(value, "boundary") for value in seconds]
|
||||
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
|
||||
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
|
||||
stamps[0] != first or stamps[-1] != last or
|
||||
any(a >= b for a, b in zip(seconds, seconds[1:])) or
|
||||
any(a >= b for a, b in zip(stamps, stamps[1:]))):
|
||||
raise RuntimeError("v4 invalid hardware boundary mapping")
|
||||
def timestamp(second):
|
||||
if abs(second) < 1e-7: return first
|
||||
if abs(second - (end-start)) < 1e-7: return last
|
||||
hits = [stamps[i] for i, value in enumerate(seconds) if abs(_v4_number(value,"boundary") - second) < 1e-7]
|
||||
if len(hits) != 1: raise RuntimeError("v4 action boundary has no original PTS mapping")
|
||||
return _v4_timestamp(hits[0], "boundary timestamp")
|
||||
actions = []
|
||||
for action in item["segments"]:
|
||||
a = _v4_number(action.get("clipStartMs"), "action clipStartMs") / 1000
|
||||
b = _v4_number(action.get("clipEndMs"), "action clipEndMs") / 1000
|
||||
if a < -1e-7 or b <= a or b > end-start+1e-7: raise RuntimeError("v4 action escapes its parent slice")
|
||||
ta, tb = (_v4_timestamp(action.get(k), k) for k in ("startTimestampUs", "endTimestampUs"))
|
||||
if not first <= ta < tb <= last: raise RuntimeError("v4 action timestamps escape its parent slice")
|
||||
if seconds and (timestamp(a) != ta or timestamp(b) != tb): raise RuntimeError("v4 action seconds disagree with original PTS")
|
||||
action.update(startSecond=a, endSecond=b, partStartSecond=start+a, partEndSecond=start+b)
|
||||
own = action.get("dictionary")
|
||||
action["tags"] = _v4_tags(own, "action") if own is not None else list(parent_tags)
|
||||
action["tagSource"] = "action" if own is not None else "slice"
|
||||
actions.append(action)
|
||||
actions.sort(key=lambda a: a["startSecond"])
|
||||
complete, previous = [], 0.0
|
||||
def gap(a,b):
|
||||
ta,tb=timestamp(a),timestamp(b)
|
||||
return {"id":f"{item['sliceId']}:gap:{ta}:{tb}", "description":"", "isInvalid":False,
|
||||
"contentKind":"gap_placeholder", "startSecond":a,"endSecond":b,
|
||||
"clipStartMs":a*1000,"clipEndMs":b*1000,"partStartSecond":start+a,"partEndSecond":start+b,
|
||||
"startTimestampUs":ta,"endTimestampUs":tb,"tags":list(parent_tags),"tagSource":"slice"}
|
||||
for action in actions:
|
||||
if action["startSecond"] < previous-1e-7: raise RuntimeError("v4 overlapping actions")
|
||||
if action["startSecond"] > previous+1e-7: complete.append(gap(previous,action["startSecond"]))
|
||||
complete.append(action);previous=action["endSecond"]
|
||||
if previous < end-start-1e-7: complete.append(gap(previous,end-start))
|
||||
if not complete: raise RuntimeError("v4 kept slice has no classified action coverage")
|
||||
previous_stamp = first
|
||||
for action in complete:
|
||||
if action["startTimestampUs"] != previous_stamp or action["endTimestampUs"] <= previous_stamp:
|
||||
raise RuntimeError("v4 action PTS coverage is discontinuous")
|
||||
previous_stamp = action["endTimestampUs"]
|
||||
if previous_stamp != last: raise RuntimeError("v4 action PTS coverage is incomplete")
|
||||
item["segments"] = complete
|
||||
counts = {kind:sum(a["contentKind"]==kind for a in complete) for kind in ("annotated","gap_placeholder","invalid_marked")}
|
||||
item["classification"]["contentCounts"] = counts
|
||||
item["classification"]["descriptionStatus"] = _description_status(complete)
|
||||
if not counts["annotated"] and counts["invalid_marked"] and counts["gap_placeholder"]: item["classification"]["descriptionStatus"]="invalid_and_blank"
|
||||
item["synchronization"]["rightBoundarySeconds"] = sync.get("rightBoundarySeconds", [])
|
||||
|
||||
|
||||
def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None, revision_manifest=None, source_root=None):
|
||||
"""Use the established immutable publisher for versioned customer JSON.
|
||||
|
||||
V3 retains its historical sibling review file; v4 stores that audit in a
|
||||
@ -929,6 +1097,8 @@ def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payl
|
||||
for key in ("source", "confidence", "phrases", "labelId"):
|
||||
action.pop(key, None)
|
||||
if action.get("dictionary") is None: action.pop("dictionary", None)
|
||||
audit["materializedSources"] = payload.get("sources", {})
|
||||
_v4_consumer_payload(payload, revision_manifest, source_root)
|
||||
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8")
|
||||
audit["partSha256"] = hashlib.sha256(body).hexdigest()
|
||||
if len(json.dumps(audit, ensure_ascii=False, indent=2).encode("utf8")) > 8 * 1024 * 1024:
|
||||
@ -1782,7 +1952,7 @@ def _run_accepted_slice_directory(
|
||||
item["sourceFrames"] = frames[item["sliceId"]]
|
||||
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
|
||||
if _bool(params, "DELIVERY_JSON_V4", False):
|
||||
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload)
|
||||
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload, revision_manifest, source_root)
|
||||
if _bool(params, "DELIVERY_JSON_V3", False):
|
||||
return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
|
||||
68
tests/backend/consume_delivery_v4.py
Normal file
68
tests/backend/consume_delivery_v4.py
Normal file
@ -0,0 +1,68 @@
|
||||
"""Read-only independent fixture consumer: stdlib, JSON and original files only.
|
||||
|
||||
Produces a processing plan; never cuts, copies, uploads or modifies source data.
|
||||
"""
|
||||
import argparse
|
||||
import bisect
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def inspect(part, original):
|
||||
original=original.resolve()
|
||||
def file(relative):
|
||||
p=(original/relative).resolve()
|
||||
if not p.is_relative_to(original) or not p.is_file():raise ValueError("original file unavailable or outside source root")
|
||||
return p
|
||||
assert part["consumerContract"]=="droboticflow.classification-consumer/v1"
|
||||
assert part["timeBases"]["timestampUs"]=="original_hardware_pts"
|
||||
assert part["timeBases"]["interval"]=="half-open"
|
||||
source=part["originalSource"]
|
||||
assert source["provider"]=="BOS" and source["bucket"]
|
||||
assert source["prefix"]==part["capture"]["prefix"]
|
||||
assert source["partPrefix"]==source["prefix"]+"/"+part["partId"]
|
||||
assert part["pathBases"]["sources"]=="original_capture"
|
||||
assert part["sources"]=={k:source["files"][k] for k in ("leftVideo","rightVideo","leftPts","rightPts","imu")}
|
||||
clocks={}
|
||||
for side in ("left","right"):
|
||||
file(part["sources"][side+"Video"]["path"])
|
||||
with file(part["sources"][side+"Pts"]["path"]).open() as f:clocks[side]=[int(row["timestamp_us"]) for row in csv.DictReader(f)]
|
||||
assert clocks[side] and all(a<b for a,b in zip(clocks[side],clocks[side][1:]))
|
||||
for c in part["calibration"]:assert hashlib.sha256(file(c["path"]).read_bytes()).hexdigest()==c["sha256"]
|
||||
file(part["sourceMetadata"]["path"])
|
||||
with file(part["sources"]["imu"]["path"]).open() as f:imu=[int(r["timestamp_us"]) for r in csv.DictReader(f)]
|
||||
covered={side:[] for side in clocks};actions=[];discard=[]
|
||||
def range_for(a,b):return {side:[bisect.bisect_left(pts,a),bisect.bisect_left(pts,b)] for side,pts in clocks.items()}
|
||||
for item in sorted(part["slices"]+part["discardedSlices"],key=lambda s:s["taskSplit"]["startTimestampUs"]):
|
||||
parent=item["taskSplit"];a,b=parent["startTimestampUs"],parent["endTimestampUs"]
|
||||
if item["classification"]["deliveryDisposition"]=="discard":
|
||||
assert item["segments"]==[]
|
||||
reason=item["classification"]["discardEvidence"];assert reason["reason"]
|
||||
frames=range_for(a,b);discard.append({"sliceId":item["sliceId"],"reason":reason,"frames":frames})
|
||||
for side,(start,end) in frames.items():covered[side]+=list(range(start,end))
|
||||
continue
|
||||
previous=a
|
||||
for action in item["segments"]:
|
||||
start,end=action["startTimestampUs"],action["endTimestampUs"]
|
||||
assert start==previous and start<end<=b
|
||||
previous=end
|
||||
assert abs(action["partStartSecond"]-parent["startSecond"]-action["startSecond"])<1e-6
|
||||
assert abs(action["partEndSecond"]-parent["startSecond"]-action["endSecond"])<1e-6
|
||||
dictionary=action.get("dictionary") if action["tagSource"]=="action" else item["classification"]["dictionary"]
|
||||
assert dictionary["tags"]==action["tags"] and action["tags"]
|
||||
frames=range_for(start,end)
|
||||
for side,(first,last) in frames.items():
|
||||
assert last>first
|
||||
covered[side]+=list(range(first,last))
|
||||
actions.append({"slice":item["sliceId"],"tags":action["tags"],"tagSource":action["tagSource"],"dictionaryId":dictionary["id"],"dictionaryVersion":dictionary["version"],"invalid":action["isInvalid"],"range":[action["partStartSecond"],action["partEndSecond"]],"timestampsUs":[start,end],"frames":frames,"imuRows":bisect.bisect_left(imu,end)-bisect.bisect_left(imu,start)})
|
||||
assert previous==b
|
||||
low=part["coverage"]["startTimestampUs"];high=part["coverage"]["endTimestampUs"]
|
||||
for side,pts in clocks.items():assert covered[side]==[i for i,t in enumerate(pts) if low<=t<high]
|
||||
return {"passed":True,"actions":actions,"discarded":discard,"coveredFrames":{side:len(frames) for side,frames in covered.items()},"sourceModified":False}
|
||||
|
||||
|
||||
if __name__=="__main__":
|
||||
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument("part",type=Path);parser.add_argument("original",type=Path)
|
||||
args=parser.parse_args();print(json.dumps(inspect(json.loads(args.part.read_text()),args.original),ensure_ascii=False))
|
||||
@ -6,7 +6,7 @@ import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from backend.operators.builtin.package import operator as package
|
||||
from tests.backend.test_delivery_v4_package import NODE, ACTION, prepare
|
||||
from tests.backend.test_delivery_v4_package import NODE, ACTION, prepare, original_source
|
||||
from tests.backend.test_part_index_package import helpers
|
||||
|
||||
|
||||
@ -18,7 +18,7 @@ def export(destination):
|
||||
inputs=prepare(work,[{"id":"blank","description":"","clipStartMs":0,"clipEndMs":500,"startTimestampUs":1000000,"endTimestampUs":1500000},{"id":"pick","description":"拿起杯子","dictionary":ACTION,"clipStartMs":500,"clipEndMs":1000,"startTimestampUs":1500000,"endTimestampUs":2000000}])
|
||||
inputs[1]["data"]["capture"].update(captureId="capture-A/"+part,prefix="incoming/capture-A/"+part)
|
||||
manifest_path=Path(inputs[0]["path"])/"revision-manifest.json"
|
||||
m=json.loads(manifest_path.read_text());m["runId"]="source-run-"+part
|
||||
m=json.loads(manifest_path.read_text());m["runId"]="source-run-"+part;m["originalSource"]=original_source(part=part)
|
||||
if part=="part0002":
|
||||
m["items"][0]["deliveryDisposition"]="discard"
|
||||
m["items"][0]["discardEvidence"]=m["items"][1]["discardEvidence"]
|
||||
|
||||
124
tests/backend/test_delivery_v4_consumer.py
Normal file
124
tests/backend/test_delivery_v4_consumer.py
Normal file
@ -0,0 +1,124 @@
|
||||
"""Consumers must classify and locate every interval without platform workspaces."""
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from backend.operators.builtin.package import operator as package
|
||||
from tests.backend.test_delivery_v4_package import ACTION, BASE, NODE, prepare, read_outputs
|
||||
from tests.backend.test_part_index_package import helpers
|
||||
from tests.backend.test_slice_discard_package import accepted_inputs
|
||||
|
||||
|
||||
def test_tags_inherit_by_dictionary_selection_not_description_text(tmp_path):
|
||||
inputs=prepare(tmp_path,[{"description":""},{"description":"人工补充文字"},{"description":"","dictionary":ACTION},{"description":"无效","isInvalid":True}])
|
||||
doc,_,_=read_outputs(tmp_path,inputs)
|
||||
actions=doc["slices"][0]["segments"]
|
||||
assert [a["tags"] for a in actions]==[BASE["tags"],BASE["tags"],ACTION["tags"],BASE["tags"]]
|
||||
assert [a["tagSource"] for a in actions]==["slice","slice","action","slice"]
|
||||
assert actions[-1]["isInvalid"] is True
|
||||
assert "dictionary" not in actions[0] # Inheritance must not invent an action selection.
|
||||
assert actions[2]["dictionary"]==ACTION
|
||||
assert doc["consumerContract"]=="droboticflow.classification-consumer/v1"
|
||||
assert doc["pathBases"]["sources"]=="original_capture"
|
||||
assert "workspaceKey" not in json.dumps(doc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure,match",[("parent-tags","slice tags"),("action-tags","action tags"),("source","original BOS"),("timestamps","startTimestampUs"),("overlap","overlapping"),("clock","disagree"),("missing-file","missing original files"),("unsafe-key","unsafe"),("boundary","boundary mapping"),("pts-gap","discontinuous"),("discard-dictionary","discard reason disagrees")])
|
||||
def test_v4_refuses_incomplete_consumer_instructions(tmp_path,failure,match):
|
||||
inputs=prepare(tmp_path,[{"description":"one"},{"description":"two","dictionary":dict(ACTION)}])
|
||||
root=Path(inputs[0]["path"])
|
||||
manifest_path=root/"revision-manifest.json"
|
||||
manifest=json.loads(manifest_path.read_text())
|
||||
sync_path=root/"slices/slice-000001/synchronized-slice.json"
|
||||
sync=json.loads(sync_path.read_text())
|
||||
ann_path=root/"slices/slice-000001/annotation.json"
|
||||
ann=json.loads(ann_path.read_text())
|
||||
if failure=="parent-tags":sync["dictionary"]["tags"]=[]
|
||||
elif failure=="action-tags":ann["actions"][1]["dictionary"]["tags"]=[]
|
||||
elif failure=="source":manifest.pop("originalSource")
|
||||
elif failure=="missing-file":manifest["originalSource"]["objectKeys"]=[k for k in manifest["originalSource"]["objectKeys"] if "camera_left" not in k]
|
||||
elif failure=="timestamps":ann["actions"][0].pop("startTimestampUs")
|
||||
elif failure=="overlap":ann["actions"][1].update(clipStartMs=0,startTimestampUs=1000000)
|
||||
elif failure=="clock":ann["actions"][0]["endTimestampUs"]-=1
|
||||
elif failure=="unsafe-key":manifest["originalSource"]["objectKeys"][0]=manifest["originalSource"]["objectKeys"][0].replace("/part0001/", "//part0001/")
|
||||
elif failure=="boundary":sync["boundaryTimestampUs"][-1]-=1
|
||||
elif failure=="discard-dictionary":manifest["items"][1]["discardEvidence"]["reasonVersion"]=999
|
||||
elif failure=="pts-gap":
|
||||
sync.pop("boundarySeconds");sync.pop("boundaryTimestampUs")
|
||||
ann["actions"][1]["startTimestampUs"]+=1
|
||||
sync_path.write_text(json.dumps(sync));ann_path.write_text(json.dumps(ann));manifest_path.write_text(json.dumps(manifest))
|
||||
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(manifest_path.read_bytes()).hexdigest()
|
||||
with pytest.raises(RuntimeError,match=match):read_outputs(tmp_path,inputs)
|
||||
|
||||
|
||||
def test_legacy_partial_annotations_keep_inherited_gap_coverage(tmp_path):
|
||||
inputs=prepare(tmp_path,[{"description":"拿起","dictionary":ACTION,"clipStartMs":250,"clipEndMs":750}])
|
||||
doc,_,_=read_outputs(tmp_path,inputs)
|
||||
actions=doc["slices"][0]["segments"]
|
||||
assert [(a["startSecond"],a["endSecond"],a["tagSource"]) for a in actions]==[(0,.25,"slice"),(.25,.75,"action"),(.75,1,"slice")]
|
||||
assert [(a["startTimestampUs"],a["endTimestampUs"]) for a in actions]==[(1000000,1250000),(1250000,1750000),(1750000,2000000)]
|
||||
|
||||
|
||||
def test_discard_aliases_use_frozen_dictionary_without_zero_version(tmp_path):
|
||||
inputs=prepare(tmp_path,[])
|
||||
manifest_path=Path(inputs[0]["path"])/"revision-manifest.json"
|
||||
manifest=json.loads(manifest_path.read_text())
|
||||
reason=manifest["items"][1]["discardEvidence"]
|
||||
reason.update(reasonLabelId="",reasonName="",reasonVersion=0)
|
||||
manifest_path.write_text(json.dumps(manifest))
|
||||
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(manifest_path.read_bytes()).hexdigest()
|
||||
doc,_,_=read_outputs(tmp_path,inputs)
|
||||
reason=doc["discardedSlices"][0]["classification"]["discardEvidence"]
|
||||
assert reason["reasonLabelId"]==reason["dictionary"]["id"]
|
||||
assert reason["reasonVersion"]==reason["dictionary"]["version"]==4
|
||||
assert reason["reasonName"]==reason["dictionary"]["name"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode",["physical","source-index"])
|
||||
def test_real_v4_roundtrip_uses_only_customer_json_and_original_data(tmp_path,monkeypatch,mode):
|
||||
root=os.environ.get("FANYU_DELIVERY_CONSUMER_FIXTURE_DIR")
|
||||
if not root:pytest.skip("generate retained local Go source fixture first")
|
||||
context=json.loads((Path(root)/mode/"context.json").read_text())
|
||||
artifact=context["artifact"]
|
||||
inputs=accepted_inputs(artifact)[:2]
|
||||
inputs[1]["data"].update(trace=[],qualityReviews=[])
|
||||
inputs[1]["data"]["finalAcceptance"]["acceptedAt"]="2026-10-21T00:00:00Z"
|
||||
original=Path(context["original"])
|
||||
for kind in ("camera","imu"):
|
||||
p=original/f"capture-A_calibration_{kind}.yaml"
|
||||
inputs.append({"portId":"calibrations","name":p.name,"path":str(p)})
|
||||
node={**NODE,"hyperparams":NODE["hyperparams"]+[{"key":"SOURCE_INDEX_DELIVERY","value":"true"}]}
|
||||
with monkeypatch.context() as guard:
|
||||
def no_subprocess(*args, **kwargs):
|
||||
raise AssertionError("classification export must not transcode or recut source data")
|
||||
original_sha = package._sha256_file
|
||||
def metadata_only(path):
|
||||
assert Path(path).suffix not in {".mp4", ".mov", ".mkv"}, "classification export rehashed video bodies"
|
||||
return original_sha(path)
|
||||
guard.setattr(package.subprocess, "run", no_subprocess)
|
||||
guard.setattr(package.subprocess, "Popen", no_subprocess)
|
||||
guard.setattr(package, "_sha256_file", metadata_only)
|
||||
result=package.run("consumer","package",tmp_path/"out",node,inputs,helpers())
|
||||
doc=json.loads((Path(result[0]["path"])/"part.json").read_text())
|
||||
assert not any(word in json.dumps(doc) for word in ("workspaceKey","system/runs/","PRIVATE_CONNECTION"))
|
||||
# A fresh process can consume copies of only the customer file and original source.
|
||||
consumer=tmp_path/"downstream";shutil.copytree(original,consumer/"original")
|
||||
(consumer/"part.json").write_text(json.dumps(doc))
|
||||
before={p.relative_to(consumer/"original"):hashlib.sha256(p.read_bytes()).hexdigest() for p in (consumer/"original").rglob("*") if p.is_file()}
|
||||
result=subprocess.run([sys.executable,str(Path(__file__).with_name("consume_delivery_v4.py").resolve()),str(consumer/"part.json"),str(consumer/"original")],cwd=consumer,capture_output=True,text=True,check=True)
|
||||
assert before=={p.relative_to(consumer/"original"):hashlib.sha256(p.read_bytes()).hexdigest() for p in (consumer/"original").rglob("*") if p.is_file()}
|
||||
report=json.loads(result.stdout)
|
||||
assert report["passed"] and report["sourceModified"] is False
|
||||
assert report["coveredFrames"]=={"left":32,"right":32}
|
||||
groups=report["actions"]
|
||||
assert [g["tags"] for g in groups]==[["商品","手部"],["抓取"],["商品","手部"],["商品","手部"]]
|
||||
assert [g["invalid"] for g in groups]==[False,False,True,False]
|
||||
assert [g["range"] for g in groups]==[[0,.5],[.5,1],[3,3.5],[3.5,4]]
|
||||
assert doc["slices"][1]["segments"][0]["startSecond"]==0
|
||||
assert doc["slices"][1]["segments"][0]["partStartSecond"]==3
|
||||
@ -19,16 +19,34 @@ BASE = {"id": "slice-word", "kind": "slice", "version": 2, "code": "base", "name
|
||||
ACTION = {"id": "action-word", "kind": "action", "version": 3, "code": "pick", "name": "拿取", "tags": ["抓取"]}
|
||||
|
||||
|
||||
def original_source(capture="capture-A", part="part0001"):
|
||||
root=f"incoming/{capture}"
|
||||
paths=[f"{part}/{capture}_camera_{side}_{part}{suffix}" for side in ("left","right") for suffix in (".mp4","_pts.csv")]
|
||||
paths += [f"{part}/{capture}_imu_{part}.csv",f"{capture}_calibration_camera.yaml",f"{capture}_calibration_imu.yaml",f"meta_{capture}.json"]
|
||||
return {"provider":"BOS","bucket":"test-bucket","prefix":root,"partPrefix":root+"/"+part,"captureUnit":capture+"/"+part,"objectKeys":[root+"/"+p for p in paths],"snapshotFingerprint":"synthetic-source-version"}
|
||||
|
||||
|
||||
def prepare(tmp_path, actions):
|
||||
inputs = inputs_for(tmp_path, actions)
|
||||
root = Path(inputs[0]["path"])
|
||||
sync = root / "slices/slice-000001/synchronized-slice.json"
|
||||
doc = json.loads(sync.read_text())
|
||||
doc["dictionary"] = BASE
|
||||
n=max(1,len(actions))
|
||||
for i,action in enumerate(actions):
|
||||
action.setdefault("id",f"action-{i}")
|
||||
action.setdefault("clipStartMs",i*1000/n)
|
||||
action.setdefault("clipEndMs",(i+1)*1000/n)
|
||||
action.setdefault("startTimestampUs",1000000+round(action["clipStartMs"]*1000))
|
||||
action.setdefault("endTimestampUs",1000000+round(action["clipEndMs"]*1000))
|
||||
(root/"slices/slice-000001/annotation.json").write_text(json.dumps({"actions":actions}))
|
||||
points=sorted({0.0,1.0,*[a[k]/1000 for a in actions for k in ("clipStartMs","clipEndMs")]})
|
||||
doc.update(boundarySeconds=points,boundaryTimestampUs=[1000000+round(p*1000000) for p in points],rightBoundarySeconds=points)
|
||||
sync.write_text(json.dumps(doc))
|
||||
manifest_path = root / "revision-manifest.json"
|
||||
m = json.loads(manifest_path.read_text())
|
||||
m["runId"] = "source-run"
|
||||
m["originalSource"] = original_source()
|
||||
m["items"][0]["sliceName"] = BASE["name"]
|
||||
reason = m["items"][1]["discardEvidence"]
|
||||
reason.update(reasonLabelId="discard-word", reasonVersion=4, reasonName="遮挡", reasonNote="持续遮挡", at="2026-10-20T00:00:00Z", dictionary={"id":"discard-word","kind":"discard_reason","version":4,"code":"occluded","name":"遮挡","tags":["不可用"]})
|
||||
@ -86,6 +104,7 @@ def test_v4_all_discard_outputs_classification_and_legacy_stays_unchanged(tmp_pa
|
||||
inputs = fixture(tmp_path,["discard","discard"])
|
||||
inputs[1]["data"].update(capture={"captureId":"capture-A/part0001","prefix":"incoming/capture-A/part0001"},trace=[])
|
||||
inputs[1]["data"]["finalAcceptance"]["acceptedAt"] = "2026-10-20T01:00:00Z"
|
||||
root=Path(inputs[0]["path"]);p=root/"revision-manifest.json";m=json.loads(p.read_text());m["originalSource"]=original_source();p.write_text(json.dumps(m));inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
doc, _, _ = read_outputs(tmp_path, inputs)
|
||||
assert doc["slices"] == [] and len(doc["discardedSlices"]) == 2
|
||||
assert doc["deliveryClassification"]["discardedSliceCount"] == 2
|
||||
|
||||
@ -1,22 +1,28 @@
|
||||
"""Consume real Go materialization revisions, including metadata-only reuse."""
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from backend.operators.builtin.package import operator as package
|
||||
from tests.backend.test_part_index_real_materialization import load_context
|
||||
from tests.backend.test_part_index_package import helpers
|
||||
from tests.backend.test_delivery_v4_package import NODE
|
||||
from tests.backend.test_delivery_v4_package import NODE, original_source
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["keep", "mixed", "discard", "blank"])
|
||||
def test_v4_real_materialization_retains_final_dictionary_snapshots(tmp_path, mode):
|
||||
context=load_context()
|
||||
artifact=context[mode]["artifact"]
|
||||
root=Path(artifact["path"])
|
||||
root=tmp_path/"accepted"
|
||||
shutil.copytree(artifact["path"],root)
|
||||
artifact={**artifact,"path":str(root)}
|
||||
mpath=root/"revision-manifest.json"
|
||||
manifest=json.loads(mpath.read_text())
|
||||
# Explicit synthetic original reference for this historical, local-only fixture.
|
||||
manifest["originalSource"]=original_source("real-capture")
|
||||
mpath.write_text(json.dumps(manifest))
|
||||
summary={"decision":"approved","processingProfile":"raw_human","capture":{"captureId":"real-capture/part0001","prefix":"incoming/real-capture/part0001"},"trace":[],"qualityReviews":[],"acceptedMaterializations":[{"artifactId":artifact["id"],"revision":manifest["revision"],"manifestSha256":hashlib.sha256(mpath.read_bytes()).hexdigest()}],"finalAcceptance":{"confirmed":True,"acceptedAt":"2026-10-21T00:00:00Z","processingProfile":"raw_human","confirmedByUserId":"internal-acceptor"}}
|
||||
inputs=[{**artifact,"portId":"acceptedSliceDirectory"},{"portId":"acceptanceSummary","data":summary}]
|
||||
for kind in ("camera","imu"):
|
||||
@ -37,7 +43,12 @@ def test_v4_real_materialization_retains_final_dictionary_snapshots(tmp_path, mo
|
||||
assert snap["version"]==2 and snap["tags"]==["基础分类","商品"]
|
||||
if mode=="blank": assert item["classification"]["descriptionStatus"]=="empty_confirmed"
|
||||
for action in item["segments"]:
|
||||
assert action["dictionary"]["version"]==4 and action["dictionary"]["tags"]==["抓取"]
|
||||
if "dictionary" in action:
|
||||
assert action["dictionary"]["version"]==4 and action["dictionary"]["tags"]==["抓取"]
|
||||
assert action["tags"]==["抓取"] and action["tagSource"]=="action"
|
||||
else:
|
||||
assert mode=="blank"
|
||||
assert action["tags"]==snap["tags"] and action["tagSource"]=="slice"
|
||||
for item in part["discardedSlices"]:
|
||||
reason=item["classification"]["discardEvidence"]
|
||||
assert reason["note"]=="画面被遮挡" and reason["dictionary"]["tags"]==["不可用"]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user