📦 Publish immutable delivery index batches with native v3 packages

This commit is contained in:
d-robotics 2026-10-20 12:14:56 +08:00
parent d10c148b21
commit 7be23604d7
23 changed files with 1912 additions and 15 deletions

View File

@ -0,0 +1,29 @@
# 平台调优与交付反馈修复(2026-10-20)
范围来自平台调优群及马迪历史交付反馈;BOS 拷贝超时按用户要求暂缓。
用户已授权本地修改、验证和提交;未授权 PR、push、部署或客户数据写入。
| 问题 | 修复与验收标准 |
| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
| 1 描述未保存、其他片草稿拦切片应用 | 完整草稿与切片改动一并进入冲突处置及同一保存事务;失败保留、确认成功才清编辑器;取消保留草稿 |
| 2 第三轮意见跳回旧轮、滚动回弹 | 意见身份为轮次 + ID;仅明确定位事件触发滚动;旧轮同 ID 不可误判最新轮 |
| 3 审查字幕不可见 | 专注布局显式占行;留空、无效、描述、读取失败分别显示;字幕开关有效 |
| 4 账号“我的任务/我已审核”重叠 | 同一采集不同修订可以同时待办及留历史;显示当前/历史版本及复查含义,不改分配权限 |
| 5 原生总 JSON 交付未启用 | 原生 v3 打包 → 冻结 part 目录条件发布 → 管理员预览/发布不可变批次 → 现有校验下载链路;默认保持关闭 |
| 6 审核历史混在主 JSON | qualityReviews 移到独立、钉住同一修订的 review-history 文件;主 JSON 保留最终结论 |
| 7 采集清单缺 meta | annotation_manifest 引用原始 meta,API 读取验证并记录 SHA/大小 |
| 8 路径基准混淆 | 总索引相对批次项目根;采集清单相对采集交付目录;meta/标定相对原始采集目录;workspace 来源引用另行标明 |
| 9 空任务标签 | 省略空 taskLabel/taskLabelId,保留有效旧标签 |
| 10 描述汇总误报 | 由非无效且非空文本判断 described;v3 区分纯留空、纯无效、无效与留空混合,并带内容数量 |
| 11 多 part 样例和汇总验证 | 同一采集两个 part 经真实打包入口及 API、Mongo 副本验证范围、修订、验收聚合、SHA、重试和旧批次保留;合成样例不得充当客户真实业务验收 |
## 已确认决定
2026-10-20 用户选择不可变交付批次目录。批次根为冻结项目交付根下
`index-batches/<snapshotId>/`;总索引最后发布。以后新增 part 生成另一批次,旧批次保持可校验。
旧已完成交付不原地补文件;重新交付、工作流开关和线上写测试需要单独计划与授权。
## 尚待线上完成
部署前必须按 `docs/release/README.md` 准备具体镜像和变更计划,取得紧邻部署的授权。
本次没有客户真实的同采集双 part 已验收样例;本地合成双 part 已验证,真实样例与客户确认仍待具备数据后完成。

View File

@ -168,3 +168,7 @@ _Avoid_:供应商任务副本(不代表给每家生产供应商复制客户
**撤销供应商项目授权**:停止供应商领取该项目的新任务,但保留其既有任务的标注、质检与返工处理资格,直至存量完成;不等于停用账号或释放已领取任务。
**生产统计异常**:仅指当前正式处理中的执行故障,包括运行失败、源采集处理失败、切片或送检处理作业失败、交付打包失败。供应商归属不同、部分未领取、归属待核对、审核退回、业务升级处理及正常等待不单独计为异常;这些状态保留在明细,不增加红色异常数量。
## 交付批次
**交付批次快照**:一次明确选定、已验收的 part 修订集合及其交付清单。批次只表示本次交付范围;新增 part 或替换修订形成新批次,已交付批次保留。批次完成不代表原始采集的全部 part 已交齐。

View File

@ -836,11 +836,67 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
``empty_confirmed`` keeps its platform meaning: no description was written
and the submitter confirmed that as a deliberate semantic blank.
"""
if not actions:
return "empty_confirmed"
if all(bool(action.get("isInvalid")) for action in actions):
if any(not action.get("isInvalid") and str(action.get("description") or "").strip() for action in actions):
return "described"
if actions and all(bool(action.get("isInvalid")) for action in actions):
return "invalid_only"
return "described"
return "empty_confirmed"
def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload):
"""Use the established immutable directory publisher for the two JSON files.
Legacy single-file publishers deliberately accept only v1/v2. A directory
artifact preserves their ownership/conditional-publication protocol without
teaching an old worker to misinterpret a new single-file schema.
"""
capture = dict(payload["capture"])
prefix = pathlib.PurePosixPath(str(capture.get("prefix") or ""))
part_id = prefix.name
capture_id = prefix.parent.name
recorded = str(capture.get("captureId") or "")
if not re.fullmatch(r"part\d{4,}", part_id) or recorded not in {capture_id, capture_id + "/" + part_id}:
raise RuntimeError("v3 capture identity disagrees with the original part prefix")
if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3:
raise RuntimeError("unsafe v3 capture identity prefix")
capture.update(captureId=capture_id, prefix=str(prefix.parent))
payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id)
payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"}
payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"}
history = {
"schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": capture_id,
"partId": part_id, "revision": payload["revision"],
"qualityReviews": payload["acceptance"].pop("qualityReviews", []),
}
payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"}
payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None)
for item in payload["calibration"]:
if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]:
raise RuntimeError("unsafe calibration filename")
item["path"] = item["name"]
for item in payload["slices"] + payload["discardedSlices"]:
classification = item["classification"]
for key in ("taskLabel", "taskLabelId"):
if not str(classification.get(key) or "").strip():
classification.pop(key, None)
classification.get("discardEvidence", {}).pop("discardedByUserId", None)
item.get("synchronization", {}).pop("calibration", None)
for action in item["segments"]:
action.pop("status", None)
action["contentKind"] = "invalid_marked" if action["isInvalid"] else "described" if action["description"].strip() else "gap_placeholder"
if classification["deliveryDisposition"] == "keep":
counts = {kind: sum(action["contentKind"] == kind for action in item["segments"]) for kind in ("described", "invalid_marked", "gap_placeholder")}
classification["contentCounts"] = counts
if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]:
classification["descriptionStatus"] = "invalid_and_blank"
staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output))
for name, document in (("part.json", payload), ("review-history.json", history)):
(staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8")
return [helpers["build_virtual_artifact"](
run_id, node_id, "delivery.directory.index", str(staging), "file", 0,
data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"])},
output=_output(node_data, helpers, "deliveryDirectory"),
)]
def _part_index_slice(
@ -1659,6 +1715,8 @@ def _run_accepted_slice_directory(
for item in payload["slices"] + payload["discardedSlices"]:
item["sourceFrames"] = frames[item["sliceId"]]
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
if _bool(params, "DELIVERY_JSON_V3", False):
return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload)
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
if source_index:
source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output)
@ -1894,6 +1952,9 @@ def _run_multiport(
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False):
raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY")
# A part-index delivery intentionally has no directory port; it must still
# reach the accepted-slice path instead of the legacy inline contract.
if (

View File

@ -31,3 +31,15 @@ status: accepted
## 审核层不受影响
物化修订里的逐片视频仍是质检与验收工作台的输入(`MaterializedSliceWorkbench`、`FinalAcceptanceWorkspace` 依赖 `slices/slice-*/left.mp4`)。本决定只改变**交付层**的输出形态,不改变内部审核层。
## 2026-10-20 修订:不可变批次及独立审核历史
用户明确选择**不可变交付批次目录**,拒绝在固定项目根反复覆盖总索引。每次选择已验收 part 修订后,内容身份生成新的 `index-batches/<snapshotId>/`,其中依次为 part JSON/独立审核历史、采集 `annotation_manifest.json`、最后写入的 `delivery_index.json`。重试只能接受相同内容,不能覆盖冲突对象或接管无归属的非空目录;旧批次继续可下载校验。批次标识绑定规范化源 JSON、原始 meta 校验信息和原始 part 目录清单,不包含生成时间或签名 URL。
新格式用默认关闭的 `DELIVERY_JSON_V3`,同时要求 `PART_INDEX_DELIVERY`;源索引修订还必须显式启用 `SOURCE_INDEX_DELIVERY`。原生算子输出 `part.json` 与 `review-history.json`,复用现有 directory artifact 发布通道和归属/条件创建规则,不要求旧 Worker 识别新的单文件 schema。此处 `manifest.json.kind=directory` 表示两个 JSON 的传输集合,不表示包含 clips;旧 v1/v2 单文件协议与旧目录交付保留。
主 JSON 移除 `acceptance.qualityReviews`,独立历史文件钉住 capture/part/revision;主文件保留最终验收与抽检结论。空任务标签省略,有效标签保留。描述状态依据实际有效文本而非 actions 数量;v3 新增 `invalid_and_blank` 表达无效和留空混合,保留 `invalid_only` 的“全部区间无效”含义。
路径基准显式写入:总索引相对本批次项目根,采集清单相对采集交付目录,原始 meta 和标定文件相对原始采集目录;已固定 workspace 来源另标基准。采集清单记录原始 `meta_<captureId>.json` 的 SHA/大小,expected parts 来自原始目录完整分页列举,不用已验收 part 数推定交齐。最终验收结论及最大 acceptedAt 的聚合仅覆盖本批次选择的修订。
总索引生成由管理员显式预览、按预览 snapshotId 条件发布,不随普通 GET/下载自动写对象存储。旧已完成 part 目录不能原地补发 v3,历史迁移仍需另行规划新交付目标。已有调用者和在途任务默认沿用旧协议;回退 API/开关不删除新批次,也不能把新格式任务改成旧格式后重用已完成目录。

View File

@ -1,5 +1,7 @@
# Fanyu 最终交付批量下载(Rust)
2026-10-20 新增 [不可变总 JSON 批次](delivery-index-batches.md) 的管理员预览/显式发布,以及按批次目录下载。旧下载协议保持兼容;批次以 `delivery_index.json` 和归属标记识别完成。
只增加后端读取接口和 `tools/drobotic-pull` 命令行客户端,不修改标注界面、任务分配、调度或交付格式。原 POC 的目录工具不是 Fanyu 的已上线能力;本版本使用独立的、只面向最终交付的协议。
## 使用

View File

@ -0,0 +1,56 @@
# 不可变总 JSON 交付批次
此能力为本地实现,尚未上线;部署及工作流启用以具体发布记录和授权为准。
原生打包需要新 package 镜像,且 `PART_INDEX_DELIVERY=true`、`DELIVERY_JSON_V3=true`;源索引修订另需 `SOURCE_INDEX_DELIVERY=true`。三个开关默认关闭。旧已交付目录保持不变,不能把 v3 补入已完成的旧目录。原生产物通过已有目录发布协议生成两个 JSON 及 worker 完成清单;它不复制媒体。
```text
<冻结项目交付根>/index-batches/<snapshotId>/
delivery_index.json # 本批次最后写入
<captureId>/
annotation_manifest.json # 此采集 part/history 完成后写入
part0001.json
part0001.review-history.json
part0002.json
part0002.review-history.json
```
`delivery_index.json` 的 `entries[].path`、`captures[].manifestPath` 相对批次项目根。
`annotation_manifest.json` 的 `entries[].path`、`parts[].path`、`reviewHistoryPath` 相对采集交付目录。
`sourceMetadata.path` 与 `calibration[].path` 相对 `capture.prefix` 指向的原始采集目录,标定只带真实文件名,不再额外拼 calibration 或采集名。
源索引 `sources[].workspaceKey` 保留固定修订与 SHA 引用,解析基准为 workspace,不冒充原始 meta 的路径。
主 part 的 `captureId` 为原始采集文件夹名,`partId` 为 part 文件夹名,`revision` 钉住已验收修订。
审核历史只在独立文件。有效 taskLabel/taskLabelId 保留,空值省略。
`segments[].contentKind` 为 `described`、`invalid_marked`、`gap_placeholder`;内部流转 status 不导出。
`classification.descriptionStatus` 为:有非无效非空文本 `described`;全部无效 `invalid_only`;无效与留空混合且无有效描述 `invalid_and_blank`;否则 `empty_confirmed`。`contentCounts` 给出三种内容数量,绝不把动作数组非空当作有描述。
`summary` 的片数、时长、验收与最大 `acceptedAt` 只针对本批次。时长按 taskSplit 的秒区间汇总,验收时间按实际时间值比较。`expectedPartCount` 来自原始目录完整分页列举,`undeliveredParts` 和 `partial_run/full_run` 不等同于“本批次发布已完成”。全舍弃且无交付物的 part 继续走既有 no_deliverables 契约,本批次不把它冒充已交付 part。
## 管理操作
沿用 `tools/drobotic-pull` 登录、CSRF 和退出,不在命令参数中放密码。显式选择 package ID,绝不自动猜最新修订或从不同交付目标混合数据。
```sh
# 只读预览:打印 snapshotId、根目录和总索引,不写对象存储。
drobotic-pull --api https://platform.example --email admin@example.com \
--project project-xxx --index-package package-part1 --index-package package-part2
# 审阅预览后用完全相同的选择和 snapshotId 发布;这是明确的存储写操作。
drobotic-pull --api https://platform.example --email admin@example.com \
--project project-xxx --index-package package-part1 --index-package package-part2 \
--publish-index <预览的snapshotId>
# 下载指定批次,沿用 SHA 校验、签名刷新与文件级续传。
drobotic-pull --api https://platform.example --email admin@example.com \
--directory 'bos://bucket/<冻结项目根>/index-batches/<snapshotId>' \
--connection storage-xxx --output ./deliveries
```
接口为 `GET /api/delivery-index-batches?projectId=...&packageId=...&packageId=...` 和
`POST /api/delivery-index-batches`,请求体 `projectId/packageIds/snapshotId`。
仅组织管理员可操作,发布连接必须属于当前组织,或具备既有部署映射中明确的交付写授权(连接归属、deliveryPrefixes 和每个来源运行的 writePrefixes 同时匹配);只读委派不提供写权限,原始输入前缀始终不写。目标从冻结 part 目标推导,不能由调用者指定;与原始采集、其他交付目标、启用的接入监听前缀重叠即拒绝。
最多 100 个 part、单 JSON 8 MiB、批次 64 MiB、原始目录列举 50000 对象;超限报错,不静默截断。发布逐对象条件创建并读回逐字节校验;中断后使用同一 snapshotId 重试。没有总索引的半成品不能下载;不自动清理、覆盖旧批次。证据变化返回 409,重新预览。旧批次下载不重新推导“当前最新包”。
合成双 part 示例可用 `PYTHONPATH=. python tests/backend/export_delivery_v3_fixture.py <新的本地输出目录>` 生成。它使用真实算子入口,明确标记 synthetic,不能当客户真实业务验收。

View File

@ -2,7 +2,9 @@
适用范围:`fanyu-deliveries/` 下的最终交付目录。一条目录对应一个采集(Capture)的一个 part。
交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定:
2026-10-20 新实现的不可变 v3 批次参见 [总 JSON 批次说明](../delivery-index-batches.md):默认关闭,尚待发布与工作流启用。它将审核历史分离、补全 meta 引用,并新增采集/项目清单。下文保持旧 v1/v2 与媒体目录契约,供历史交付及在途任务解析。
旧交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定:
| 形态 | 开关 | 交付内容 |
| --- | --- | --- |

View File

@ -0,0 +1,45 @@
# 平台调优反馈修复记录(2026-10-20)
实现分支 `codex/platform-feedback-fixes-20261020`,基于当前生产 API 来源 `630a95c8`。
工作在独立 worktree,未修改主工作区和 dev-mainline 的在途改动;没有 Git push、PR、部署或客户数据写入。
BOS 拷贝超时按用户要求排除。
## 修复范围
完整问题与验收标准见 [spec](../../../.scratch/platform-feedback-20261020/spec.md)。
| 范围 | 本地结果 | 线上状态 |
| --- | --- | --- |
| 描述草稿与切片应用 | 完整草稿进入同一冲突处置与保存事务;失败、取消保留,确认后才清除;保存状态不再把待保存修改显示成已保存 | 待发布 |
| 跨轮意见定位、滚动回弹 | 轮次 + ID 定位;滚动只响应定位事件;判定键拒绝旧轮同 ID | 待发布 |
| 审查字幕 | 专注布局固定行;留空/无效/有效文本/读取异常清楚显示 | 待发布 |
| 账号任务与历史重叠 | 保留既有分配语义;解释当前待办与历史版本,提示新修订复查 | 待发布 |
| 总 JSON 与审核历史/meta/路径/标签/描述状态 | 原生 v3 两 JSON 交付、不可变批次、独立历史、原始 meta 校验引用、显式路径基准、空标签省略、内容驱动汇总 | 待新镜像与工作流开关 |
| 多 part | 同采集双 part 通过真实打包入口、Worker 发布协议、API、独立本地 Mongo 数据库、恢复与校验下载 | 合成验证完成;真实客户双 part 样例及验收仍待具备输入 |
草稿问题的原因是切片应用沿用全局草稿守卫,其他片的停泊草稿会阻断应用;修复将完整草稿纳入事务和冲突检查。意见定位只用 ID,跨轮复用 ID 会同时命中;内联 ref 又在重新渲染时重复滚动。描述汇总将非空动作数组直接当作 described,未检查空白占位/无效内容。回归测试分别钉住这些触发条件,不能只验证“页面能打开”。
## 验证证据
| 检查 | 结果 |
| --- | --- |
| 新浏览器回归(真实组件 + Playwright + 合成媒体) | 通过:草稿冲突应用、刷新保留、取消、保存拒绝后重试、三轮同 ID、手动滚动后重渲染、字幕布局/状态/开关、当前/历史版本 |
| 前端完整门禁 | 通过基线比较:13 项既有失败,无新增;逐用例和原因对照 `630a95c8`,未修改 known-failures |
| TypeScript / Web 构建 | 通过;构建有原有大 bundle 提示 |
| Python 交付回归 | 59 passed,1 skipped;跳过的是原有可选 60 秒物化夹具用例(缺 FANYU_DISCARD_MATERIALIZATION_CONTEXT) |
| 源索引媒体夹具 | 通过 Go 本地生成及 Python v2/v3 不解码/不重算整视频验证,固定 sources/sourceFrames 保留 |
| Go 新交付与本地副本 | API、deliveryindex、Worker 通过;本地 Mongo 独立 `delivery_index_batch_test_*` 库已清理,不读写快照客户集合 |
| 切片保存、边界证据、撤销/重做、历史兼容 | 定向 API 回归通过 |
| 交付兼容范围 | 46 个原有/新增用例通过;`TestDeliveryDownloadScopeAndCompletion` 失败在基线同样可复现(原夹具运行产物未放入 nodeExecutions),未修改其断言掩盖失败 |
| Rust 客户端 | 6 个 library + 7 个 CLI 测试通过;clippy `--all-targets -- -D warnings` 通过 |
| 发布流程工具与切片恢复 | 37 个 release unittest 通过;TestDurableAnnotationOutbox/TestSliceRecovery 通过 |
| 算子生成文件 | generate_operator_repositories.py --check 通过 |
浏览器命令:`BROWSER_EXECUTABLE=<本机 Chromium 系浏览器> node tests/browser/run-platform-feedback.mjs`。
原生双 part 生成:`PYTHONPATH=. python tests/backend/export_delivery_v3_fixture.py <新的本地目录>`。
Go 串联验证:设置 `DELIVERY_V3_FIXTURE_DIR=<该目录>`,执行 API/Worker 的 `TestIndexBatch` 与 `TestCaptureDeliveryNativeV3`;`DELIVERY_INDEX_MONGO_HTTP=1` 仅连接硬编码的本地 27018,使用新建合成库。
## 发布边界
见 [发布计划](rollout-plan.md)。成功 rollout 不是业务验收。
本地合成/组件回归不证明现网已修好;本次不把旧聊天 v3 ZIP 或合成双 part 当成正式客户交付验收。

View File

@ -0,0 +1,29 @@
# 平台反馈修复的分步发布计划
本文件是待审阅计划,不是部署授权。执行时先读 `docs/release/README.md`,存在正式 release 记录后先运行 `python3 scripts/release.py status <id>`。候选必须绑定提交、镜像 digest、目标身份和测试证据;不得用本地 tag 代替正式 digest。
## 1. API / annotation processor
- 候选代码包含前端草稿/轮次/字幕/版本说明修复及默认未启用的 v3 交付能力。无数据库迁移,无账号/权限/客户任务修复脚本,无启动数据改写新增。
- API 和 annotation processor 按 release.py 使用同一不可变镜像。保留现网资源、副本、调度、存储、ConfigMap、Secret、Service、外部 source-index 处理模式和 verify-existing 启动设置。
- 前端和 API 同一镜像更新;回退到原 API/processor 镜像,已保存的工作区仍是原协议。回退后草稿全局守卫与跨轮定位旧行为可能再现,必须先让使用者保存并关闭任务。
- 重新核实 Fanyu API server、context、namespace UID,检查处理中的提交/物化状态后安排切换。只读检查优先;客户保存、领取、送检等验证必须另得精确授权。
- 完成本地 prepare/check、镜像来源和启动安全审查,展示 release.py plan 后取得紧邻部署的明确授权,才运行 deploy。业务验收前不把发布记录标成完成。
## 2. package 算子及工作流(单独配置计划)
这部分不属于 release.py 首版镜像更新范围,不随第 1 步自动修改。
- 新 package 镜像固定 digest,使用本分支生成后的 `operator-repos/builtin/package/src/operator.py`。Worker、validator、coordinator 镜像不变;现网 Worker 的目录发布实现与基线相同,新 v3 复用其协议,新增回归已通过。
- 在获准的工作流修订中显式配置 `PART_INDEX_DELIVERY=true`、`SOURCE_INDEX_DELIVERY=true`、`DELIVERY_JSON_V3=true`,输出仍用现有 deliveryDirectory。默认值均 false。
- 仅新建且冻结到未占用目标的交付使用新格式。在途工作流/任务保持原镜像、原开关、原修订;不得直接给已完成目录补 v3 文件。
- 管理员 GET 预览明确选定的 package IDs,核对范围、修订、expected parts、meta 与交付前缀。POST 发布会创建 BOS 对象,必须另行取得该批次和精确前缀的写授权。
- 委派连接只复用已有明确的交付写授权,不扩大 mapping;批次目标或来源运行不在批准范围则拒绝,需先审查单独配置变更。
- 客户真实的同采集双 part 验收后,生成正式样例,分别核对 part/采集/项目汇总和下载 SHA。当前合成验证不能替代这一项。
- 回退只关闭后续新任务开关或恢复旧镜像;已发布批次保留,已开始的新格式任务不可改写成旧格式后覆盖同一冻结目录。
## 3. 历史交付
四个旧已交付采集仍保留原目录、manifest 和归属。需要正式 v3 重交时,另列源修订、全新交付目标、兼容和回退影响,经授权再执行;本次不改写历史客户文件。
BOS 拷贝超时没有纳入本次变更。

View File

@ -7848,6 +7848,18 @@
"key": "PART_INDEX_DELIVERY",
"type": "boolean",
"value": "false"
},
{
"key": "SOURCE_INDEX_DELIVERY",
"type": "boolean",
"value": "false",
"description": "显式允许源索引修订的 JSON 交付;与 PART_INDEX_DELIVERY 同时启用"
},
{
"key": "DELIVERY_JSON_V3",
"type": "boolean",
"value": "false",
"description": "原生 v3:part.json 与独立审核历史,通过目录发布;须同时启用 PART_INDEX_DELIVERY"
}
],
"id": "operator-builtin-package",

View File

@ -109,9 +109,13 @@ func (s *server) handleDeliveryDownloadProjects(w http.ResponseWriter, r *http.R
}
func (s *server) downloadBrowser(ctx context.Context, org, id, bucket, prefix string) (objectStorageBrowser, func(), error) {
return s.deliveryReadBrowser(ctx, org, id, bucket, prefix, true)
}
func (s *server) deliveryReadBrowser(ctx context.Context, org, id, bucket, prefix string, deliveryOnly bool) (objectStorageBrowser, func(), error) {
c, e := s.store.GetStorageConnection(ctx, id)
if e == nil && c != nil && c.OrganizationID != org {
c, e = s.delegatedStorageRead(ctx, org, id, bucket, prefix, true)
c, e = s.delegatedStorageRead(ctx, org, id, bucket, prefix, deliveryOnly)
}
if e != nil || c == nil || c.Provider != domain.StorageProviderBOS {
return nil, nil, fmt.Errorf("交付存储不存在或不可访问")
@ -127,6 +131,10 @@ func (s *server) projectDownloadTargets(r *http.Request, id string) ([]deliveryD
if e != nil {
return nil, e
}
return s.packageDownloadTargets(r, id, packages)
}
func (s *server) packageDownloadTargets(r *http.Request, id string, packages []domain.DeliveryPackageRecord) ([]deliveryDownloadTarget, error) {
targets := []deliveryDownloadTarget{}
seen := map[string]bool{}
for _, pkg := range packages {
@ -229,7 +237,7 @@ func discoverDeliveryDirectories(ctx context.Context, b objectStorageBrowser, bu
}
if item.Kind == "directory" {
children = append(children, key)
} else if key == dir+"/manifest.json" {
} else if key == dir+"/manifest.json" || key == dir+"/delivery_index.json" {
complete = true
}
}
@ -270,7 +278,7 @@ func readDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliver
}
raw, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/manifest.json", deliveryDownloadMaxManifestBytes+1)
if e != nil || len(raw) > deliveryDownloadMaxManifestBytes {
return nil, fmt.Errorf("交付清单不可读或过大")
return readIndexBatchDownloadManifest(ctx, b, t)
}
var m struct {
SchemaVersion int `json:"schemaVersion"`

View File

@ -0,0 +1,446 @@
package api
// Batch publication is an explicit admin operation. GET is a read-only preview;
// POST requires the exact preview ID and only conditionally creates new objects.
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"path"
"sort"
"strings"
"time"
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/workspacescope"
)
type indexBatchRequest struct {
ProjectID string `json:"projectId"`
PackageIDs []string `json:"packageIds"`
SnapshotID string `json:"snapshotId"`
}
type conditionalIndexWriter interface {
PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
}
type batchPlan struct {
snapshot deliveryindex.Snapshot
browser objectStorageBrowser
target deliveryDownloadTarget
cleanup func()
}
func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPlan, error) {
if q.ProjectID == "" || len(q.PackageIDs) == 0 || len(q.PackageIDs) > 100 {
return nil, fmt.Errorf("选择 1–100 个正式验收交付包")
}
project, e := s.store.GetProject(r.Context(), q.ProjectID)
if e != nil || project == nil || project.OrganizationID != s.requestOrganizationID(r) {
return nil, fmt.Errorf("项目不存在或不可访问")
}
packages, e := s.deliveryDownloadPackages(r, q.ProjectID)
if e != nil {
return nil, e
}
selected := []domain.DeliveryPackageRecord{}
wanted := map[string]bool{}
for _, id := range q.PackageIDs {
if id == "" || wanted[id] {
return nil, fmt.Errorf("交付包选择重复或为空")
}
wanted[id] = true
}
for _, pkg := range packages {
if wanted[pkg.ID] {
selected = append(selected, pkg)
}
}
if len(selected) != len(wanted) {
return nil, fmt.Errorf("仅可选择本项目已正式验收且有最终交付物的包")
}
targets, e := s.packageDownloadTargets(r, q.ProjectID, selected)
if e != nil {
return nil, e
}
if len(targets) != len(selected) {
return nil, fmt.Errorf("每个交付包必须有唯一且已完成的最终目标")
}
t := targets[0]
root := path.Dir(path.Dir(t.prefix))
if !strings.HasSuffix(path.Base(root), "__"+q.ProjectID) || !domain.ValidDeliveryPrefix(root) {
return nil, fmt.Errorf("交付路径缺少冻结项目层")
}
c, e := s.store.GetStorageConnection(r.Context(), t.connectionID)
if e != nil || c == nil {
return nil, fmt.Errorf("批次交付连接不存在")
}
b, cleanup, e := s.downloadBrowser(r.Context(), s.requestOrganizationID(r), t.connectionID, t.bucket, t.prefix)
if e != nil {
return nil, e
}
success := false
defer func() {
if !success && cleanup != nil {
cleanup()
}
}()
sources := []deliveryindex.Source{}
var sourceBytes int64
sourceRuns := []string{}
for _, target := range targets {
if target.connectionID != t.connectionID || target.bucket != t.bucket || path.Dir(path.Dir(target.prefix)) != root {
return nil, fmt.Errorf("选择的包不在同一冻结项目交付根")
}
files, e := readDownloadManifest(r.Context(), b, target)
if e != nil {
return nil, e
}
if len(files) != 3 {
return nil, fmt.Errorf("原生 v3 part 包必须只含 part.json 与独立审核历史")
}
var partBytes, historyBytes []byte
for _, f := range files {
if f.key != target.prefix+"/part.json" && f.key != target.prefix+"/review-history.json" {
continue
}
if f.SizeBytes > 8<<20 {
return nil, fmt.Errorf("单份 JSON 超过 8 MiB")
}
raw, _, e := b.ReadObjectPreview(r.Context(), f.bucket, f.key, 8<<20+1)
if e != nil || int64(len(raw)) != f.SizeBytes || deliveryindex.Hash(raw) != f.SHA256 {
return nil, fmt.Errorf("交付 JSON 与完成清单校验不一致")
}
sourceBytes += int64(len(raw))
if sourceBytes > 64<<20 {
return nil, fmt.Errorf("选择的源 JSON 合计超过 64 MiB,请缩小批次")
}
if strings.HasSuffix(f.key, "/part.json") {
partBytes = raw
} else {
historyBytes = raw
}
}
if len(partBytes) == 0 || len(historyBytes) == 0 {
return nil, fmt.Errorf("选择的包尚未由原生 v3 算子交付;旧格式必须另行重交,不会覆盖旧文件")
}
var part struct {
Capture struct {
ID string `json:"captureId"`
Prefix string `json:"prefix"`
} `json:"capture"`
PartID string `json:"partId"`
Lineage struct {
RunID string `json:"sourceRunId"`
} `json:"lineage"`
}
if json.Unmarshal(partBytes, &part) != nil || target.prefix != root+"/"+part.Capture.ID+"/"+part.PartID {
return nil, fmt.Errorf("part 身份与冻结交付目标不一致")
}
record, e := s.store.GetCaptureRecordByRunID(r.Context(), part.Lineage.RunID)
if e != nil || record == nil || record.OrganizationID != project.OrganizationID || record.ProjectID != project.ID || record.Prefix != part.Capture.Prefix+"/"+part.PartID || record.Delivery == nil || record.Delivery.Prefix != target.prefix || record.Delivery.ConnectionID != t.connectionID || record.Delivery.Bucket != t.bucket {
return nil, fmt.Errorf("part 来源与采集记录不一致")
}
sourceRuns = append(sourceRuns, part.Lineage.RunID)
sourceBrowser, closeSource, e := s.deliveryReadBrowser(r.Context(), project.OrganizationID, record.ConnectionID, record.Bucket, part.Capture.Prefix, false)
if e != nil {
return nil, e
}
expected, meta, e := indexBatchSourceEvidence(r.Context(), sourceBrowser, record.Bucket, part.Capture.Prefix, part.Capture.ID)
if closeSource != nil {
closeSource()
}
if e != nil {
return nil, e
}
sources = append(sources, deliveryindex.Source{Part: partBytes, History: historyBytes, ExpectedParts: expected, MetadataSHA256: deliveryindex.Hash(meta), MetadataBytes: int64(len(meta))})
}
snapshot, e := deliveryindex.Build(project.ID, project.Name, root, sources)
if e != nil {
return nil, e
}
if e := indexBatchWriteScope(project.OrganizationID, c, t.bucket, snapshot.Root, sourceRuns); e != nil {
return nil, e
}
validationFiles := make([]deliveryDownloadFile, 0, len(snapshot.Files))
for name, body := range snapshot.Files {
if len(body) > 8<<20 {
return nil, fmt.Errorf("规范化后的单份 JSON 超过 8 MiB,请缩小交付内容")
}
validationFiles = append(validationFiles, deliveryDownloadFile{RelativePath: name, SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)})
}
if _, e := finalizeDownloadManifest(validationFiles); e != nil {
return nil, e
}
// Share the existing publisher's overlap rules; destination is server-derived.
captures, e := s.store.ListCaptureRecords(r.Context(), project.OrganizationID, "", 0)
if e != nil {
return nil, e
}
for _, record := range captures {
if record.Delivery != nil && record.Delivery.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Delivery.Prefix, snapshot.Root) {
return nil, fmt.Errorf("批次目录与已有采集交付目标重叠")
}
if record.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Prefix, snapshot.Root) {
return nil, fmt.Errorf("批次目录与来源采集重叠")
}
}
listeners, e := s.store.ListEnabledCaptureAutomations(r.Context(), project.OrganizationID)
if e != nil {
return nil, e
}
for _, workflow := range listeners {
if c := workflow.CaptureAutomation; c != nil && c.Bucket == t.bucket && domain.StoragePrefixesOverlap(c.Prefix, snapshot.Root) {
return nil, fmt.Errorf("批次目录与已启用接入监听目录重叠")
}
}
t.prefix = snapshot.Root
t.relative = path.Base(root) + "/index-batches/" + snapshot.ID
success = true
return &batchPlan{snapshot, b, t, cleanup}, nil
}
// Borrowed connections require the deployment's existing explicit final-output
// grant and approved source runs. Read delegation never becomes write access.
func indexBatchWriteScope(org string, c *domain.StorageConnectionRecord, bucket, prefix string, sourceRuns []string) error {
m, err := workspacescope.ForOrganization(org)
if err != nil {
return err
}
if c.OrganizationID == org && (m == nil || m.ConnectionID != c.ID) {
return nil
}
if m == nil || m.ConnectionID != c.ID || m.Bucket != bucket || !workspacescope.PrefixAllows(prefix, m.DeliveryPrefixes) {
return fmt.Errorf("批次目录超出已授权交付前缀")
}
if c.OrganizationID != org {
if m.WriteConnectionOrganizationID != c.OrganizationID {
return fmt.Errorf("委派连接没有明确交付写授权")
}
for _, runID := range sourceRuns {
if !workspacescope.PrefixAllows("system/runs/"+runID, m.WritePrefixes) {
return fmt.Errorf("批次来源运行不在已批准委派范围")
}
}
}
return nil
}
func indexBatchSourceEvidence(ctx context.Context, b objectStorageBrowser, bucket, prefix, captureID string) ([]string, []byte, error) {
paged, ok := b.(pagedObjectStorageBrowser)
if !ok {
return nil, nil, fmt.Errorf("原始存储不支持完整分页列举")
}
expected := map[string]bool{}
marker := ""
markers := map[string]bool{}
count := 0
for {
page, e := paged.ListObjectsPage(ctx, bucket, prefix+"/", marker, 1000)
if e != nil {
return nil, nil, fmt.Errorf("列举原始 part 失败")
}
for _, item := range page.Items {
count++
if count > 50000 {
return nil, nil, fmt.Errorf("原始采集目录超过 50000 对象")
}
if !strings.HasPrefix(item.Key, prefix+"/") {
return nil, nil, fmt.Errorf("原始列举返回越界对象")
}
first := strings.SplitN(strings.TrimPrefix(item.Key, prefix+"/"), "/", 2)[0]
if isIndexPartName(first) {
expected[first] = true
}
}
if !page.HasMore {
break
}
if page.NextMarker == "" || page.NextMarker == marker || markers[page.NextMarker] {
return nil, nil, fmt.Errorf("原始存储分页无进展")
}
marker = page.NextMarker
markers[marker] = true
}
meta, _, e := b.ReadObjectPreview(ctx, bucket, prefix+"/meta_"+captureID+".json", 8<<20+1)
if e != nil || len(meta) == 0 || len(meta) > 8<<20 || !json.Valid(meta) {
return nil, nil, fmt.Errorf("原始 meta JSON 不存在、不可读或过大")
}
parts := []string{}
for p := range expected {
parts = append(parts, p)
}
sort.Strings(parts)
if len(parts) == 0 {
return nil, nil, fmt.Errorf("原始采集没有 part 清单")
}
return parts, meta, nil
}
func isIndexPartName(s string) bool {
if !strings.HasPrefix(s, "part") || len(s) < 8 {
return false
}
for _, c := range s[4:] {
if c < '0' || c > '9' {
return false
}
}
return true
}
// A failed/ambiguous conditional PUT is successful only when a subsequent read
// verifies the exact bytes. No overwrite, deletion or rollback of old batches.
func createIndexObject(ctx context.Context, b objectStorageBrowser, writer conditionalIndexWriter, bucket, key string, body []byte) error {
_, err := writer.PutObjectConditional(ctx, bucket, key, body, "application/json", "", "*", map[string]string{"sha256": deliveryindex.Hash(body)})
got, _, e := b.ReadObjectPreview(ctx, bucket, key, int64(len(body))+1)
if e == nil && bytes.Equal(got, body) {
return nil
}
if err != nil {
return fmt.Errorf("批次对象条件创建失败或存在冲突")
}
return fmt.Errorf("批次对象写入后校验失败")
}
func publishIndexBatch(ctx context.Context, plan *batchPlan) error {
writer, ok := plan.browser.(conditionalIndexWriter)
if !ok {
return fmt.Errorf("存储不支持条件创建,拒绝发布")
}
owner, _ := json.Marshal(map[string]string{"owner": "index-batch:" + plan.snapshot.ID})
// Empty-prefix proof before taking ownership. Existing marker must match;
// read failures never count as absence: a successful complete listing proves it.
markerKey := plan.target.prefix + "/.delivery-owner.json"
existing, _, err := plan.browser.ReadObjectPreview(ctx, plan.target.bucket, markerKey, 4096)
if err == nil {
if !bytes.Equal(existing, owner) {
return fmt.Errorf("批次归属冲突")
}
} else {
paged, ok := plan.browser.(pagedObjectStorageBrowser)
if !ok {
return fmt.Errorf("存储无法校验空目录")
}
page, e := paged.ListObjectsPage(ctx, plan.target.bucket, plan.target.prefix+"/", "", 1)
if e != nil || len(page.Items) > 0 || page.HasMore {
return fmt.Errorf("批次目录非空或无法确认,拒绝接管")
}
}
if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, markerKey, owner); e != nil {
return e
}
names := []string{}
for name := range plan.snapshot.Files {
if !validDeliveryDownloadPath(name) {
return fmt.Errorf("不安全批次路径")
}
if name != "delivery_index.json" {
names = append(names, name)
}
}
sort.Strings(names)
// Capture manifests follow their part/history files; total index is last.
sort.SliceStable(names, func(i, j int) bool {
a := strings.HasSuffix(names[i], "/annotation_manifest.json")
b := strings.HasSuffix(names[j], "/annotation_manifest.json")
return !a && b
})
for _, name := range append(names, "delivery_index.json") {
if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, plan.target.prefix+"/"+name, plan.snapshot.Files[name]); e != nil {
return e
}
}
return nil
}
func (s *server) handleDeliveryIndexBatch(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Referrer-Policy", "no-referrer")
principal, ok := currentPrincipal(r)
if !ok || principal == nil || principal.User.Role != domain.UserRoleAdmin {
renderJSON(w, 403, map[string]string{"error": "交付批次生成仅限组织管理员"})
return
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
defer cancel()
r = r.WithContext(ctx)
q := indexBatchRequest{ProjectID: r.URL.Query().Get("projectId"), PackageIDs: r.URL.Query()["packageId"]}
if r.Method == http.MethodPost {
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
decoder.DisallowUnknownFields()
if decoder.Decode(&q) != nil || q.SnapshotID == "" {
renderJSON(w, 400, map[string]string{"error": "发布必须提交预览的 snapshotId、projectId 和 packageIds"})
return
}
}
plan, e := s.planIndexBatch(r, q)
if e != nil {
renderJSON(w, 409, map[string]string{"error": e.Error()})
return
}
if plan.cleanup != nil {
defer plan.cleanup()
}
if r.Method == http.MethodPost {
if q.SnapshotID != plan.snapshot.ID {
renderJSON(w, 409, map[string]string{"error": "交付证据已变化,请重新预览", "code": "index_snapshot_changed"})
return
}
if e := publishIndexBatch(ctx, plan); e != nil {
renderJSON(w, 409, map[string]string{"error": e.Error()})
return
}
}
renderJSON(w, 200, map[string]any{"snapshotId": plan.snapshot.ID, "rootKey": plan.snapshot.Root, "index": plan.snapshot.Index, "published": r.Method == http.MethodPost})
}
// Immutable batches participate in the existing directory-download protocol,
// including per-file hash checks, resume snapshots and refreshed signed URLs.
func readIndexBatchDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliveryDownloadTarget) ([]deliveryDownloadFile, error) {
id := path.Base(t.prefix)
digest, digestErr := hex.DecodeString(id)
if digestErr != nil || len(digest) != 32 || path.Base(path.Dir(t.prefix)) != "index-batches" {
return nil, fmt.Errorf("交付清单不可读或过大")
}
marker, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/.delivery-owner.json", 4096)
var owner struct {
Owner string `json:"owner"`
}
if e != nil || json.Unmarshal(marker, &owner) != nil || owner.Owner != "index-batch:"+id {
return nil, fmt.Errorf("批次归属标记无效")
}
raw, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/delivery_index.json", deliveryDownloadMaxManifestBytes+1)
if e != nil || len(raw) > deliveryDownloadMaxManifestBytes {
return nil, fmt.Errorf("批次尚未完成或清单过大")
}
var index struct {
Schema int `json:"schemaVersion"`
Kind string `json:"kind"`
ID string `json:"snapshotId"`
Count int `json:"fileCount"`
Bytes int64 `json:"totalBytes"`
Project struct {
Root string `json:"rootKey"`
} `json:"project"`
Entries []deliveryindex.Entry `json:"entries"`
}
if json.Unmarshal(raw, &index) != nil || index.Schema != 3 || index.Kind != "part-index" || index.ID != id || index.Project.Root != t.prefix || index.Count != len(index.Entries) || index.Count < 1 || index.Count > 400 {
return nil, fmt.Errorf("批次总索引身份或成员数量无效")
}
files := []deliveryDownloadFile{}
var size int64
for _, entry := range index.Entries {
sha, shaErr := hex.DecodeString(entry.SHA256)
if !validDeliveryDownloadPath(entry.Path) || entry.Path == "delivery_index.json" || strings.HasPrefix(entry.Path, ".") || entry.Key != t.prefix+"/"+entry.Path || entry.SizeBytes < 0 || entry.SizeBytes > 8<<20 || shaErr != nil || len(sha) != 32 {
return nil, fmt.Errorf("批次成员校验信息无效")
}
size += entry.SizeBytes
files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/" + entry.Path, SizeBytes: entry.SizeBytes, SHA256: entry.SHA256, key: entry.Key, bucket: t.bucket, connectionID: t.connectionID})
}
if size != index.Bytes || size > 64<<20 {
return nil, fmt.Errorf("批次大小校验不一致")
}
files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/delivery_index.json", SizeBytes: int64(len(raw)), SHA256: deliveryindex.Hash(raw), key: t.prefix + "/delivery_index.json", bucket: t.bucket, connectionID: t.connectionID})
return files, nil
}

View File

@ -0,0 +1,40 @@
package api
import (
"context"
"fmt"
"os"
"testing"
"time"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// Local replica only. A fresh synthetic database is created and removed; no
// snapshot/customer collection is opened and the URI cannot be overridden.
func TestIndexBatchMongoReplicaRecovery(t *testing.T) {
if os.Getenv("DELIVERY_INDEX_MONGO_HTTP") != "1" {
t.Skip("opt-in isolated local Mongo replica verification")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(context.Background())
if e = client.Ping(ctx, nil); e != nil {
t.Fatal(e)
}
db := client.Database(fmt.Sprintf("delivery_index_batch_test_%d", time.Now().UnixNano()))
defer func() {
cleanup, done := context.WithTimeout(context.Background(), 10*time.Second)
defer done()
if e := db.Drop(cleanup); e != nil {
t.Error(e)
}
}()
exerciseIndexBatch(t, store.NewMongoStore(db))
}

View File

@ -0,0 +1,301 @@
package api
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
type indexTestBrowser struct {
*downloadTestBrowser
puts []string
fail string
ambiguous bool
}
func (b *indexTestBrowser) PutObjectConditional(_ context.Context, _ string, key string, body []byte, _ string, match, none string, _ map[string]string) (domain.ObjectStorageObjectSummary, error) {
if match != "" || none != "*" {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("unconditional write")
}
b.puts = append(b.puts, key)
if strings.HasSuffix(key, b.fail) && b.fail != "" {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("injected interruption")
}
if _, ok := b.objects[key]; ok {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("precondition failed")
}
b.objects[key] = append([]byte(nil), body...)
if b.ambiguous {
b.ambiguous = false
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("response lost after commit")
}
return domain.ObjectStorageObjectSummary{}, nil
}
func (b *indexTestBrowser) ListObjectsPage(_ context.Context, _ string, prefix, marker string, limit int) (objectStoragePage, error) {
if p, ok := b.pages[prefix+"|"+marker]; ok {
return p, nil
}
keys := []string{}
seen := map[string]bool{}
for key := range b.objects {
if strings.HasPrefix(key, prefix) {
rest := strings.TrimPrefix(key, prefix)
first, _, nested := strings.Cut(rest, "/")
item := prefix + first
if nested {
item += "/"
}
if !seen[item] && item > marker {
seen[item] = true
keys = append(keys, item)
}
}
}
sort.Strings(keys)
page := objectStoragePage{}
if limit <= 0 {
limit = 1000
}
if len(keys) > limit {
page.HasMore = true
keys = keys[:limit]
page.NextMarker = keys[len(keys)-1]
}
for _, key := range keys {
kind := "file"
if strings.HasSuffix(key, "/") {
kind = "directory"
}
page.Items = append(page.Items, domain.ObjectStorageObjectSummary{Key: key, Kind: kind})
}
return page, nil
}
func indexCall(h http.Handler, p *authenticatedPrincipal, method, target string, body any) *httptest.ResponseRecorder {
raw, _ := json.Marshal(body)
r := httptest.NewRequest(method, target, bytes.NewReader(raw))
r.Header.Set("Content-Type", "application/json")
r.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "index-test-csrf"})
r.Header.Set(csrfHeaderName, "index-test-csrf")
r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, p))
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return w
}
func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authenticatedPrincipal, string) {
t.Helper()
root := "deliveries/项目A__project-A"
org := "index-test-org"
ctx := t.Context()
b := &indexTestBrowser{downloadTestBrowser: &downloadTestBrowser{objects: map[string][]byte{}, pages: map[string]objectStoragePage{}}}
must := func(e error) {
t.Helper()
if e != nil {
t.Fatal(e)
}
}
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: "project-A", OrganizationID: org, Name: "项目A"}))
must(ds.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "index-storage", OrganizationID: org, Provider: domain.StorageProviderBOS, Bucket: "test-bucket", Endpoint: "https://bj.bcebos.com"}))
dir := os.Getenv("DELIVERY_V3_FIXTURE_DIR")
for _, part := range []string{"part0001", "part0002"} {
prefix := root + "/capture-A/" + part
sourceRun := "source-run-" + part
runID := "package-run-" + part
artifactID := "directory-" + part
body := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.part-index/v3","capture":{"captureId":"capture-A","prefix":"incoming/capture-A"},"partId":%q,"revision":1,"sourceMetadata":{"path":"meta_capture-A.json"},"lineage":{"sourceRunId":%q},"acceptance":{"finalAcceptance":{"confirmed":true,"acceptedAt":"2026-10-20T01:00:00Z"}},"slices":[{"sliceId":"slice-1","taskSplit":{"startSecond":0,"endSecond":1,"semantics":"half-open"},"classification":{"deliveryDisposition":"keep","descriptionStatus":"empty_confirmed"},"segments":[{"description":"","isInvalid":false,"contentKind":"gap_placeholder"}]}],"discardedSlices":[],"deliveryClassification":{"keptSliceCount":1,"discardedSliceCount":0,"keptDurationSeconds":1,"discardedDurationSeconds":0}}`, part, sourceRun))
history := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.review-history/v1","captureId":"capture-A","partId":%q,"revision":1,"qualityReviews":[]}`, part))
if dir != "" {
var e error
body, e = os.ReadFile(filepath.Join(dir, part, "part.json"))
must(e)
history, e = os.ReadFile(filepath.Join(dir, part, "review-history.json"))
must(e)
}
b.objects[prefix+"/.delivery-owner.json"] = []byte(`{"owner":"synthetic-source"}`)
b.objects[prefix+"/part.json"] = body
b.objects[prefix+"/review-history.json"] = history
entries := []deliveryindex.Entry{{Path: "part.json", Key: prefix + "/part.json", SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)}, {Path: "review-history.json", Key: prefix + "/review-history.json", SizeBytes: int64(len(history)), SHA256: deliveryindex.Hash(history)}}
b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 2, "totalBytes": len(body) + len(history), "entries": entries})
b.objects["incoming/capture-A/"+part+"/left.mp4"] = []byte("synthetic")
run := domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: "project-A", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: artifactID, RunID: runID, Name: "delivery.directory.index", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindDirectory, Provider: domain.StorageProviderBOS, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix, ManifestKey: prefix + "/manifest.json"}}}}}
run.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "package", Artifacts: run.Artifacts}}
must(ds.UpsertRun(ctx, run))
must(ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "package-" + part, OrganizationID: org, ProjectID: "project-A", ChildRunID: runID, Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{artifactID}}))
must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, OrganizationID: org, ProjectID: "project-A", RunID: sourceRun, CaptureID: "capture-A/" + part, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A/" + part, SourcePrefix: "incoming/capture-A", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix}}))
}
b.objects["incoming/capture-A/meta_capture-A.json"] = []byte(`{"synthetic":true}`)
old := storageConnectionBrowserResolver
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
if c.OrganizationID != org {
return nil, nil, fmt.Errorf("foreign storage")
}
return b, nil, nil
}
t.Cleanup(func() { storageConnectionBrowserResolver = old })
p := &authenticatedPrincipal{User: domain.UserRecord{ID: "index-admin", OrganizationID: org, Role: domain.UserRoleAdmin}, Organization: domain.OrganizationRecord{ID: org}}
return b, p, root
}
func exerciseIndexBatch(t *testing.T, ds store.Store) {
t.Helper()
b, p, root := seedIndexFixture(t, ds)
h := NewHandler(ds, nil)
selection := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001&packageId=package-part0002"
preview := indexCall(h, p, "GET", selection, nil)
if preview.Code != 200 {
t.Fatal(preview.Code, preview.Body.String())
}
if len(b.puts) > 0 {
t.Fatal("preview wrote storage")
}
var plan struct {
ID string `json:"snapshotId"`
Root string `json:"rootKey"`
}
json.Unmarshal(preview.Body.Bytes(), &plan)
post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001", "package-part0002"}, SnapshotID: plan.ID}
stale := post
stale.SnapshotID = "old"
if w := indexCall(h, p, "POST", "/api/delivery-index-batches", stale); w.Code != 409 || len(b.puts) > 0 {
t.Fatal("stale preview wrote")
}
b.fail = "part0002.json"
w := indexCall(h, p, "POST", "/api/delivery-index-batches", post)
if w.Code != 409 {
t.Fatal(w.Code, w.Body.String())
}
if _, ok := b.objects[plan.Root+"/delivery_index.json"]; ok {
t.Fatal("partial batch marked complete")
}
b.fail = ""
b.ambiguous = true
h = NewHandler(ds, nil) // API restart; all progress lives in immutable storage.
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
if b.puts[len(b.puts)-1] != plan.Root+"/delivery_index.json" {
t.Fatal("total index must be last")
}
before := append([]byte(nil), b.objects[plan.Root+"/delivery_index.json"]...)
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
if w.Code != 200 || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) {
t.Fatal("idempotent retry")
}
target := deliveryDownloadTarget{"index-storage", "test-bucket", plan.Root, pathBase(plan.Root)}
files, e := readDownloadManifest(t.Context(), b, target)
if e != nil {
t.Fatal(e)
}
for _, f := range files {
if deliveryindex.Hash(b.objects[f.key]) != f.SHA256 {
t.Fatal("download hash", f.RelativePath)
}
}
// Exercise the existing public download route, including refreshed signatures.
download := downloadRequest(h, p, "/api/delivery-downloads/manifest?directory=bos://test-bucket/"+plan.Root+"&connectionId=index-storage")
if download.Code != 200 || !strings.Contains(download.Body.String(), "delivery_index.json") {
t.Fatal(download.Code, download.Body.String())
}
// A later partial snapshot does not alter the old full snapshot.
first := indexCall(h, p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
var partial struct {
ID string `json:"snapshotId"`
}
json.Unmarshal(first.Body.Bytes(), &partial)
w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID})
if w.Code != 200 || partial.ID == plan.ID || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) {
t.Fatal("immutable snapshots", w.Code, w.Body.String())
}
// Occupied, unowned batch directories can never be adopted.
conflictRoot := root + "/index-batches/" + partial.ID
delete(b.objects, conflictRoot+"/.delivery-owner.json")
if w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID}); w.Code != 409 {
t.Fatal("adopted unowned directory")
}
}
func pathBase(s string) string { p := strings.Split(s, "/"); return p[len(p)-1] }
func TestIndexBatchHTTPPublicationRecoveryAndDownload(t *testing.T) {
exerciseIndexBatch(t, store.NewMemoryStore())
}
func TestIndexBatchRejectsScopeAndTampering(t *testing.T) {
ds := store.NewMemoryStore()
b, p, _ := seedIndexFixture(t, ds)
h := NewHandler(ds, nil)
target := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001"
other := *p
other.User.Role = domain.UserRoleQualityReviewer
if w := indexCall(h, &other, "GET", target, nil); w.Code != 403 {
t.Fatal("non-admin")
}
other = *p
other.Organization.ID = "other"
other.User.OrganizationID = "other"
if w := indexCall(h, &other, "GET", target, nil); w.Code == 200 {
t.Fatal("foreign org")
}
b.objects["deliveries/项目A__project-A/capture-A/part0001/part.json"] = []byte(`{"tampered":true}`)
if w := indexCall(h, p, "GET", target, nil); w.Code != 409 || len(b.puts) > 0 {
t.Fatal("tampered source accepted")
}
}
func TestIndexBatchDelegationKeepsSourceAndDeliveryScopesSeparate(t *testing.T) {
ds := store.NewMemoryStore()
b, p, root := seedIndexFixture(t, ds)
c, _ := ds.GetStorageConnection(t.Context(), "index-storage")
c.OrganizationID = "source-owner"
if e := ds.UpsertStorageConnection(t.Context(), *c); e != nil {
t.Fatal(e)
}
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
if c.OrganizationID != "source-owner" {
return nil, nil, fmt.Errorf("wrong owner")
}
return b, nil, nil
}
mapping := func(writes []string, deliveries []string) string {
raw, _ := json.Marshal(map[string]any{p.Organization.ID: map[string]any{"connectionId": "index-storage", "bucket": "test-bucket", "readConnectionOrganizationId": "source-owner", "writeConnectionOrganizationId": "source-owner", "readPrefixes": []string{"incoming"}, "writePrefixes": writes, "deliveryPrefixes": deliveries}})
return string(raw)
}
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001", "system/runs/source-run-part0002"}, []string{root}))
h := NewHandler(ds, nil)
query := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001"
w := indexCall(h, p, "GET", query, nil)
if w.Code != 200 {
t.Fatal("explicit source read/final-output write grant", w.Code, w.Body.String())
}
var preview struct {
ID string `json:"snapshotId"`
}
json.Unmarshal(w.Body.Bytes(), &preview)
post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: preview.ID}
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
for _, key := range b.puts {
if !strings.HasPrefix(key, root+"/index-batches/") {
t.Fatal("write escaped final-output scope", key)
}
}
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/some-other-run"}, []string{root}))
before := len(b.puts)
if w = indexCall(h, p, "POST", "/api/delivery-index-batches", post); w.Code != 409 || len(b.puts) != before {
t.Fatal("unapproved source run wrote")
}
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001"}, []string{root + "/capture-A"}))
if w = indexCall(h, p, "GET", query, nil); w.Code != 409 {
t.Fatal("sibling batch prefix escaped delivery grant", w.Code, w.Body.String())
}
}

View File

@ -16,6 +16,8 @@ func (server *server) mountDeliveryRoutes(router chi.Router) {
router.Get("/api/delivery-packages", server.handleDeliveryPackagesList)
router.Get("/api/delivery-downloads/projects", server.handleDeliveryDownloadProjects)
router.Get("/api/delivery-downloads/manifest", server.handleDeliveryDownloadManifest)
router.Get("/api/delivery-index-batches", server.handleDeliveryIndexBatch)
router.Post("/api/delivery-index-batches", server.handleDeliveryIndexBatch)
router.Get("/api/delivery-packages/{id}", server.handleDeliveryPackageGet)
router.Get("/api/delivery-packages/{id}/contents", server.handleDeliveryPackageContents)
router.Get("/api/delivery-packages/{id}/artifacts/{artifactId}/content", server.handleDeliveryPackageArtifactContent)

View File

@ -0,0 +1,366 @@
// Package deliveryindex builds deterministic immutable delivery snapshots. It
// performs no storage operations; callers must verify the accepted source bytes.
package deliveryindex
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"math"
"path"
"regexp"
"sort"
"strings"
"time"
)
const PartSchema = "droboticflow.delivery.part-index/v3"
var partName = regexp.MustCompile(`^part[0-9]{4,}$`)
type Source struct {
Part json.RawMessage `json:"part"`
History json.RawMessage `json:"history"`
ExpectedParts []string `json:"expectedParts"`
MetadataSHA256 string `json:"metadataSha256"`
MetadataBytes int64 `json:"metadataBytes"`
}
type Snapshot struct {
ID string `json:"snapshotId"`
Root string `json:"rootKey"`
Files map[string][]byte `json:"-"`
Index json.RawMessage `json:"index"`
}
type Entry struct {
Path string `json:"path"`
Key string `json:"key"`
SizeBytes int64 `json:"sizeBytes"`
SHA256 string `json:"sha256"`
}
type Summary struct {
CaptureCount int `json:"captureCount"`
PartCount int `json:"partCount"`
ExpectedPartCount int `json:"expectedPartCount"`
UndeliveredParts []string `json:"undeliveredParts,omitempty"`
Completion string `json:"deliveryCompletion"`
KeptCount int `json:"keptSliceCount"`
DiscardedCount int `json:"discardedSliceCount"`
KeptDuration float64 `json:"keptDurationSeconds"`
DiscardedDuration float64 `json:"discardedDurationSeconds"`
Acceptance map[string]string `json:"acceptance"`
}
type part struct {
Schema string `json:"schemaVersion"`
Capture struct {
ID string `json:"captureId"`
Prefix string `json:"prefix"`
} `json:"capture"`
PartID string `json:"partId"`
Revision int `json:"revision"`
SourceMetadata struct {
Path string `json:"path"`
} `json:"sourceMetadata"`
Slices []slice `json:"slices"`
Discarded []slice `json:"discardedSlices"`
Classification *struct {
KeptCount int `json:"keptSliceCount"`
DiscardedCount int `json:"discardedSliceCount"`
KeptDuration float64 `json:"keptDurationSeconds"`
DiscardedDuration float64 `json:"discardedDurationSeconds"`
} `json:"deliveryClassification"`
Acceptance struct {
Final struct {
Confirmed bool `json:"confirmed"`
AcceptedAt string `json:"acceptedAt"`
} `json:"finalAcceptance"`
Reviews json.RawMessage `json:"qualityReviews"`
} `json:"acceptance"`
}
type slice struct {
ID string `json:"sliceId"`
Split struct {
Start float64 `json:"startSecond"`
End float64 `json:"endSecond"`
Semantics string `json:"semantics"`
} `json:"taskSplit"`
Classification struct {
Disposition string `json:"deliveryDisposition"`
DescriptionStatus string `json:"descriptionStatus"`
Discard struct {
Reason string `json:"reason"`
} `json:"discardEvidence"`
} `json:"classification"`
Segments []struct {
Description string `json:"description"`
Invalid bool `json:"isInvalid"`
Kind string `json:"contentKind"`
Status *string `json:"status"`
} `json:"segments"`
}
func Hash(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
func encode(v any) []byte {
b, e := json.MarshalIndent(v, "", " ")
if e != nil {
panic(e)
}
return append(b, '\n')
}
func safeName(s string) bool {
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\:\x00") && !strings.HasPrefix(s, ".")
}
func addSummary(a *Summary, b Summary) {
a.PartCount += b.PartCount
a.ExpectedPartCount += b.ExpectedPartCount
a.KeptCount += b.KeptCount
a.DiscardedCount += b.DiscardedCount
a.KeptDuration += b.KeptDuration
a.DiscardedDuration += b.DiscardedDuration
if a.Acceptance == nil {
a.Acceptance = map[string]string{"decision": "approved"}
}
before, _ := time.Parse(time.RFC3339Nano, a.Acceptance["acceptedAt"])
after, _ := time.Parse(time.RFC3339Nano, b.Acceptance["acceptedAt"])
if after.After(before) {
a.Acceptance["acceptedAt"] = b.Acceptance["acceptedAt"]
}
}
func checkPart(p part) (Summary, error) {
s := Summary{PartCount: 1, Acceptance: map[string]string{"decision": "approved", "acceptedAt": p.Acceptance.Final.AcceptedAt}}
if p.Schema != PartSchema || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 {
return s, fmt.Errorf("part must pin an accepted native v3 revision without embedded review history")
}
if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil {
return s, fmt.Errorf("missing acceptance timestamp")
}
if p.SourceMetadata.Path != "meta_"+p.Capture.ID+".json" || path.Base(p.Capture.Prefix) != p.Capture.ID || path.Clean(p.Capture.Prefix) != p.Capture.Prefix || strings.HasPrefix(p.Capture.Prefix, "/") || strings.Contains(p.Capture.Prefix, "../") {
return s, fmt.Errorf("capture/source metadata identity mismatch")
}
seen := map[string]bool{}
var intervals [][2]float64
for group, items := range [][]slice{p.Slices, p.Discarded} {
for _, v := range items {
if v.ID == "" || seen[v.ID] || v.Split.Semantics != "half-open" || math.IsNaN(v.Split.Start) || v.Split.Start < 0 || v.Split.End <= v.Split.Start {
return s, fmt.Errorf("invalid slice identity or interval")
}
seen[v.ID] = true
intervals = append(intervals, [2]float64{v.Split.Start, v.Split.End})
duration := v.Split.End - v.Split.Start
if group == 1 {
if v.Classification.Disposition != "discard" || strings.TrimSpace(v.Classification.Discard.Reason) == "" || len(v.Segments) > 0 {
return s, fmt.Errorf("invalid discarded evidence")
}
s.DiscardedCount++
s.DiscardedDuration += duration
continue
}
if v.Classification.Disposition != "keep" {
return s, fmt.Errorf("invalid kept disposition")
}
s.KeptCount++
s.KeptDuration += duration
described, invalid, blank := false, false, false
for _, a := range v.Segments {
kind := "gap_placeholder"
if a.Invalid {
kind = "invalid_marked"
invalid = true
} else if strings.TrimSpace(a.Description) != "" {
kind = "described"
described = true
} else {
blank = true
}
if a.Kind != kind || a.Status != nil {
return s, fmt.Errorf("action content classification mismatch")
}
}
status := "empty_confirmed"
if invalid {
status = "invalid_only"
}
if invalid && blank {
status = "invalid_and_blank"
}
if described {
status = "described"
}
if v.Classification.DescriptionStatus != status {
return s, fmt.Errorf("descriptionStatus mismatch")
}
}
}
if s.KeptCount == 0 {
return s, fmt.Errorf("no deliverable slices")
}
if math.IsInf(s.KeptDuration, 0) || math.IsInf(s.DiscardedDuration, 0) {
return s, fmt.Errorf("slice duration overflow")
}
if c := p.Classification; c == nil || c.KeptCount != s.KeptCount || c.DiscardedCount != s.DiscardedCount || math.Abs(c.KeptDuration-s.KeptDuration) > 0.000001 || math.Abs(c.DiscardedDuration-s.DiscardedDuration) > 0.000001 {
return s, fmt.Errorf("part summary disagrees with its selected slices")
}
sort.Slice(intervals, func(i, j int) bool { return intervals[i][0] < intervals[j][0] })
for i := 1; i < len(intervals); i++ {
if math.Abs(intervals[i][0]-intervals[i-1][1]) > 0.000001 {
return s, fmt.Errorf("slice coverage gap or overlap")
}
}
return s, nil
}
// Build uses a hash of canonical source documents, source metadata and expected
// part inventory as its identity. Signed URLs and build time never affect it.
func Build(projectID, projectName, projectRoot string, sources []Source) (Snapshot, error) {
out := Snapshot{Files: map[string][]byte{}}
if projectID == "" || projectRoot == "." || path.Clean(projectRoot) != projectRoot || strings.HasPrefix(projectRoot, "/") || strings.Contains(projectRoot, "../") || len(sources) == 0 || len(sources) > 100 {
return out, fmt.Errorf("select 1–100 accepted part packages")
}
var sourceBytes int
for _, source := range sources {
sourceBytes += len(source.Part) + len(source.History)
if sourceBytes > 64<<20 {
return out, fmt.Errorf("source JSON exceeds 64 MiB")
}
}
// Normalize JSON before hashing to ignore transport whitespace/map ordering.
for i := range sources {
var d, h any
if json.Unmarshal(sources[i].Part, &d) != nil || json.Unmarshal(sources[i].History, &h) != nil {
return out, fmt.Errorf("invalid source JSON")
}
sources[i].Part = encode(d)
sources[i].History = encode(h)
sources[i].ExpectedParts = append([]string(nil), sources[i].ExpectedParts...)
sort.Strings(sources[i].ExpectedParts)
}
sort.Slice(sources, func(i, j int) bool { return string(sources[i].Part) < string(sources[j].Part) })
identity := encode([]any{"droboticflow.delivery.batch/v1", projectID, projectName, projectRoot, sources})
out.ID = Hash(identity)
out.Root = projectRoot + "/index-batches/" + out.ID
type group struct {
capture map[string]string
expected []string
metadata Source
parts []map[string]any
summary Summary
entries []Entry
}
groups := map[string]*group{}
seen := map[string]bool{}
var totalBytes int
add := func(name string, body []byte) Entry {
out.Files[name] = body
totalBytes += len(body)
return Entry{name, out.Root + "/" + name, int64(len(body)), Hash(body)}
}
for _, src := range sources {
var p part
if e := json.Unmarshal(src.Part, &p); e != nil {
return out, e
}
s, e := checkPart(p)
if e != nil {
return out, e
}
name := p.Capture.ID + "/" + p.PartID
if seen[strings.ToLower(name)] {
return out, fmt.Errorf("duplicate capture/part, select exactly one revision")
}
seen[strings.ToLower(name)] = true
if len(src.ExpectedParts) == 0 || len(src.ExpectedParts) > 10000 || src.MetadataBytes <= 0 || len(src.MetadataSHA256) != 64 {
return out, fmt.Errorf("missing original source inventory/metadata evidence")
}
expectedSeen := map[string]bool{}
found := false
for _, n := range src.ExpectedParts {
if !partName.MatchString(n) || expectedSeen[n] {
return out, fmt.Errorf("invalid expected part inventory")
}
expectedSeen[n] = true
found = found || n == p.PartID
}
if !found {
return out, fmt.Errorf("delivered part missing from original inventory")
}
var history struct {
Schema string `json:"schemaVersion"`
CaptureID string `json:"captureId"`
PartID string `json:"partId"`
Revision int `json:"revision"`
Reviews []json.RawMessage `json:"qualityReviews"`
}
if json.Unmarshal(src.History, &history) != nil || history.Schema != "droboticflow.delivery.review-history/v1" || history.CaptureID != p.Capture.ID || history.PartID != p.PartID || history.Revision != p.Revision || history.Reviews == nil {
return out, fmt.Errorf("review history revision mismatch")
}
g := groups[p.Capture.ID]
if g == nil {
g = &group{capture: map[string]string{"captureId": p.Capture.ID, "prefix": p.Capture.Prefix}, expected: src.ExpectedParts, metadata: src}
groups[p.Capture.ID] = g
} else if g.capture["prefix"] != p.Capture.Prefix || strings.Join(g.expected, ",") != strings.Join(src.ExpectedParts, ",") || g.metadata.MetadataSHA256 != src.MetadataSHA256 {
return out, fmt.Errorf("inconsistent capture source evidence")
}
var doc map[string]any
json.Unmarshal(src.Part, &doc)
doc["reviewHistory"] = map[string]string{"path": p.PartID + ".review-history.json", "pathBase": "capture_delivery"}
doc["delivery"] = map[string]string{"kind": "part_index", "fileName": p.PartID + ".json", "artifactName": "delivery.part-index"}
g.entries = append(g.entries, add(name+".json", encode(doc)), add(name+".review-history.json", src.History))
g.parts = append(g.parts, map[string]any{"part": p.PartID, "revision": p.Revision, "path": p.PartID + ".json", "reviewHistoryPath": p.PartID + ".review-history.json"})
addSummary(&g.summary, s)
if math.IsInf(g.summary.KeptDuration, 0) || math.IsInf(g.summary.DiscardedDuration, 0) {
return out, fmt.Errorf("capture duration overflow")
}
}
names := make([]string, 0, len(groups))
for n := range groups {
names = append(names, n)
}
sort.Strings(names)
allEntries := []Entry{}
captures := []map[string]any{}
total := Summary{Completion: "full_run", CaptureCount: len(groups)}
for _, n := range names {
g := groups[n]
sort.Slice(g.parts, func(i, j int) bool { return g.parts[i]["part"].(string) < g.parts[j]["part"].(string) })
g.summary.CaptureCount = 1
g.summary.ExpectedPartCount = len(g.expected)
g.summary.Completion = "full_run"
g.summary.UndeliveredParts = []string{}
for _, pn := range g.expected {
if !seen[strings.ToLower(n+"/"+pn)] {
g.summary.UndeliveredParts = append(g.summary.UndeliveredParts, pn)
}
}
if len(g.summary.UndeliveredParts) > 0 {
g.summary.Completion = "partial_run"
total.Completion = "partial_run"
}
sort.Slice(g.entries, func(i, j int) bool { return g.entries[i].Path < g.entries[j].Path })
local := append([]Entry(nil), g.entries...)
var size int64
for i := range local {
local[i].Path = strings.TrimPrefix(local[i].Path, n+"/")
size += local[i].SizeBytes
}
metadata := map[string]any{"path": "meta_" + n + ".json", "pathBase": "original_capture", "sha256": g.metadata.MetadataSHA256, "sizeBytes": g.metadata.MetadataBytes}
manifest := map[string]any{"schemaVersion": 3, "kind": "part-index", "capture": g.capture, "pathBase": "capture_delivery", "sourceMetadata": metadata, "summary": g.summary, "parts": g.parts, "entries": local, "fileCount": len(local), "totalBytes": size}
allEntries = append(allEntries, g.entries...)
allEntries = append(allEntries, add(n+"/annotation_manifest.json", encode(manifest)))
captures = append(captures, map[string]any{"captureId": n, "prefix": g.capture["prefix"], "manifestPath": n + "/annotation_manifest.json", "summary": g.summary})
addSummary(&total, g.summary)
if math.IsInf(total.KeptDuration, 0) || math.IsInf(total.DiscardedDuration, 0) {
return out, fmt.Errorf("batch duration overflow")
}
}
if totalBytes > 64<<20 {
return out, fmt.Errorf("batch exceeds 64 MiB")
}
sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Path < allEntries[j].Path })
var size int64
for _, v := range allEntries {
size += v.SizeBytes
}
out.Index = encode(map[string]any{"schemaVersion": 3, "kind": "part-index", "snapshotId": out.ID, "pathBase": "batch_project", "project": map[string]string{"projectId": projectID, "deliveryName": projectName, "rootKey": out.Root}, "summary": total, "captures": captures, "entries": allEntries, "fileCount": len(allEntries), "totalBytes": size})
out.Files["delivery_index.json"] = out.Index
return out, nil
}

View File

@ -0,0 +1,127 @@
package deliveryindex
import (
"bytes"
"encoding/json"
"strings"
"testing"
)
func fixture(partID string, revision int) Source {
p := map[string]any{"schemaVersion": PartSchema, "capture": map[string]any{"captureId": "capture-A", "prefix": "incoming/capture-A"}, "partId": partID, "revision": revision, "sourceMetadata": map[string]string{"path": "meta_capture-A.json"}, "acceptance": map[string]any{"finalAcceptance": map[string]any{"confirmed": true, "acceptedAt": "2026-10-20T01:00:00Z"}}, "slices": []any{map[string]any{"sliceId": "s1", "taskSplit": map[string]any{"startSecond": 0, "endSecond": 2, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "keep", "descriptionStatus": "invalid_and_blank"}, "segments": []any{map[string]any{"description": "", "contentKind": "gap_placeholder", "isInvalid": false}, map[string]any{"description": "无效", "contentKind": "invalid_marked", "isInvalid": true}}}}, "discardedSlices": []any{map[string]any{"sliceId": "s2", "taskSplit": map[string]any{"startSecond": 2, "endSecond": 3, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "discard", "discardEvidence": map[string]string{"reason": "不交付"}}, "segments": []any{}}}}
p["deliveryClassification"] = map[string]any{"keptSliceCount": 1, "discardedSliceCount": 1, "keptDurationSeconds": 2, "discardedDurationSeconds": 1}
h := map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": "capture-A", "partId": partID, "revision": revision, "qualityReviews": []any{map[string]any{"revision": revision, "decision": "approved"}}}
return Source{Part: encode(p), History: encode(h), ExpectedParts: []string{"part0001", "part0002"}, MetadataSHA256: strings.Repeat("a", 64), MetadataBytes: 10}
}
func build(t *testing.T, sources ...Source) Snapshot {
t.Helper()
s, e := Build("project-A", "项目A", "delivery/项目A__project-A", sources)
if e != nil {
t.Fatal(e)
}
return s
}
func TestBatchMultipartIntegrityAndImmutableScopes(t *testing.T) {
a, b := fixture("part0001", 3), fixture("part0002", 8)
first := build(t, a)
full := build(t, a, b)
reversed := build(t, b, a)
if first.ID == full.ID || full.ID != reversed.ID {
t.Fatal("snapshot identity")
}
var index struct {
Summary Summary
Entries []Entry
Bytes int64 `json:"totalBytes"`
}
json.Unmarshal(full.Index, &index)
if index.Summary.PartCount != 2 || index.Summary.ExpectedPartCount != 2 || index.Summary.Completion != "full_run" || index.Summary.KeptCount != 2 || index.Summary.DiscardedCount != 2 || index.Summary.KeptDuration != 4 || index.Summary.DiscardedDuration != 2 {
t.Fatalf("summary %+v", index.Summary)
}
var size int64
for _, e := range index.Entries {
body := full.Files[e.Path]
if int64(len(body)) != e.SizeBytes || Hash(body) != e.SHA256 || e.Key != full.Root+"/"+e.Path {
t.Fatal("entry integrity", e)
}
size += e.SizeBytes
}
if size != index.Bytes {
t.Fatal("total bytes")
}
var manifest struct {
Entries []Entry
Summary Summary
Metadata struct {
Path string
Base string `json:"pathBase"`
} `json:"sourceMetadata"`
}
json.Unmarshal(full.Files["capture-A/annotation_manifest.json"], &manifest)
for _, e := range manifest.Entries {
if strings.Contains(e.Path, "/") || !bytes.Equal(full.Files["capture-A/"+e.Path], reversed.Files["capture-A/"+e.Path]) {
t.Fatal("capture-relative entry", e)
}
}
if manifest.Metadata.Path != "meta_capture-A.json" || manifest.Metadata.Base != "original_capture" {
t.Fatal("metadata reference")
}
again := build(t, a)
if !bytes.Equal(first.Index, again.Index) {
t.Fatal("old batch changed")
}
if !bytes.Contains(first.Index, []byte(`"partial_run"`)) || !bytes.Contains(first.Index, []byte(`"part0002"`)) {
t.Fatal("partial scope")
}
}
func TestBatchRejectsConflictingEvidence(t *testing.T) {
for _, tc := range []string{"duplicate", "revision-history", "missing-inventory", "description", "coverage", "acceptance", "embedded-history", "part-summary"} {
t.Run(tc, func(t *testing.T) {
a := fixture("part0001", 1)
sources := []Source{a}
var d map[string]any
json.Unmarshal(a.Part, &d)
switch tc {
case "duplicate":
sources = append(sources, fixture("part0001", 2))
case "revision-history":
sources[0].History = fixture("part0001", 2).History
case "missing-inventory":
sources[0].ExpectedParts = []string{"part0002"}
case "description":
d["slices"].([]any)[0].(map[string]any)["classification"].(map[string]any)["descriptionStatus"] = "described"
case "coverage":
d["discardedSlices"].([]any)[0].(map[string]any)["taskSplit"].(map[string]any)["startSecond"] = 2.5
case "acceptance":
d["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["confirmed"] = false
case "part-summary":
d["deliveryClassification"].(map[string]any)["keptSliceCount"] = 99
case "embedded-history":
d["acceptance"].(map[string]any)["qualityReviews"] = []any{}
}
if tc == "description" || tc == "coverage" || tc == "acceptance" || tc == "embedded-history" || tc == "part-summary" {
sources[0].Part = encode(d)
}
if _, e := Build("project-A", "项目A", "delivery/项目A__project-A", sources); e == nil {
t.Fatal("accepted", tc)
}
})
}
}
func TestBatchAcceptanceUsesInstantsRatherThanTimestampText(t *testing.T) {
a, b := fixture("part0001", 1), fixture("part0002", 2)
var doc map[string]any
json.Unmarshal(a.Part, &doc)
doc["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["acceptedAt"] = "2026-10-20T10:00:00+08:00"
a.Part = encode(doc)
json.Unmarshal(b.Part, &doc)
doc["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["acceptedAt"] = "2026-10-20T03:00:00Z"
b.Part = encode(doc)
s := build(t, a, b)
var index struct{ Summary Summary }
json.Unmarshal(s.Index, &index)
if index.Summary.Acceptance["acceptedAt"] != "2026-10-20T03:00:00Z" {
t.Fatal("timezone aggregation", index.Summary.Acceptance)
}
}

View File

@ -0,0 +1,56 @@
package worker
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// The production worker's directory protocol must publish the real native v3
// artifact without routing it through its intentionally v1/v2-only file parser.
func TestCaptureDeliveryNativeV3UsesDirectoryProtocol(t *testing.T) {
fixture := os.Getenv("DELIVERY_V3_FIXTURE_DIR")
if fixture == "" {
t.Skip("generate native Python v3 fixture first")
}
db := store.NewMemoryStore()
publisher := &deliveryPublisher{}
runtime := &Runtime{store: db, artifactPublisher: publisher}
ctx := t.Context()
if e := db.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "storage-v3", OrganizationID: "org-v3", Provider: domain.StorageProviderBOS, Bucket: "b"}); e != nil {
t.Fatal(e)
}
for _, part := range []string{"part0001", "part0002"} {
prefix := "delivery/project-v3__project/capture-A/" + part
run := &domain.RunRecord{ID: "run-" + part, OrganizationID: "org-v3"}
if e := db.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, RunID: run.ID, OrganizationID: run.OrganizationID, Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "storage-v3", Bucket: "b", Prefix: prefix}}); e != nil {
t.Fatal(e)
}
artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Path: filepath.Join(fixture, part), Type: domain.PortTypeFile}
for attempt := 0; attempt < 2; attempt++ {
if _, e := runtime.publishCaptureDelivery(ctx, run, "package", artifact); e != nil {
t.Fatal(e)
}
}
var manifest struct {
Kind string
FileCount int
Entries []directoryManifestEntry
}
if e := json.Unmarshal(publisher.objects["b/"+prefix+"/manifest.json"], &manifest); e != nil {
t.Fatal(e)
}
if manifest.Kind != "directory" || manifest.FileCount != 2 {
t.Fatal("native v3 directory protocol", manifest)
}
for _, name := range []string{"part.json", "review-history.json"} {
if len(publisher.objects["b/"+prefix+"/"+name]) == 0 {
t.Fatal("missing", part, name)
}
}
}
}

View File

@ -836,11 +836,67 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
``empty_confirmed`` keeps its platform meaning: no description was written
and the submitter confirmed that as a deliberate semantic blank.
"""
if not actions:
return "empty_confirmed"
if all(bool(action.get("isInvalid")) for action in actions):
if any(not action.get("isInvalid") and str(action.get("description") or "").strip() for action in actions):
return "described"
if actions and all(bool(action.get("isInvalid")) for action in actions):
return "invalid_only"
return "described"
return "empty_confirmed"
def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload):
"""Use the established immutable directory publisher for the two JSON files.
Legacy single-file publishers deliberately accept only v1/v2. A directory
artifact preserves their ownership/conditional-publication protocol without
teaching an old worker to misinterpret a new single-file schema.
"""
capture = dict(payload["capture"])
prefix = pathlib.PurePosixPath(str(capture.get("prefix") or ""))
part_id = prefix.name
capture_id = prefix.parent.name
recorded = str(capture.get("captureId") or "")
if not re.fullmatch(r"part\d{4,}", part_id) or recorded not in {capture_id, capture_id + "/" + part_id}:
raise RuntimeError("v3 capture identity disagrees with the original part prefix")
if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3:
raise RuntimeError("unsafe v3 capture identity prefix")
capture.update(captureId=capture_id, prefix=str(prefix.parent))
payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id)
payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"}
payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"}
history = {
"schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": capture_id,
"partId": part_id, "revision": payload["revision"],
"qualityReviews": payload["acceptance"].pop("qualityReviews", []),
}
payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"}
payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None)
for item in payload["calibration"]:
if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]:
raise RuntimeError("unsafe calibration filename")
item["path"] = item["name"]
for item in payload["slices"] + payload["discardedSlices"]:
classification = item["classification"]
for key in ("taskLabel", "taskLabelId"):
if not str(classification.get(key) or "").strip():
classification.pop(key, None)
classification.get("discardEvidence", {}).pop("discardedByUserId", None)
item.get("synchronization", {}).pop("calibration", None)
for action in item["segments"]:
action.pop("status", None)
action["contentKind"] = "invalid_marked" if action["isInvalid"] else "described" if action["description"].strip() else "gap_placeholder"
if classification["deliveryDisposition"] == "keep":
counts = {kind: sum(action["contentKind"] == kind for action in item["segments"]) for kind in ("described", "invalid_marked", "gap_placeholder")}
classification["contentCounts"] = counts
if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]:
classification["descriptionStatus"] = "invalid_and_blank"
staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output))
for name, document in (("part.json", payload), ("review-history.json", history)):
(staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8")
return [helpers["build_virtual_artifact"](
run_id, node_id, "delivery.directory.index", str(staging), "file", 0,
data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"])},
output=_output(node_data, helpers, "deliveryDirectory"),
)]
def _part_index_slice(
@ -1659,6 +1715,8 @@ def _run_accepted_slice_directory(
for item in payload["slices"] + payload["discardedSlices"]:
item["sourceFrames"] = frames[item["sliceId"]]
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
if _bool(params, "DELIVERY_JSON_V3", False):
return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload)
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
if source_index:
source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output)
@ -1894,6 +1952,9 @@ def _run_multiport(
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False):
raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY")
# A part-index delivery intentionally has no directory port; it must still
# reach the accepted-slice path instead of the legacy inline contract.
if (

View File

@ -0,0 +1,38 @@
"""Produce explicitly synthetic multi-part examples using the real package entrypoint."""
import argparse
import hashlib
import json
import shutil
from pathlib import Path
from backend.operators.builtin.package import operator as package
from tests.backend.test_delivery_v3_package import NODE, inputs_for
from tests.backend.test_part_index_package import helpers
def export(destination):
destination.mkdir(parents=True, exist_ok=True)
for part in ("part0001", "part0002"):
work = destination / ("work-" + part)
work.mkdir()
inputs = inputs_for(work, [
{"id": "blank", "description": "", "clipStartMs": 0, "clipEndMs": 500},
{"id": "invalid", "description": "无效", "isInvalid": True, "clipStartMs": 500, "clipEndMs": 1000},
])
inputs[1]["data"]["capture"]["captureId"] = "capture-A/" + part
inputs[1]["data"]["capture"]["prefix"] = "incoming/capture-A/" + part
manifest_path = Path(inputs[0]["path"]) / "revision-manifest.json"
manifest = json.loads(manifest_path.read_text())
manifest["runId"] = "source-run-" + part
manifest_path.write_text(json.dumps(manifest))
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
artifacts = package.run("package-" + part, "package", work / "out", NODE, inputs, helpers())
shutil.copytree(artifacts[0]["path"], destination / part)
shutil.rmtree(work)
(destination / "meta_capture-A.json").write_text('{"synthetic":true,"parts":["part0001","part0002"]}\n')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("output", type=Path)
export(parser.parse_args().output)

View File

@ -0,0 +1,79 @@
"""Native v3 packaging, rather than postprocessing a customer sample ZIP."""
import json
from pathlib import Path
import pytest
from backend.operators.builtin.package import operator as package
from tests.backend.test_part_index_package import helpers
from tests.backend.test_slice_discard_package import fixture
NODE = {
"outputs": [{"id": n, "name": n} for n in ("deliveryDirectory", "deliverySummary")],
"hyperparams": [{"key": k, "value": v} for k, v in {
"PART_INDEX_DELIVERY": "true", "DELIVERY_JSON_V3": "true", "PROCESSING_PROFILE": "raw_human"
}.items()],
}
def inputs_for(tmp_path, actions):
inputs = fixture(tmp_path, ["keep", "discard"])
inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001", "captureRecordId": "record-A1"}
inputs[1]["data"]["finalAcceptance"]["acceptedAt"] = "2026-10-20T01:00:00Z"
root = Path(inputs[0]["path"])
(root / "slices/slice-000001/annotation.json").write_text(json.dumps({"actions": actions}))
return inputs
@pytest.mark.parametrize("actions,expected", [
([{"description": "", "status": "described"}], "empty_confirmed"),
([{"description": "无效", "isInvalid": True}, {"description": " "}], "invalid_and_blank"),
([{"description": "拿起杯子", "status": "pending"}], "described"),
])
def test_native_v3_semantics_and_separate_history(tmp_path, actions, expected):
inputs = inputs_for(tmp_path, actions)
artifacts = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())
artifact = artifacts[0]
assert artifact["portId"] == "deliveryDirectory"
root = Path(artifact["path"])
assert {p.name for p in root.iterdir()} == {"part.json", "review-history.json"}
doc = json.loads((root / "part.json").read_text())
assert doc["schemaVersion"] == "droboticflow.delivery.part-index/v3"
assert doc["partId"] == "part0001"
assert doc["capture"]["captureId"] == "capture-A"
assert doc["capture"]["prefix"] == "incoming/capture-A"
assert doc["sourceMetadata"]["path"] == "meta_capture-A.json"
assert doc["slices"][0]["classification"]["descriptionStatus"] == expected
assert "qualityReviews" not in doc["acceptance"]
assert json.loads((root / "review-history.json").read_text())["qualityReviews"]
for entry in doc["slices"]:
assert "taskLabel" not in entry["classification"]
assert "taskLabelId" not in entry["classification"]
assert "calibration" not in entry["synchronization"]
assert all("status" not in a for a in entry["segments"])
assert all(a["contentKind"] in {"gap_placeholder", "invalid_marked", "described"} for a in entry["segments"])
assert all(c["path"] == c["name"] for c in doc["calibration"])
first = {p.name: p.read_bytes() for p in root.iterdir()}
again = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())
assert {p.name: p.read_bytes() for p in Path(again[0]["path"]).iterdir()} == first
assert not list((tmp_path / "out").rglob("*.mp4"))
def test_v3_refuses_ambiguous_capture_identity(tmp_path):
inputs = inputs_for(tmp_path, [])
inputs[1]["data"]["capture"]["prefix"] = "incoming/another-capture/part0001"
with pytest.raises(RuntimeError, match="capture identity"):
package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())
def test_v3_keeps_valid_legacy_labels_and_rejects_incomplete_configuration(tmp_path):
inputs = inputs_for(tmp_path, [])
sync = Path(inputs[0]["path"]) / "slices/slice-000001/synchronized-slice.json"
doc = json.loads(sync.read_text())
doc.update(taskLabel="拿取物品", taskLabelId="label-1")
sync.write_text(json.dumps(doc))
artifact = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())[0]
classification = json.loads((Path(artifact["path"]) / "part.json").read_text())["slices"][0]["classification"]
assert classification["taskLabel"] == "拿取物品" and classification["taskLabelId"] == "label-1"
bad = {**NODE, "hyperparams": [{"key": "DELIVERY_JSON_V3", "value": "true"}]}
with pytest.raises(RuntimeError, match="requires PART_INDEX_DELIVERY"):
package.run("r", "p", tmp_path / "refused", bad, inputs, helpers())

View File

@ -53,9 +53,13 @@ def test_source_index_real_media_legacy_export_keeps_exact_frames_and_discards(t
assert all("synthetic-discard" not in p.read_text() for p in root.rglob("*.json"))
def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch):
@pytest.mark.parametrize("v3", [False, True])
def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch, v3):
node = {"outputs": NODE["outputs"] + [{"id": "deliveryPartIndex", "name": "deliveryPartIndex"}],
"hyperparams": NODE["hyperparams"] + [{"key": "PART_INDEX_DELIVERY", "value": "true"}, {"key": "SOURCE_INDEX_DELIVERY", "value": "true"}]}
if v3:
node["hyperparams"].append({"key": "DELIVERY_JSON_V3", "value": "true"})
source_inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001"}
def forbidden(*args, **kwargs):
raise AssertionError("JSON-only export must not transcode")
monkeypatch.setattr(package.subprocess, "run", forbidden)
@ -66,8 +70,9 @@ def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path
monkeypatch.setattr(package, "_sha256_file", metadata_only)
artifacts = package.run("test", "package", tmp_path / "output", node, source_inputs, helpers())
assert len(artifacts) == 1
payload = json.loads(Path(artifacts[0]["path"]).read_text())
assert payload["schemaVersion"] == "droboticflow.delivery.part-index/v2"
index_path = Path(artifacts[0]["path"]) / "part.json" if v3 else Path(artifacts[0]["path"])
payload = json.loads(index_path.read_text())
assert payload["schemaVersion"] == ("droboticflow.delivery.part-index/v3" if v3 else "droboticflow.delivery.part-index/v2")
assert len(payload["sources"]) == 5
assert [item["sliceId"] for item in payload["slices"]] == ["slice-000001", "slice-000003"]
assert payload["discardedSlices"][0]["sourceFrames"] == {"left": [8, 24], "right": [8, 24]}

View File

@ -38,6 +38,12 @@ struct Args {
allow_http: bool,
#[arg(long, conflicts_with = "directory")]
project: Option<String>,
/// 预览所选原生 v3 交付包的不可变批次;可重复,须同时指定 --project。
#[arg(long, requires = "project", conflicts_with_all = ["directory", "list", "report"])]
index_package: Vec<String>,
/// 发布与预览完全一致的批次;参数为预览返回的 snapshotId。
#[arg(long, requires = "index_package")]
publish_index: Option<String>,
/// 例如 bos://drobodata-fanyu/yanshou/项目__ID (仅管理员)。
#[arg(long, requires = "connection", conflicts_with = "project")]
directory: Option<String>,
@ -125,6 +131,49 @@ async fn portal_error(r: reqwest::Response) -> anyhow::Error {
}
}
impl Portal {
async fn index_batch(
&self,
project: &str,
packages: &[String],
publish: Option<&str>,
) -> Result<serde_json::Value> {
let u = self.base.join("api/delivery-index-batches")?;
let request = if let Some(snapshot) = publish {
let cookie = self
.jar
.cookies(&u)
.and_then(|c| c.to_str().ok().map(str::to_owned))
.unwrap_or_default();
let csrf = cookie
.split(';')
.find_map(|c| c.trim().strip_prefix("droboticflow_csrf="))
.unwrap_or("");
ensure!(!csrf.is_empty(), "缺少发布所需的 CSRF 会话,请重新登录");
self.client.post(u)
.header("Origin", self.base.origin().ascii_serialization())
.header("X-Droboticflow-CSRF-Token", csrf)
.json(&serde_json::json!({"projectId":project,"packageIds":packages,"snapshotId":snapshot}))
} else {
let mut q = vec![("projectId", project.to_owned())];
q.extend(packages.iter().map(|p| ("packageId", p.clone())));
self.client.get(u).query(&q)
};
let r = request
.send()
.await
.map_err(|_| anyhow::anyhow!("交付批次请求中断;可用相同 snapshotId 重试"))?;
if !r.status().is_success() {
return Err(portal_error(r).await);
}
let mut stream = r.bytes_stream();
let mut bytes = Vec::new();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|_| anyhow::anyhow!("批次响应中断"))?;
ensure!(bytes.len() + chunk.len() <= 8 * 1024 * 1024, "批次响应过大");
bytes.extend_from_slice(&chunk);
}
serde_json::from_slice(&bytes).context("批次响应无效")
}
async fn projects(&self) -> Result<Vec<Project>> {
let r = self
.client
@ -289,6 +338,17 @@ async fn download_one(
}
async fn execute(a: &Args, portal: &Portal) -> Result<()> {
if !a.index_package.is_empty() {
let result = portal
.index_batch(
a.project.as_deref().context("请指定 --project")?,
&a.index_package,
a.publish_index.as_deref(),
)
.await?;
println!("{}", serde_json::to_string_pretty(&result)?);
return Ok(());
}
let selection = if let Some(project) = &a.project {
Selection(vec![("projectId", project.clone())])
} else if let Some(directory) = &a.directory {
@ -481,6 +541,62 @@ mod tests {
use super::*;
use std::{io::Read, net::TcpListener, thread};
#[test]
fn index_publication_requires_explicit_package_selection() {
assert!(Args::try_parse_from([
"pull",
"--api",
"https://portal.test",
"--publish-index",
"snapshot"
])
.is_err());
assert!(Args::try_parse_from([
"pull",
"--api",
"https://portal.test",
"--index-package",
"p1"
])
.is_err());
let a = Args::try_parse_from([
"pull",
"--api",
"https://portal.test",
"--project",
"project-A",
"--index-package",
"p1",
"--index-package",
"p2",
"--publish-index",
"snapshot",
])
.unwrap();
assert_eq!(a.index_package, ["p1", "p2"]);
assert_eq!(a.publish_index.as_deref(), Some("snapshot"));
}
#[tokio::test]
async fn index_preview_is_read_only_and_publication_uses_csrf() {
let response = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}";
let (url, worker) = server(vec![response, response]);
let p = portal(&url);
p.jar
.add_cookie_str("droboticflow_csrf=batch-csrf; Path=/", &p.base);
p.index_batch("project-A", &["p1".into()], None)
.await
.unwrap();
p.index_batch("project-A", &["p1".into()], Some("snapshot"))
.await
.unwrap();
let requests = worker.join().unwrap();
assert!(requests[0]
.starts_with("GET /api/delivery-index-batches?projectId=project-A&packageId=p1 "));
assert!(requests[1].starts_with("POST /api/delivery-index-batches "));
assert!(requests[1]
.to_ascii_lowercase()
.contains("x-droboticflow-csrf-token: batch-csrf"));
}
#[test]
fn error_messages_never_log_signed_urls() {
assert!(
safe_error_detail(br#"{"error":"https://storage/?authorization=secret"}"#).is_none()