📦 Publish immutable delivery index batches with native v3 packages
This commit is contained in:
parent
d10c148b21
commit
7be23604d7
29
.scratch/platform-feedback-20261020/spec.md
Normal file
29
.scratch/platform-feedback-20261020/spec.md
Normal file
@ -0,0 +1,29 @@
|
||||
# 平台调优与交付反馈修复(2026-10-20)
|
||||
|
||||
范围来自平台调优群及马迪历史交付反馈;BOS 拷贝超时按用户要求暂缓。
|
||||
用户已授权本地修改、验证和提交;未授权 PR、push、部署或客户数据写入。
|
||||
|
||||
| 问题 | 修复与验收标准 |
|
||||
| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| 1 描述未保存、其他片草稿拦切片应用 | 完整草稿与切片改动一并进入冲突处置及同一保存事务;失败保留、确认成功才清编辑器;取消保留草稿 |
|
||||
| 2 第三轮意见跳回旧轮、滚动回弹 | 意见身份为轮次 + ID;仅明确定位事件触发滚动;旧轮同 ID 不可误判最新轮 |
|
||||
| 3 审查字幕不可见 | 专注布局显式占行;留空、无效、描述、读取失败分别显示;字幕开关有效 |
|
||||
| 4 账号“我的任务/我已审核”重叠 | 同一采集不同修订可以同时待办及留历史;显示当前/历史版本及复查含义,不改分配权限 |
|
||||
| 5 原生总 JSON 交付未启用 | 原生 v3 打包 → 冻结 part 目录条件发布 → 管理员预览/发布不可变批次 → 现有校验下载链路;默认保持关闭 |
|
||||
| 6 审核历史混在主 JSON | qualityReviews 移到独立、钉住同一修订的 review-history 文件;主 JSON 保留最终结论 |
|
||||
| 7 采集清单缺 meta | annotation_manifest 引用原始 meta,API 读取验证并记录 SHA/大小 |
|
||||
| 8 路径基准混淆 | 总索引相对批次项目根;采集清单相对采集交付目录;meta/标定相对原始采集目录;workspace 来源引用另行标明 |
|
||||
| 9 空任务标签 | 省略空 taskLabel/taskLabelId,保留有效旧标签 |
|
||||
| 10 描述汇总误报 | 由非无效且非空文本判断 described;v3 区分纯留空、纯无效、无效与留空混合,并带内容数量 |
|
||||
| 11 多 part 样例和汇总验证 | 同一采集两个 part 经真实打包入口及 API、Mongo 副本验证范围、修订、验收聚合、SHA、重试和旧批次保留;合成样例不得充当客户真实业务验收 |
|
||||
|
||||
## 已确认决定
|
||||
|
||||
2026-10-20 用户选择不可变交付批次目录。批次根为冻结项目交付根下
|
||||
`index-batches/<snapshotId>/`;总索引最后发布。以后新增 part 生成另一批次,旧批次保持可校验。
|
||||
旧已完成交付不原地补文件;重新交付、工作流开关和线上写测试需要单独计划与授权。
|
||||
|
||||
## 尚待线上完成
|
||||
|
||||
部署前必须按 `docs/release/README.md` 准备具体镜像和变更计划,取得紧邻部署的授权。
|
||||
本次没有客户真实的同采集双 part 已验收样例;本地合成双 part 已验证,真实样例与客户确认仍待具备数据后完成。
|
||||
@ -168,3 +168,7 @@ _Avoid_:供应商任务副本(不代表给每家生产供应商复制客户
|
||||
**撤销供应商项目授权**:停止供应商领取该项目的新任务,但保留其既有任务的标注、质检与返工处理资格,直至存量完成;不等于停用账号或释放已领取任务。
|
||||
|
||||
**生产统计异常**:仅指当前正式处理中的执行故障,包括运行失败、源采集处理失败、切片或送检处理作业失败、交付打包失败。供应商归属不同、部分未领取、归属待核对、审核退回、业务升级处理及正常等待不单独计为异常;这些状态保留在明细,不增加红色异常数量。
|
||||
|
||||
## 交付批次
|
||||
|
||||
**交付批次快照**:一次明确选定、已验收的 part 修订集合及其交付清单。批次只表示本次交付范围;新增 part 或替换修订形成新批次,已交付批次保留。批次完成不代表原始采集的全部 part 已交齐。
|
||||
|
||||
@ -836,11 +836,67 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
``empty_confirmed`` keeps its platform meaning: no description was written
|
||||
and the submitter confirmed that as a deliberate semantic blank.
|
||||
"""
|
||||
if not actions:
|
||||
return "empty_confirmed"
|
||||
if all(bool(action.get("isInvalid")) for action in actions):
|
||||
if any(not action.get("isInvalid") and str(action.get("description") or "").strip() for action in actions):
|
||||
return "described"
|
||||
if actions and all(bool(action.get("isInvalid")) for action in actions):
|
||||
return "invalid_only"
|
||||
return "described"
|
||||
return "empty_confirmed"
|
||||
|
||||
|
||||
def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload):
|
||||
"""Use the established immutable directory publisher for the two JSON files.
|
||||
|
||||
Legacy single-file publishers deliberately accept only v1/v2. A directory
|
||||
artifact preserves their ownership/conditional-publication protocol without
|
||||
teaching an old worker to misinterpret a new single-file schema.
|
||||
"""
|
||||
capture = dict(payload["capture"])
|
||||
prefix = pathlib.PurePosixPath(str(capture.get("prefix") or ""))
|
||||
part_id = prefix.name
|
||||
capture_id = prefix.parent.name
|
||||
recorded = str(capture.get("captureId") or "")
|
||||
if not re.fullmatch(r"part\d{4,}", part_id) or recorded not in {capture_id, capture_id + "/" + part_id}:
|
||||
raise RuntimeError("v3 capture identity disagrees with the original part prefix")
|
||||
if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3:
|
||||
raise RuntimeError("unsafe v3 capture identity prefix")
|
||||
capture.update(captureId=capture_id, prefix=str(prefix.parent))
|
||||
payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id)
|
||||
payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"}
|
||||
payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"}
|
||||
history = {
|
||||
"schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": capture_id,
|
||||
"partId": part_id, "revision": payload["revision"],
|
||||
"qualityReviews": payload["acceptance"].pop("qualityReviews", []),
|
||||
}
|
||||
payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"}
|
||||
payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None)
|
||||
for item in payload["calibration"]:
|
||||
if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]:
|
||||
raise RuntimeError("unsafe calibration filename")
|
||||
item["path"] = item["name"]
|
||||
for item in payload["slices"] + payload["discardedSlices"]:
|
||||
classification = item["classification"]
|
||||
for key in ("taskLabel", "taskLabelId"):
|
||||
if not str(classification.get(key) or "").strip():
|
||||
classification.pop(key, None)
|
||||
classification.get("discardEvidence", {}).pop("discardedByUserId", None)
|
||||
item.get("synchronization", {}).pop("calibration", None)
|
||||
for action in item["segments"]:
|
||||
action.pop("status", None)
|
||||
action["contentKind"] = "invalid_marked" if action["isInvalid"] else "described" if action["description"].strip() else "gap_placeholder"
|
||||
if classification["deliveryDisposition"] == "keep":
|
||||
counts = {kind: sum(action["contentKind"] == kind for action in item["segments"]) for kind in ("described", "invalid_marked", "gap_placeholder")}
|
||||
classification["contentCounts"] = counts
|
||||
if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]:
|
||||
classification["descriptionStatus"] = "invalid_and_blank"
|
||||
staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output))
|
||||
for name, document in (("part.json", payload), ("review-history.json", history)):
|
||||
(staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8")
|
||||
return [helpers["build_virtual_artifact"](
|
||||
run_id, node_id, "delivery.directory.index", str(staging), "file", 0,
|
||||
data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"])},
|
||||
output=_output(node_data, helpers, "deliveryDirectory"),
|
||||
)]
|
||||
|
||||
|
||||
def _part_index_slice(
|
||||
@ -1659,6 +1715,8 @@ def _run_accepted_slice_directory(
|
||||
for item in payload["slices"] + payload["discardedSlices"]:
|
||||
item["sourceFrames"] = frames[item["sliceId"]]
|
||||
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
|
||||
if _bool(params, "DELIVERY_JSON_V3", False):
|
||||
return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
if source_index:
|
||||
source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output)
|
||||
@ -1894,6 +1952,9 @@ def _run_multiport(
|
||||
|
||||
|
||||
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
|
||||
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
|
||||
if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False):
|
||||
raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY")
|
||||
# A part-index delivery intentionally has no directory port; it must still
|
||||
# reach the accepted-slice path instead of the legacy inline contract.
|
||||
if (
|
||||
|
||||
@ -31,3 +31,15 @@ status: accepted
|
||||
## 审核层不受影响
|
||||
|
||||
物化修订里的逐片视频仍是质检与验收工作台的输入(`MaterializedSliceWorkbench`、`FinalAcceptanceWorkspace` 依赖 `slices/slice-*/left.mp4`)。本决定只改变**交付层**的输出形态,不改变内部审核层。
|
||||
|
||||
## 2026-10-20 修订:不可变批次及独立审核历史
|
||||
|
||||
用户明确选择**不可变交付批次目录**,拒绝在固定项目根反复覆盖总索引。每次选择已验收 part 修订后,内容身份生成新的 `index-batches/<snapshotId>/`,其中依次为 part JSON/独立审核历史、采集 `annotation_manifest.json`、最后写入的 `delivery_index.json`。重试只能接受相同内容,不能覆盖冲突对象或接管无归属的非空目录;旧批次继续可下载校验。批次标识绑定规范化源 JSON、原始 meta 校验信息和原始 part 目录清单,不包含生成时间或签名 URL。
|
||||
|
||||
新格式用默认关闭的 `DELIVERY_JSON_V3`,同时要求 `PART_INDEX_DELIVERY`;源索引修订还必须显式启用 `SOURCE_INDEX_DELIVERY`。原生算子输出 `part.json` 与 `review-history.json`,复用现有 directory artifact 发布通道和归属/条件创建规则,不要求旧 Worker 识别新的单文件 schema。此处 `manifest.json.kind=directory` 表示两个 JSON 的传输集合,不表示包含 clips;旧 v1/v2 单文件协议与旧目录交付保留。
|
||||
|
||||
主 JSON 移除 `acceptance.qualityReviews`,独立历史文件钉住 capture/part/revision;主文件保留最终验收与抽检结论。空任务标签省略,有效标签保留。描述状态依据实际有效文本而非 actions 数量;v3 新增 `invalid_and_blank` 表达无效和留空混合,保留 `invalid_only` 的“全部区间无效”含义。
|
||||
|
||||
路径基准显式写入:总索引相对本批次项目根,采集清单相对采集交付目录,原始 meta 和标定文件相对原始采集目录;已固定 workspace 来源另标基准。采集清单记录原始 `meta_<captureId>.json` 的 SHA/大小,expected parts 来自原始目录完整分页列举,不用已验收 part 数推定交齐。最终验收结论及最大 acceptedAt 的聚合仅覆盖本批次选择的修订。
|
||||
|
||||
总索引生成由管理员显式预览、按预览 snapshotId 条件发布,不随普通 GET/下载自动写对象存储。旧已完成 part 目录不能原地补发 v3,历史迁移仍需另行规划新交付目标。已有调用者和在途任务默认沿用旧协议;回退 API/开关不删除新批次,也不能把新格式任务改成旧格式后重用已完成目录。
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
# Fanyu 最终交付批量下载(Rust)
|
||||
|
||||
2026-10-20 新增 [不可变总 JSON 批次](delivery-index-batches.md) 的管理员预览/显式发布,以及按批次目录下载。旧下载协议保持兼容;批次以 `delivery_index.json` 和归属标记识别完成。
|
||||
|
||||
只增加后端读取接口和 `tools/drobotic-pull` 命令行客户端,不修改标注界面、任务分配、调度或交付格式。原 POC 的目录工具不是 Fanyu 的已上线能力;本版本使用独立的、只面向最终交付的协议。
|
||||
|
||||
## 使用
|
||||
|
||||
56
docs/delivery-index-batches.md
Normal file
56
docs/delivery-index-batches.md
Normal file
@ -0,0 +1,56 @@
|
||||
# 不可变总 JSON 交付批次
|
||||
|
||||
此能力为本地实现,尚未上线;部署及工作流启用以具体发布记录和授权为准。
|
||||
|
||||
原生打包需要新 package 镜像,且 `PART_INDEX_DELIVERY=true`、`DELIVERY_JSON_V3=true`;源索引修订另需 `SOURCE_INDEX_DELIVERY=true`。三个开关默认关闭。旧已交付目录保持不变,不能把 v3 补入已完成的旧目录。原生产物通过已有目录发布协议生成两个 JSON 及 worker 完成清单;它不复制媒体。
|
||||
|
||||
```text
|
||||
<冻结项目交付根>/index-batches/<snapshotId>/
|
||||
delivery_index.json # 本批次最后写入
|
||||
<captureId>/
|
||||
annotation_manifest.json # 此采集 part/history 完成后写入
|
||||
part0001.json
|
||||
part0001.review-history.json
|
||||
part0002.json
|
||||
part0002.review-history.json
|
||||
```
|
||||
|
||||
`delivery_index.json` 的 `entries[].path`、`captures[].manifestPath` 相对批次项目根。
|
||||
`annotation_manifest.json` 的 `entries[].path`、`parts[].path`、`reviewHistoryPath` 相对采集交付目录。
|
||||
`sourceMetadata.path` 与 `calibration[].path` 相对 `capture.prefix` 指向的原始采集目录,标定只带真实文件名,不再额外拼 calibration 或采集名。
|
||||
源索引 `sources[].workspaceKey` 保留固定修订与 SHA 引用,解析基准为 workspace,不冒充原始 meta 的路径。
|
||||
|
||||
主 part 的 `captureId` 为原始采集文件夹名,`partId` 为 part 文件夹名,`revision` 钉住已验收修订。
|
||||
审核历史只在独立文件。有效 taskLabel/taskLabelId 保留,空值省略。
|
||||
`segments[].contentKind` 为 `described`、`invalid_marked`、`gap_placeholder`;内部流转 status 不导出。
|
||||
`classification.descriptionStatus` 为:有非无效非空文本 `described`;全部无效 `invalid_only`;无效与留空混合且无有效描述 `invalid_and_blank`;否则 `empty_confirmed`。`contentCounts` 给出三种内容数量,绝不把动作数组非空当作有描述。
|
||||
|
||||
`summary` 的片数、时长、验收与最大 `acceptedAt` 只针对本批次。时长按 taskSplit 的秒区间汇总,验收时间按实际时间值比较。`expectedPartCount` 来自原始目录完整分页列举,`undeliveredParts` 和 `partial_run/full_run` 不等同于“本批次发布已完成”。全舍弃且无交付物的 part 继续走既有 no_deliverables 契约,本批次不把它冒充已交付 part。
|
||||
|
||||
## 管理操作
|
||||
|
||||
沿用 `tools/drobotic-pull` 登录、CSRF 和退出,不在命令参数中放密码。显式选择 package ID,绝不自动猜最新修订或从不同交付目标混合数据。
|
||||
|
||||
```sh
|
||||
# 只读预览:打印 snapshotId、根目录和总索引,不写对象存储。
|
||||
drobotic-pull --api https://platform.example --email admin@example.com \
|
||||
--project project-xxx --index-package package-part1 --index-package package-part2
|
||||
|
||||
# 审阅预览后用完全相同的选择和 snapshotId 发布;这是明确的存储写操作。
|
||||
drobotic-pull --api https://platform.example --email admin@example.com \
|
||||
--project project-xxx --index-package package-part1 --index-package package-part2 \
|
||||
--publish-index <预览的snapshotId>
|
||||
|
||||
# 下载指定批次,沿用 SHA 校验、签名刷新与文件级续传。
|
||||
drobotic-pull --api https://platform.example --email admin@example.com \
|
||||
--directory 'bos://bucket/<冻结项目根>/index-batches/<snapshotId>' \
|
||||
--connection storage-xxx --output ./deliveries
|
||||
```
|
||||
|
||||
接口为 `GET /api/delivery-index-batches?projectId=...&packageId=...&packageId=...` 和
|
||||
`POST /api/delivery-index-batches`,请求体 `projectId/packageIds/snapshotId`。
|
||||
仅组织管理员可操作,发布连接必须属于当前组织,或具备既有部署映射中明确的交付写授权(连接归属、deliveryPrefixes 和每个来源运行的 writePrefixes 同时匹配);只读委派不提供写权限,原始输入前缀始终不写。目标从冻结 part 目标推导,不能由调用者指定;与原始采集、其他交付目标、启用的接入监听前缀重叠即拒绝。
|
||||
|
||||
最多 100 个 part、单 JSON 8 MiB、批次 64 MiB、原始目录列举 50000 对象;超限报错,不静默截断。发布逐对象条件创建并读回逐字节校验;中断后使用同一 snapshotId 重试。没有总索引的半成品不能下载;不自动清理、覆盖旧批次。证据变化返回 409,重新预览。旧批次下载不重新推导“当前最新包”。
|
||||
|
||||
合成双 part 示例可用 `PYTHONPATH=. python tests/backend/export_delivery_v3_fixture.py <新的本地输出目录>` 生成。它使用真实算子入口,明确标记 synthetic,不能当客户真实业务验收。
|
||||
@ -2,7 +2,9 @@
|
||||
|
||||
适用范围:`fanyu-deliveries/` 下的最终交付目录。一条目录对应一个采集(Capture)的一个 part。
|
||||
|
||||
交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定:
|
||||
2026-10-20 新实现的不可变 v3 批次参见 [总 JSON 批次说明](../delivery-index-batches.md):默认关闭,尚待发布与工作流启用。它将审核历史分离、补全 meta 引用,并新增采集/项目清单。下文保持旧 v1/v2 与媒体目录契约,供历史交付及在途任务解析。
|
||||
|
||||
旧交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定:
|
||||
|
||||
| 形态 | 开关 | 交付内容 |
|
||||
| --- | --- | --- |
|
||||
|
||||
45
docs/verification/platform-feedback-20261020/report.md
Normal file
45
docs/verification/platform-feedback-20261020/report.md
Normal file
@ -0,0 +1,45 @@
|
||||
# 平台调优反馈修复记录(2026-10-20)
|
||||
|
||||
实现分支 `codex/platform-feedback-fixes-20261020`,基于当前生产 API 来源 `630a95c8`。
|
||||
工作在独立 worktree,未修改主工作区和 dev-mainline 的在途改动;没有 Git push、PR、部署或客户数据写入。
|
||||
BOS 拷贝超时按用户要求排除。
|
||||
|
||||
## 修复范围
|
||||
|
||||
完整问题与验收标准见 [spec](../../../.scratch/platform-feedback-20261020/spec.md)。
|
||||
|
||||
| 范围 | 本地结果 | 线上状态 |
|
||||
| --- | --- | --- |
|
||||
| 描述草稿与切片应用 | 完整草稿进入同一冲突处置与保存事务;失败、取消保留,确认后才清除;保存状态不再把待保存修改显示成已保存 | 待发布 |
|
||||
| 跨轮意见定位、滚动回弹 | 轮次 + ID 定位;滚动只响应定位事件;判定键拒绝旧轮同 ID | 待发布 |
|
||||
| 审查字幕 | 专注布局固定行;留空/无效/有效文本/读取异常清楚显示 | 待发布 |
|
||||
| 账号任务与历史重叠 | 保留既有分配语义;解释当前待办与历史版本,提示新修订复查 | 待发布 |
|
||||
| 总 JSON 与审核历史/meta/路径/标签/描述状态 | 原生 v3 两 JSON 交付、不可变批次、独立历史、原始 meta 校验引用、显式路径基准、空标签省略、内容驱动汇总 | 待新镜像与工作流开关 |
|
||||
| 多 part | 同采集双 part 通过真实打包入口、Worker 发布协议、API、独立本地 Mongo 数据库、恢复与校验下载 | 合成验证完成;真实客户双 part 样例及验收仍待具备输入 |
|
||||
|
||||
草稿问题的原因是切片应用沿用全局草稿守卫,其他片的停泊草稿会阻断应用;修复将完整草稿纳入事务和冲突检查。意见定位只用 ID,跨轮复用 ID 会同时命中;内联 ref 又在重新渲染时重复滚动。描述汇总将非空动作数组直接当作 described,未检查空白占位/无效内容。回归测试分别钉住这些触发条件,不能只验证“页面能打开”。
|
||||
|
||||
## 验证证据
|
||||
|
||||
| 检查 | 结果 |
|
||||
| --- | --- |
|
||||
| 新浏览器回归(真实组件 + Playwright + 合成媒体) | 通过:草稿冲突应用、刷新保留、取消、保存拒绝后重试、三轮同 ID、手动滚动后重渲染、字幕布局/状态/开关、当前/历史版本 |
|
||||
| 前端完整门禁 | 通过基线比较:13 项既有失败,无新增;逐用例和原因对照 `630a95c8`,未修改 known-failures |
|
||||
| TypeScript / Web 构建 | 通过;构建有原有大 bundle 提示 |
|
||||
| Python 交付回归 | 59 passed,1 skipped;跳过的是原有可选 60 秒物化夹具用例(缺 FANYU_DISCARD_MATERIALIZATION_CONTEXT) |
|
||||
| 源索引媒体夹具 | 通过 Go 本地生成及 Python v2/v3 不解码/不重算整视频验证,固定 sources/sourceFrames 保留 |
|
||||
| Go 新交付与本地副本 | API、deliveryindex、Worker 通过;本地 Mongo 独立 `delivery_index_batch_test_*` 库已清理,不读写快照客户集合 |
|
||||
| 切片保存、边界证据、撤销/重做、历史兼容 | 定向 API 回归通过 |
|
||||
| 交付兼容范围 | 46 个原有/新增用例通过;`TestDeliveryDownloadScopeAndCompletion` 失败在基线同样可复现(原夹具运行产物未放入 nodeExecutions),未修改其断言掩盖失败 |
|
||||
| Rust 客户端 | 6 个 library + 7 个 CLI 测试通过;clippy `--all-targets -- -D warnings` 通过 |
|
||||
| 发布流程工具与切片恢复 | 37 个 release unittest 通过;TestDurableAnnotationOutbox/TestSliceRecovery 通过 |
|
||||
| 算子生成文件 | generate_operator_repositories.py --check 通过 |
|
||||
|
||||
浏览器命令:`BROWSER_EXECUTABLE=<本机 Chromium 系浏览器> node tests/browser/run-platform-feedback.mjs`。
|
||||
原生双 part 生成:`PYTHONPATH=. python tests/backend/export_delivery_v3_fixture.py <新的本地目录>`。
|
||||
Go 串联验证:设置 `DELIVERY_V3_FIXTURE_DIR=<该目录>`,执行 API/Worker 的 `TestIndexBatch` 与 `TestCaptureDeliveryNativeV3`;`DELIVERY_INDEX_MONGO_HTTP=1` 仅连接硬编码的本地 27018,使用新建合成库。
|
||||
|
||||
## 发布边界
|
||||
|
||||
见 [发布计划](rollout-plan.md)。成功 rollout 不是业务验收。
|
||||
本地合成/组件回归不证明现网已修好;本次不把旧聊天 v3 ZIP 或合成双 part 当成正式客户交付验收。
|
||||
29
docs/verification/platform-feedback-20261020/rollout-plan.md
Normal file
29
docs/verification/platform-feedback-20261020/rollout-plan.md
Normal file
@ -0,0 +1,29 @@
|
||||
# 平台反馈修复的分步发布计划
|
||||
|
||||
本文件是待审阅计划,不是部署授权。执行时先读 `docs/release/README.md`,存在正式 release 记录后先运行 `python3 scripts/release.py status <id>`。候选必须绑定提交、镜像 digest、目标身份和测试证据;不得用本地 tag 代替正式 digest。
|
||||
|
||||
## 1. API / annotation processor
|
||||
|
||||
- 候选代码包含前端草稿/轮次/字幕/版本说明修复及默认未启用的 v3 交付能力。无数据库迁移,无账号/权限/客户任务修复脚本,无启动数据改写新增。
|
||||
- API 和 annotation processor 按 release.py 使用同一不可变镜像。保留现网资源、副本、调度、存储、ConfigMap、Secret、Service、外部 source-index 处理模式和 verify-existing 启动设置。
|
||||
- 前端和 API 同一镜像更新;回退到原 API/processor 镜像,已保存的工作区仍是原协议。回退后草稿全局守卫与跨轮定位旧行为可能再现,必须先让使用者保存并关闭任务。
|
||||
- 重新核实 Fanyu API server、context、namespace UID,检查处理中的提交/物化状态后安排切换。只读检查优先;客户保存、领取、送检等验证必须另得精确授权。
|
||||
- 完成本地 prepare/check、镜像来源和启动安全审查,展示 release.py plan 后取得紧邻部署的明确授权,才运行 deploy。业务验收前不把发布记录标成完成。
|
||||
|
||||
## 2. package 算子及工作流(单独配置计划)
|
||||
|
||||
这部分不属于 release.py 首版镜像更新范围,不随第 1 步自动修改。
|
||||
|
||||
- 新 package 镜像固定 digest,使用本分支生成后的 `operator-repos/builtin/package/src/operator.py`。Worker、validator、coordinator 镜像不变;现网 Worker 的目录发布实现与基线相同,新 v3 复用其协议,新增回归已通过。
|
||||
- 在获准的工作流修订中显式配置 `PART_INDEX_DELIVERY=true`、`SOURCE_INDEX_DELIVERY=true`、`DELIVERY_JSON_V3=true`,输出仍用现有 deliveryDirectory。默认值均 false。
|
||||
- 仅新建且冻结到未占用目标的交付使用新格式。在途工作流/任务保持原镜像、原开关、原修订;不得直接给已完成目录补 v3 文件。
|
||||
- 管理员 GET 预览明确选定的 package IDs,核对范围、修订、expected parts、meta 与交付前缀。POST 发布会创建 BOS 对象,必须另行取得该批次和精确前缀的写授权。
|
||||
- 委派连接只复用已有明确的交付写授权,不扩大 mapping;批次目标或来源运行不在批准范围则拒绝,需先审查单独配置变更。
|
||||
- 客户真实的同采集双 part 验收后,生成正式样例,分别核对 part/采集/项目汇总和下载 SHA。当前合成验证不能替代这一项。
|
||||
- 回退只关闭后续新任务开关或恢复旧镜像;已发布批次保留,已开始的新格式任务不可改写成旧格式后覆盖同一冻结目录。
|
||||
|
||||
## 3. 历史交付
|
||||
|
||||
四个旧已交付采集仍保留原目录、manifest 和归属。需要正式 v3 重交时,另列源修订、全新交付目标、兼容和回退影响,经授权再执行;本次不改写历史客户文件。
|
||||
|
||||
BOS 拷贝超时没有纳入本次变更。
|
||||
@ -7848,6 +7848,18 @@
|
||||
"key": "PART_INDEX_DELIVERY",
|
||||
"type": "boolean",
|
||||
"value": "false"
|
||||
},
|
||||
{
|
||||
"key": "SOURCE_INDEX_DELIVERY",
|
||||
"type": "boolean",
|
||||
"value": "false",
|
||||
"description": "显式允许源索引修订的 JSON 交付;与 PART_INDEX_DELIVERY 同时启用"
|
||||
},
|
||||
{
|
||||
"key": "DELIVERY_JSON_V3",
|
||||
"type": "boolean",
|
||||
"value": "false",
|
||||
"description": "原生 v3:part.json 与独立审核历史,通过目录发布;须同时启用 PART_INDEX_DELIVERY"
|
||||
}
|
||||
],
|
||||
"id": "operator-builtin-package",
|
||||
|
||||
@ -109,9 +109,13 @@ func (s *server) handleDeliveryDownloadProjects(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
func (s *server) downloadBrowser(ctx context.Context, org, id, bucket, prefix string) (objectStorageBrowser, func(), error) {
|
||||
return s.deliveryReadBrowser(ctx, org, id, bucket, prefix, true)
|
||||
}
|
||||
|
||||
func (s *server) deliveryReadBrowser(ctx context.Context, org, id, bucket, prefix string, deliveryOnly bool) (objectStorageBrowser, func(), error) {
|
||||
c, e := s.store.GetStorageConnection(ctx, id)
|
||||
if e == nil && c != nil && c.OrganizationID != org {
|
||||
c, e = s.delegatedStorageRead(ctx, org, id, bucket, prefix, true)
|
||||
c, e = s.delegatedStorageRead(ctx, org, id, bucket, prefix, deliveryOnly)
|
||||
}
|
||||
if e != nil || c == nil || c.Provider != domain.StorageProviderBOS {
|
||||
return nil, nil, fmt.Errorf("交付存储不存在或不可访问")
|
||||
@ -127,6 +131,10 @@ func (s *server) projectDownloadTargets(r *http.Request, id string) ([]deliveryD
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return s.packageDownloadTargets(r, id, packages)
|
||||
}
|
||||
|
||||
func (s *server) packageDownloadTargets(r *http.Request, id string, packages []domain.DeliveryPackageRecord) ([]deliveryDownloadTarget, error) {
|
||||
targets := []deliveryDownloadTarget{}
|
||||
seen := map[string]bool{}
|
||||
for _, pkg := range packages {
|
||||
@ -229,7 +237,7 @@ func discoverDeliveryDirectories(ctx context.Context, b objectStorageBrowser, bu
|
||||
}
|
||||
if item.Kind == "directory" {
|
||||
children = append(children, key)
|
||||
} else if key == dir+"/manifest.json" {
|
||||
} else if key == dir+"/manifest.json" || key == dir+"/delivery_index.json" {
|
||||
complete = true
|
||||
}
|
||||
}
|
||||
@ -270,7 +278,7 @@ func readDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliver
|
||||
}
|
||||
raw, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/manifest.json", deliveryDownloadMaxManifestBytes+1)
|
||||
if e != nil || len(raw) > deliveryDownloadMaxManifestBytes {
|
||||
return nil, fmt.Errorf("交付清单不可读或过大")
|
||||
return readIndexBatchDownloadManifest(ctx, b, t)
|
||||
}
|
||||
var m struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
|
||||
446
internal/api/delivery_index_batches.go
Normal file
446
internal/api/delivery_index_batches.go
Normal file
@ -0,0 +1,446 @@
|
||||
package api
|
||||
|
||||
// Batch publication is an explicit admin operation. GET is a read-only preview;
|
||||
// POST requires the exact preview ID and only conditionally creates new objects.
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/deliveryindex"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workspacescope"
|
||||
)
|
||||
|
||||
type indexBatchRequest struct {
|
||||
ProjectID string `json:"projectId"`
|
||||
PackageIDs []string `json:"packageIds"`
|
||||
SnapshotID string `json:"snapshotId"`
|
||||
}
|
||||
type conditionalIndexWriter interface {
|
||||
PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
|
||||
}
|
||||
type batchPlan struct {
|
||||
snapshot deliveryindex.Snapshot
|
||||
browser objectStorageBrowser
|
||||
target deliveryDownloadTarget
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPlan, error) {
|
||||
if q.ProjectID == "" || len(q.PackageIDs) == 0 || len(q.PackageIDs) > 100 {
|
||||
return nil, fmt.Errorf("选择 1–100 个正式验收交付包")
|
||||
}
|
||||
project, e := s.store.GetProject(r.Context(), q.ProjectID)
|
||||
if e != nil || project == nil || project.OrganizationID != s.requestOrganizationID(r) {
|
||||
return nil, fmt.Errorf("项目不存在或不可访问")
|
||||
}
|
||||
packages, e := s.deliveryDownloadPackages(r, q.ProjectID)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
selected := []domain.DeliveryPackageRecord{}
|
||||
wanted := map[string]bool{}
|
||||
for _, id := range q.PackageIDs {
|
||||
if id == "" || wanted[id] {
|
||||
return nil, fmt.Errorf("交付包选择重复或为空")
|
||||
}
|
||||
wanted[id] = true
|
||||
}
|
||||
for _, pkg := range packages {
|
||||
if wanted[pkg.ID] {
|
||||
selected = append(selected, pkg)
|
||||
}
|
||||
}
|
||||
if len(selected) != len(wanted) {
|
||||
return nil, fmt.Errorf("仅可选择本项目已正式验收且有最终交付物的包")
|
||||
}
|
||||
targets, e := s.packageDownloadTargets(r, q.ProjectID, selected)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(targets) != len(selected) {
|
||||
return nil, fmt.Errorf("每个交付包必须有唯一且已完成的最终目标")
|
||||
}
|
||||
t := targets[0]
|
||||
root := path.Dir(path.Dir(t.prefix))
|
||||
if !strings.HasSuffix(path.Base(root), "__"+q.ProjectID) || !domain.ValidDeliveryPrefix(root) {
|
||||
return nil, fmt.Errorf("交付路径缺少冻结项目层")
|
||||
}
|
||||
c, e := s.store.GetStorageConnection(r.Context(), t.connectionID)
|
||||
if e != nil || c == nil {
|
||||
return nil, fmt.Errorf("批次交付连接不存在")
|
||||
}
|
||||
b, cleanup, e := s.downloadBrowser(r.Context(), s.requestOrganizationID(r), t.connectionID, t.bucket, t.prefix)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
success := false
|
||||
defer func() {
|
||||
if !success && cleanup != nil {
|
||||
cleanup()
|
||||
}
|
||||
}()
|
||||
sources := []deliveryindex.Source{}
|
||||
var sourceBytes int64
|
||||
sourceRuns := []string{}
|
||||
for _, target := range targets {
|
||||
if target.connectionID != t.connectionID || target.bucket != t.bucket || path.Dir(path.Dir(target.prefix)) != root {
|
||||
return nil, fmt.Errorf("选择的包不在同一冻结项目交付根")
|
||||
}
|
||||
files, e := readDownloadManifest(r.Context(), b, target)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if len(files) != 3 {
|
||||
return nil, fmt.Errorf("原生 v3 part 包必须只含 part.json 与独立审核历史")
|
||||
}
|
||||
var partBytes, historyBytes []byte
|
||||
for _, f := range files {
|
||||
if f.key != target.prefix+"/part.json" && f.key != target.prefix+"/review-history.json" {
|
||||
continue
|
||||
}
|
||||
if f.SizeBytes > 8<<20 {
|
||||
return nil, fmt.Errorf("单份 JSON 超过 8 MiB")
|
||||
}
|
||||
raw, _, e := b.ReadObjectPreview(r.Context(), f.bucket, f.key, 8<<20+1)
|
||||
if e != nil || int64(len(raw)) != f.SizeBytes || deliveryindex.Hash(raw) != f.SHA256 {
|
||||
return nil, fmt.Errorf("交付 JSON 与完成清单校验不一致")
|
||||
}
|
||||
sourceBytes += int64(len(raw))
|
||||
if sourceBytes > 64<<20 {
|
||||
return nil, fmt.Errorf("选择的源 JSON 合计超过 64 MiB,请缩小批次")
|
||||
}
|
||||
if strings.HasSuffix(f.key, "/part.json") {
|
||||
partBytes = raw
|
||||
} else {
|
||||
historyBytes = raw
|
||||
}
|
||||
}
|
||||
if len(partBytes) == 0 || len(historyBytes) == 0 {
|
||||
return nil, fmt.Errorf("选择的包尚未由原生 v3 算子交付;旧格式必须另行重交,不会覆盖旧文件")
|
||||
}
|
||||
var part struct {
|
||||
Capture struct {
|
||||
ID string `json:"captureId"`
|
||||
Prefix string `json:"prefix"`
|
||||
} `json:"capture"`
|
||||
PartID string `json:"partId"`
|
||||
Lineage struct {
|
||||
RunID string `json:"sourceRunId"`
|
||||
} `json:"lineage"`
|
||||
}
|
||||
if json.Unmarshal(partBytes, &part) != nil || target.prefix != root+"/"+part.Capture.ID+"/"+part.PartID {
|
||||
return nil, fmt.Errorf("part 身份与冻结交付目标不一致")
|
||||
}
|
||||
record, e := s.store.GetCaptureRecordByRunID(r.Context(), part.Lineage.RunID)
|
||||
if e != nil || record == nil || record.OrganizationID != project.OrganizationID || record.ProjectID != project.ID || record.Prefix != part.Capture.Prefix+"/"+part.PartID || record.Delivery == nil || record.Delivery.Prefix != target.prefix || record.Delivery.ConnectionID != t.connectionID || record.Delivery.Bucket != t.bucket {
|
||||
return nil, fmt.Errorf("part 来源与采集记录不一致")
|
||||
}
|
||||
sourceRuns = append(sourceRuns, part.Lineage.RunID)
|
||||
sourceBrowser, closeSource, e := s.deliveryReadBrowser(r.Context(), project.OrganizationID, record.ConnectionID, record.Bucket, part.Capture.Prefix, false)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
expected, meta, e := indexBatchSourceEvidence(r.Context(), sourceBrowser, record.Bucket, part.Capture.Prefix, part.Capture.ID)
|
||||
if closeSource != nil {
|
||||
closeSource()
|
||||
}
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
sources = append(sources, deliveryindex.Source{Part: partBytes, History: historyBytes, ExpectedParts: expected, MetadataSHA256: deliveryindex.Hash(meta), MetadataBytes: int64(len(meta))})
|
||||
}
|
||||
snapshot, e := deliveryindex.Build(project.ID, project.Name, root, sources)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if e := indexBatchWriteScope(project.OrganizationID, c, t.bucket, snapshot.Root, sourceRuns); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
validationFiles := make([]deliveryDownloadFile, 0, len(snapshot.Files))
|
||||
for name, body := range snapshot.Files {
|
||||
if len(body) > 8<<20 {
|
||||
return nil, fmt.Errorf("规范化后的单份 JSON 超过 8 MiB,请缩小交付内容")
|
||||
}
|
||||
validationFiles = append(validationFiles, deliveryDownloadFile{RelativePath: name, SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)})
|
||||
}
|
||||
if _, e := finalizeDownloadManifest(validationFiles); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
// Share the existing publisher's overlap rules; destination is server-derived.
|
||||
captures, e := s.store.ListCaptureRecords(r.Context(), project.OrganizationID, "", 0)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
for _, record := range captures {
|
||||
if record.Delivery != nil && record.Delivery.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Delivery.Prefix, snapshot.Root) {
|
||||
return nil, fmt.Errorf("批次目录与已有采集交付目标重叠")
|
||||
}
|
||||
if record.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Prefix, snapshot.Root) {
|
||||
return nil, fmt.Errorf("批次目录与来源采集重叠")
|
||||
}
|
||||
}
|
||||
listeners, e := s.store.ListEnabledCaptureAutomations(r.Context(), project.OrganizationID)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
for _, workflow := range listeners {
|
||||
if c := workflow.CaptureAutomation; c != nil && c.Bucket == t.bucket && domain.StoragePrefixesOverlap(c.Prefix, snapshot.Root) {
|
||||
return nil, fmt.Errorf("批次目录与已启用接入监听目录重叠")
|
||||
}
|
||||
}
|
||||
t.prefix = snapshot.Root
|
||||
t.relative = path.Base(root) + "/index-batches/" + snapshot.ID
|
||||
success = true
|
||||
return &batchPlan{snapshot, b, t, cleanup}, nil
|
||||
}
|
||||
|
||||
// Borrowed connections require the deployment's existing explicit final-output
|
||||
// grant and approved source runs. Read delegation never becomes write access.
|
||||
func indexBatchWriteScope(org string, c *domain.StorageConnectionRecord, bucket, prefix string, sourceRuns []string) error {
|
||||
m, err := workspacescope.ForOrganization(org)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.OrganizationID == org && (m == nil || m.ConnectionID != c.ID) {
|
||||
return nil
|
||||
}
|
||||
if m == nil || m.ConnectionID != c.ID || m.Bucket != bucket || !workspacescope.PrefixAllows(prefix, m.DeliveryPrefixes) {
|
||||
return fmt.Errorf("批次目录超出已授权交付前缀")
|
||||
}
|
||||
if c.OrganizationID != org {
|
||||
if m.WriteConnectionOrganizationID != c.OrganizationID {
|
||||
return fmt.Errorf("委派连接没有明确交付写授权")
|
||||
}
|
||||
for _, runID := range sourceRuns {
|
||||
if !workspacescope.PrefixAllows("system/runs/"+runID, m.WritePrefixes) {
|
||||
return fmt.Errorf("批次来源运行不在已批准委派范围")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func indexBatchSourceEvidence(ctx context.Context, b objectStorageBrowser, bucket, prefix, captureID string) ([]string, []byte, error) {
|
||||
paged, ok := b.(pagedObjectStorageBrowser)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("原始存储不支持完整分页列举")
|
||||
}
|
||||
expected := map[string]bool{}
|
||||
marker := ""
|
||||
markers := map[string]bool{}
|
||||
count := 0
|
||||
for {
|
||||
page, e := paged.ListObjectsPage(ctx, bucket, prefix+"/", marker, 1000)
|
||||
if e != nil {
|
||||
return nil, nil, fmt.Errorf("列举原始 part 失败")
|
||||
}
|
||||
for _, item := range page.Items {
|
||||
count++
|
||||
if count > 50000 {
|
||||
return nil, nil, fmt.Errorf("原始采集目录超过 50000 对象")
|
||||
}
|
||||
if !strings.HasPrefix(item.Key, prefix+"/") {
|
||||
return nil, nil, fmt.Errorf("原始列举返回越界对象")
|
||||
}
|
||||
first := strings.SplitN(strings.TrimPrefix(item.Key, prefix+"/"), "/", 2)[0]
|
||||
if isIndexPartName(first) {
|
||||
expected[first] = true
|
||||
}
|
||||
}
|
||||
if !page.HasMore {
|
||||
break
|
||||
}
|
||||
if page.NextMarker == "" || page.NextMarker == marker || markers[page.NextMarker] {
|
||||
return nil, nil, fmt.Errorf("原始存储分页无进展")
|
||||
}
|
||||
marker = page.NextMarker
|
||||
markers[marker] = true
|
||||
}
|
||||
meta, _, e := b.ReadObjectPreview(ctx, bucket, prefix+"/meta_"+captureID+".json", 8<<20+1)
|
||||
if e != nil || len(meta) == 0 || len(meta) > 8<<20 || !json.Valid(meta) {
|
||||
return nil, nil, fmt.Errorf("原始 meta JSON 不存在、不可读或过大")
|
||||
}
|
||||
parts := []string{}
|
||||
for p := range expected {
|
||||
parts = append(parts, p)
|
||||
}
|
||||
sort.Strings(parts)
|
||||
if len(parts) == 0 {
|
||||
return nil, nil, fmt.Errorf("原始采集没有 part 清单")
|
||||
}
|
||||
return parts, meta, nil
|
||||
}
|
||||
func isIndexPartName(s string) bool {
|
||||
if !strings.HasPrefix(s, "part") || len(s) < 8 {
|
||||
return false
|
||||
}
|
||||
for _, c := range s[4:] {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// A failed/ambiguous conditional PUT is successful only when a subsequent read
|
||||
// verifies the exact bytes. No overwrite, deletion or rollback of old batches.
|
||||
func createIndexObject(ctx context.Context, b objectStorageBrowser, writer conditionalIndexWriter, bucket, key string, body []byte) error {
|
||||
_, err := writer.PutObjectConditional(ctx, bucket, key, body, "application/json", "", "*", map[string]string{"sha256": deliveryindex.Hash(body)})
|
||||
got, _, e := b.ReadObjectPreview(ctx, bucket, key, int64(len(body))+1)
|
||||
if e == nil && bytes.Equal(got, body) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("批次对象条件创建失败或存在冲突")
|
||||
}
|
||||
return fmt.Errorf("批次对象写入后校验失败")
|
||||
}
|
||||
func publishIndexBatch(ctx context.Context, plan *batchPlan) error {
|
||||
writer, ok := plan.browser.(conditionalIndexWriter)
|
||||
if !ok {
|
||||
return fmt.Errorf("存储不支持条件创建,拒绝发布")
|
||||
}
|
||||
owner, _ := json.Marshal(map[string]string{"owner": "index-batch:" + plan.snapshot.ID})
|
||||
// Empty-prefix proof before taking ownership. Existing marker must match;
|
||||
// read failures never count as absence: a successful complete listing proves it.
|
||||
markerKey := plan.target.prefix + "/.delivery-owner.json"
|
||||
existing, _, err := plan.browser.ReadObjectPreview(ctx, plan.target.bucket, markerKey, 4096)
|
||||
if err == nil {
|
||||
if !bytes.Equal(existing, owner) {
|
||||
return fmt.Errorf("批次归属冲突")
|
||||
}
|
||||
} else {
|
||||
paged, ok := plan.browser.(pagedObjectStorageBrowser)
|
||||
if !ok {
|
||||
return fmt.Errorf("存储无法校验空目录")
|
||||
}
|
||||
page, e := paged.ListObjectsPage(ctx, plan.target.bucket, plan.target.prefix+"/", "", 1)
|
||||
if e != nil || len(page.Items) > 0 || page.HasMore {
|
||||
return fmt.Errorf("批次目录非空或无法确认,拒绝接管")
|
||||
}
|
||||
}
|
||||
if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, markerKey, owner); e != nil {
|
||||
return e
|
||||
}
|
||||
names := []string{}
|
||||
for name := range plan.snapshot.Files {
|
||||
if !validDeliveryDownloadPath(name) {
|
||||
return fmt.Errorf("不安全批次路径")
|
||||
}
|
||||
if name != "delivery_index.json" {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
// Capture manifests follow their part/history files; total index is last.
|
||||
sort.SliceStable(names, func(i, j int) bool {
|
||||
a := strings.HasSuffix(names[i], "/annotation_manifest.json")
|
||||
b := strings.HasSuffix(names[j], "/annotation_manifest.json")
|
||||
return !a && b
|
||||
})
|
||||
for _, name := range append(names, "delivery_index.json") {
|
||||
if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, plan.target.prefix+"/"+name, plan.snapshot.Files[name]); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *server) handleDeliveryIndexBatch(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Referrer-Policy", "no-referrer")
|
||||
principal, ok := currentPrincipal(r)
|
||||
if !ok || principal == nil || principal.User.Role != domain.UserRoleAdmin {
|
||||
renderJSON(w, 403, map[string]string{"error": "交付批次生成仅限组织管理员"})
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
r = r.WithContext(ctx)
|
||||
q := indexBatchRequest{ProjectID: r.URL.Query().Get("projectId"), PackageIDs: r.URL.Query()["packageId"]}
|
||||
if r.Method == http.MethodPost {
|
||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&q) != nil || q.SnapshotID == "" {
|
||||
renderJSON(w, 400, map[string]string{"error": "发布必须提交预览的 snapshotId、projectId 和 packageIds"})
|
||||
return
|
||||
}
|
||||
}
|
||||
plan, e := s.planIndexBatch(r, q)
|
||||
if e != nil {
|
||||
renderJSON(w, 409, map[string]string{"error": e.Error()})
|
||||
return
|
||||
}
|
||||
if plan.cleanup != nil {
|
||||
defer plan.cleanup()
|
||||
}
|
||||
if r.Method == http.MethodPost {
|
||||
if q.SnapshotID != plan.snapshot.ID {
|
||||
renderJSON(w, 409, map[string]string{"error": "交付证据已变化,请重新预览", "code": "index_snapshot_changed"})
|
||||
return
|
||||
}
|
||||
if e := publishIndexBatch(ctx, plan); e != nil {
|
||||
renderJSON(w, 409, map[string]string{"error": e.Error()})
|
||||
return
|
||||
}
|
||||
}
|
||||
renderJSON(w, 200, map[string]any{"snapshotId": plan.snapshot.ID, "rootKey": plan.snapshot.Root, "index": plan.snapshot.Index, "published": r.Method == http.MethodPost})
|
||||
}
|
||||
|
||||
// Immutable batches participate in the existing directory-download protocol,
|
||||
// including per-file hash checks, resume snapshots and refreshed signed URLs.
|
||||
func readIndexBatchDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliveryDownloadTarget) ([]deliveryDownloadFile, error) {
|
||||
id := path.Base(t.prefix)
|
||||
digest, digestErr := hex.DecodeString(id)
|
||||
if digestErr != nil || len(digest) != 32 || path.Base(path.Dir(t.prefix)) != "index-batches" {
|
||||
return nil, fmt.Errorf("交付清单不可读或过大")
|
||||
}
|
||||
marker, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/.delivery-owner.json", 4096)
|
||||
var owner struct {
|
||||
Owner string `json:"owner"`
|
||||
}
|
||||
if e != nil || json.Unmarshal(marker, &owner) != nil || owner.Owner != "index-batch:"+id {
|
||||
return nil, fmt.Errorf("批次归属标记无效")
|
||||
}
|
||||
raw, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/delivery_index.json", deliveryDownloadMaxManifestBytes+1)
|
||||
if e != nil || len(raw) > deliveryDownloadMaxManifestBytes {
|
||||
return nil, fmt.Errorf("批次尚未完成或清单过大")
|
||||
}
|
||||
var index struct {
|
||||
Schema int `json:"schemaVersion"`
|
||||
Kind string `json:"kind"`
|
||||
ID string `json:"snapshotId"`
|
||||
Count int `json:"fileCount"`
|
||||
Bytes int64 `json:"totalBytes"`
|
||||
Project struct {
|
||||
Root string `json:"rootKey"`
|
||||
} `json:"project"`
|
||||
Entries []deliveryindex.Entry `json:"entries"`
|
||||
}
|
||||
if json.Unmarshal(raw, &index) != nil || index.Schema != 3 || index.Kind != "part-index" || index.ID != id || index.Project.Root != t.prefix || index.Count != len(index.Entries) || index.Count < 1 || index.Count > 400 {
|
||||
return nil, fmt.Errorf("批次总索引身份或成员数量无效")
|
||||
}
|
||||
files := []deliveryDownloadFile{}
|
||||
var size int64
|
||||
for _, entry := range index.Entries {
|
||||
sha, shaErr := hex.DecodeString(entry.SHA256)
|
||||
if !validDeliveryDownloadPath(entry.Path) || entry.Path == "delivery_index.json" || strings.HasPrefix(entry.Path, ".") || entry.Key != t.prefix+"/"+entry.Path || entry.SizeBytes < 0 || entry.SizeBytes > 8<<20 || shaErr != nil || len(sha) != 32 {
|
||||
return nil, fmt.Errorf("批次成员校验信息无效")
|
||||
}
|
||||
size += entry.SizeBytes
|
||||
files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/" + entry.Path, SizeBytes: entry.SizeBytes, SHA256: entry.SHA256, key: entry.Key, bucket: t.bucket, connectionID: t.connectionID})
|
||||
}
|
||||
if size != index.Bytes || size > 64<<20 {
|
||||
return nil, fmt.Errorf("批次大小校验不一致")
|
||||
}
|
||||
files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/delivery_index.json", SizeBytes: int64(len(raw)), SHA256: deliveryindex.Hash(raw), key: t.prefix + "/delivery_index.json", bucket: t.bucket, connectionID: t.connectionID})
|
||||
return files, nil
|
||||
}
|
||||
40
internal/api/delivery_index_batches_mongo_test.go
Normal file
40
internal/api/delivery_index_batches_mongo_test.go
Normal file
@ -0,0 +1,40 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/store"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// Local replica only. A fresh synthetic database is created and removed; no
|
||||
// snapshot/customer collection is opened and the URI cannot be overridden.
|
||||
func TestIndexBatchMongoReplicaRecovery(t *testing.T) {
|
||||
if os.Getenv("DELIVERY_INDEX_MONGO_HTTP") != "1" {
|
||||
t.Skip("opt-in isolated local Mongo replica verification")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
if e = client.Ping(ctx, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
db := client.Database(fmt.Sprintf("delivery_index_batch_test_%d", time.Now().UnixNano()))
|
||||
defer func() {
|
||||
cleanup, done := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer done()
|
||||
if e := db.Drop(cleanup); e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
}()
|
||||
exerciseIndexBatch(t, store.NewMongoStore(db))
|
||||
}
|
||||
301
internal/api/delivery_index_batches_test.go
Normal file
301
internal/api/delivery_index_batches_test.go
Normal file
@ -0,0 +1,301 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/deliveryindex"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
type indexTestBrowser struct {
|
||||
*downloadTestBrowser
|
||||
puts []string
|
||||
fail string
|
||||
ambiguous bool
|
||||
}
|
||||
|
||||
func (b *indexTestBrowser) PutObjectConditional(_ context.Context, _ string, key string, body []byte, _ string, match, none string, _ map[string]string) (domain.ObjectStorageObjectSummary, error) {
|
||||
if match != "" || none != "*" {
|
||||
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("unconditional write")
|
||||
}
|
||||
b.puts = append(b.puts, key)
|
||||
if strings.HasSuffix(key, b.fail) && b.fail != "" {
|
||||
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("injected interruption")
|
||||
}
|
||||
if _, ok := b.objects[key]; ok {
|
||||
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("precondition failed")
|
||||
}
|
||||
b.objects[key] = append([]byte(nil), body...)
|
||||
if b.ambiguous {
|
||||
b.ambiguous = false
|
||||
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("response lost after commit")
|
||||
}
|
||||
return domain.ObjectStorageObjectSummary{}, nil
|
||||
}
|
||||
func (b *indexTestBrowser) ListObjectsPage(_ context.Context, _ string, prefix, marker string, limit int) (objectStoragePage, error) {
|
||||
if p, ok := b.pages[prefix+"|"+marker]; ok {
|
||||
return p, nil
|
||||
}
|
||||
keys := []string{}
|
||||
seen := map[string]bool{}
|
||||
for key := range b.objects {
|
||||
if strings.HasPrefix(key, prefix) {
|
||||
rest := strings.TrimPrefix(key, prefix)
|
||||
first, _, nested := strings.Cut(rest, "/")
|
||||
item := prefix + first
|
||||
if nested {
|
||||
item += "/"
|
||||
}
|
||||
if !seen[item] && item > marker {
|
||||
seen[item] = true
|
||||
keys = append(keys, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
page := objectStoragePage{}
|
||||
if limit <= 0 {
|
||||
limit = 1000
|
||||
}
|
||||
if len(keys) > limit {
|
||||
page.HasMore = true
|
||||
keys = keys[:limit]
|
||||
page.NextMarker = keys[len(keys)-1]
|
||||
}
|
||||
for _, key := range keys {
|
||||
kind := "file"
|
||||
if strings.HasSuffix(key, "/") {
|
||||
kind = "directory"
|
||||
}
|
||||
page.Items = append(page.Items, domain.ObjectStorageObjectSummary{Key: key, Kind: kind})
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
func indexCall(h http.Handler, p *authenticatedPrincipal, method, target string, body any) *httptest.ResponseRecorder {
|
||||
raw, _ := json.Marshal(body)
|
||||
r := httptest.NewRequest(method, target, bytes.NewReader(raw))
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
r.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "index-test-csrf"})
|
||||
r.Header.Set(csrfHeaderName, "index-test-csrf")
|
||||
r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, p))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authenticatedPrincipal, string) {
|
||||
t.Helper()
|
||||
root := "deliveries/项目A__project-A"
|
||||
org := "index-test-org"
|
||||
ctx := t.Context()
|
||||
b := &indexTestBrowser{downloadTestBrowser: &downloadTestBrowser{objects: map[string][]byte{}, pages: map[string]objectStoragePage{}}}
|
||||
must := func(e error) {
|
||||
t.Helper()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: "project-A", OrganizationID: org, Name: "项目A"}))
|
||||
must(ds.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "index-storage", OrganizationID: org, Provider: domain.StorageProviderBOS, Bucket: "test-bucket", Endpoint: "https://bj.bcebos.com"}))
|
||||
dir := os.Getenv("DELIVERY_V3_FIXTURE_DIR")
|
||||
for _, part := range []string{"part0001", "part0002"} {
|
||||
prefix := root + "/capture-A/" + part
|
||||
sourceRun := "source-run-" + part
|
||||
runID := "package-run-" + part
|
||||
artifactID := "directory-" + part
|
||||
body := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.part-index/v3","capture":{"captureId":"capture-A","prefix":"incoming/capture-A"},"partId":%q,"revision":1,"sourceMetadata":{"path":"meta_capture-A.json"},"lineage":{"sourceRunId":%q},"acceptance":{"finalAcceptance":{"confirmed":true,"acceptedAt":"2026-10-20T01:00:00Z"}},"slices":[{"sliceId":"slice-1","taskSplit":{"startSecond":0,"endSecond":1,"semantics":"half-open"},"classification":{"deliveryDisposition":"keep","descriptionStatus":"empty_confirmed"},"segments":[{"description":"","isInvalid":false,"contentKind":"gap_placeholder"}]}],"discardedSlices":[],"deliveryClassification":{"keptSliceCount":1,"discardedSliceCount":0,"keptDurationSeconds":1,"discardedDurationSeconds":0}}`, part, sourceRun))
|
||||
history := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.review-history/v1","captureId":"capture-A","partId":%q,"revision":1,"qualityReviews":[]}`, part))
|
||||
if dir != "" {
|
||||
var e error
|
||||
body, e = os.ReadFile(filepath.Join(dir, part, "part.json"))
|
||||
must(e)
|
||||
history, e = os.ReadFile(filepath.Join(dir, part, "review-history.json"))
|
||||
must(e)
|
||||
}
|
||||
b.objects[prefix+"/.delivery-owner.json"] = []byte(`{"owner":"synthetic-source"}`)
|
||||
b.objects[prefix+"/part.json"] = body
|
||||
b.objects[prefix+"/review-history.json"] = history
|
||||
entries := []deliveryindex.Entry{{Path: "part.json", Key: prefix + "/part.json", SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)}, {Path: "review-history.json", Key: prefix + "/review-history.json", SizeBytes: int64(len(history)), SHA256: deliveryindex.Hash(history)}}
|
||||
b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 2, "totalBytes": len(body) + len(history), "entries": entries})
|
||||
b.objects["incoming/capture-A/"+part+"/left.mp4"] = []byte("synthetic")
|
||||
run := domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: "project-A", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: artifactID, RunID: runID, Name: "delivery.directory.index", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindDirectory, Provider: domain.StorageProviderBOS, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix, ManifestKey: prefix + "/manifest.json"}}}}}
|
||||
run.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "package", Artifacts: run.Artifacts}}
|
||||
must(ds.UpsertRun(ctx, run))
|
||||
must(ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "package-" + part, OrganizationID: org, ProjectID: "project-A", ChildRunID: runID, Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{artifactID}}))
|
||||
must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, OrganizationID: org, ProjectID: "project-A", RunID: sourceRun, CaptureID: "capture-A/" + part, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A/" + part, SourcePrefix: "incoming/capture-A", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix}}))
|
||||
}
|
||||
b.objects["incoming/capture-A/meta_capture-A.json"] = []byte(`{"synthetic":true}`)
|
||||
old := storageConnectionBrowserResolver
|
||||
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
if c.OrganizationID != org {
|
||||
return nil, nil, fmt.Errorf("foreign storage")
|
||||
}
|
||||
return b, nil, nil
|
||||
}
|
||||
t.Cleanup(func() { storageConnectionBrowserResolver = old })
|
||||
p := &authenticatedPrincipal{User: domain.UserRecord{ID: "index-admin", OrganizationID: org, Role: domain.UserRoleAdmin}, Organization: domain.OrganizationRecord{ID: org}}
|
||||
return b, p, root
|
||||
}
|
||||
func exerciseIndexBatch(t *testing.T, ds store.Store) {
|
||||
t.Helper()
|
||||
b, p, root := seedIndexFixture(t, ds)
|
||||
h := NewHandler(ds, nil)
|
||||
selection := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001&packageId=package-part0002"
|
||||
preview := indexCall(h, p, "GET", selection, nil)
|
||||
if preview.Code != 200 {
|
||||
t.Fatal(preview.Code, preview.Body.String())
|
||||
}
|
||||
if len(b.puts) > 0 {
|
||||
t.Fatal("preview wrote storage")
|
||||
}
|
||||
var plan struct {
|
||||
ID string `json:"snapshotId"`
|
||||
Root string `json:"rootKey"`
|
||||
}
|
||||
json.Unmarshal(preview.Body.Bytes(), &plan)
|
||||
post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001", "package-part0002"}, SnapshotID: plan.ID}
|
||||
stale := post
|
||||
stale.SnapshotID = "old"
|
||||
if w := indexCall(h, p, "POST", "/api/delivery-index-batches", stale); w.Code != 409 || len(b.puts) > 0 {
|
||||
t.Fatal("stale preview wrote")
|
||||
}
|
||||
b.fail = "part0002.json"
|
||||
w := indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
||||
if w.Code != 409 {
|
||||
t.Fatal(w.Code, w.Body.String())
|
||||
}
|
||||
if _, ok := b.objects[plan.Root+"/delivery_index.json"]; ok {
|
||||
t.Fatal("partial batch marked complete")
|
||||
}
|
||||
b.fail = ""
|
||||
b.ambiguous = true
|
||||
h = NewHandler(ds, nil) // API restart; all progress lives in immutable storage.
|
||||
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
||||
if w.Code != 200 {
|
||||
t.Fatal(w.Code, w.Body.String())
|
||||
}
|
||||
if b.puts[len(b.puts)-1] != plan.Root+"/delivery_index.json" {
|
||||
t.Fatal("total index must be last")
|
||||
}
|
||||
before := append([]byte(nil), b.objects[plan.Root+"/delivery_index.json"]...)
|
||||
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
||||
if w.Code != 200 || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) {
|
||||
t.Fatal("idempotent retry")
|
||||
}
|
||||
target := deliveryDownloadTarget{"index-storage", "test-bucket", plan.Root, pathBase(plan.Root)}
|
||||
files, e := readDownloadManifest(t.Context(), b, target)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, f := range files {
|
||||
if deliveryindex.Hash(b.objects[f.key]) != f.SHA256 {
|
||||
t.Fatal("download hash", f.RelativePath)
|
||||
}
|
||||
}
|
||||
// Exercise the existing public download route, including refreshed signatures.
|
||||
download := downloadRequest(h, p, "/api/delivery-downloads/manifest?directory=bos://test-bucket/"+plan.Root+"&connectionId=index-storage")
|
||||
if download.Code != 200 || !strings.Contains(download.Body.String(), "delivery_index.json") {
|
||||
t.Fatal(download.Code, download.Body.String())
|
||||
}
|
||||
// A later partial snapshot does not alter the old full snapshot.
|
||||
first := indexCall(h, p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
|
||||
var partial struct {
|
||||
ID string `json:"snapshotId"`
|
||||
}
|
||||
json.Unmarshal(first.Body.Bytes(), &partial)
|
||||
w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID})
|
||||
if w.Code != 200 || partial.ID == plan.ID || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) {
|
||||
t.Fatal("immutable snapshots", w.Code, w.Body.String())
|
||||
}
|
||||
// Occupied, unowned batch directories can never be adopted.
|
||||
conflictRoot := root + "/index-batches/" + partial.ID
|
||||
delete(b.objects, conflictRoot+"/.delivery-owner.json")
|
||||
if w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID}); w.Code != 409 {
|
||||
t.Fatal("adopted unowned directory")
|
||||
}
|
||||
}
|
||||
func pathBase(s string) string { p := strings.Split(s, "/"); return p[len(p)-1] }
|
||||
func TestIndexBatchHTTPPublicationRecoveryAndDownload(t *testing.T) {
|
||||
exerciseIndexBatch(t, store.NewMemoryStore())
|
||||
}
|
||||
func TestIndexBatchRejectsScopeAndTampering(t *testing.T) {
|
||||
ds := store.NewMemoryStore()
|
||||
b, p, _ := seedIndexFixture(t, ds)
|
||||
h := NewHandler(ds, nil)
|
||||
target := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001"
|
||||
other := *p
|
||||
other.User.Role = domain.UserRoleQualityReviewer
|
||||
if w := indexCall(h, &other, "GET", target, nil); w.Code != 403 {
|
||||
t.Fatal("non-admin")
|
||||
}
|
||||
other = *p
|
||||
other.Organization.ID = "other"
|
||||
other.User.OrganizationID = "other"
|
||||
if w := indexCall(h, &other, "GET", target, nil); w.Code == 200 {
|
||||
t.Fatal("foreign org")
|
||||
}
|
||||
b.objects["deliveries/项目A__project-A/capture-A/part0001/part.json"] = []byte(`{"tampered":true}`)
|
||||
if w := indexCall(h, p, "GET", target, nil); w.Code != 409 || len(b.puts) > 0 {
|
||||
t.Fatal("tampered source accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexBatchDelegationKeepsSourceAndDeliveryScopesSeparate(t *testing.T) {
|
||||
ds := store.NewMemoryStore()
|
||||
b, p, root := seedIndexFixture(t, ds)
|
||||
c, _ := ds.GetStorageConnection(t.Context(), "index-storage")
|
||||
c.OrganizationID = "source-owner"
|
||||
if e := ds.UpsertStorageConnection(t.Context(), *c); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
if c.OrganizationID != "source-owner" {
|
||||
return nil, nil, fmt.Errorf("wrong owner")
|
||||
}
|
||||
return b, nil, nil
|
||||
}
|
||||
mapping := func(writes []string, deliveries []string) string {
|
||||
raw, _ := json.Marshal(map[string]any{p.Organization.ID: map[string]any{"connectionId": "index-storage", "bucket": "test-bucket", "readConnectionOrganizationId": "source-owner", "writeConnectionOrganizationId": "source-owner", "readPrefixes": []string{"incoming"}, "writePrefixes": writes, "deliveryPrefixes": deliveries}})
|
||||
return string(raw)
|
||||
}
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001", "system/runs/source-run-part0002"}, []string{root}))
|
||||
h := NewHandler(ds, nil)
|
||||
query := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001"
|
||||
w := indexCall(h, p, "GET", query, nil)
|
||||
if w.Code != 200 {
|
||||
t.Fatal("explicit source read/final-output write grant", w.Code, w.Body.String())
|
||||
}
|
||||
var preview struct {
|
||||
ID string `json:"snapshotId"`
|
||||
}
|
||||
json.Unmarshal(w.Body.Bytes(), &preview)
|
||||
post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: preview.ID}
|
||||
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
||||
if w.Code != 200 {
|
||||
t.Fatal(w.Code, w.Body.String())
|
||||
}
|
||||
for _, key := range b.puts {
|
||||
if !strings.HasPrefix(key, root+"/index-batches/") {
|
||||
t.Fatal("write escaped final-output scope", key)
|
||||
}
|
||||
}
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/some-other-run"}, []string{root}))
|
||||
before := len(b.puts)
|
||||
if w = indexCall(h, p, "POST", "/api/delivery-index-batches", post); w.Code != 409 || len(b.puts) != before {
|
||||
t.Fatal("unapproved source run wrote")
|
||||
}
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001"}, []string{root + "/capture-A"}))
|
||||
if w = indexCall(h, p, "GET", query, nil); w.Code != 409 {
|
||||
t.Fatal("sibling batch prefix escaped delivery grant", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
@ -16,6 +16,8 @@ func (server *server) mountDeliveryRoutes(router chi.Router) {
|
||||
router.Get("/api/delivery-packages", server.handleDeliveryPackagesList)
|
||||
router.Get("/api/delivery-downloads/projects", server.handleDeliveryDownloadProjects)
|
||||
router.Get("/api/delivery-downloads/manifest", server.handleDeliveryDownloadManifest)
|
||||
router.Get("/api/delivery-index-batches", server.handleDeliveryIndexBatch)
|
||||
router.Post("/api/delivery-index-batches", server.handleDeliveryIndexBatch)
|
||||
router.Get("/api/delivery-packages/{id}", server.handleDeliveryPackageGet)
|
||||
router.Get("/api/delivery-packages/{id}/contents", server.handleDeliveryPackageContents)
|
||||
router.Get("/api/delivery-packages/{id}/artifacts/{artifactId}/content", server.handleDeliveryPackageArtifactContent)
|
||||
|
||||
366
internal/deliveryindex/batch.go
Normal file
366
internal/deliveryindex/batch.go
Normal file
@ -0,0 +1,366 @@
|
||||
// Package deliveryindex builds deterministic immutable delivery snapshots. It
|
||||
// performs no storage operations; callers must verify the accepted source bytes.
|
||||
package deliveryindex
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const PartSchema = "droboticflow.delivery.part-index/v3"
|
||||
|
||||
var partName = regexp.MustCompile(`^part[0-9]{4,}$`)
|
||||
|
||||
type Source struct {
|
||||
Part json.RawMessage `json:"part"`
|
||||
History json.RawMessage `json:"history"`
|
||||
ExpectedParts []string `json:"expectedParts"`
|
||||
MetadataSHA256 string `json:"metadataSha256"`
|
||||
MetadataBytes int64 `json:"metadataBytes"`
|
||||
}
|
||||
type Snapshot struct {
|
||||
ID string `json:"snapshotId"`
|
||||
Root string `json:"rootKey"`
|
||||
Files map[string][]byte `json:"-"`
|
||||
Index json.RawMessage `json:"index"`
|
||||
}
|
||||
type Entry struct {
|
||||
Path string `json:"path"`
|
||||
Key string `json:"key"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
type Summary struct {
|
||||
CaptureCount int `json:"captureCount"`
|
||||
PartCount int `json:"partCount"`
|
||||
ExpectedPartCount int `json:"expectedPartCount"`
|
||||
UndeliveredParts []string `json:"undeliveredParts,omitempty"`
|
||||
Completion string `json:"deliveryCompletion"`
|
||||
KeptCount int `json:"keptSliceCount"`
|
||||
DiscardedCount int `json:"discardedSliceCount"`
|
||||
KeptDuration float64 `json:"keptDurationSeconds"`
|
||||
DiscardedDuration float64 `json:"discardedDurationSeconds"`
|
||||
Acceptance map[string]string `json:"acceptance"`
|
||||
}
|
||||
type part struct {
|
||||
Schema string `json:"schemaVersion"`
|
||||
Capture struct {
|
||||
ID string `json:"captureId"`
|
||||
Prefix string `json:"prefix"`
|
||||
} `json:"capture"`
|
||||
PartID string `json:"partId"`
|
||||
Revision int `json:"revision"`
|
||||
SourceMetadata struct {
|
||||
Path string `json:"path"`
|
||||
} `json:"sourceMetadata"`
|
||||
Slices []slice `json:"slices"`
|
||||
Discarded []slice `json:"discardedSlices"`
|
||||
Classification *struct {
|
||||
KeptCount int `json:"keptSliceCount"`
|
||||
DiscardedCount int `json:"discardedSliceCount"`
|
||||
KeptDuration float64 `json:"keptDurationSeconds"`
|
||||
DiscardedDuration float64 `json:"discardedDurationSeconds"`
|
||||
} `json:"deliveryClassification"`
|
||||
Acceptance struct {
|
||||
Final struct {
|
||||
Confirmed bool `json:"confirmed"`
|
||||
AcceptedAt string `json:"acceptedAt"`
|
||||
} `json:"finalAcceptance"`
|
||||
Reviews json.RawMessage `json:"qualityReviews"`
|
||||
} `json:"acceptance"`
|
||||
}
|
||||
type slice struct {
|
||||
ID string `json:"sliceId"`
|
||||
Split struct {
|
||||
Start float64 `json:"startSecond"`
|
||||
End float64 `json:"endSecond"`
|
||||
Semantics string `json:"semantics"`
|
||||
} `json:"taskSplit"`
|
||||
Classification struct {
|
||||
Disposition string `json:"deliveryDisposition"`
|
||||
DescriptionStatus string `json:"descriptionStatus"`
|
||||
Discard struct {
|
||||
Reason string `json:"reason"`
|
||||
} `json:"discardEvidence"`
|
||||
} `json:"classification"`
|
||||
Segments []struct {
|
||||
Description string `json:"description"`
|
||||
Invalid bool `json:"isInvalid"`
|
||||
Kind string `json:"contentKind"`
|
||||
Status *string `json:"status"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
|
||||
func Hash(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
func encode(v any) []byte {
|
||||
b, e := json.MarshalIndent(v, "", " ")
|
||||
if e != nil {
|
||||
panic(e)
|
||||
}
|
||||
return append(b, '\n')
|
||||
}
|
||||
func safeName(s string) bool {
|
||||
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\:\x00") && !strings.HasPrefix(s, ".")
|
||||
}
|
||||
func addSummary(a *Summary, b Summary) {
|
||||
a.PartCount += b.PartCount
|
||||
a.ExpectedPartCount += b.ExpectedPartCount
|
||||
a.KeptCount += b.KeptCount
|
||||
a.DiscardedCount += b.DiscardedCount
|
||||
a.KeptDuration += b.KeptDuration
|
||||
a.DiscardedDuration += b.DiscardedDuration
|
||||
if a.Acceptance == nil {
|
||||
a.Acceptance = map[string]string{"decision": "approved"}
|
||||
}
|
||||
before, _ := time.Parse(time.RFC3339Nano, a.Acceptance["acceptedAt"])
|
||||
after, _ := time.Parse(time.RFC3339Nano, b.Acceptance["acceptedAt"])
|
||||
if after.After(before) {
|
||||
a.Acceptance["acceptedAt"] = b.Acceptance["acceptedAt"]
|
||||
}
|
||||
}
|
||||
func checkPart(p part) (Summary, error) {
|
||||
s := Summary{PartCount: 1, Acceptance: map[string]string{"decision": "approved", "acceptedAt": p.Acceptance.Final.AcceptedAt}}
|
||||
if p.Schema != PartSchema || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 {
|
||||
return s, fmt.Errorf("part must pin an accepted native v3 revision without embedded review history")
|
||||
}
|
||||
if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil {
|
||||
return s, fmt.Errorf("missing acceptance timestamp")
|
||||
}
|
||||
if p.SourceMetadata.Path != "meta_"+p.Capture.ID+".json" || path.Base(p.Capture.Prefix) != p.Capture.ID || path.Clean(p.Capture.Prefix) != p.Capture.Prefix || strings.HasPrefix(p.Capture.Prefix, "/") || strings.Contains(p.Capture.Prefix, "../") {
|
||||
return s, fmt.Errorf("capture/source metadata identity mismatch")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var intervals [][2]float64
|
||||
for group, items := range [][]slice{p.Slices, p.Discarded} {
|
||||
for _, v := range items {
|
||||
if v.ID == "" || seen[v.ID] || v.Split.Semantics != "half-open" || math.IsNaN(v.Split.Start) || v.Split.Start < 0 || v.Split.End <= v.Split.Start {
|
||||
return s, fmt.Errorf("invalid slice identity or interval")
|
||||
}
|
||||
seen[v.ID] = true
|
||||
intervals = append(intervals, [2]float64{v.Split.Start, v.Split.End})
|
||||
duration := v.Split.End - v.Split.Start
|
||||
if group == 1 {
|
||||
if v.Classification.Disposition != "discard" || strings.TrimSpace(v.Classification.Discard.Reason) == "" || len(v.Segments) > 0 {
|
||||
return s, fmt.Errorf("invalid discarded evidence")
|
||||
}
|
||||
s.DiscardedCount++
|
||||
s.DiscardedDuration += duration
|
||||
continue
|
||||
}
|
||||
if v.Classification.Disposition != "keep" {
|
||||
return s, fmt.Errorf("invalid kept disposition")
|
||||
}
|
||||
s.KeptCount++
|
||||
s.KeptDuration += duration
|
||||
described, invalid, blank := false, false, false
|
||||
for _, a := range v.Segments {
|
||||
kind := "gap_placeholder"
|
||||
if a.Invalid {
|
||||
kind = "invalid_marked"
|
||||
invalid = true
|
||||
} else if strings.TrimSpace(a.Description) != "" {
|
||||
kind = "described"
|
||||
described = true
|
||||
} else {
|
||||
blank = true
|
||||
}
|
||||
if a.Kind != kind || a.Status != nil {
|
||||
return s, fmt.Errorf("action content classification mismatch")
|
||||
}
|
||||
}
|
||||
status := "empty_confirmed"
|
||||
if invalid {
|
||||
status = "invalid_only"
|
||||
}
|
||||
if invalid && blank {
|
||||
status = "invalid_and_blank"
|
||||
}
|
||||
if described {
|
||||
status = "described"
|
||||
}
|
||||
if v.Classification.DescriptionStatus != status {
|
||||
return s, fmt.Errorf("descriptionStatus mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.KeptCount == 0 {
|
||||
return s, fmt.Errorf("no deliverable slices")
|
||||
}
|
||||
if math.IsInf(s.KeptDuration, 0) || math.IsInf(s.DiscardedDuration, 0) {
|
||||
return s, fmt.Errorf("slice duration overflow")
|
||||
}
|
||||
if c := p.Classification; c == nil || c.KeptCount != s.KeptCount || c.DiscardedCount != s.DiscardedCount || math.Abs(c.KeptDuration-s.KeptDuration) > 0.000001 || math.Abs(c.DiscardedDuration-s.DiscardedDuration) > 0.000001 {
|
||||
return s, fmt.Errorf("part summary disagrees with its selected slices")
|
||||
}
|
||||
sort.Slice(intervals, func(i, j int) bool { return intervals[i][0] < intervals[j][0] })
|
||||
for i := 1; i < len(intervals); i++ {
|
||||
if math.Abs(intervals[i][0]-intervals[i-1][1]) > 0.000001 {
|
||||
return s, fmt.Errorf("slice coverage gap or overlap")
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Build uses a hash of canonical source documents, source metadata and expected
|
||||
// part inventory as its identity. Signed URLs and build time never affect it.
|
||||
func Build(projectID, projectName, projectRoot string, sources []Source) (Snapshot, error) {
|
||||
out := Snapshot{Files: map[string][]byte{}}
|
||||
if projectID == "" || projectRoot == "." || path.Clean(projectRoot) != projectRoot || strings.HasPrefix(projectRoot, "/") || strings.Contains(projectRoot, "../") || len(sources) == 0 || len(sources) > 100 {
|
||||
return out, fmt.Errorf("select 1–100 accepted part packages")
|
||||
}
|
||||
var sourceBytes int
|
||||
for _, source := range sources {
|
||||
sourceBytes += len(source.Part) + len(source.History)
|
||||
if sourceBytes > 64<<20 {
|
||||
return out, fmt.Errorf("source JSON exceeds 64 MiB")
|
||||
}
|
||||
}
|
||||
// Normalize JSON before hashing to ignore transport whitespace/map ordering.
|
||||
for i := range sources {
|
||||
var d, h any
|
||||
if json.Unmarshal(sources[i].Part, &d) != nil || json.Unmarshal(sources[i].History, &h) != nil {
|
||||
return out, fmt.Errorf("invalid source JSON")
|
||||
}
|
||||
sources[i].Part = encode(d)
|
||||
sources[i].History = encode(h)
|
||||
sources[i].ExpectedParts = append([]string(nil), sources[i].ExpectedParts...)
|
||||
sort.Strings(sources[i].ExpectedParts)
|
||||
}
|
||||
sort.Slice(sources, func(i, j int) bool { return string(sources[i].Part) < string(sources[j].Part) })
|
||||
identity := encode([]any{"droboticflow.delivery.batch/v1", projectID, projectName, projectRoot, sources})
|
||||
out.ID = Hash(identity)
|
||||
out.Root = projectRoot + "/index-batches/" + out.ID
|
||||
type group struct {
|
||||
capture map[string]string
|
||||
expected []string
|
||||
metadata Source
|
||||
parts []map[string]any
|
||||
summary Summary
|
||||
entries []Entry
|
||||
}
|
||||
groups := map[string]*group{}
|
||||
seen := map[string]bool{}
|
||||
var totalBytes int
|
||||
add := func(name string, body []byte) Entry {
|
||||
out.Files[name] = body
|
||||
totalBytes += len(body)
|
||||
return Entry{name, out.Root + "/" + name, int64(len(body)), Hash(body)}
|
||||
}
|
||||
for _, src := range sources {
|
||||
var p part
|
||||
if e := json.Unmarshal(src.Part, &p); e != nil {
|
||||
return out, e
|
||||
}
|
||||
s, e := checkPart(p)
|
||||
if e != nil {
|
||||
return out, e
|
||||
}
|
||||
name := p.Capture.ID + "/" + p.PartID
|
||||
if seen[strings.ToLower(name)] {
|
||||
return out, fmt.Errorf("duplicate capture/part, select exactly one revision")
|
||||
}
|
||||
seen[strings.ToLower(name)] = true
|
||||
if len(src.ExpectedParts) == 0 || len(src.ExpectedParts) > 10000 || src.MetadataBytes <= 0 || len(src.MetadataSHA256) != 64 {
|
||||
return out, fmt.Errorf("missing original source inventory/metadata evidence")
|
||||
}
|
||||
expectedSeen := map[string]bool{}
|
||||
found := false
|
||||
for _, n := range src.ExpectedParts {
|
||||
if !partName.MatchString(n) || expectedSeen[n] {
|
||||
return out, fmt.Errorf("invalid expected part inventory")
|
||||
}
|
||||
expectedSeen[n] = true
|
||||
found = found || n == p.PartID
|
||||
}
|
||||
if !found {
|
||||
return out, fmt.Errorf("delivered part missing from original inventory")
|
||||
}
|
||||
var history struct {
|
||||
Schema string `json:"schemaVersion"`
|
||||
CaptureID string `json:"captureId"`
|
||||
PartID string `json:"partId"`
|
||||
Revision int `json:"revision"`
|
||||
Reviews []json.RawMessage `json:"qualityReviews"`
|
||||
}
|
||||
if json.Unmarshal(src.History, &history) != nil || history.Schema != "droboticflow.delivery.review-history/v1" || history.CaptureID != p.Capture.ID || history.PartID != p.PartID || history.Revision != p.Revision || history.Reviews == nil {
|
||||
return out, fmt.Errorf("review history revision mismatch")
|
||||
}
|
||||
g := groups[p.Capture.ID]
|
||||
if g == nil {
|
||||
g = &group{capture: map[string]string{"captureId": p.Capture.ID, "prefix": p.Capture.Prefix}, expected: src.ExpectedParts, metadata: src}
|
||||
groups[p.Capture.ID] = g
|
||||
} else if g.capture["prefix"] != p.Capture.Prefix || strings.Join(g.expected, ",") != strings.Join(src.ExpectedParts, ",") || g.metadata.MetadataSHA256 != src.MetadataSHA256 {
|
||||
return out, fmt.Errorf("inconsistent capture source evidence")
|
||||
}
|
||||
var doc map[string]any
|
||||
json.Unmarshal(src.Part, &doc)
|
||||
doc["reviewHistory"] = map[string]string{"path": p.PartID + ".review-history.json", "pathBase": "capture_delivery"}
|
||||
doc["delivery"] = map[string]string{"kind": "part_index", "fileName": p.PartID + ".json", "artifactName": "delivery.part-index"}
|
||||
g.entries = append(g.entries, add(name+".json", encode(doc)), add(name+".review-history.json", src.History))
|
||||
g.parts = append(g.parts, map[string]any{"part": p.PartID, "revision": p.Revision, "path": p.PartID + ".json", "reviewHistoryPath": p.PartID + ".review-history.json"})
|
||||
addSummary(&g.summary, s)
|
||||
if math.IsInf(g.summary.KeptDuration, 0) || math.IsInf(g.summary.DiscardedDuration, 0) {
|
||||
return out, fmt.Errorf("capture duration overflow")
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(groups))
|
||||
for n := range groups {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
allEntries := []Entry{}
|
||||
captures := []map[string]any{}
|
||||
total := Summary{Completion: "full_run", CaptureCount: len(groups)}
|
||||
for _, n := range names {
|
||||
g := groups[n]
|
||||
sort.Slice(g.parts, func(i, j int) bool { return g.parts[i]["part"].(string) < g.parts[j]["part"].(string) })
|
||||
g.summary.CaptureCount = 1
|
||||
g.summary.ExpectedPartCount = len(g.expected)
|
||||
g.summary.Completion = "full_run"
|
||||
g.summary.UndeliveredParts = []string{}
|
||||
for _, pn := range g.expected {
|
||||
if !seen[strings.ToLower(n+"/"+pn)] {
|
||||
g.summary.UndeliveredParts = append(g.summary.UndeliveredParts, pn)
|
||||
}
|
||||
}
|
||||
if len(g.summary.UndeliveredParts) > 0 {
|
||||
g.summary.Completion = "partial_run"
|
||||
total.Completion = "partial_run"
|
||||
}
|
||||
sort.Slice(g.entries, func(i, j int) bool { return g.entries[i].Path < g.entries[j].Path })
|
||||
local := append([]Entry(nil), g.entries...)
|
||||
var size int64
|
||||
for i := range local {
|
||||
local[i].Path = strings.TrimPrefix(local[i].Path, n+"/")
|
||||
size += local[i].SizeBytes
|
||||
}
|
||||
metadata := map[string]any{"path": "meta_" + n + ".json", "pathBase": "original_capture", "sha256": g.metadata.MetadataSHA256, "sizeBytes": g.metadata.MetadataBytes}
|
||||
manifest := map[string]any{"schemaVersion": 3, "kind": "part-index", "capture": g.capture, "pathBase": "capture_delivery", "sourceMetadata": metadata, "summary": g.summary, "parts": g.parts, "entries": local, "fileCount": len(local), "totalBytes": size}
|
||||
allEntries = append(allEntries, g.entries...)
|
||||
allEntries = append(allEntries, add(n+"/annotation_manifest.json", encode(manifest)))
|
||||
captures = append(captures, map[string]any{"captureId": n, "prefix": g.capture["prefix"], "manifestPath": n + "/annotation_manifest.json", "summary": g.summary})
|
||||
addSummary(&total, g.summary)
|
||||
if math.IsInf(total.KeptDuration, 0) || math.IsInf(total.DiscardedDuration, 0) {
|
||||
return out, fmt.Errorf("batch duration overflow")
|
||||
}
|
||||
}
|
||||
if totalBytes > 64<<20 {
|
||||
return out, fmt.Errorf("batch exceeds 64 MiB")
|
||||
}
|
||||
sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Path < allEntries[j].Path })
|
||||
var size int64
|
||||
for _, v := range allEntries {
|
||||
size += v.SizeBytes
|
||||
}
|
||||
out.Index = encode(map[string]any{"schemaVersion": 3, "kind": "part-index", "snapshotId": out.ID, "pathBase": "batch_project", "project": map[string]string{"projectId": projectID, "deliveryName": projectName, "rootKey": out.Root}, "summary": total, "captures": captures, "entries": allEntries, "fileCount": len(allEntries), "totalBytes": size})
|
||||
out.Files["delivery_index.json"] = out.Index
|
||||
return out, nil
|
||||
}
|
||||
127
internal/deliveryindex/batch_test.go
Normal file
127
internal/deliveryindex/batch_test.go
Normal file
@ -0,0 +1,127 @@
|
||||
package deliveryindex
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func fixture(partID string, revision int) Source {
|
||||
p := map[string]any{"schemaVersion": PartSchema, "capture": map[string]any{"captureId": "capture-A", "prefix": "incoming/capture-A"}, "partId": partID, "revision": revision, "sourceMetadata": map[string]string{"path": "meta_capture-A.json"}, "acceptance": map[string]any{"finalAcceptance": map[string]any{"confirmed": true, "acceptedAt": "2026-10-20T01:00:00Z"}}, "slices": []any{map[string]any{"sliceId": "s1", "taskSplit": map[string]any{"startSecond": 0, "endSecond": 2, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "keep", "descriptionStatus": "invalid_and_blank"}, "segments": []any{map[string]any{"description": "", "contentKind": "gap_placeholder", "isInvalid": false}, map[string]any{"description": "无效", "contentKind": "invalid_marked", "isInvalid": true}}}}, "discardedSlices": []any{map[string]any{"sliceId": "s2", "taskSplit": map[string]any{"startSecond": 2, "endSecond": 3, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "discard", "discardEvidence": map[string]string{"reason": "不交付"}}, "segments": []any{}}}}
|
||||
p["deliveryClassification"] = map[string]any{"keptSliceCount": 1, "discardedSliceCount": 1, "keptDurationSeconds": 2, "discardedDurationSeconds": 1}
|
||||
h := map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": "capture-A", "partId": partID, "revision": revision, "qualityReviews": []any{map[string]any{"revision": revision, "decision": "approved"}}}
|
||||
return Source{Part: encode(p), History: encode(h), ExpectedParts: []string{"part0001", "part0002"}, MetadataSHA256: strings.Repeat("a", 64), MetadataBytes: 10}
|
||||
}
|
||||
func build(t *testing.T, sources ...Source) Snapshot {
|
||||
t.Helper()
|
||||
s, e := Build("project-A", "项目A", "delivery/项目A__project-A", sources)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
return s
|
||||
}
|
||||
func TestBatchMultipartIntegrityAndImmutableScopes(t *testing.T) {
|
||||
a, b := fixture("part0001", 3), fixture("part0002", 8)
|
||||
first := build(t, a)
|
||||
full := build(t, a, b)
|
||||
reversed := build(t, b, a)
|
||||
if first.ID == full.ID || full.ID != reversed.ID {
|
||||
t.Fatal("snapshot identity")
|
||||
}
|
||||
var index struct {
|
||||
Summary Summary
|
||||
Entries []Entry
|
||||
Bytes int64 `json:"totalBytes"`
|
||||
}
|
||||
json.Unmarshal(full.Index, &index)
|
||||
if index.Summary.PartCount != 2 || index.Summary.ExpectedPartCount != 2 || index.Summary.Completion != "full_run" || index.Summary.KeptCount != 2 || index.Summary.DiscardedCount != 2 || index.Summary.KeptDuration != 4 || index.Summary.DiscardedDuration != 2 {
|
||||
t.Fatalf("summary %+v", index.Summary)
|
||||
}
|
||||
var size int64
|
||||
for _, e := range index.Entries {
|
||||
body := full.Files[e.Path]
|
||||
if int64(len(body)) != e.SizeBytes || Hash(body) != e.SHA256 || e.Key != full.Root+"/"+e.Path {
|
||||
t.Fatal("entry integrity", e)
|
||||
}
|
||||
size += e.SizeBytes
|
||||
}
|
||||
if size != index.Bytes {
|
||||
t.Fatal("total bytes")
|
||||
}
|
||||
var manifest struct {
|
||||
Entries []Entry
|
||||
Summary Summary
|
||||
Metadata struct {
|
||||
Path string
|
||||
Base string `json:"pathBase"`
|
||||
} `json:"sourceMetadata"`
|
||||
}
|
||||
json.Unmarshal(full.Files["capture-A/annotation_manifest.json"], &manifest)
|
||||
for _, e := range manifest.Entries {
|
||||
if strings.Contains(e.Path, "/") || !bytes.Equal(full.Files["capture-A/"+e.Path], reversed.Files["capture-A/"+e.Path]) {
|
||||
t.Fatal("capture-relative entry", e)
|
||||
}
|
||||
}
|
||||
if manifest.Metadata.Path != "meta_capture-A.json" || manifest.Metadata.Base != "original_capture" {
|
||||
t.Fatal("metadata reference")
|
||||
}
|
||||
again := build(t, a)
|
||||
if !bytes.Equal(first.Index, again.Index) {
|
||||
t.Fatal("old batch changed")
|
||||
}
|
||||
if !bytes.Contains(first.Index, []byte(`"partial_run"`)) || !bytes.Contains(first.Index, []byte(`"part0002"`)) {
|
||||
t.Fatal("partial scope")
|
||||
}
|
||||
}
|
||||
func TestBatchRejectsConflictingEvidence(t *testing.T) {
|
||||
for _, tc := range []string{"duplicate", "revision-history", "missing-inventory", "description", "coverage", "acceptance", "embedded-history", "part-summary"} {
|
||||
t.Run(tc, func(t *testing.T) {
|
||||
a := fixture("part0001", 1)
|
||||
sources := []Source{a}
|
||||
var d map[string]any
|
||||
json.Unmarshal(a.Part, &d)
|
||||
switch tc {
|
||||
case "duplicate":
|
||||
sources = append(sources, fixture("part0001", 2))
|
||||
case "revision-history":
|
||||
sources[0].History = fixture("part0001", 2).History
|
||||
case "missing-inventory":
|
||||
sources[0].ExpectedParts = []string{"part0002"}
|
||||
case "description":
|
||||
d["slices"].([]any)[0].(map[string]any)["classification"].(map[string]any)["descriptionStatus"] = "described"
|
||||
case "coverage":
|
||||
d["discardedSlices"].([]any)[0].(map[string]any)["taskSplit"].(map[string]any)["startSecond"] = 2.5
|
||||
case "acceptance":
|
||||
d["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["confirmed"] = false
|
||||
case "part-summary":
|
||||
d["deliveryClassification"].(map[string]any)["keptSliceCount"] = 99
|
||||
case "embedded-history":
|
||||
d["acceptance"].(map[string]any)["qualityReviews"] = []any{}
|
||||
}
|
||||
if tc == "description" || tc == "coverage" || tc == "acceptance" || tc == "embedded-history" || tc == "part-summary" {
|
||||
sources[0].Part = encode(d)
|
||||
}
|
||||
if _, e := Build("project-A", "项目A", "delivery/项目A__project-A", sources); e == nil {
|
||||
t.Fatal("accepted", tc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchAcceptanceUsesInstantsRatherThanTimestampText(t *testing.T) {
|
||||
a, b := fixture("part0001", 1), fixture("part0002", 2)
|
||||
var doc map[string]any
|
||||
json.Unmarshal(a.Part, &doc)
|
||||
doc["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["acceptedAt"] = "2026-10-20T10:00:00+08:00"
|
||||
a.Part = encode(doc)
|
||||
json.Unmarshal(b.Part, &doc)
|
||||
doc["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["acceptedAt"] = "2026-10-20T03:00:00Z"
|
||||
b.Part = encode(doc)
|
||||
s := build(t, a, b)
|
||||
var index struct{ Summary Summary }
|
||||
json.Unmarshal(s.Index, &index)
|
||||
if index.Summary.Acceptance["acceptedAt"] != "2026-10-20T03:00:00Z" {
|
||||
t.Fatal("timezone aggregation", index.Summary.Acceptance)
|
||||
}
|
||||
}
|
||||
56
internal/worker/capture_delivery_v3_test.go
Normal file
56
internal/worker/capture_delivery_v3_test.go
Normal file
@ -0,0 +1,56 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
// The production worker's directory protocol must publish the real native v3
|
||||
// artifact without routing it through its intentionally v1/v2-only file parser.
|
||||
func TestCaptureDeliveryNativeV3UsesDirectoryProtocol(t *testing.T) {
|
||||
fixture := os.Getenv("DELIVERY_V3_FIXTURE_DIR")
|
||||
if fixture == "" {
|
||||
t.Skip("generate native Python v3 fixture first")
|
||||
}
|
||||
db := store.NewMemoryStore()
|
||||
publisher := &deliveryPublisher{}
|
||||
runtime := &Runtime{store: db, artifactPublisher: publisher}
|
||||
ctx := t.Context()
|
||||
if e := db.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "storage-v3", OrganizationID: "org-v3", Provider: domain.StorageProviderBOS, Bucket: "b"}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
for _, part := range []string{"part0001", "part0002"} {
|
||||
prefix := "delivery/project-v3__project/capture-A/" + part
|
||||
run := &domain.RunRecord{ID: "run-" + part, OrganizationID: "org-v3"}
|
||||
if e := db.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, RunID: run.ID, OrganizationID: run.OrganizationID, Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "storage-v3", Bucket: "b", Prefix: prefix}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Path: filepath.Join(fixture, part), Type: domain.PortTypeFile}
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if _, e := runtime.publishCaptureDelivery(ctx, run, "package", artifact); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
var manifest struct {
|
||||
Kind string
|
||||
FileCount int
|
||||
Entries []directoryManifestEntry
|
||||
}
|
||||
if e := json.Unmarshal(publisher.objects["b/"+prefix+"/manifest.json"], &manifest); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if manifest.Kind != "directory" || manifest.FileCount != 2 {
|
||||
t.Fatal("native v3 directory protocol", manifest)
|
||||
}
|
||||
for _, name := range []string{"part.json", "review-history.json"} {
|
||||
if len(publisher.objects["b/"+prefix+"/"+name]) == 0 {
|
||||
t.Fatal("missing", part, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -836,11 +836,67 @@ def _description_status(actions: list[dict[str, Any]]) -> str:
|
||||
``empty_confirmed`` keeps its platform meaning: no description was written
|
||||
and the submitter confirmed that as a deliberate semantic blank.
|
||||
"""
|
||||
if not actions:
|
||||
return "empty_confirmed"
|
||||
if all(bool(action.get("isInvalid")) for action in actions):
|
||||
if any(not action.get("isInvalid") and str(action.get("description") or "").strip() for action in actions):
|
||||
return "described"
|
||||
if actions and all(bool(action.get("isInvalid")) for action in actions):
|
||||
return "invalid_only"
|
||||
return "described"
|
||||
return "empty_confirmed"
|
||||
|
||||
|
||||
def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload):
|
||||
"""Use the established immutable directory publisher for the two JSON files.
|
||||
|
||||
Legacy single-file publishers deliberately accept only v1/v2. A directory
|
||||
artifact preserves their ownership/conditional-publication protocol without
|
||||
teaching an old worker to misinterpret a new single-file schema.
|
||||
"""
|
||||
capture = dict(payload["capture"])
|
||||
prefix = pathlib.PurePosixPath(str(capture.get("prefix") or ""))
|
||||
part_id = prefix.name
|
||||
capture_id = prefix.parent.name
|
||||
recorded = str(capture.get("captureId") or "")
|
||||
if not re.fullmatch(r"part\d{4,}", part_id) or recorded not in {capture_id, capture_id + "/" + part_id}:
|
||||
raise RuntimeError("v3 capture identity disagrees with the original part prefix")
|
||||
if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3:
|
||||
raise RuntimeError("unsafe v3 capture identity prefix")
|
||||
capture.update(captureId=capture_id, prefix=str(prefix.parent))
|
||||
payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id)
|
||||
payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"}
|
||||
payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"}
|
||||
history = {
|
||||
"schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": capture_id,
|
||||
"partId": part_id, "revision": payload["revision"],
|
||||
"qualityReviews": payload["acceptance"].pop("qualityReviews", []),
|
||||
}
|
||||
payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"}
|
||||
payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None)
|
||||
for item in payload["calibration"]:
|
||||
if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]:
|
||||
raise RuntimeError("unsafe calibration filename")
|
||||
item["path"] = item["name"]
|
||||
for item in payload["slices"] + payload["discardedSlices"]:
|
||||
classification = item["classification"]
|
||||
for key in ("taskLabel", "taskLabelId"):
|
||||
if not str(classification.get(key) or "").strip():
|
||||
classification.pop(key, None)
|
||||
classification.get("discardEvidence", {}).pop("discardedByUserId", None)
|
||||
item.get("synchronization", {}).pop("calibration", None)
|
||||
for action in item["segments"]:
|
||||
action.pop("status", None)
|
||||
action["contentKind"] = "invalid_marked" if action["isInvalid"] else "described" if action["description"].strip() else "gap_placeholder"
|
||||
if classification["deliveryDisposition"] == "keep":
|
||||
counts = {kind: sum(action["contentKind"] == kind for action in item["segments"]) for kind in ("described", "invalid_marked", "gap_placeholder")}
|
||||
classification["contentCounts"] = counts
|
||||
if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]:
|
||||
classification["descriptionStatus"] = "invalid_and_blank"
|
||||
staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output))
|
||||
for name, document in (("part.json", payload), ("review-history.json", history)):
|
||||
(staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8")
|
||||
return [helpers["build_virtual_artifact"](
|
||||
run_id, node_id, "delivery.directory.index", str(staging), "file", 0,
|
||||
data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"])},
|
||||
output=_output(node_data, helpers, "deliveryDirectory"),
|
||||
)]
|
||||
|
||||
|
||||
def _part_index_slice(
|
||||
@ -1659,6 +1715,8 @@ def _run_accepted_slice_directory(
|
||||
for item in payload["slices"] + payload["discardedSlices"]:
|
||||
item["sourceFrames"] = frames[item["sliceId"]]
|
||||
body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8")
|
||||
if _bool(params, "DELIVERY_JSON_V3", False):
|
||||
return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload)
|
||||
return _emit_part_index(run_id, node_id, output, node_data, helpers, body)
|
||||
if source_index:
|
||||
source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output)
|
||||
@ -1894,6 +1952,9 @@ def _run_multiport(
|
||||
|
||||
|
||||
def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]:
|
||||
params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {}
|
||||
if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False):
|
||||
raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY")
|
||||
# A part-index delivery intentionally has no directory port; it must still
|
||||
# reach the accepted-slice path instead of the legacy inline contract.
|
||||
if (
|
||||
|
||||
38
tests/backend/export_delivery_v3_fixture.py
Normal file
38
tests/backend/export_delivery_v3_fixture.py
Normal file
@ -0,0 +1,38 @@
|
||||
"""Produce explicitly synthetic multi-part examples using the real package entrypoint."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from backend.operators.builtin.package import operator as package
|
||||
from tests.backend.test_delivery_v3_package import NODE, inputs_for
|
||||
from tests.backend.test_part_index_package import helpers
|
||||
|
||||
|
||||
def export(destination):
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
for part in ("part0001", "part0002"):
|
||||
work = destination / ("work-" + part)
|
||||
work.mkdir()
|
||||
inputs = inputs_for(work, [
|
||||
{"id": "blank", "description": "", "clipStartMs": 0, "clipEndMs": 500},
|
||||
{"id": "invalid", "description": "无效", "isInvalid": True, "clipStartMs": 500, "clipEndMs": 1000},
|
||||
])
|
||||
inputs[1]["data"]["capture"]["captureId"] = "capture-A/" + part
|
||||
inputs[1]["data"]["capture"]["prefix"] = "incoming/capture-A/" + part
|
||||
manifest_path = Path(inputs[0]["path"]) / "revision-manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
manifest["runId"] = "source-run-" + part
|
||||
manifest_path.write_text(json.dumps(manifest))
|
||||
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
|
||||
artifacts = package.run("package-" + part, "package", work / "out", NODE, inputs, helpers())
|
||||
shutil.copytree(artifacts[0]["path"], destination / part)
|
||||
shutil.rmtree(work)
|
||||
(destination / "meta_capture-A.json").write_text('{"synthetic":true,"parts":["part0001","part0002"]}\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("output", type=Path)
|
||||
export(parser.parse_args().output)
|
||||
79
tests/backend/test_delivery_v3_package.py
Normal file
79
tests/backend/test_delivery_v3_package.py
Normal file
@ -0,0 +1,79 @@
|
||||
"""Native v3 packaging, rather than postprocessing a customer sample ZIP."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from backend.operators.builtin.package import operator as package
|
||||
from tests.backend.test_part_index_package import helpers
|
||||
from tests.backend.test_slice_discard_package import fixture
|
||||
|
||||
NODE = {
|
||||
"outputs": [{"id": n, "name": n} for n in ("deliveryDirectory", "deliverySummary")],
|
||||
"hyperparams": [{"key": k, "value": v} for k, v in {
|
||||
"PART_INDEX_DELIVERY": "true", "DELIVERY_JSON_V3": "true", "PROCESSING_PROFILE": "raw_human"
|
||||
}.items()],
|
||||
}
|
||||
|
||||
|
||||
def inputs_for(tmp_path, actions):
|
||||
inputs = fixture(tmp_path, ["keep", "discard"])
|
||||
inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001", "captureRecordId": "record-A1"}
|
||||
inputs[1]["data"]["finalAcceptance"]["acceptedAt"] = "2026-10-20T01:00:00Z"
|
||||
root = Path(inputs[0]["path"])
|
||||
(root / "slices/slice-000001/annotation.json").write_text(json.dumps({"actions": actions}))
|
||||
return inputs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("actions,expected", [
|
||||
([{"description": "", "status": "described"}], "empty_confirmed"),
|
||||
([{"description": "无效", "isInvalid": True}, {"description": " "}], "invalid_and_blank"),
|
||||
([{"description": "拿起杯子", "status": "pending"}], "described"),
|
||||
])
|
||||
def test_native_v3_semantics_and_separate_history(tmp_path, actions, expected):
|
||||
inputs = inputs_for(tmp_path, actions)
|
||||
artifacts = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())
|
||||
artifact = artifacts[0]
|
||||
assert artifact["portId"] == "deliveryDirectory"
|
||||
root = Path(artifact["path"])
|
||||
assert {p.name for p in root.iterdir()} == {"part.json", "review-history.json"}
|
||||
doc = json.loads((root / "part.json").read_text())
|
||||
assert doc["schemaVersion"] == "droboticflow.delivery.part-index/v3"
|
||||
assert doc["partId"] == "part0001"
|
||||
assert doc["capture"]["captureId"] == "capture-A"
|
||||
assert doc["capture"]["prefix"] == "incoming/capture-A"
|
||||
assert doc["sourceMetadata"]["path"] == "meta_capture-A.json"
|
||||
assert doc["slices"][0]["classification"]["descriptionStatus"] == expected
|
||||
assert "qualityReviews" not in doc["acceptance"]
|
||||
assert json.loads((root / "review-history.json").read_text())["qualityReviews"]
|
||||
for entry in doc["slices"]:
|
||||
assert "taskLabel" not in entry["classification"]
|
||||
assert "taskLabelId" not in entry["classification"]
|
||||
assert "calibration" not in entry["synchronization"]
|
||||
assert all("status" not in a for a in entry["segments"])
|
||||
assert all(a["contentKind"] in {"gap_placeholder", "invalid_marked", "described"} for a in entry["segments"])
|
||||
assert all(c["path"] == c["name"] for c in doc["calibration"])
|
||||
first = {p.name: p.read_bytes() for p in root.iterdir()}
|
||||
again = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())
|
||||
assert {p.name: p.read_bytes() for p in Path(again[0]["path"]).iterdir()} == first
|
||||
assert not list((tmp_path / "out").rglob("*.mp4"))
|
||||
|
||||
|
||||
def test_v3_refuses_ambiguous_capture_identity(tmp_path):
|
||||
inputs = inputs_for(tmp_path, [])
|
||||
inputs[1]["data"]["capture"]["prefix"] = "incoming/another-capture/part0001"
|
||||
with pytest.raises(RuntimeError, match="capture identity"):
|
||||
package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())
|
||||
|
||||
|
||||
def test_v3_keeps_valid_legacy_labels_and_rejects_incomplete_configuration(tmp_path):
|
||||
inputs = inputs_for(tmp_path, [])
|
||||
sync = Path(inputs[0]["path"]) / "slices/slice-000001/synchronized-slice.json"
|
||||
doc = json.loads(sync.read_text())
|
||||
doc.update(taskLabel="拿取物品", taskLabelId="label-1")
|
||||
sync.write_text(json.dumps(doc))
|
||||
artifact = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())[0]
|
||||
classification = json.loads((Path(artifact["path"]) / "part.json").read_text())["slices"][0]["classification"]
|
||||
assert classification["taskLabel"] == "拿取物品" and classification["taskLabelId"] == "label-1"
|
||||
bad = {**NODE, "hyperparams": [{"key": "DELIVERY_JSON_V3", "value": "true"}]}
|
||||
with pytest.raises(RuntimeError, match="requires PART_INDEX_DELIVERY"):
|
||||
package.run("r", "p", tmp_path / "refused", bad, inputs, helpers())
|
||||
@ -53,9 +53,13 @@ def test_source_index_real_media_legacy_export_keeps_exact_frames_and_discards(t
|
||||
assert all("synthetic-discard" not in p.read_text() for p in root.rglob("*.json"))
|
||||
|
||||
|
||||
def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch):
|
||||
@pytest.mark.parametrize("v3", [False, True])
|
||||
def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch, v3):
|
||||
node = {"outputs": NODE["outputs"] + [{"id": "deliveryPartIndex", "name": "deliveryPartIndex"}],
|
||||
"hyperparams": NODE["hyperparams"] + [{"key": "PART_INDEX_DELIVERY", "value": "true"}, {"key": "SOURCE_INDEX_DELIVERY", "value": "true"}]}
|
||||
if v3:
|
||||
node["hyperparams"].append({"key": "DELIVERY_JSON_V3", "value": "true"})
|
||||
source_inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001"}
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError("JSON-only export must not transcode")
|
||||
monkeypatch.setattr(package.subprocess, "run", forbidden)
|
||||
@ -66,8 +70,9 @@ def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path
|
||||
monkeypatch.setattr(package, "_sha256_file", metadata_only)
|
||||
artifacts = package.run("test", "package", tmp_path / "output", node, source_inputs, helpers())
|
||||
assert len(artifacts) == 1
|
||||
payload = json.loads(Path(artifacts[0]["path"]).read_text())
|
||||
assert payload["schemaVersion"] == "droboticflow.delivery.part-index/v2"
|
||||
index_path = Path(artifacts[0]["path"]) / "part.json" if v3 else Path(artifacts[0]["path"])
|
||||
payload = json.loads(index_path.read_text())
|
||||
assert payload["schemaVersion"] == ("droboticflow.delivery.part-index/v3" if v3 else "droboticflow.delivery.part-index/v2")
|
||||
assert len(payload["sources"]) == 5
|
||||
assert [item["sliceId"] for item in payload["slices"]] == ["slice-000001", "slice-000003"]
|
||||
assert payload["discardedSlices"][0]["sourceFrames"] == {"left": [8, 24], "right": [8, 24]}
|
||||
|
||||
@ -38,6 +38,12 @@ struct Args {
|
||||
allow_http: bool,
|
||||
#[arg(long, conflicts_with = "directory")]
|
||||
project: Option<String>,
|
||||
/// 预览所选原生 v3 交付包的不可变批次;可重复,须同时指定 --project。
|
||||
#[arg(long, requires = "project", conflicts_with_all = ["directory", "list", "report"])]
|
||||
index_package: Vec<String>,
|
||||
/// 发布与预览完全一致的批次;参数为预览返回的 snapshotId。
|
||||
#[arg(long, requires = "index_package")]
|
||||
publish_index: Option<String>,
|
||||
/// 例如 bos://drobodata-fanyu/yanshou/项目__ID (仅管理员)。
|
||||
#[arg(long, requires = "connection", conflicts_with = "project")]
|
||||
directory: Option<String>,
|
||||
@ -125,6 +131,49 @@ async fn portal_error(r: reqwest::Response) -> anyhow::Error {
|
||||
}
|
||||
}
|
||||
impl Portal {
|
||||
async fn index_batch(
|
||||
&self,
|
||||
project: &str,
|
||||
packages: &[String],
|
||||
publish: Option<&str>,
|
||||
) -> Result<serde_json::Value> {
|
||||
let u = self.base.join("api/delivery-index-batches")?;
|
||||
let request = if let Some(snapshot) = publish {
|
||||
let cookie = self
|
||||
.jar
|
||||
.cookies(&u)
|
||||
.and_then(|c| c.to_str().ok().map(str::to_owned))
|
||||
.unwrap_or_default();
|
||||
let csrf = cookie
|
||||
.split(';')
|
||||
.find_map(|c| c.trim().strip_prefix("droboticflow_csrf="))
|
||||
.unwrap_or("");
|
||||
ensure!(!csrf.is_empty(), "缺少发布所需的 CSRF 会话,请重新登录");
|
||||
self.client.post(u)
|
||||
.header("Origin", self.base.origin().ascii_serialization())
|
||||
.header("X-Droboticflow-CSRF-Token", csrf)
|
||||
.json(&serde_json::json!({"projectId":project,"packageIds":packages,"snapshotId":snapshot}))
|
||||
} else {
|
||||
let mut q = vec![("projectId", project.to_owned())];
|
||||
q.extend(packages.iter().map(|p| ("packageId", p.clone())));
|
||||
self.client.get(u).query(&q)
|
||||
};
|
||||
let r = request
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("交付批次请求中断;可用相同 snapshotId 重试"))?;
|
||||
if !r.status().is_success() {
|
||||
return Err(portal_error(r).await);
|
||||
}
|
||||
let mut stream = r.bytes_stream();
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|_| anyhow::anyhow!("批次响应中断"))?;
|
||||
ensure!(bytes.len() + chunk.len() <= 8 * 1024 * 1024, "批次响应过大");
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
serde_json::from_slice(&bytes).context("批次响应无效")
|
||||
}
|
||||
async fn projects(&self) -> Result<Vec<Project>> {
|
||||
let r = self
|
||||
.client
|
||||
@ -289,6 +338,17 @@ async fn download_one(
|
||||
}
|
||||
|
||||
async fn execute(a: &Args, portal: &Portal) -> Result<()> {
|
||||
if !a.index_package.is_empty() {
|
||||
let result = portal
|
||||
.index_batch(
|
||||
a.project.as_deref().context("请指定 --project")?,
|
||||
&a.index_package,
|
||||
a.publish_index.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
println!("{}", serde_json::to_string_pretty(&result)?);
|
||||
return Ok(());
|
||||
}
|
||||
let selection = if let Some(project) = &a.project {
|
||||
Selection(vec![("projectId", project.clone())])
|
||||
} else if let Some(directory) = &a.directory {
|
||||
@ -481,6 +541,62 @@ mod tests {
|
||||
use super::*;
|
||||
use std::{io::Read, net::TcpListener, thread};
|
||||
#[test]
|
||||
fn index_publication_requires_explicit_package_selection() {
|
||||
assert!(Args::try_parse_from([
|
||||
"pull",
|
||||
"--api",
|
||||
"https://portal.test",
|
||||
"--publish-index",
|
||||
"snapshot"
|
||||
])
|
||||
.is_err());
|
||||
assert!(Args::try_parse_from([
|
||||
"pull",
|
||||
"--api",
|
||||
"https://portal.test",
|
||||
"--index-package",
|
||||
"p1"
|
||||
])
|
||||
.is_err());
|
||||
let a = Args::try_parse_from([
|
||||
"pull",
|
||||
"--api",
|
||||
"https://portal.test",
|
||||
"--project",
|
||||
"project-A",
|
||||
"--index-package",
|
||||
"p1",
|
||||
"--index-package",
|
||||
"p2",
|
||||
"--publish-index",
|
||||
"snapshot",
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(a.index_package, ["p1", "p2"]);
|
||||
assert_eq!(a.publish_index.as_deref(), Some("snapshot"));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn index_preview_is_read_only_and_publication_uses_csrf() {
|
||||
let response = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}";
|
||||
let (url, worker) = server(vec![response, response]);
|
||||
let p = portal(&url);
|
||||
p.jar
|
||||
.add_cookie_str("droboticflow_csrf=batch-csrf; Path=/", &p.base);
|
||||
p.index_batch("project-A", &["p1".into()], None)
|
||||
.await
|
||||
.unwrap();
|
||||
p.index_batch("project-A", &["p1".into()], Some("snapshot"))
|
||||
.await
|
||||
.unwrap();
|
||||
let requests = worker.join().unwrap();
|
||||
assert!(requests[0]
|
||||
.starts_with("GET /api/delivery-index-batches?projectId=project-A&packageId=p1 "));
|
||||
assert!(requests[1].starts_with("POST /api/delivery-index-batches "));
|
||||
assert!(requests[1]
|
||||
.to_ascii_lowercase()
|
||||
.contains("x-droboticflow-csrf-token: batch-csrf"));
|
||||
}
|
||||
#[test]
|
||||
fn error_messages_never_log_signed_urls() {
|
||||
assert!(
|
||||
safe_error_detail(br#"{"error":"https://storage/?authorization=secret"}"#).is_none()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user