From 7be23604d70f94a4a71c9e1dbeee611ac255ec1d Mon Sep 17 00:00:00 2001 From: d-robotics Date: Tue, 20 Oct 2026 12:14:56 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=A6=20Publish=20immutable=20delivery?= =?UTF-8?q?=20index=20batches=20with=20native=20v3=20packages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/platform-feedback-20261020/spec.md | 29 ++ CONTEXT.md | 4 + backend/operators/builtin/package/operator.py | 69 ++- docs/adr/0005-part-index-delivery-shape.md | 12 + docs/delivery-downloads.md | 2 + docs/delivery-index-batches.md | 56 +++ docs/manuals/delivery-tree-reference.zh-CN.md | 4 +- .../platform-feedback-20261020/report.md | 45 ++ .../rollout-plan.md | 29 ++ internal/api/builtin_operators_catalog.json | 12 + internal/api/delivery_downloads.go | 14 +- internal/api/delivery_index_batches.go | 446 ++++++++++++++++++ .../api/delivery_index_batches_mongo_test.go | 40 ++ internal/api/delivery_index_batches_test.go | 301 ++++++++++++ internal/api/routes_delivery.go | 2 + internal/deliveryindex/batch.go | 366 ++++++++++++++ internal/deliveryindex/batch_test.go | 127 +++++ internal/worker/capture_delivery_v3_test.go | 56 +++ .../builtin/package/src/operator.py | 69 ++- tests/backend/export_delivery_v3_fixture.py | 38 ++ tests/backend/test_delivery_v3_package.py | 79 ++++ tests/backend/test_source_index_package.py | 11 +- tools/drobotic-pull/src/main.rs | 116 +++++ 23 files changed, 1912 insertions(+), 15 deletions(-) create mode 100644 .scratch/platform-feedback-20261020/spec.md create mode 100644 docs/delivery-index-batches.md create mode 100644 docs/verification/platform-feedback-20261020/report.md create mode 100644 docs/verification/platform-feedback-20261020/rollout-plan.md create mode 100644 internal/api/delivery_index_batches.go create mode 100644 internal/api/delivery_index_batches_mongo_test.go create mode 100644 internal/api/delivery_index_batches_test.go create mode 100644 internal/deliveryindex/batch.go create mode 100644 internal/deliveryindex/batch_test.go create mode 100644 internal/worker/capture_delivery_v3_test.go create mode 100644 tests/backend/export_delivery_v3_fixture.py create mode 100644 tests/backend/test_delivery_v3_package.py diff --git a/.scratch/platform-feedback-20261020/spec.md b/.scratch/platform-feedback-20261020/spec.md new file mode 100644 index 00000000..1084db70 --- /dev/null +++ b/.scratch/platform-feedback-20261020/spec.md @@ -0,0 +1,29 @@ +# 平台调优与交付反馈修复(2026-10-20) + +范围来自平台调优群及马迪历史交付反馈;BOS 拷贝超时按用户要求暂缓。 +用户已授权本地修改、验证和提交;未授权 PR、push、部署或客户数据写入。 + +| 问题 | 修复与验收标准 | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| 1 描述未保存、其他片草稿拦切片应用 | 完整草稿与切片改动一并进入冲突处置及同一保存事务;失败保留、确认成功才清编辑器;取消保留草稿 | +| 2 第三轮意见跳回旧轮、滚动回弹 | 意见身份为轮次 + ID;仅明确定位事件触发滚动;旧轮同 ID 不可误判最新轮 | +| 3 审查字幕不可见 | 专注布局显式占行;留空、无效、描述、读取失败分别显示;字幕开关有效 | +| 4 账号“我的任务/我已审核”重叠 | 同一采集不同修订可以同时待办及留历史;显示当前/历史版本及复查含义,不改分配权限 | +| 5 原生总 JSON 交付未启用 | 原生 v3 打包 → 冻结 part 目录条件发布 → 管理员预览/发布不可变批次 → 现有校验下载链路;默认保持关闭 | +| 6 审核历史混在主 JSON | qualityReviews 移到独立、钉住同一修订的 review-history 文件;主 JSON 保留最终结论 | +| 7 采集清单缺 meta | annotation_manifest 引用原始 meta,API 读取验证并记录 SHA/大小 | +| 8 路径基准混淆 | 总索引相对批次项目根;采集清单相对采集交付目录;meta/标定相对原始采集目录;workspace 来源引用另行标明 | +| 9 空任务标签 | 省略空 taskLabel/taskLabelId,保留有效旧标签 | +| 10 描述汇总误报 | 由非无效且非空文本判断 described;v3 区分纯留空、纯无效、无效与留空混合,并带内容数量 | +| 11 多 part 样例和汇总验证 | 同一采集两个 part 经真实打包入口及 API、Mongo 副本验证范围、修订、验收聚合、SHA、重试和旧批次保留;合成样例不得充当客户真实业务验收 | + +## 已确认决定 + +2026-10-20 用户选择不可变交付批次目录。批次根为冻结项目交付根下 +`index-batches//`;总索引最后发布。以后新增 part 生成另一批次,旧批次保持可校验。 +旧已完成交付不原地补文件;重新交付、工作流开关和线上写测试需要单独计划与授权。 + +## 尚待线上完成 + +部署前必须按 `docs/release/README.md` 准备具体镜像和变更计划,取得紧邻部署的授权。 +本次没有客户真实的同采集双 part 已验收样例;本地合成双 part 已验证,真实样例与客户确认仍待具备数据后完成。 diff --git a/CONTEXT.md b/CONTEXT.md index d1cbd0bb..9d87430d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -168,3 +168,7 @@ _Avoid_:供应商任务副本(不代表给每家生产供应商复制客户 **撤销供应商项目授权**:停止供应商领取该项目的新任务,但保留其既有任务的标注、质检与返工处理资格,直至存量完成;不等于停用账号或释放已领取任务。 **生产统计异常**:仅指当前正式处理中的执行故障,包括运行失败、源采集处理失败、切片或送检处理作业失败、交付打包失败。供应商归属不同、部分未领取、归属待核对、审核退回、业务升级处理及正常等待不单独计为异常;这些状态保留在明细,不增加红色异常数量。 + +## 交付批次 + +**交付批次快照**:一次明确选定、已验收的 part 修订集合及其交付清单。批次只表示本次交付范围;新增 part 或替换修订形成新批次,已交付批次保留。批次完成不代表原始采集的全部 part 已交齐。 diff --git a/backend/operators/builtin/package/operator.py b/backend/operators/builtin/package/operator.py index fb3e73f6..cb8ae7e9 100644 --- a/backend/operators/builtin/package/operator.py +++ b/backend/operators/builtin/package/operator.py @@ -836,11 +836,67 @@ def _description_status(actions: list[dict[str, Any]]) -> str: ``empty_confirmed`` keeps its platform meaning: no description was written and the submitter confirmed that as a deliberate semantic blank. """ - if not actions: - return "empty_confirmed" - if all(bool(action.get("isInvalid")) for action in actions): + if any(not action.get("isInvalid") and str(action.get("description") or "").strip() for action in actions): + return "described" + if actions and all(bool(action.get("isInvalid")) for action in actions): return "invalid_only" - return "described" + return "empty_confirmed" + + +def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): + """Use the established immutable directory publisher for the two JSON files. + + Legacy single-file publishers deliberately accept only v1/v2. A directory + artifact preserves their ownership/conditional-publication protocol without + teaching an old worker to misinterpret a new single-file schema. + """ + capture = dict(payload["capture"]) + prefix = pathlib.PurePosixPath(str(capture.get("prefix") or "")) + part_id = prefix.name + capture_id = prefix.parent.name + recorded = str(capture.get("captureId") or "") + if not re.fullmatch(r"part\d{4,}", part_id) or recorded not in {capture_id, capture_id + "/" + part_id}: + raise RuntimeError("v3 capture identity disagrees with the original part prefix") + if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3: + raise RuntimeError("unsafe v3 capture identity prefix") + capture.update(captureId=capture_id, prefix=str(prefix.parent)) + payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id) + payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"} + payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"} + history = { + "schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": capture_id, + "partId": part_id, "revision": payload["revision"], + "qualityReviews": payload["acceptance"].pop("qualityReviews", []), + } + payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"} + payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None) + for item in payload["calibration"]: + if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]: + raise RuntimeError("unsafe calibration filename") + item["path"] = item["name"] + for item in payload["slices"] + payload["discardedSlices"]: + classification = item["classification"] + for key in ("taskLabel", "taskLabelId"): + if not str(classification.get(key) or "").strip(): + classification.pop(key, None) + classification.get("discardEvidence", {}).pop("discardedByUserId", None) + item.get("synchronization", {}).pop("calibration", None) + for action in item["segments"]: + action.pop("status", None) + action["contentKind"] = "invalid_marked" if action["isInvalid"] else "described" if action["description"].strip() else "gap_placeholder" + if classification["deliveryDisposition"] == "keep": + counts = {kind: sum(action["contentKind"] == kind for action in item["segments"]) for kind in ("described", "invalid_marked", "gap_placeholder")} + classification["contentCounts"] = counts + if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]: + classification["descriptionStatus"] = "invalid_and_blank" + staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output)) + for name, document in (("part.json", payload), ("review-history.json", history)): + (staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8") + return [helpers["build_virtual_artifact"]( + run_id, node_id, "delivery.directory.index", str(staging), "file", 0, + data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"])}, + output=_output(node_data, helpers, "deliveryDirectory"), + )] def _part_index_slice( @@ -1659,6 +1715,8 @@ def _run_accepted_slice_directory( for item in payload["slices"] + payload["discardedSlices"]: item["sourceFrames"] = frames[item["sliceId"]] body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8") + if _bool(params, "DELIVERY_JSON_V3", False): + return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload) return _emit_part_index(run_id, node_id, output, node_data, helpers, body) if source_index: source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output) @@ -1894,6 +1952,9 @@ def _run_multiport( def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]: + params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False): + raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY") # A part-index delivery intentionally has no directory port; it must still # reach the accepted-slice path instead of the legacy inline contract. if ( diff --git a/docs/adr/0005-part-index-delivery-shape.md b/docs/adr/0005-part-index-delivery-shape.md index 2655c3b2..f2f816df 100644 --- a/docs/adr/0005-part-index-delivery-shape.md +++ b/docs/adr/0005-part-index-delivery-shape.md @@ -31,3 +31,15 @@ status: accepted ## 审核层不受影响 物化修订里的逐片视频仍是质检与验收工作台的输入(`MaterializedSliceWorkbench`、`FinalAcceptanceWorkspace` 依赖 `slices/slice-*/left.mp4`)。本决定只改变**交付层**的输出形态,不改变内部审核层。 + +## 2026-10-20 修订:不可变批次及独立审核历史 + +用户明确选择**不可变交付批次目录**,拒绝在固定项目根反复覆盖总索引。每次选择已验收 part 修订后,内容身份生成新的 `index-batches//`,其中依次为 part JSON/独立审核历史、采集 `annotation_manifest.json`、最后写入的 `delivery_index.json`。重试只能接受相同内容,不能覆盖冲突对象或接管无归属的非空目录;旧批次继续可下载校验。批次标识绑定规范化源 JSON、原始 meta 校验信息和原始 part 目录清单,不包含生成时间或签名 URL。 + +新格式用默认关闭的 `DELIVERY_JSON_V3`,同时要求 `PART_INDEX_DELIVERY`;源索引修订还必须显式启用 `SOURCE_INDEX_DELIVERY`。原生算子输出 `part.json` 与 `review-history.json`,复用现有 directory artifact 发布通道和归属/条件创建规则,不要求旧 Worker 识别新的单文件 schema。此处 `manifest.json.kind=directory` 表示两个 JSON 的传输集合,不表示包含 clips;旧 v1/v2 单文件协议与旧目录交付保留。 + +主 JSON 移除 `acceptance.qualityReviews`,独立历史文件钉住 capture/part/revision;主文件保留最终验收与抽检结论。空任务标签省略,有效标签保留。描述状态依据实际有效文本而非 actions 数量;v3 新增 `invalid_and_blank` 表达无效和留空混合,保留 `invalid_only` 的“全部区间无效”含义。 + +路径基准显式写入:总索引相对本批次项目根,采集清单相对采集交付目录,原始 meta 和标定文件相对原始采集目录;已固定 workspace 来源另标基准。采集清单记录原始 `meta_.json` 的 SHA/大小,expected parts 来自原始目录完整分页列举,不用已验收 part 数推定交齐。最终验收结论及最大 acceptedAt 的聚合仅覆盖本批次选择的修订。 + +总索引生成由管理员显式预览、按预览 snapshotId 条件发布,不随普通 GET/下载自动写对象存储。旧已完成 part 目录不能原地补发 v3,历史迁移仍需另行规划新交付目标。已有调用者和在途任务默认沿用旧协议;回退 API/开关不删除新批次,也不能把新格式任务改成旧格式后重用已完成目录。 diff --git a/docs/delivery-downloads.md b/docs/delivery-downloads.md index 9baef846..f58eed01 100644 --- a/docs/delivery-downloads.md +++ b/docs/delivery-downloads.md @@ -1,5 +1,7 @@ # Fanyu 最终交付批量下载(Rust) +2026-10-20 新增 [不可变总 JSON 批次](delivery-index-batches.md) 的管理员预览/显式发布,以及按批次目录下载。旧下载协议保持兼容;批次以 `delivery_index.json` 和归属标记识别完成。 + 只增加后端读取接口和 `tools/drobotic-pull` 命令行客户端,不修改标注界面、任务分配、调度或交付格式。原 POC 的目录工具不是 Fanyu 的已上线能力;本版本使用独立的、只面向最终交付的协议。 ## 使用 diff --git a/docs/delivery-index-batches.md b/docs/delivery-index-batches.md new file mode 100644 index 00000000..9187ed7e --- /dev/null +++ b/docs/delivery-index-batches.md @@ -0,0 +1,56 @@ +# 不可变总 JSON 交付批次 + +此能力为本地实现,尚未上线;部署及工作流启用以具体发布记录和授权为准。 + +原生打包需要新 package 镜像,且 `PART_INDEX_DELIVERY=true`、`DELIVERY_JSON_V3=true`;源索引修订另需 `SOURCE_INDEX_DELIVERY=true`。三个开关默认关闭。旧已交付目录保持不变,不能把 v3 补入已完成的旧目录。原生产物通过已有目录发布协议生成两个 JSON 及 worker 完成清单;它不复制媒体。 + +```text +<冻结项目交付根>/index-batches// + delivery_index.json # 本批次最后写入 + / + annotation_manifest.json # 此采集 part/history 完成后写入 + part0001.json + part0001.review-history.json + part0002.json + part0002.review-history.json +``` + +`delivery_index.json` 的 `entries[].path`、`captures[].manifestPath` 相对批次项目根。 +`annotation_manifest.json` 的 `entries[].path`、`parts[].path`、`reviewHistoryPath` 相对采集交付目录。 +`sourceMetadata.path` 与 `calibration[].path` 相对 `capture.prefix` 指向的原始采集目录,标定只带真实文件名,不再额外拼 calibration 或采集名。 +源索引 `sources[].workspaceKey` 保留固定修订与 SHA 引用,解析基准为 workspace,不冒充原始 meta 的路径。 + +主 part 的 `captureId` 为原始采集文件夹名,`partId` 为 part 文件夹名,`revision` 钉住已验收修订。 +审核历史只在独立文件。有效 taskLabel/taskLabelId 保留,空值省略。 +`segments[].contentKind` 为 `described`、`invalid_marked`、`gap_placeholder`;内部流转 status 不导出。 +`classification.descriptionStatus` 为:有非无效非空文本 `described`;全部无效 `invalid_only`;无效与留空混合且无有效描述 `invalid_and_blank`;否则 `empty_confirmed`。`contentCounts` 给出三种内容数量,绝不把动作数组非空当作有描述。 + +`summary` 的片数、时长、验收与最大 `acceptedAt` 只针对本批次。时长按 taskSplit 的秒区间汇总,验收时间按实际时间值比较。`expectedPartCount` 来自原始目录完整分页列举,`undeliveredParts` 和 `partial_run/full_run` 不等同于“本批次发布已完成”。全舍弃且无交付物的 part 继续走既有 no_deliverables 契约,本批次不把它冒充已交付 part。 + +## 管理操作 + +沿用 `tools/drobotic-pull` 登录、CSRF 和退出,不在命令参数中放密码。显式选择 package ID,绝不自动猜最新修订或从不同交付目标混合数据。 + +```sh +# 只读预览:打印 snapshotId、根目录和总索引,不写对象存储。 +drobotic-pull --api https://platform.example --email admin@example.com \ + --project project-xxx --index-package package-part1 --index-package package-part2 + +# 审阅预览后用完全相同的选择和 snapshotId 发布;这是明确的存储写操作。 +drobotic-pull --api https://platform.example --email admin@example.com \ + --project project-xxx --index-package package-part1 --index-package package-part2 \ + --publish-index <预览的snapshotId> + +# 下载指定批次,沿用 SHA 校验、签名刷新与文件级续传。 +drobotic-pull --api https://platform.example --email admin@example.com \ + --directory 'bos://bucket/<冻结项目根>/index-batches/' \ + --connection storage-xxx --output ./deliveries +``` + +接口为 `GET /api/delivery-index-batches?projectId=...&packageId=...&packageId=...` 和 +`POST /api/delivery-index-batches`,请求体 `projectId/packageIds/snapshotId`。 +仅组织管理员可操作,发布连接必须属于当前组织,或具备既有部署映射中明确的交付写授权(连接归属、deliveryPrefixes 和每个来源运行的 writePrefixes 同时匹配);只读委派不提供写权限,原始输入前缀始终不写。目标从冻结 part 目标推导,不能由调用者指定;与原始采集、其他交付目标、启用的接入监听前缀重叠即拒绝。 + +最多 100 个 part、单 JSON 8 MiB、批次 64 MiB、原始目录列举 50000 对象;超限报错,不静默截断。发布逐对象条件创建并读回逐字节校验;中断后使用同一 snapshotId 重试。没有总索引的半成品不能下载;不自动清理、覆盖旧批次。证据变化返回 409,重新预览。旧批次下载不重新推导“当前最新包”。 + +合成双 part 示例可用 `PYTHONPATH=. python tests/backend/export_delivery_v3_fixture.py <新的本地输出目录>` 生成。它使用真实算子入口,明确标记 synthetic,不能当客户真实业务验收。 diff --git a/docs/manuals/delivery-tree-reference.zh-CN.md b/docs/manuals/delivery-tree-reference.zh-CN.md index 4a6218bf..3b91c30f 100644 --- a/docs/manuals/delivery-tree-reference.zh-CN.md +++ b/docs/manuals/delivery-tree-reference.zh-CN.md @@ -2,7 +2,9 @@ 适用范围:`fanyu-deliveries/` 下的最终交付目录。一条目录对应一个采集(Capture)的一个 part。 -交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定: +2026-10-20 新实现的不可变 v3 批次参见 [总 JSON 批次说明](../delivery-index-batches.md):默认关闭,尚待发布与工作流启用。它将审核历史分离、补全 meta 引用,并新增采集/项目清单。下文保持旧 v1/v2 与媒体目录契约,供历史交付及在途任务解析。 + +旧交付有两种形态,由打包节点的 `PART_INDEX_DELIVERY` 超参数决定: | 形态 | 开关 | 交付内容 | | --- | --- | --- | diff --git a/docs/verification/platform-feedback-20261020/report.md b/docs/verification/platform-feedback-20261020/report.md new file mode 100644 index 00000000..0bb5c469 --- /dev/null +++ b/docs/verification/platform-feedback-20261020/report.md @@ -0,0 +1,45 @@ +# 平台调优反馈修复记录(2026-10-20) + +实现分支 `codex/platform-feedback-fixes-20261020`,基于当前生产 API 来源 `630a95c8`。 +工作在独立 worktree,未修改主工作区和 dev-mainline 的在途改动;没有 Git push、PR、部署或客户数据写入。 +BOS 拷贝超时按用户要求排除。 + +## 修复范围 + +完整问题与验收标准见 [spec](../../../.scratch/platform-feedback-20261020/spec.md)。 + +| 范围 | 本地结果 | 线上状态 | +| --- | --- | --- | +| 描述草稿与切片应用 | 完整草稿进入同一冲突处置与保存事务;失败、取消保留,确认后才清除;保存状态不再把待保存修改显示成已保存 | 待发布 | +| 跨轮意见定位、滚动回弹 | 轮次 + ID 定位;滚动只响应定位事件;判定键拒绝旧轮同 ID | 待发布 | +| 审查字幕 | 专注布局固定行;留空/无效/有效文本/读取异常清楚显示 | 待发布 | +| 账号任务与历史重叠 | 保留既有分配语义;解释当前待办与历史版本,提示新修订复查 | 待发布 | +| 总 JSON 与审核历史/meta/路径/标签/描述状态 | 原生 v3 两 JSON 交付、不可变批次、独立历史、原始 meta 校验引用、显式路径基准、空标签省略、内容驱动汇总 | 待新镜像与工作流开关 | +| 多 part | 同采集双 part 通过真实打包入口、Worker 发布协议、API、独立本地 Mongo 数据库、恢复与校验下载 | 合成验证完成;真实客户双 part 样例及验收仍待具备输入 | + +草稿问题的原因是切片应用沿用全局草稿守卫,其他片的停泊草稿会阻断应用;修复将完整草稿纳入事务和冲突检查。意见定位只用 ID,跨轮复用 ID 会同时命中;内联 ref 又在重新渲染时重复滚动。描述汇总将非空动作数组直接当作 described,未检查空白占位/无效内容。回归测试分别钉住这些触发条件,不能只验证“页面能打开”。 + +## 验证证据 + +| 检查 | 结果 | +| --- | --- | +| 新浏览器回归(真实组件 + Playwright + 合成媒体) | 通过:草稿冲突应用、刷新保留、取消、保存拒绝后重试、三轮同 ID、手动滚动后重渲染、字幕布局/状态/开关、当前/历史版本 | +| 前端完整门禁 | 通过基线比较:13 项既有失败,无新增;逐用例和原因对照 `630a95c8`,未修改 known-failures | +| TypeScript / Web 构建 | 通过;构建有原有大 bundle 提示 | +| Python 交付回归 | 59 passed,1 skipped;跳过的是原有可选 60 秒物化夹具用例(缺 FANYU_DISCARD_MATERIALIZATION_CONTEXT) | +| 源索引媒体夹具 | 通过 Go 本地生成及 Python v2/v3 不解码/不重算整视频验证,固定 sources/sourceFrames 保留 | +| Go 新交付与本地副本 | API、deliveryindex、Worker 通过;本地 Mongo 独立 `delivery_index_batch_test_*` 库已清理,不读写快照客户集合 | +| 切片保存、边界证据、撤销/重做、历史兼容 | 定向 API 回归通过 | +| 交付兼容范围 | 46 个原有/新增用例通过;`TestDeliveryDownloadScopeAndCompletion` 失败在基线同样可复现(原夹具运行产物未放入 nodeExecutions),未修改其断言掩盖失败 | +| Rust 客户端 | 6 个 library + 7 个 CLI 测试通过;clippy `--all-targets -- -D warnings` 通过 | +| 发布流程工具与切片恢复 | 37 个 release unittest 通过;TestDurableAnnotationOutbox/TestSliceRecovery 通过 | +| 算子生成文件 | generate_operator_repositories.py --check 通过 | + +浏览器命令:`BROWSER_EXECUTABLE=<本机 Chromium 系浏览器> node tests/browser/run-platform-feedback.mjs`。 +原生双 part 生成:`PYTHONPATH=. python tests/backend/export_delivery_v3_fixture.py <新的本地目录>`。 +Go 串联验证:设置 `DELIVERY_V3_FIXTURE_DIR=<该目录>`,执行 API/Worker 的 `TestIndexBatch` 与 `TestCaptureDeliveryNativeV3`;`DELIVERY_INDEX_MONGO_HTTP=1` 仅连接硬编码的本地 27018,使用新建合成库。 + +## 发布边界 + +见 [发布计划](rollout-plan.md)。成功 rollout 不是业务验收。 +本地合成/组件回归不证明现网已修好;本次不把旧聊天 v3 ZIP 或合成双 part 当成正式客户交付验收。 diff --git a/docs/verification/platform-feedback-20261020/rollout-plan.md b/docs/verification/platform-feedback-20261020/rollout-plan.md new file mode 100644 index 00000000..620900aa --- /dev/null +++ b/docs/verification/platform-feedback-20261020/rollout-plan.md @@ -0,0 +1,29 @@ +# 平台反馈修复的分步发布计划 + +本文件是待审阅计划,不是部署授权。执行时先读 `docs/release/README.md`,存在正式 release 记录后先运行 `python3 scripts/release.py status `。候选必须绑定提交、镜像 digest、目标身份和测试证据;不得用本地 tag 代替正式 digest。 + +## 1. API / annotation processor + +- 候选代码包含前端草稿/轮次/字幕/版本说明修复及默认未启用的 v3 交付能力。无数据库迁移,无账号/权限/客户任务修复脚本,无启动数据改写新增。 +- API 和 annotation processor 按 release.py 使用同一不可变镜像。保留现网资源、副本、调度、存储、ConfigMap、Secret、Service、外部 source-index 处理模式和 verify-existing 启动设置。 +- 前端和 API 同一镜像更新;回退到原 API/processor 镜像,已保存的工作区仍是原协议。回退后草稿全局守卫与跨轮定位旧行为可能再现,必须先让使用者保存并关闭任务。 +- 重新核实 Fanyu API server、context、namespace UID,检查处理中的提交/物化状态后安排切换。只读检查优先;客户保存、领取、送检等验证必须另得精确授权。 +- 完成本地 prepare/check、镜像来源和启动安全审查,展示 release.py plan 后取得紧邻部署的明确授权,才运行 deploy。业务验收前不把发布记录标成完成。 + +## 2. package 算子及工作流(单独配置计划) + +这部分不属于 release.py 首版镜像更新范围,不随第 1 步自动修改。 + +- 新 package 镜像固定 digest,使用本分支生成后的 `operator-repos/builtin/package/src/operator.py`。Worker、validator、coordinator 镜像不变;现网 Worker 的目录发布实现与基线相同,新 v3 复用其协议,新增回归已通过。 +- 在获准的工作流修订中显式配置 `PART_INDEX_DELIVERY=true`、`SOURCE_INDEX_DELIVERY=true`、`DELIVERY_JSON_V3=true`,输出仍用现有 deliveryDirectory。默认值均 false。 +- 仅新建且冻结到未占用目标的交付使用新格式。在途工作流/任务保持原镜像、原开关、原修订;不得直接给已完成目录补 v3 文件。 +- 管理员 GET 预览明确选定的 package IDs,核对范围、修订、expected parts、meta 与交付前缀。POST 发布会创建 BOS 对象,必须另行取得该批次和精确前缀的写授权。 +- 委派连接只复用已有明确的交付写授权,不扩大 mapping;批次目标或来源运行不在批准范围则拒绝,需先审查单独配置变更。 +- 客户真实的同采集双 part 验收后,生成正式样例,分别核对 part/采集/项目汇总和下载 SHA。当前合成验证不能替代这一项。 +- 回退只关闭后续新任务开关或恢复旧镜像;已发布批次保留,已开始的新格式任务不可改写成旧格式后覆盖同一冻结目录。 + +## 3. 历史交付 + +四个旧已交付采集仍保留原目录、manifest 和归属。需要正式 v3 重交时,另列源修订、全新交付目标、兼容和回退影响,经授权再执行;本次不改写历史客户文件。 + +BOS 拷贝超时没有纳入本次变更。 diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index 098d3f78..4c72fb03 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -7848,6 +7848,18 @@ "key": "PART_INDEX_DELIVERY", "type": "boolean", "value": "false" + }, + { + "key": "SOURCE_INDEX_DELIVERY", + "type": "boolean", + "value": "false", + "description": "显式允许源索引修订的 JSON 交付;与 PART_INDEX_DELIVERY 同时启用" + }, + { + "key": "DELIVERY_JSON_V3", + "type": "boolean", + "value": "false", + "description": "原生 v3:part.json 与独立审核历史,通过目录发布;须同时启用 PART_INDEX_DELIVERY" } ], "id": "operator-builtin-package", diff --git a/internal/api/delivery_downloads.go b/internal/api/delivery_downloads.go index 0783199d..1bdc7028 100644 --- a/internal/api/delivery_downloads.go +++ b/internal/api/delivery_downloads.go @@ -109,9 +109,13 @@ func (s *server) handleDeliveryDownloadProjects(w http.ResponseWriter, r *http.R } func (s *server) downloadBrowser(ctx context.Context, org, id, bucket, prefix string) (objectStorageBrowser, func(), error) { + return s.deliveryReadBrowser(ctx, org, id, bucket, prefix, true) +} + +func (s *server) deliveryReadBrowser(ctx context.Context, org, id, bucket, prefix string, deliveryOnly bool) (objectStorageBrowser, func(), error) { c, e := s.store.GetStorageConnection(ctx, id) if e == nil && c != nil && c.OrganizationID != org { - c, e = s.delegatedStorageRead(ctx, org, id, bucket, prefix, true) + c, e = s.delegatedStorageRead(ctx, org, id, bucket, prefix, deliveryOnly) } if e != nil || c == nil || c.Provider != domain.StorageProviderBOS { return nil, nil, fmt.Errorf("交付存储不存在或不可访问") @@ -127,6 +131,10 @@ func (s *server) projectDownloadTargets(r *http.Request, id string) ([]deliveryD if e != nil { return nil, e } + return s.packageDownloadTargets(r, id, packages) +} + +func (s *server) packageDownloadTargets(r *http.Request, id string, packages []domain.DeliveryPackageRecord) ([]deliveryDownloadTarget, error) { targets := []deliveryDownloadTarget{} seen := map[string]bool{} for _, pkg := range packages { @@ -229,7 +237,7 @@ func discoverDeliveryDirectories(ctx context.Context, b objectStorageBrowser, bu } if item.Kind == "directory" { children = append(children, key) - } else if key == dir+"/manifest.json" { + } else if key == dir+"/manifest.json" || key == dir+"/delivery_index.json" { complete = true } } @@ -270,7 +278,7 @@ func readDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliver } raw, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/manifest.json", deliveryDownloadMaxManifestBytes+1) if e != nil || len(raw) > deliveryDownloadMaxManifestBytes { - return nil, fmt.Errorf("交付清单不可读或过大") + return readIndexBatchDownloadManifest(ctx, b, t) } var m struct { SchemaVersion int `json:"schemaVersion"` diff --git a/internal/api/delivery_index_batches.go b/internal/api/delivery_index_batches.go new file mode 100644 index 00000000..c9ee435f --- /dev/null +++ b/internal/api/delivery_index_batches.go @@ -0,0 +1,446 @@ +package api + +// Batch publication is an explicit admin operation. GET is a read-only preview; +// POST requires the exact preview ID and only conditionally creates new objects. +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "path" + "sort" + "strings" + "time" + + "droboticflow/internal/deliveryindex" + "droboticflow/internal/domain" + "droboticflow/internal/workspacescope" +) + +type indexBatchRequest struct { + ProjectID string `json:"projectId"` + PackageIDs []string `json:"packageIds"` + SnapshotID string `json:"snapshotId"` +} +type conditionalIndexWriter interface { + PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error) +} +type batchPlan struct { + snapshot deliveryindex.Snapshot + browser objectStorageBrowser + target deliveryDownloadTarget + cleanup func() +} + +func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPlan, error) { + if q.ProjectID == "" || len(q.PackageIDs) == 0 || len(q.PackageIDs) > 100 { + return nil, fmt.Errorf("选择 1–100 个正式验收交付包") + } + project, e := s.store.GetProject(r.Context(), q.ProjectID) + if e != nil || project == nil || project.OrganizationID != s.requestOrganizationID(r) { + return nil, fmt.Errorf("项目不存在或不可访问") + } + packages, e := s.deliveryDownloadPackages(r, q.ProjectID) + if e != nil { + return nil, e + } + selected := []domain.DeliveryPackageRecord{} + wanted := map[string]bool{} + for _, id := range q.PackageIDs { + if id == "" || wanted[id] { + return nil, fmt.Errorf("交付包选择重复或为空") + } + wanted[id] = true + } + for _, pkg := range packages { + if wanted[pkg.ID] { + selected = append(selected, pkg) + } + } + if len(selected) != len(wanted) { + return nil, fmt.Errorf("仅可选择本项目已正式验收且有最终交付物的包") + } + targets, e := s.packageDownloadTargets(r, q.ProjectID, selected) + if e != nil { + return nil, e + } + if len(targets) != len(selected) { + return nil, fmt.Errorf("每个交付包必须有唯一且已完成的最终目标") + } + t := targets[0] + root := path.Dir(path.Dir(t.prefix)) + if !strings.HasSuffix(path.Base(root), "__"+q.ProjectID) || !domain.ValidDeliveryPrefix(root) { + return nil, fmt.Errorf("交付路径缺少冻结项目层") + } + c, e := s.store.GetStorageConnection(r.Context(), t.connectionID) + if e != nil || c == nil { + return nil, fmt.Errorf("批次交付连接不存在") + } + b, cleanup, e := s.downloadBrowser(r.Context(), s.requestOrganizationID(r), t.connectionID, t.bucket, t.prefix) + if e != nil { + return nil, e + } + success := false + defer func() { + if !success && cleanup != nil { + cleanup() + } + }() + sources := []deliveryindex.Source{} + var sourceBytes int64 + sourceRuns := []string{} + for _, target := range targets { + if target.connectionID != t.connectionID || target.bucket != t.bucket || path.Dir(path.Dir(target.prefix)) != root { + return nil, fmt.Errorf("选择的包不在同一冻结项目交付根") + } + files, e := readDownloadManifest(r.Context(), b, target) + if e != nil { + return nil, e + } + if len(files) != 3 { + return nil, fmt.Errorf("原生 v3 part 包必须只含 part.json 与独立审核历史") + } + var partBytes, historyBytes []byte + for _, f := range files { + if f.key != target.prefix+"/part.json" && f.key != target.prefix+"/review-history.json" { + continue + } + if f.SizeBytes > 8<<20 { + return nil, fmt.Errorf("单份 JSON 超过 8 MiB") + } + raw, _, e := b.ReadObjectPreview(r.Context(), f.bucket, f.key, 8<<20+1) + if e != nil || int64(len(raw)) != f.SizeBytes || deliveryindex.Hash(raw) != f.SHA256 { + return nil, fmt.Errorf("交付 JSON 与完成清单校验不一致") + } + sourceBytes += int64(len(raw)) + if sourceBytes > 64<<20 { + return nil, fmt.Errorf("选择的源 JSON 合计超过 64 MiB,请缩小批次") + } + if strings.HasSuffix(f.key, "/part.json") { + partBytes = raw + } else { + historyBytes = raw + } + } + if len(partBytes) == 0 || len(historyBytes) == 0 { + return nil, fmt.Errorf("选择的包尚未由原生 v3 算子交付;旧格式必须另行重交,不会覆盖旧文件") + } + var part struct { + Capture struct { + ID string `json:"captureId"` + Prefix string `json:"prefix"` + } `json:"capture"` + PartID string `json:"partId"` + Lineage struct { + RunID string `json:"sourceRunId"` + } `json:"lineage"` + } + if json.Unmarshal(partBytes, &part) != nil || target.prefix != root+"/"+part.Capture.ID+"/"+part.PartID { + return nil, fmt.Errorf("part 身份与冻结交付目标不一致") + } + record, e := s.store.GetCaptureRecordByRunID(r.Context(), part.Lineage.RunID) + if e != nil || record == nil || record.OrganizationID != project.OrganizationID || record.ProjectID != project.ID || record.Prefix != part.Capture.Prefix+"/"+part.PartID || record.Delivery == nil || record.Delivery.Prefix != target.prefix || record.Delivery.ConnectionID != t.connectionID || record.Delivery.Bucket != t.bucket { + return nil, fmt.Errorf("part 来源与采集记录不一致") + } + sourceRuns = append(sourceRuns, part.Lineage.RunID) + sourceBrowser, closeSource, e := s.deliveryReadBrowser(r.Context(), project.OrganizationID, record.ConnectionID, record.Bucket, part.Capture.Prefix, false) + if e != nil { + return nil, e + } + expected, meta, e := indexBatchSourceEvidence(r.Context(), sourceBrowser, record.Bucket, part.Capture.Prefix, part.Capture.ID) + if closeSource != nil { + closeSource() + } + if e != nil { + return nil, e + } + sources = append(sources, deliveryindex.Source{Part: partBytes, History: historyBytes, ExpectedParts: expected, MetadataSHA256: deliveryindex.Hash(meta), MetadataBytes: int64(len(meta))}) + } + snapshot, e := deliveryindex.Build(project.ID, project.Name, root, sources) + if e != nil { + return nil, e + } + if e := indexBatchWriteScope(project.OrganizationID, c, t.bucket, snapshot.Root, sourceRuns); e != nil { + return nil, e + } + validationFiles := make([]deliveryDownloadFile, 0, len(snapshot.Files)) + for name, body := range snapshot.Files { + if len(body) > 8<<20 { + return nil, fmt.Errorf("规范化后的单份 JSON 超过 8 MiB,请缩小交付内容") + } + validationFiles = append(validationFiles, deliveryDownloadFile{RelativePath: name, SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)}) + } + if _, e := finalizeDownloadManifest(validationFiles); e != nil { + return nil, e + } + // Share the existing publisher's overlap rules; destination is server-derived. + captures, e := s.store.ListCaptureRecords(r.Context(), project.OrganizationID, "", 0) + if e != nil { + return nil, e + } + for _, record := range captures { + if record.Delivery != nil && record.Delivery.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Delivery.Prefix, snapshot.Root) { + return nil, fmt.Errorf("批次目录与已有采集交付目标重叠") + } + if record.Bucket == t.bucket && domain.StoragePrefixesOverlap(record.Prefix, snapshot.Root) { + return nil, fmt.Errorf("批次目录与来源采集重叠") + } + } + listeners, e := s.store.ListEnabledCaptureAutomations(r.Context(), project.OrganizationID) + if e != nil { + return nil, e + } + for _, workflow := range listeners { + if c := workflow.CaptureAutomation; c != nil && c.Bucket == t.bucket && domain.StoragePrefixesOverlap(c.Prefix, snapshot.Root) { + return nil, fmt.Errorf("批次目录与已启用接入监听目录重叠") + } + } + t.prefix = snapshot.Root + t.relative = path.Base(root) + "/index-batches/" + snapshot.ID + success = true + return &batchPlan{snapshot, b, t, cleanup}, nil +} + +// Borrowed connections require the deployment's existing explicit final-output +// grant and approved source runs. Read delegation never becomes write access. +func indexBatchWriteScope(org string, c *domain.StorageConnectionRecord, bucket, prefix string, sourceRuns []string) error { + m, err := workspacescope.ForOrganization(org) + if err != nil { + return err + } + if c.OrganizationID == org && (m == nil || m.ConnectionID != c.ID) { + return nil + } + if m == nil || m.ConnectionID != c.ID || m.Bucket != bucket || !workspacescope.PrefixAllows(prefix, m.DeliveryPrefixes) { + return fmt.Errorf("批次目录超出已授权交付前缀") + } + if c.OrganizationID != org { + if m.WriteConnectionOrganizationID != c.OrganizationID { + return fmt.Errorf("委派连接没有明确交付写授权") + } + for _, runID := range sourceRuns { + if !workspacescope.PrefixAllows("system/runs/"+runID, m.WritePrefixes) { + return fmt.Errorf("批次来源运行不在已批准委派范围") + } + } + } + return nil +} + +func indexBatchSourceEvidence(ctx context.Context, b objectStorageBrowser, bucket, prefix, captureID string) ([]string, []byte, error) { + paged, ok := b.(pagedObjectStorageBrowser) + if !ok { + return nil, nil, fmt.Errorf("原始存储不支持完整分页列举") + } + expected := map[string]bool{} + marker := "" + markers := map[string]bool{} + count := 0 + for { + page, e := paged.ListObjectsPage(ctx, bucket, prefix+"/", marker, 1000) + if e != nil { + return nil, nil, fmt.Errorf("列举原始 part 失败") + } + for _, item := range page.Items { + count++ + if count > 50000 { + return nil, nil, fmt.Errorf("原始采集目录超过 50000 对象") + } + if !strings.HasPrefix(item.Key, prefix+"/") { + return nil, nil, fmt.Errorf("原始列举返回越界对象") + } + first := strings.SplitN(strings.TrimPrefix(item.Key, prefix+"/"), "/", 2)[0] + if isIndexPartName(first) { + expected[first] = true + } + } + if !page.HasMore { + break + } + if page.NextMarker == "" || page.NextMarker == marker || markers[page.NextMarker] { + return nil, nil, fmt.Errorf("原始存储分页无进展") + } + marker = page.NextMarker + markers[marker] = true + } + meta, _, e := b.ReadObjectPreview(ctx, bucket, prefix+"/meta_"+captureID+".json", 8<<20+1) + if e != nil || len(meta) == 0 || len(meta) > 8<<20 || !json.Valid(meta) { + return nil, nil, fmt.Errorf("原始 meta JSON 不存在、不可读或过大") + } + parts := []string{} + for p := range expected { + parts = append(parts, p) + } + sort.Strings(parts) + if len(parts) == 0 { + return nil, nil, fmt.Errorf("原始采集没有 part 清单") + } + return parts, meta, nil +} +func isIndexPartName(s string) bool { + if !strings.HasPrefix(s, "part") || len(s) < 8 { + return false + } + for _, c := range s[4:] { + if c < '0' || c > '9' { + return false + } + } + return true +} + +// A failed/ambiguous conditional PUT is successful only when a subsequent read +// verifies the exact bytes. No overwrite, deletion or rollback of old batches. +func createIndexObject(ctx context.Context, b objectStorageBrowser, writer conditionalIndexWriter, bucket, key string, body []byte) error { + _, err := writer.PutObjectConditional(ctx, bucket, key, body, "application/json", "", "*", map[string]string{"sha256": deliveryindex.Hash(body)}) + got, _, e := b.ReadObjectPreview(ctx, bucket, key, int64(len(body))+1) + if e == nil && bytes.Equal(got, body) { + return nil + } + if err != nil { + return fmt.Errorf("批次对象条件创建失败或存在冲突") + } + return fmt.Errorf("批次对象写入后校验失败") +} +func publishIndexBatch(ctx context.Context, plan *batchPlan) error { + writer, ok := plan.browser.(conditionalIndexWriter) + if !ok { + return fmt.Errorf("存储不支持条件创建,拒绝发布") + } + owner, _ := json.Marshal(map[string]string{"owner": "index-batch:" + plan.snapshot.ID}) + // Empty-prefix proof before taking ownership. Existing marker must match; + // read failures never count as absence: a successful complete listing proves it. + markerKey := plan.target.prefix + "/.delivery-owner.json" + existing, _, err := plan.browser.ReadObjectPreview(ctx, plan.target.bucket, markerKey, 4096) + if err == nil { + if !bytes.Equal(existing, owner) { + return fmt.Errorf("批次归属冲突") + } + } else { + paged, ok := plan.browser.(pagedObjectStorageBrowser) + if !ok { + return fmt.Errorf("存储无法校验空目录") + } + page, e := paged.ListObjectsPage(ctx, plan.target.bucket, plan.target.prefix+"/", "", 1) + if e != nil || len(page.Items) > 0 || page.HasMore { + return fmt.Errorf("批次目录非空或无法确认,拒绝接管") + } + } + if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, markerKey, owner); e != nil { + return e + } + names := []string{} + for name := range plan.snapshot.Files { + if !validDeliveryDownloadPath(name) { + return fmt.Errorf("不安全批次路径") + } + if name != "delivery_index.json" { + names = append(names, name) + } + } + sort.Strings(names) + // Capture manifests follow their part/history files; total index is last. + sort.SliceStable(names, func(i, j int) bool { + a := strings.HasSuffix(names[i], "/annotation_manifest.json") + b := strings.HasSuffix(names[j], "/annotation_manifest.json") + return !a && b + }) + for _, name := range append(names, "delivery_index.json") { + if e := createIndexObject(ctx, plan.browser, writer, plan.target.bucket, plan.target.prefix+"/"+name, plan.snapshot.Files[name]); e != nil { + return e + } + } + return nil +} +func (s *server) handleDeliveryIndexBatch(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Referrer-Policy", "no-referrer") + principal, ok := currentPrincipal(r) + if !ok || principal == nil || principal.User.Role != domain.UserRoleAdmin { + renderJSON(w, 403, map[string]string{"error": "交付批次生成仅限组织管理员"}) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute) + defer cancel() + r = r.WithContext(ctx) + q := indexBatchRequest{ProjectID: r.URL.Query().Get("projectId"), PackageIDs: r.URL.Query()["packageId"]} + if r.Method == http.MethodPost { + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)) + decoder.DisallowUnknownFields() + if decoder.Decode(&q) != nil || q.SnapshotID == "" { + renderJSON(w, 400, map[string]string{"error": "发布必须提交预览的 snapshotId、projectId 和 packageIds"}) + return + } + } + plan, e := s.planIndexBatch(r, q) + if e != nil { + renderJSON(w, 409, map[string]string{"error": e.Error()}) + return + } + if plan.cleanup != nil { + defer plan.cleanup() + } + if r.Method == http.MethodPost { + if q.SnapshotID != plan.snapshot.ID { + renderJSON(w, 409, map[string]string{"error": "交付证据已变化,请重新预览", "code": "index_snapshot_changed"}) + return + } + if e := publishIndexBatch(ctx, plan); e != nil { + renderJSON(w, 409, map[string]string{"error": e.Error()}) + return + } + } + renderJSON(w, 200, map[string]any{"snapshotId": plan.snapshot.ID, "rootKey": plan.snapshot.Root, "index": plan.snapshot.Index, "published": r.Method == http.MethodPost}) +} + +// Immutable batches participate in the existing directory-download protocol, +// including per-file hash checks, resume snapshots and refreshed signed URLs. +func readIndexBatchDownloadManifest(ctx context.Context, b objectStorageBrowser, t deliveryDownloadTarget) ([]deliveryDownloadFile, error) { + id := path.Base(t.prefix) + digest, digestErr := hex.DecodeString(id) + if digestErr != nil || len(digest) != 32 || path.Base(path.Dir(t.prefix)) != "index-batches" { + return nil, fmt.Errorf("交付清单不可读或过大") + } + marker, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/.delivery-owner.json", 4096) + var owner struct { + Owner string `json:"owner"` + } + if e != nil || json.Unmarshal(marker, &owner) != nil || owner.Owner != "index-batch:"+id { + return nil, fmt.Errorf("批次归属标记无效") + } + raw, _, e := b.ReadObjectPreview(ctx, t.bucket, t.prefix+"/delivery_index.json", deliveryDownloadMaxManifestBytes+1) + if e != nil || len(raw) > deliveryDownloadMaxManifestBytes { + return nil, fmt.Errorf("批次尚未完成或清单过大") + } + var index struct { + Schema int `json:"schemaVersion"` + Kind string `json:"kind"` + ID string `json:"snapshotId"` + Count int `json:"fileCount"` + Bytes int64 `json:"totalBytes"` + Project struct { + Root string `json:"rootKey"` + } `json:"project"` + Entries []deliveryindex.Entry `json:"entries"` + } + if json.Unmarshal(raw, &index) != nil || index.Schema != 3 || index.Kind != "part-index" || index.ID != id || index.Project.Root != t.prefix || index.Count != len(index.Entries) || index.Count < 1 || index.Count > 400 { + return nil, fmt.Errorf("批次总索引身份或成员数量无效") + } + files := []deliveryDownloadFile{} + var size int64 + for _, entry := range index.Entries { + sha, shaErr := hex.DecodeString(entry.SHA256) + if !validDeliveryDownloadPath(entry.Path) || entry.Path == "delivery_index.json" || strings.HasPrefix(entry.Path, ".") || entry.Key != t.prefix+"/"+entry.Path || entry.SizeBytes < 0 || entry.SizeBytes > 8<<20 || shaErr != nil || len(sha) != 32 { + return nil, fmt.Errorf("批次成员校验信息无效") + } + size += entry.SizeBytes + files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/" + entry.Path, SizeBytes: entry.SizeBytes, SHA256: entry.SHA256, key: entry.Key, bucket: t.bucket, connectionID: t.connectionID}) + } + if size != index.Bytes || size > 64<<20 { + return nil, fmt.Errorf("批次大小校验不一致") + } + files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/delivery_index.json", SizeBytes: int64(len(raw)), SHA256: deliveryindex.Hash(raw), key: t.prefix + "/delivery_index.json", bucket: t.bucket, connectionID: t.connectionID}) + return files, nil +} diff --git a/internal/api/delivery_index_batches_mongo_test.go b/internal/api/delivery_index_batches_mongo_test.go new file mode 100644 index 00000000..7bb02164 --- /dev/null +++ b/internal/api/delivery_index_batches_mongo_test.go @@ -0,0 +1,40 @@ +package api + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "droboticflow/internal/store" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Local replica only. A fresh synthetic database is created and removed; no +// snapshot/customer collection is opened and the URI cannot be overridden. +func TestIndexBatchMongoReplicaRecovery(t *testing.T) { + if os.Getenv("DELIVERY_INDEX_MONGO_HTTP") != "1" { + t.Skip("opt-in isolated local Mongo replica verification") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true")) + if e != nil { + t.Fatal(e) + } + defer client.Disconnect(context.Background()) + if e = client.Ping(ctx, nil); e != nil { + t.Fatal(e) + } + db := client.Database(fmt.Sprintf("delivery_index_batch_test_%d", time.Now().UnixNano())) + defer func() { + cleanup, done := context.WithTimeout(context.Background(), 10*time.Second) + defer done() + if e := db.Drop(cleanup); e != nil { + t.Error(e) + } + }() + exerciseIndexBatch(t, store.NewMongoStore(db)) +} diff --git a/internal/api/delivery_index_batches_test.go b/internal/api/delivery_index_batches_test.go new file mode 100644 index 00000000..191355c2 --- /dev/null +++ b/internal/api/delivery_index_batches_test.go @@ -0,0 +1,301 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "droboticflow/internal/deliveryindex" + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +type indexTestBrowser struct { + *downloadTestBrowser + puts []string + fail string + ambiguous bool +} + +func (b *indexTestBrowser) PutObjectConditional(_ context.Context, _ string, key string, body []byte, _ string, match, none string, _ map[string]string) (domain.ObjectStorageObjectSummary, error) { + if match != "" || none != "*" { + return domain.ObjectStorageObjectSummary{}, fmt.Errorf("unconditional write") + } + b.puts = append(b.puts, key) + if strings.HasSuffix(key, b.fail) && b.fail != "" { + return domain.ObjectStorageObjectSummary{}, fmt.Errorf("injected interruption") + } + if _, ok := b.objects[key]; ok { + return domain.ObjectStorageObjectSummary{}, fmt.Errorf("precondition failed") + } + b.objects[key] = append([]byte(nil), body...) + if b.ambiguous { + b.ambiguous = false + return domain.ObjectStorageObjectSummary{}, fmt.Errorf("response lost after commit") + } + return domain.ObjectStorageObjectSummary{}, nil +} +func (b *indexTestBrowser) ListObjectsPage(_ context.Context, _ string, prefix, marker string, limit int) (objectStoragePage, error) { + if p, ok := b.pages[prefix+"|"+marker]; ok { + return p, nil + } + keys := []string{} + seen := map[string]bool{} + for key := range b.objects { + if strings.HasPrefix(key, prefix) { + rest := strings.TrimPrefix(key, prefix) + first, _, nested := strings.Cut(rest, "/") + item := prefix + first + if nested { + item += "/" + } + if !seen[item] && item > marker { + seen[item] = true + keys = append(keys, item) + } + } + } + sort.Strings(keys) + page := objectStoragePage{} + if limit <= 0 { + limit = 1000 + } + if len(keys) > limit { + page.HasMore = true + keys = keys[:limit] + page.NextMarker = keys[len(keys)-1] + } + for _, key := range keys { + kind := "file" + if strings.HasSuffix(key, "/") { + kind = "directory" + } + page.Items = append(page.Items, domain.ObjectStorageObjectSummary{Key: key, Kind: kind}) + } + return page, nil +} +func indexCall(h http.Handler, p *authenticatedPrincipal, method, target string, body any) *httptest.ResponseRecorder { + raw, _ := json.Marshal(body) + r := httptest.NewRequest(method, target, bytes.NewReader(raw)) + r.Header.Set("Content-Type", "application/json") + r.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "index-test-csrf"}) + r.Header.Set(csrfHeaderName, "index-test-csrf") + r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, p)) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w +} +func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authenticatedPrincipal, string) { + t.Helper() + root := "deliveries/项目A__project-A" + org := "index-test-org" + ctx := t.Context() + b := &indexTestBrowser{downloadTestBrowser: &downloadTestBrowser{objects: map[string][]byte{}, pages: map[string]objectStoragePage{}}} + must := func(e error) { + t.Helper() + if e != nil { + t.Fatal(e) + } + } + must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: "project-A", OrganizationID: org, Name: "项目A"})) + must(ds.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "index-storage", OrganizationID: org, Provider: domain.StorageProviderBOS, Bucket: "test-bucket", Endpoint: "https://bj.bcebos.com"})) + dir := os.Getenv("DELIVERY_V3_FIXTURE_DIR") + for _, part := range []string{"part0001", "part0002"} { + prefix := root + "/capture-A/" + part + sourceRun := "source-run-" + part + runID := "package-run-" + part + artifactID := "directory-" + part + body := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.part-index/v3","capture":{"captureId":"capture-A","prefix":"incoming/capture-A"},"partId":%q,"revision":1,"sourceMetadata":{"path":"meta_capture-A.json"},"lineage":{"sourceRunId":%q},"acceptance":{"finalAcceptance":{"confirmed":true,"acceptedAt":"2026-10-20T01:00:00Z"}},"slices":[{"sliceId":"slice-1","taskSplit":{"startSecond":0,"endSecond":1,"semantics":"half-open"},"classification":{"deliveryDisposition":"keep","descriptionStatus":"empty_confirmed"},"segments":[{"description":"","isInvalid":false,"contentKind":"gap_placeholder"}]}],"discardedSlices":[],"deliveryClassification":{"keptSliceCount":1,"discardedSliceCount":0,"keptDurationSeconds":1,"discardedDurationSeconds":0}}`, part, sourceRun)) + history := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.review-history/v1","captureId":"capture-A","partId":%q,"revision":1,"qualityReviews":[]}`, part)) + if dir != "" { + var e error + body, e = os.ReadFile(filepath.Join(dir, part, "part.json")) + must(e) + history, e = os.ReadFile(filepath.Join(dir, part, "review-history.json")) + must(e) + } + b.objects[prefix+"/.delivery-owner.json"] = []byte(`{"owner":"synthetic-source"}`) + b.objects[prefix+"/part.json"] = body + b.objects[prefix+"/review-history.json"] = history + entries := []deliveryindex.Entry{{Path: "part.json", Key: prefix + "/part.json", SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)}, {Path: "review-history.json", Key: prefix + "/review-history.json", SizeBytes: int64(len(history)), SHA256: deliveryindex.Hash(history)}} + b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 2, "totalBytes": len(body) + len(history), "entries": entries}) + b.objects["incoming/capture-A/"+part+"/left.mp4"] = []byte("synthetic") + run := domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: "project-A", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: artifactID, RunID: runID, Name: "delivery.directory.index", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindDirectory, Provider: domain.StorageProviderBOS, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix, ManifestKey: prefix + "/manifest.json"}}}}} + run.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "package", Artifacts: run.Artifacts}} + must(ds.UpsertRun(ctx, run)) + must(ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "package-" + part, OrganizationID: org, ProjectID: "project-A", ChildRunID: runID, Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{artifactID}})) + must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, OrganizationID: org, ProjectID: "project-A", RunID: sourceRun, CaptureID: "capture-A/" + part, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A/" + part, SourcePrefix: "incoming/capture-A", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix}})) + } + b.objects["incoming/capture-A/meta_capture-A.json"] = []byte(`{"synthetic":true}`) + old := storageConnectionBrowserResolver + storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + if c.OrganizationID != org { + return nil, nil, fmt.Errorf("foreign storage") + } + return b, nil, nil + } + t.Cleanup(func() { storageConnectionBrowserResolver = old }) + p := &authenticatedPrincipal{User: domain.UserRecord{ID: "index-admin", OrganizationID: org, Role: domain.UserRoleAdmin}, Organization: domain.OrganizationRecord{ID: org}} + return b, p, root +} +func exerciseIndexBatch(t *testing.T, ds store.Store) { + t.Helper() + b, p, root := seedIndexFixture(t, ds) + h := NewHandler(ds, nil) + selection := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001&packageId=package-part0002" + preview := indexCall(h, p, "GET", selection, nil) + if preview.Code != 200 { + t.Fatal(preview.Code, preview.Body.String()) + } + if len(b.puts) > 0 { + t.Fatal("preview wrote storage") + } + var plan struct { + ID string `json:"snapshotId"` + Root string `json:"rootKey"` + } + json.Unmarshal(preview.Body.Bytes(), &plan) + post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001", "package-part0002"}, SnapshotID: plan.ID} + stale := post + stale.SnapshotID = "old" + if w := indexCall(h, p, "POST", "/api/delivery-index-batches", stale); w.Code != 409 || len(b.puts) > 0 { + t.Fatal("stale preview wrote") + } + b.fail = "part0002.json" + w := indexCall(h, p, "POST", "/api/delivery-index-batches", post) + if w.Code != 409 { + t.Fatal(w.Code, w.Body.String()) + } + if _, ok := b.objects[plan.Root+"/delivery_index.json"]; ok { + t.Fatal("partial batch marked complete") + } + b.fail = "" + b.ambiguous = true + h = NewHandler(ds, nil) // API restart; all progress lives in immutable storage. + w = indexCall(h, p, "POST", "/api/delivery-index-batches", post) + if w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + if b.puts[len(b.puts)-1] != plan.Root+"/delivery_index.json" { + t.Fatal("total index must be last") + } + before := append([]byte(nil), b.objects[plan.Root+"/delivery_index.json"]...) + w = indexCall(h, p, "POST", "/api/delivery-index-batches", post) + if w.Code != 200 || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) { + t.Fatal("idempotent retry") + } + target := deliveryDownloadTarget{"index-storage", "test-bucket", plan.Root, pathBase(plan.Root)} + files, e := readDownloadManifest(t.Context(), b, target) + if e != nil { + t.Fatal(e) + } + for _, f := range files { + if deliveryindex.Hash(b.objects[f.key]) != f.SHA256 { + t.Fatal("download hash", f.RelativePath) + } + } + // Exercise the existing public download route, including refreshed signatures. + download := downloadRequest(h, p, "/api/delivery-downloads/manifest?directory=bos://test-bucket/"+plan.Root+"&connectionId=index-storage") + if download.Code != 200 || !strings.Contains(download.Body.String(), "delivery_index.json") { + t.Fatal(download.Code, download.Body.String()) + } + // A later partial snapshot does not alter the old full snapshot. + first := indexCall(h, p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil) + var partial struct { + ID string `json:"snapshotId"` + } + json.Unmarshal(first.Body.Bytes(), &partial) + w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID}) + if w.Code != 200 || partial.ID == plan.ID || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) { + t.Fatal("immutable snapshots", w.Code, w.Body.String()) + } + // Occupied, unowned batch directories can never be adopted. + conflictRoot := root + "/index-batches/" + partial.ID + delete(b.objects, conflictRoot+"/.delivery-owner.json") + if w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID}); w.Code != 409 { + t.Fatal("adopted unowned directory") + } +} +func pathBase(s string) string { p := strings.Split(s, "/"); return p[len(p)-1] } +func TestIndexBatchHTTPPublicationRecoveryAndDownload(t *testing.T) { + exerciseIndexBatch(t, store.NewMemoryStore()) +} +func TestIndexBatchRejectsScopeAndTampering(t *testing.T) { + ds := store.NewMemoryStore() + b, p, _ := seedIndexFixture(t, ds) + h := NewHandler(ds, nil) + target := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001" + other := *p + other.User.Role = domain.UserRoleQualityReviewer + if w := indexCall(h, &other, "GET", target, nil); w.Code != 403 { + t.Fatal("non-admin") + } + other = *p + other.Organization.ID = "other" + other.User.OrganizationID = "other" + if w := indexCall(h, &other, "GET", target, nil); w.Code == 200 { + t.Fatal("foreign org") + } + b.objects["deliveries/项目A__project-A/capture-A/part0001/part.json"] = []byte(`{"tampered":true}`) + if w := indexCall(h, p, "GET", target, nil); w.Code != 409 || len(b.puts) > 0 { + t.Fatal("tampered source accepted") + } +} + +func TestIndexBatchDelegationKeepsSourceAndDeliveryScopesSeparate(t *testing.T) { + ds := store.NewMemoryStore() + b, p, root := seedIndexFixture(t, ds) + c, _ := ds.GetStorageConnection(t.Context(), "index-storage") + c.OrganizationID = "source-owner" + if e := ds.UpsertStorageConnection(t.Context(), *c); e != nil { + t.Fatal(e) + } + storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + if c.OrganizationID != "source-owner" { + return nil, nil, fmt.Errorf("wrong owner") + } + return b, nil, nil + } + mapping := func(writes []string, deliveries []string) string { + raw, _ := json.Marshal(map[string]any{p.Organization.ID: map[string]any{"connectionId": "index-storage", "bucket": "test-bucket", "readConnectionOrganizationId": "source-owner", "writeConnectionOrganizationId": "source-owner", "readPrefixes": []string{"incoming"}, "writePrefixes": writes, "deliveryPrefixes": deliveries}}) + return string(raw) + } + t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001", "system/runs/source-run-part0002"}, []string{root})) + h := NewHandler(ds, nil) + query := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001" + w := indexCall(h, p, "GET", query, nil) + if w.Code != 200 { + t.Fatal("explicit source read/final-output write grant", w.Code, w.Body.String()) + } + var preview struct { + ID string `json:"snapshotId"` + } + json.Unmarshal(w.Body.Bytes(), &preview) + post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: preview.ID} + w = indexCall(h, p, "POST", "/api/delivery-index-batches", post) + if w.Code != 200 { + t.Fatal(w.Code, w.Body.String()) + } + for _, key := range b.puts { + if !strings.HasPrefix(key, root+"/index-batches/") { + t.Fatal("write escaped final-output scope", key) + } + } + t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/some-other-run"}, []string{root})) + before := len(b.puts) + if w = indexCall(h, p, "POST", "/api/delivery-index-batches", post); w.Code != 409 || len(b.puts) != before { + t.Fatal("unapproved source run wrote") + } + t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001"}, []string{root + "/capture-A"})) + if w = indexCall(h, p, "GET", query, nil); w.Code != 409 { + t.Fatal("sibling batch prefix escaped delivery grant", w.Code, w.Body.String()) + } +} diff --git a/internal/api/routes_delivery.go b/internal/api/routes_delivery.go index bae6c820..39770ab0 100644 --- a/internal/api/routes_delivery.go +++ b/internal/api/routes_delivery.go @@ -16,6 +16,8 @@ func (server *server) mountDeliveryRoutes(router chi.Router) { router.Get("/api/delivery-packages", server.handleDeliveryPackagesList) router.Get("/api/delivery-downloads/projects", server.handleDeliveryDownloadProjects) router.Get("/api/delivery-downloads/manifest", server.handleDeliveryDownloadManifest) + router.Get("/api/delivery-index-batches", server.handleDeliveryIndexBatch) + router.Post("/api/delivery-index-batches", server.handleDeliveryIndexBatch) router.Get("/api/delivery-packages/{id}", server.handleDeliveryPackageGet) router.Get("/api/delivery-packages/{id}/contents", server.handleDeliveryPackageContents) router.Get("/api/delivery-packages/{id}/artifacts/{artifactId}/content", server.handleDeliveryPackageArtifactContent) diff --git a/internal/deliveryindex/batch.go b/internal/deliveryindex/batch.go new file mode 100644 index 00000000..84463db5 --- /dev/null +++ b/internal/deliveryindex/batch.go @@ -0,0 +1,366 @@ +// Package deliveryindex builds deterministic immutable delivery snapshots. It +// performs no storage operations; callers must verify the accepted source bytes. +package deliveryindex + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "path" + "regexp" + "sort" + "strings" + "time" +) + +const PartSchema = "droboticflow.delivery.part-index/v3" + +var partName = regexp.MustCompile(`^part[0-9]{4,}$`) + +type Source struct { + Part json.RawMessage `json:"part"` + History json.RawMessage `json:"history"` + ExpectedParts []string `json:"expectedParts"` + MetadataSHA256 string `json:"metadataSha256"` + MetadataBytes int64 `json:"metadataBytes"` +} +type Snapshot struct { + ID string `json:"snapshotId"` + Root string `json:"rootKey"` + Files map[string][]byte `json:"-"` + Index json.RawMessage `json:"index"` +} +type Entry struct { + Path string `json:"path"` + Key string `json:"key"` + SizeBytes int64 `json:"sizeBytes"` + SHA256 string `json:"sha256"` +} +type Summary struct { + CaptureCount int `json:"captureCount"` + PartCount int `json:"partCount"` + ExpectedPartCount int `json:"expectedPartCount"` + UndeliveredParts []string `json:"undeliveredParts,omitempty"` + Completion string `json:"deliveryCompletion"` + KeptCount int `json:"keptSliceCount"` + DiscardedCount int `json:"discardedSliceCount"` + KeptDuration float64 `json:"keptDurationSeconds"` + DiscardedDuration float64 `json:"discardedDurationSeconds"` + Acceptance map[string]string `json:"acceptance"` +} +type part struct { + Schema string `json:"schemaVersion"` + Capture struct { + ID string `json:"captureId"` + Prefix string `json:"prefix"` + } `json:"capture"` + PartID string `json:"partId"` + Revision int `json:"revision"` + SourceMetadata struct { + Path string `json:"path"` + } `json:"sourceMetadata"` + Slices []slice `json:"slices"` + Discarded []slice `json:"discardedSlices"` + Classification *struct { + KeptCount int `json:"keptSliceCount"` + DiscardedCount int `json:"discardedSliceCount"` + KeptDuration float64 `json:"keptDurationSeconds"` + DiscardedDuration float64 `json:"discardedDurationSeconds"` + } `json:"deliveryClassification"` + Acceptance struct { + Final struct { + Confirmed bool `json:"confirmed"` + AcceptedAt string `json:"acceptedAt"` + } `json:"finalAcceptance"` + Reviews json.RawMessage `json:"qualityReviews"` + } `json:"acceptance"` +} +type slice struct { + ID string `json:"sliceId"` + Split struct { + Start float64 `json:"startSecond"` + End float64 `json:"endSecond"` + Semantics string `json:"semantics"` + } `json:"taskSplit"` + Classification struct { + Disposition string `json:"deliveryDisposition"` + DescriptionStatus string `json:"descriptionStatus"` + Discard struct { + Reason string `json:"reason"` + } `json:"discardEvidence"` + } `json:"classification"` + Segments []struct { + Description string `json:"description"` + Invalid bool `json:"isInvalid"` + Kind string `json:"contentKind"` + Status *string `json:"status"` + } `json:"segments"` +} + +func Hash(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } +func encode(v any) []byte { + b, e := json.MarshalIndent(v, "", " ") + if e != nil { + panic(e) + } + return append(b, '\n') +} +func safeName(s string) bool { + return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\:\x00") && !strings.HasPrefix(s, ".") +} +func addSummary(a *Summary, b Summary) { + a.PartCount += b.PartCount + a.ExpectedPartCount += b.ExpectedPartCount + a.KeptCount += b.KeptCount + a.DiscardedCount += b.DiscardedCount + a.KeptDuration += b.KeptDuration + a.DiscardedDuration += b.DiscardedDuration + if a.Acceptance == nil { + a.Acceptance = map[string]string{"decision": "approved"} + } + before, _ := time.Parse(time.RFC3339Nano, a.Acceptance["acceptedAt"]) + after, _ := time.Parse(time.RFC3339Nano, b.Acceptance["acceptedAt"]) + if after.After(before) { + a.Acceptance["acceptedAt"] = b.Acceptance["acceptedAt"] + } +} +func checkPart(p part) (Summary, error) { + s := Summary{PartCount: 1, Acceptance: map[string]string{"decision": "approved", "acceptedAt": p.Acceptance.Final.AcceptedAt}} + if p.Schema != PartSchema || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 { + return s, fmt.Errorf("part must pin an accepted native v3 revision without embedded review history") + } + if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil { + return s, fmt.Errorf("missing acceptance timestamp") + } + if p.SourceMetadata.Path != "meta_"+p.Capture.ID+".json" || path.Base(p.Capture.Prefix) != p.Capture.ID || path.Clean(p.Capture.Prefix) != p.Capture.Prefix || strings.HasPrefix(p.Capture.Prefix, "/") || strings.Contains(p.Capture.Prefix, "../") { + return s, fmt.Errorf("capture/source metadata identity mismatch") + } + seen := map[string]bool{} + var intervals [][2]float64 + for group, items := range [][]slice{p.Slices, p.Discarded} { + for _, v := range items { + if v.ID == "" || seen[v.ID] || v.Split.Semantics != "half-open" || math.IsNaN(v.Split.Start) || v.Split.Start < 0 || v.Split.End <= v.Split.Start { + return s, fmt.Errorf("invalid slice identity or interval") + } + seen[v.ID] = true + intervals = append(intervals, [2]float64{v.Split.Start, v.Split.End}) + duration := v.Split.End - v.Split.Start + if group == 1 { + if v.Classification.Disposition != "discard" || strings.TrimSpace(v.Classification.Discard.Reason) == "" || len(v.Segments) > 0 { + return s, fmt.Errorf("invalid discarded evidence") + } + s.DiscardedCount++ + s.DiscardedDuration += duration + continue + } + if v.Classification.Disposition != "keep" { + return s, fmt.Errorf("invalid kept disposition") + } + s.KeptCount++ + s.KeptDuration += duration + described, invalid, blank := false, false, false + for _, a := range v.Segments { + kind := "gap_placeholder" + if a.Invalid { + kind = "invalid_marked" + invalid = true + } else if strings.TrimSpace(a.Description) != "" { + kind = "described" + described = true + } else { + blank = true + } + if a.Kind != kind || a.Status != nil { + return s, fmt.Errorf("action content classification mismatch") + } + } + status := "empty_confirmed" + if invalid { + status = "invalid_only" + } + if invalid && blank { + status = "invalid_and_blank" + } + if described { + status = "described" + } + if v.Classification.DescriptionStatus != status { + return s, fmt.Errorf("descriptionStatus mismatch") + } + } + } + if s.KeptCount == 0 { + return s, fmt.Errorf("no deliverable slices") + } + if math.IsInf(s.KeptDuration, 0) || math.IsInf(s.DiscardedDuration, 0) { + return s, fmt.Errorf("slice duration overflow") + } + if c := p.Classification; c == nil || c.KeptCount != s.KeptCount || c.DiscardedCount != s.DiscardedCount || math.Abs(c.KeptDuration-s.KeptDuration) > 0.000001 || math.Abs(c.DiscardedDuration-s.DiscardedDuration) > 0.000001 { + return s, fmt.Errorf("part summary disagrees with its selected slices") + } + sort.Slice(intervals, func(i, j int) bool { return intervals[i][0] < intervals[j][0] }) + for i := 1; i < len(intervals); i++ { + if math.Abs(intervals[i][0]-intervals[i-1][1]) > 0.000001 { + return s, fmt.Errorf("slice coverage gap or overlap") + } + } + return s, nil +} + +// Build uses a hash of canonical source documents, source metadata and expected +// part inventory as its identity. Signed URLs and build time never affect it. +func Build(projectID, projectName, projectRoot string, sources []Source) (Snapshot, error) { + out := Snapshot{Files: map[string][]byte{}} + if projectID == "" || projectRoot == "." || path.Clean(projectRoot) != projectRoot || strings.HasPrefix(projectRoot, "/") || strings.Contains(projectRoot, "../") || len(sources) == 0 || len(sources) > 100 { + return out, fmt.Errorf("select 1–100 accepted part packages") + } + var sourceBytes int + for _, source := range sources { + sourceBytes += len(source.Part) + len(source.History) + if sourceBytes > 64<<20 { + return out, fmt.Errorf("source JSON exceeds 64 MiB") + } + } + // Normalize JSON before hashing to ignore transport whitespace/map ordering. + for i := range sources { + var d, h any + if json.Unmarshal(sources[i].Part, &d) != nil || json.Unmarshal(sources[i].History, &h) != nil { + return out, fmt.Errorf("invalid source JSON") + } + sources[i].Part = encode(d) + sources[i].History = encode(h) + sources[i].ExpectedParts = append([]string(nil), sources[i].ExpectedParts...) + sort.Strings(sources[i].ExpectedParts) + } + sort.Slice(sources, func(i, j int) bool { return string(sources[i].Part) < string(sources[j].Part) }) + identity := encode([]any{"droboticflow.delivery.batch/v1", projectID, projectName, projectRoot, sources}) + out.ID = Hash(identity) + out.Root = projectRoot + "/index-batches/" + out.ID + type group struct { + capture map[string]string + expected []string + metadata Source + parts []map[string]any + summary Summary + entries []Entry + } + groups := map[string]*group{} + seen := map[string]bool{} + var totalBytes int + add := func(name string, body []byte) Entry { + out.Files[name] = body + totalBytes += len(body) + return Entry{name, out.Root + "/" + name, int64(len(body)), Hash(body)} + } + for _, src := range sources { + var p part + if e := json.Unmarshal(src.Part, &p); e != nil { + return out, e + } + s, e := checkPart(p) + if e != nil { + return out, e + } + name := p.Capture.ID + "/" + p.PartID + if seen[strings.ToLower(name)] { + return out, fmt.Errorf("duplicate capture/part, select exactly one revision") + } + seen[strings.ToLower(name)] = true + if len(src.ExpectedParts) == 0 || len(src.ExpectedParts) > 10000 || src.MetadataBytes <= 0 || len(src.MetadataSHA256) != 64 { + return out, fmt.Errorf("missing original source inventory/metadata evidence") + } + expectedSeen := map[string]bool{} + found := false + for _, n := range src.ExpectedParts { + if !partName.MatchString(n) || expectedSeen[n] { + return out, fmt.Errorf("invalid expected part inventory") + } + expectedSeen[n] = true + found = found || n == p.PartID + } + if !found { + return out, fmt.Errorf("delivered part missing from original inventory") + } + var history struct { + Schema string `json:"schemaVersion"` + CaptureID string `json:"captureId"` + PartID string `json:"partId"` + Revision int `json:"revision"` + Reviews []json.RawMessage `json:"qualityReviews"` + } + if json.Unmarshal(src.History, &history) != nil || history.Schema != "droboticflow.delivery.review-history/v1" || history.CaptureID != p.Capture.ID || history.PartID != p.PartID || history.Revision != p.Revision || history.Reviews == nil { + return out, fmt.Errorf("review history revision mismatch") + } + g := groups[p.Capture.ID] + if g == nil { + g = &group{capture: map[string]string{"captureId": p.Capture.ID, "prefix": p.Capture.Prefix}, expected: src.ExpectedParts, metadata: src} + groups[p.Capture.ID] = g + } else if g.capture["prefix"] != p.Capture.Prefix || strings.Join(g.expected, ",") != strings.Join(src.ExpectedParts, ",") || g.metadata.MetadataSHA256 != src.MetadataSHA256 { + return out, fmt.Errorf("inconsistent capture source evidence") + } + var doc map[string]any + json.Unmarshal(src.Part, &doc) + doc["reviewHistory"] = map[string]string{"path": p.PartID + ".review-history.json", "pathBase": "capture_delivery"} + doc["delivery"] = map[string]string{"kind": "part_index", "fileName": p.PartID + ".json", "artifactName": "delivery.part-index"} + g.entries = append(g.entries, add(name+".json", encode(doc)), add(name+".review-history.json", src.History)) + g.parts = append(g.parts, map[string]any{"part": p.PartID, "revision": p.Revision, "path": p.PartID + ".json", "reviewHistoryPath": p.PartID + ".review-history.json"}) + addSummary(&g.summary, s) + if math.IsInf(g.summary.KeptDuration, 0) || math.IsInf(g.summary.DiscardedDuration, 0) { + return out, fmt.Errorf("capture duration overflow") + } + } + names := make([]string, 0, len(groups)) + for n := range groups { + names = append(names, n) + } + sort.Strings(names) + allEntries := []Entry{} + captures := []map[string]any{} + total := Summary{Completion: "full_run", CaptureCount: len(groups)} + for _, n := range names { + g := groups[n] + sort.Slice(g.parts, func(i, j int) bool { return g.parts[i]["part"].(string) < g.parts[j]["part"].(string) }) + g.summary.CaptureCount = 1 + g.summary.ExpectedPartCount = len(g.expected) + g.summary.Completion = "full_run" + g.summary.UndeliveredParts = []string{} + for _, pn := range g.expected { + if !seen[strings.ToLower(n+"/"+pn)] { + g.summary.UndeliveredParts = append(g.summary.UndeliveredParts, pn) + } + } + if len(g.summary.UndeliveredParts) > 0 { + g.summary.Completion = "partial_run" + total.Completion = "partial_run" + } + sort.Slice(g.entries, func(i, j int) bool { return g.entries[i].Path < g.entries[j].Path }) + local := append([]Entry(nil), g.entries...) + var size int64 + for i := range local { + local[i].Path = strings.TrimPrefix(local[i].Path, n+"/") + size += local[i].SizeBytes + } + metadata := map[string]any{"path": "meta_" + n + ".json", "pathBase": "original_capture", "sha256": g.metadata.MetadataSHA256, "sizeBytes": g.metadata.MetadataBytes} + manifest := map[string]any{"schemaVersion": 3, "kind": "part-index", "capture": g.capture, "pathBase": "capture_delivery", "sourceMetadata": metadata, "summary": g.summary, "parts": g.parts, "entries": local, "fileCount": len(local), "totalBytes": size} + allEntries = append(allEntries, g.entries...) + allEntries = append(allEntries, add(n+"/annotation_manifest.json", encode(manifest))) + captures = append(captures, map[string]any{"captureId": n, "prefix": g.capture["prefix"], "manifestPath": n + "/annotation_manifest.json", "summary": g.summary}) + addSummary(&total, g.summary) + if math.IsInf(total.KeptDuration, 0) || math.IsInf(total.DiscardedDuration, 0) { + return out, fmt.Errorf("batch duration overflow") + } + } + if totalBytes > 64<<20 { + return out, fmt.Errorf("batch exceeds 64 MiB") + } + sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Path < allEntries[j].Path }) + var size int64 + for _, v := range allEntries { + size += v.SizeBytes + } + out.Index = encode(map[string]any{"schemaVersion": 3, "kind": "part-index", "snapshotId": out.ID, "pathBase": "batch_project", "project": map[string]string{"projectId": projectID, "deliveryName": projectName, "rootKey": out.Root}, "summary": total, "captures": captures, "entries": allEntries, "fileCount": len(allEntries), "totalBytes": size}) + out.Files["delivery_index.json"] = out.Index + return out, nil +} diff --git a/internal/deliveryindex/batch_test.go b/internal/deliveryindex/batch_test.go new file mode 100644 index 00000000..2a41b1f3 --- /dev/null +++ b/internal/deliveryindex/batch_test.go @@ -0,0 +1,127 @@ +package deliveryindex + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func fixture(partID string, revision int) Source { + p := map[string]any{"schemaVersion": PartSchema, "capture": map[string]any{"captureId": "capture-A", "prefix": "incoming/capture-A"}, "partId": partID, "revision": revision, "sourceMetadata": map[string]string{"path": "meta_capture-A.json"}, "acceptance": map[string]any{"finalAcceptance": map[string]any{"confirmed": true, "acceptedAt": "2026-10-20T01:00:00Z"}}, "slices": []any{map[string]any{"sliceId": "s1", "taskSplit": map[string]any{"startSecond": 0, "endSecond": 2, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "keep", "descriptionStatus": "invalid_and_blank"}, "segments": []any{map[string]any{"description": "", "contentKind": "gap_placeholder", "isInvalid": false}, map[string]any{"description": "无效", "contentKind": "invalid_marked", "isInvalid": true}}}}, "discardedSlices": []any{map[string]any{"sliceId": "s2", "taskSplit": map[string]any{"startSecond": 2, "endSecond": 3, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "discard", "discardEvidence": map[string]string{"reason": "不交付"}}, "segments": []any{}}}} + p["deliveryClassification"] = map[string]any{"keptSliceCount": 1, "discardedSliceCount": 1, "keptDurationSeconds": 2, "discardedDurationSeconds": 1} + h := map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": "capture-A", "partId": partID, "revision": revision, "qualityReviews": []any{map[string]any{"revision": revision, "decision": "approved"}}} + return Source{Part: encode(p), History: encode(h), ExpectedParts: []string{"part0001", "part0002"}, MetadataSHA256: strings.Repeat("a", 64), MetadataBytes: 10} +} +func build(t *testing.T, sources ...Source) Snapshot { + t.Helper() + s, e := Build("project-A", "项目A", "delivery/项目A__project-A", sources) + if e != nil { + t.Fatal(e) + } + return s +} +func TestBatchMultipartIntegrityAndImmutableScopes(t *testing.T) { + a, b := fixture("part0001", 3), fixture("part0002", 8) + first := build(t, a) + full := build(t, a, b) + reversed := build(t, b, a) + if first.ID == full.ID || full.ID != reversed.ID { + t.Fatal("snapshot identity") + } + var index struct { + Summary Summary + Entries []Entry + Bytes int64 `json:"totalBytes"` + } + json.Unmarshal(full.Index, &index) + if index.Summary.PartCount != 2 || index.Summary.ExpectedPartCount != 2 || index.Summary.Completion != "full_run" || index.Summary.KeptCount != 2 || index.Summary.DiscardedCount != 2 || index.Summary.KeptDuration != 4 || index.Summary.DiscardedDuration != 2 { + t.Fatalf("summary %+v", index.Summary) + } + var size int64 + for _, e := range index.Entries { + body := full.Files[e.Path] + if int64(len(body)) != e.SizeBytes || Hash(body) != e.SHA256 || e.Key != full.Root+"/"+e.Path { + t.Fatal("entry integrity", e) + } + size += e.SizeBytes + } + if size != index.Bytes { + t.Fatal("total bytes") + } + var manifest struct { + Entries []Entry + Summary Summary + Metadata struct { + Path string + Base string `json:"pathBase"` + } `json:"sourceMetadata"` + } + json.Unmarshal(full.Files["capture-A/annotation_manifest.json"], &manifest) + for _, e := range manifest.Entries { + if strings.Contains(e.Path, "/") || !bytes.Equal(full.Files["capture-A/"+e.Path], reversed.Files["capture-A/"+e.Path]) { + t.Fatal("capture-relative entry", e) + } + } + if manifest.Metadata.Path != "meta_capture-A.json" || manifest.Metadata.Base != "original_capture" { + t.Fatal("metadata reference") + } + again := build(t, a) + if !bytes.Equal(first.Index, again.Index) { + t.Fatal("old batch changed") + } + if !bytes.Contains(first.Index, []byte(`"partial_run"`)) || !bytes.Contains(first.Index, []byte(`"part0002"`)) { + t.Fatal("partial scope") + } +} +func TestBatchRejectsConflictingEvidence(t *testing.T) { + for _, tc := range []string{"duplicate", "revision-history", "missing-inventory", "description", "coverage", "acceptance", "embedded-history", "part-summary"} { + t.Run(tc, func(t *testing.T) { + a := fixture("part0001", 1) + sources := []Source{a} + var d map[string]any + json.Unmarshal(a.Part, &d) + switch tc { + case "duplicate": + sources = append(sources, fixture("part0001", 2)) + case "revision-history": + sources[0].History = fixture("part0001", 2).History + case "missing-inventory": + sources[0].ExpectedParts = []string{"part0002"} + case "description": + d["slices"].([]any)[0].(map[string]any)["classification"].(map[string]any)["descriptionStatus"] = "described" + case "coverage": + d["discardedSlices"].([]any)[0].(map[string]any)["taskSplit"].(map[string]any)["startSecond"] = 2.5 + case "acceptance": + d["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["confirmed"] = false + case "part-summary": + d["deliveryClassification"].(map[string]any)["keptSliceCount"] = 99 + case "embedded-history": + d["acceptance"].(map[string]any)["qualityReviews"] = []any{} + } + if tc == "description" || tc == "coverage" || tc == "acceptance" || tc == "embedded-history" || tc == "part-summary" { + sources[0].Part = encode(d) + } + if _, e := Build("project-A", "项目A", "delivery/项目A__project-A", sources); e == nil { + t.Fatal("accepted", tc) + } + }) + } +} + +func TestBatchAcceptanceUsesInstantsRatherThanTimestampText(t *testing.T) { + a, b := fixture("part0001", 1), fixture("part0002", 2) + var doc map[string]any + json.Unmarshal(a.Part, &doc) + doc["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["acceptedAt"] = "2026-10-20T10:00:00+08:00" + a.Part = encode(doc) + json.Unmarshal(b.Part, &doc) + doc["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any)["acceptedAt"] = "2026-10-20T03:00:00Z" + b.Part = encode(doc) + s := build(t, a, b) + var index struct{ Summary Summary } + json.Unmarshal(s.Index, &index) + if index.Summary.Acceptance["acceptedAt"] != "2026-10-20T03:00:00Z" { + t.Fatal("timezone aggregation", index.Summary.Acceptance) + } +} diff --git a/internal/worker/capture_delivery_v3_test.go b/internal/worker/capture_delivery_v3_test.go new file mode 100644 index 00000000..d513446a --- /dev/null +++ b/internal/worker/capture_delivery_v3_test.go @@ -0,0 +1,56 @@ +package worker + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +// The production worker's directory protocol must publish the real native v3 +// artifact without routing it through its intentionally v1/v2-only file parser. +func TestCaptureDeliveryNativeV3UsesDirectoryProtocol(t *testing.T) { + fixture := os.Getenv("DELIVERY_V3_FIXTURE_DIR") + if fixture == "" { + t.Skip("generate native Python v3 fixture first") + } + db := store.NewMemoryStore() + publisher := &deliveryPublisher{} + runtime := &Runtime{store: db, artifactPublisher: publisher} + ctx := t.Context() + if e := db.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "storage-v3", OrganizationID: "org-v3", Provider: domain.StorageProviderBOS, Bucket: "b"}); e != nil { + t.Fatal(e) + } + for _, part := range []string{"part0001", "part0002"} { + prefix := "delivery/project-v3__project/capture-A/" + part + run := &domain.RunRecord{ID: "run-" + part, OrganizationID: "org-v3"} + if e := db.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, RunID: run.ID, OrganizationID: run.OrganizationID, Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "storage-v3", Bucket: "b", Prefix: prefix}}); e != nil { + t.Fatal(e) + } + artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Path: filepath.Join(fixture, part), Type: domain.PortTypeFile} + for attempt := 0; attempt < 2; attempt++ { + if _, e := runtime.publishCaptureDelivery(ctx, run, "package", artifact); e != nil { + t.Fatal(e) + } + } + var manifest struct { + Kind string + FileCount int + Entries []directoryManifestEntry + } + if e := json.Unmarshal(publisher.objects["b/"+prefix+"/manifest.json"], &manifest); e != nil { + t.Fatal(e) + } + if manifest.Kind != "directory" || manifest.FileCount != 2 { + t.Fatal("native v3 directory protocol", manifest) + } + for _, name := range []string{"part.json", "review-history.json"} { + if len(publisher.objects["b/"+prefix+"/"+name]) == 0 { + t.Fatal("missing", part, name) + } + } + } +} diff --git a/operator-repos/builtin/package/src/operator.py b/operator-repos/builtin/package/src/operator.py index fb3e73f6..cb8ae7e9 100644 --- a/operator-repos/builtin/package/src/operator.py +++ b/operator-repos/builtin/package/src/operator.py @@ -836,11 +836,67 @@ def _description_status(actions: list[dict[str, Any]]) -> str: ``empty_confirmed`` keeps its platform meaning: no description was written and the submitter confirmed that as a deliberate semantic blank. """ - if not actions: - return "empty_confirmed" - if all(bool(action.get("isInvalid")) for action in actions): + if any(not action.get("isInvalid") and str(action.get("description") or "").strip() for action in actions): + return "described" + if actions and all(bool(action.get("isInvalid")) for action in actions): return "invalid_only" - return "described" + return "empty_confirmed" + + +def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): + """Use the established immutable directory publisher for the two JSON files. + + Legacy single-file publishers deliberately accept only v1/v2. A directory + artifact preserves their ownership/conditional-publication protocol without + teaching an old worker to misinterpret a new single-file schema. + """ + capture = dict(payload["capture"]) + prefix = pathlib.PurePosixPath(str(capture.get("prefix") or "")) + part_id = prefix.name + capture_id = prefix.parent.name + recorded = str(capture.get("captureId") or "") + if not re.fullmatch(r"part\d{4,}", part_id) or recorded not in {capture_id, capture_id + "/" + part_id}: + raise RuntimeError("v3 capture identity disagrees with the original part prefix") + if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3: + raise RuntimeError("unsafe v3 capture identity prefix") + capture.update(captureId=capture_id, prefix=str(prefix.parent)) + payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id) + payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"} + payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"} + history = { + "schemaVersion": "droboticflow.delivery.review-history/v1", "captureId": capture_id, + "partId": part_id, "revision": payload["revision"], + "qualityReviews": payload["acceptance"].pop("qualityReviews", []), + } + payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"} + payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None) + for item in payload["calibration"]: + if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]: + raise RuntimeError("unsafe calibration filename") + item["path"] = item["name"] + for item in payload["slices"] + payload["discardedSlices"]: + classification = item["classification"] + for key in ("taskLabel", "taskLabelId"): + if not str(classification.get(key) or "").strip(): + classification.pop(key, None) + classification.get("discardEvidence", {}).pop("discardedByUserId", None) + item.get("synchronization", {}).pop("calibration", None) + for action in item["segments"]: + action.pop("status", None) + action["contentKind"] = "invalid_marked" if action["isInvalid"] else "described" if action["description"].strip() else "gap_placeholder" + if classification["deliveryDisposition"] == "keep": + counts = {kind: sum(action["contentKind"] == kind for action in item["segments"]) for kind in ("described", "invalid_marked", "gap_placeholder")} + classification["contentCounts"] = counts + if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]: + classification["descriptionStatus"] = "invalid_and_blank" + staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output)) + for name, document in (("part.json", payload), ("review-history.json", history)): + (staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8") + return [helpers["build_virtual_artifact"]( + run_id, node_id, "delivery.directory.index", str(staging), "file", 0, + data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"])}, + output=_output(node_data, helpers, "deliveryDirectory"), + )] def _part_index_slice( @@ -1659,6 +1715,8 @@ def _run_accepted_slice_directory( for item in payload["slices"] + payload["discardedSlices"]: item["sourceFrames"] = frames[item["sliceId"]] body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8") + if _bool(params, "DELIVERY_JSON_V3", False): + return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload) return _emit_part_index(run_id, node_id, output, node_data, helpers, body) if source_index: source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output) @@ -1894,6 +1952,9 @@ def _run_multiport( def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]: + params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False): + raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY") # A part-index delivery intentionally has no directory port; it must still # reach the accepted-slice path instead of the legacy inline contract. if ( diff --git a/tests/backend/export_delivery_v3_fixture.py b/tests/backend/export_delivery_v3_fixture.py new file mode 100644 index 00000000..fb5df5ea --- /dev/null +++ b/tests/backend/export_delivery_v3_fixture.py @@ -0,0 +1,38 @@ +"""Produce explicitly synthetic multi-part examples using the real package entrypoint.""" +import argparse +import hashlib +import json +import shutil +from pathlib import Path + +from backend.operators.builtin.package import operator as package +from tests.backend.test_delivery_v3_package import NODE, inputs_for +from tests.backend.test_part_index_package import helpers + + +def export(destination): + destination.mkdir(parents=True, exist_ok=True) + for part in ("part0001", "part0002"): + work = destination / ("work-" + part) + work.mkdir() + inputs = inputs_for(work, [ + {"id": "blank", "description": "", "clipStartMs": 0, "clipEndMs": 500}, + {"id": "invalid", "description": "无效", "isInvalid": True, "clipStartMs": 500, "clipEndMs": 1000}, + ]) + inputs[1]["data"]["capture"]["captureId"] = "capture-A/" + part + inputs[1]["data"]["capture"]["prefix"] = "incoming/capture-A/" + part + manifest_path = Path(inputs[0]["path"]) / "revision-manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["runId"] = "source-run-" + part + manifest_path.write_text(json.dumps(manifest)) + inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + artifacts = package.run("package-" + part, "package", work / "out", NODE, inputs, helpers()) + shutil.copytree(artifacts[0]["path"], destination / part) + shutil.rmtree(work) + (destination / "meta_capture-A.json").write_text('{"synthetic":true,"parts":["part0001","part0002"]}\n') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + export(parser.parse_args().output) diff --git a/tests/backend/test_delivery_v3_package.py b/tests/backend/test_delivery_v3_package.py new file mode 100644 index 00000000..8e708a2b --- /dev/null +++ b/tests/backend/test_delivery_v3_package.py @@ -0,0 +1,79 @@ +"""Native v3 packaging, rather than postprocessing a customer sample ZIP.""" +import json +from pathlib import Path + +import pytest +from backend.operators.builtin.package import operator as package +from tests.backend.test_part_index_package import helpers +from tests.backend.test_slice_discard_package import fixture + +NODE = { + "outputs": [{"id": n, "name": n} for n in ("deliveryDirectory", "deliverySummary")], + "hyperparams": [{"key": k, "value": v} for k, v in { + "PART_INDEX_DELIVERY": "true", "DELIVERY_JSON_V3": "true", "PROCESSING_PROFILE": "raw_human" + }.items()], +} + + +def inputs_for(tmp_path, actions): + inputs = fixture(tmp_path, ["keep", "discard"]) + inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001", "captureRecordId": "record-A1"} + inputs[1]["data"]["finalAcceptance"]["acceptedAt"] = "2026-10-20T01:00:00Z" + root = Path(inputs[0]["path"]) + (root / "slices/slice-000001/annotation.json").write_text(json.dumps({"actions": actions})) + return inputs + + +@pytest.mark.parametrize("actions,expected", [ + ([{"description": "", "status": "described"}], "empty_confirmed"), + ([{"description": "无效", "isInvalid": True}, {"description": " "}], "invalid_and_blank"), + ([{"description": "拿起杯子", "status": "pending"}], "described"), +]) +def test_native_v3_semantics_and_separate_history(tmp_path, actions, expected): + inputs = inputs_for(tmp_path, actions) + artifacts = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers()) + artifact = artifacts[0] + assert artifact["portId"] == "deliveryDirectory" + root = Path(artifact["path"]) + assert {p.name for p in root.iterdir()} == {"part.json", "review-history.json"} + doc = json.loads((root / "part.json").read_text()) + assert doc["schemaVersion"] == "droboticflow.delivery.part-index/v3" + assert doc["partId"] == "part0001" + assert doc["capture"]["captureId"] == "capture-A" + assert doc["capture"]["prefix"] == "incoming/capture-A" + assert doc["sourceMetadata"]["path"] == "meta_capture-A.json" + assert doc["slices"][0]["classification"]["descriptionStatus"] == expected + assert "qualityReviews" not in doc["acceptance"] + assert json.loads((root / "review-history.json").read_text())["qualityReviews"] + for entry in doc["slices"]: + assert "taskLabel" not in entry["classification"] + assert "taskLabelId" not in entry["classification"] + assert "calibration" not in entry["synchronization"] + assert all("status" not in a for a in entry["segments"]) + assert all(a["contentKind"] in {"gap_placeholder", "invalid_marked", "described"} for a in entry["segments"]) + assert all(c["path"] == c["name"] for c in doc["calibration"]) + first = {p.name: p.read_bytes() for p in root.iterdir()} + again = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers()) + assert {p.name: p.read_bytes() for p in Path(again[0]["path"]).iterdir()} == first + assert not list((tmp_path / "out").rglob("*.mp4")) + + +def test_v3_refuses_ambiguous_capture_identity(tmp_path): + inputs = inputs_for(tmp_path, []) + inputs[1]["data"]["capture"]["prefix"] = "incoming/another-capture/part0001" + with pytest.raises(RuntimeError, match="capture identity"): + package.run("r", "p", tmp_path / "out", NODE, inputs, helpers()) + + +def test_v3_keeps_valid_legacy_labels_and_rejects_incomplete_configuration(tmp_path): + inputs = inputs_for(tmp_path, []) + sync = Path(inputs[0]["path"]) / "slices/slice-000001/synchronized-slice.json" + doc = json.loads(sync.read_text()) + doc.update(taskLabel="拿取物品", taskLabelId="label-1") + sync.write_text(json.dumps(doc)) + artifact = package.run("r", "p", tmp_path / "out", NODE, inputs, helpers())[0] + classification = json.loads((Path(artifact["path"]) / "part.json").read_text())["slices"][0]["classification"] + assert classification["taskLabel"] == "拿取物品" and classification["taskLabelId"] == "label-1" + bad = {**NODE, "hyperparams": [{"key": "DELIVERY_JSON_V3", "value": "true"}]} + with pytest.raises(RuntimeError, match="requires PART_INDEX_DELIVERY"): + package.run("r", "p", tmp_path / "refused", bad, inputs, helpers()) diff --git a/tests/backend/test_source_index_package.py b/tests/backend/test_source_index_package.py index d8e4982d..c6e1859a 100644 --- a/tests/backend/test_source_index_package.py +++ b/tests/backend/test_source_index_package.py @@ -53,9 +53,13 @@ def test_source_index_real_media_legacy_export_keeps_exact_frames_and_discards(t assert all("synthetic-discard" not in p.read_text() for p in root.rglob("*.json")) -def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch): +@pytest.mark.parametrize("v3", [False, True]) +def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch, v3): node = {"outputs": NODE["outputs"] + [{"id": "deliveryPartIndex", "name": "deliveryPartIndex"}], "hyperparams": NODE["hyperparams"] + [{"key": "PART_INDEX_DELIVERY", "value": "true"}, {"key": "SOURCE_INDEX_DELIVERY", "value": "true"}]} + if v3: + node["hyperparams"].append({"key": "DELIVERY_JSON_V3", "value": "true"}) + source_inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001"} def forbidden(*args, **kwargs): raise AssertionError("JSON-only export must not transcode") monkeypatch.setattr(package.subprocess, "run", forbidden) @@ -66,8 +70,9 @@ def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path monkeypatch.setattr(package, "_sha256_file", metadata_only) artifacts = package.run("test", "package", tmp_path / "output", node, source_inputs, helpers()) assert len(artifacts) == 1 - payload = json.loads(Path(artifacts[0]["path"]).read_text()) - assert payload["schemaVersion"] == "droboticflow.delivery.part-index/v2" + index_path = Path(artifacts[0]["path"]) / "part.json" if v3 else Path(artifacts[0]["path"]) + payload = json.loads(index_path.read_text()) + assert payload["schemaVersion"] == ("droboticflow.delivery.part-index/v3" if v3 else "droboticflow.delivery.part-index/v2") assert len(payload["sources"]) == 5 assert [item["sliceId"] for item in payload["slices"]] == ["slice-000001", "slice-000003"] assert payload["discardedSlices"][0]["sourceFrames"] == {"left": [8, 24], "right": [8, 24]} diff --git a/tools/drobotic-pull/src/main.rs b/tools/drobotic-pull/src/main.rs index 389dd097..42f701af 100644 --- a/tools/drobotic-pull/src/main.rs +++ b/tools/drobotic-pull/src/main.rs @@ -38,6 +38,12 @@ struct Args { allow_http: bool, #[arg(long, conflicts_with = "directory")] project: Option, + /// 预览所选原生 v3 交付包的不可变批次;可重复,须同时指定 --project。 + #[arg(long, requires = "project", conflicts_with_all = ["directory", "list", "report"])] + index_package: Vec, + /// 发布与预览完全一致的批次;参数为预览返回的 snapshotId。 + #[arg(long, requires = "index_package")] + publish_index: Option, /// 例如 bos://drobodata-fanyu/yanshou/项目__ID (仅管理员)。 #[arg(long, requires = "connection", conflicts_with = "project")] directory: Option, @@ -125,6 +131,49 @@ async fn portal_error(r: reqwest::Response) -> anyhow::Error { } } impl Portal { + async fn index_batch( + &self, + project: &str, + packages: &[String], + publish: Option<&str>, + ) -> Result { + let u = self.base.join("api/delivery-index-batches")?; + let request = if let Some(snapshot) = publish { + let cookie = self + .jar + .cookies(&u) + .and_then(|c| c.to_str().ok().map(str::to_owned)) + .unwrap_or_default(); + let csrf = cookie + .split(';') + .find_map(|c| c.trim().strip_prefix("droboticflow_csrf=")) + .unwrap_or(""); + ensure!(!csrf.is_empty(), "缺少发布所需的 CSRF 会话,请重新登录"); + self.client.post(u) + .header("Origin", self.base.origin().ascii_serialization()) + .header("X-Droboticflow-CSRF-Token", csrf) + .json(&serde_json::json!({"projectId":project,"packageIds":packages,"snapshotId":snapshot})) + } else { + let mut q = vec![("projectId", project.to_owned())]; + q.extend(packages.iter().map(|p| ("packageId", p.clone()))); + self.client.get(u).query(&q) + }; + let r = request + .send() + .await + .map_err(|_| anyhow::anyhow!("交付批次请求中断;可用相同 snapshotId 重试"))?; + if !r.status().is_success() { + return Err(portal_error(r).await); + } + let mut stream = r.bytes_stream(); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|_| anyhow::anyhow!("批次响应中断"))?; + ensure!(bytes.len() + chunk.len() <= 8 * 1024 * 1024, "批次响应过大"); + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice(&bytes).context("批次响应无效") + } async fn projects(&self) -> Result> { let r = self .client @@ -289,6 +338,17 @@ async fn download_one( } async fn execute(a: &Args, portal: &Portal) -> Result<()> { + if !a.index_package.is_empty() { + let result = portal + .index_batch( + a.project.as_deref().context("请指定 --project")?, + &a.index_package, + a.publish_index.as_deref(), + ) + .await?; + println!("{}", serde_json::to_string_pretty(&result)?); + return Ok(()); + } let selection = if let Some(project) = &a.project { Selection(vec![("projectId", project.clone())]) } else if let Some(directory) = &a.directory { @@ -481,6 +541,62 @@ mod tests { use super::*; use std::{io::Read, net::TcpListener, thread}; #[test] + fn index_publication_requires_explicit_package_selection() { + assert!(Args::try_parse_from([ + "pull", + "--api", + "https://portal.test", + "--publish-index", + "snapshot" + ]) + .is_err()); + assert!(Args::try_parse_from([ + "pull", + "--api", + "https://portal.test", + "--index-package", + "p1" + ]) + .is_err()); + let a = Args::try_parse_from([ + "pull", + "--api", + "https://portal.test", + "--project", + "project-A", + "--index-package", + "p1", + "--index-package", + "p2", + "--publish-index", + "snapshot", + ]) + .unwrap(); + assert_eq!(a.index_package, ["p1", "p2"]); + assert_eq!(a.publish_index.as_deref(), Some("snapshot")); + } + #[tokio::test] + async fn index_preview_is_read_only_and_publication_uses_csrf() { + let response = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"; + let (url, worker) = server(vec![response, response]); + let p = portal(&url); + p.jar + .add_cookie_str("droboticflow_csrf=batch-csrf; Path=/", &p.base); + p.index_batch("project-A", &["p1".into()], None) + .await + .unwrap(); + p.index_batch("project-A", &["p1".into()], Some("snapshot")) + .await + .unwrap(); + let requests = worker.join().unwrap(); + assert!(requests[0] + .starts_with("GET /api/delivery-index-batches?projectId=project-A&packageId=p1 ")); + assert!(requests[1].starts_with("POST /api/delivery-index-batches ")); + assert!(requests[1] + .to_ascii_lowercase() + .contains("x-droboticflow-csrf-token: batch-csrf")); + } + #[test] fn error_messages_never_log_signed_urls() { assert!( safe_error_detail(br#"{"error":"https://storage/?authorization=secret"}"#).is_none()