🐛 Preserve confirmed annotation frames and enforce v4 time evidence
Some checks failed
web-ci / checks (push) Waiting to run
web-ci / checks (pull_request) Has been cancelled

This commit is contained in:
d-robotics 2026-10-22 12:54:08 +08:00
parent 69cdbb5ad7
commit 63512da71f
34 changed files with 2049 additions and 139 deletions

View File

@ -9,8 +9,10 @@ ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
ENV DROBOTICFLOW_OPERATOR_ID=operator-builtin-package
ENV DROBOTICFLOW_OPERATOR_SLUG=package
ENV DROBOTICFLOW_OPERATOR_HANDLER=builtin_result_package
ENV DROBOTICFLOW_OPERATOR_SOURCE=/app/backend/operators/builtin/package/operator.py
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
COPY --chown=10001:10001 operator-repos/builtin/package/src/operator.py /app/backend/operators/builtin/package/operator.py
COPY --chown=10001:10001 operator-repos/builtin/package/src/operator.py /app/operator-repos/builtin/package/src/operator.py
USER 10001:10001
ENTRYPOINT ["python", "/app/backend/operator-runner/main.py"]

View File

@ -950,6 +950,26 @@ def _v4_consumer_payload(payload, manifest, source_root):
start, end = (_v4_number(split[k], k) for k in ("startSecond", "endSecond"))
first, last = (_v4_timestamp(split[k], k) for k in ("startTimestampUs", "endTimestampUs"))
if start < 0 or end <= start or last <= first: raise RuntimeError("v4 invalid accepted slice range")
row = rows[item["sliceId"]]
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
seconds, stamps = sync.get("boundarySeconds"), sync.get("boundaryTimestampUs")
# A continuous annotation is not proof of its relation to media seconds.
# Require the accepted revision's mapping for kept and discarded ranges;
# never infer hardware PTS from duration, cadence, or action endpoints.
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
raise RuntimeError("v4 missing or invalid hardware boundary mapping")
seconds = [_v4_number(value, "boundary") for value in seconds]
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
stamps[0] != first or stamps[-1] != last or
any(a >= b for a, b in zip(seconds, seconds[1:])) or
any(a >= b for a, b in zip(stamps, stamps[1:]))):
raise RuntimeError("v4 invalid hardware boundary mapping")
window = sync.get("window")
if (not isinstance(window, dict) or window.get("semantics") != "half-open" or
_v4_timestamp(window.get("startTimestampUs"), "synchronization window start") != first or
_v4_timestamp(window.get("endTimestampUs"), "synchronization window end") != last):
raise RuntimeError("v4 boundary mapping disagrees with frozen synchronization window")
if item["classification"]["deliveryDisposition"] == "discard":
item["sliceName"] = ""
evidence = item["classification"]["discardEvidence"]
@ -966,25 +986,10 @@ def _v4_consumer_payload(payload, manifest, source_root):
continue
parent_tags = _v4_tags(item["classification"].get("dictionary"), "slice")
item["classification"]["tags"] = parent_tags
row = rows[item["sliceId"]]
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
seconds, stamps = sync.get("boundarySeconds", []), sync.get("boundaryTimestampUs", [])
if seconds or stamps:
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
raise RuntimeError("v4 invalid hardware boundary mapping")
seconds = [_v4_number(value, "boundary") for value in seconds]
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
stamps[0] != first or stamps[-1] != last or
any(a >= b for a, b in zip(seconds, seconds[1:])) or
any(a >= b for a, b in zip(stamps, stamps[1:]))):
raise RuntimeError("v4 invalid hardware boundary mapping")
def timestamp(second):
if abs(second) < 1e-7: return first
if abs(second - (end-start)) < 1e-7: return last
hits = [stamps[i] for i, value in enumerate(seconds) if abs(_v4_number(value,"boundary") - second) < 1e-7]
hits = [stamps[i] for i, value in enumerate(seconds) if abs(value - second) < 1e-7]
if len(hits) != 1: raise RuntimeError("v4 action boundary has no original PTS mapping")
return _v4_timestamp(hits[0], "boundary timestamp")
return hits[0]
actions = []
for action in item["segments"]:
a = _v4_number(action.get("clipStartMs"), "action clipStartMs") / 1000
@ -992,7 +997,7 @@ def _v4_consumer_payload(payload, manifest, source_root):
if a < -1e-7 or b <= a or b > end-start+1e-7: raise RuntimeError("v4 action escapes its parent slice")
ta, tb = (_v4_timestamp(action.get(k), k) for k in ("startTimestampUs", "endTimestampUs"))
if not first <= ta < tb <= last: raise RuntimeError("v4 action timestamps escape its parent slice")
if seconds and (timestamp(a) != ta or timestamp(b) != tb): raise RuntimeError("v4 action seconds disagree with original PTS")
if timestamp(a) != ta or timestamp(b) != tb: raise RuntimeError("v4 action seconds disagree with original PTS")
action.update(startSecond=a, endSecond=b, partStartSecond=start+a, partEndSecond=start+b)
own = action.get("dictionary")
action["tags"] = _v4_tags(own, "action") if own is not None else list(parent_tags)

View File

@ -40,6 +40,8 @@ B06/part0002、B14/part0001、B18/part0003 的本地样本均为连续 30fps,p
## 成本、兼容性与回退
2026-10-22 补充:普通失败重试不能以旧确认覆盖页面恢复的较新草稿。重试核对确认内容指纹;内容不同需操作者显式按当前保存内容重新确认,保留旧快照及已有动作。后台完成时再次核对,旧队列作业亦适用,不自动回填或重算历史时间。具体契约见 [标注时间完整性](../annotation-time-integrity.zh-CN.md)。
- 固定源每 Run/源版本复制一次,不是跨 Run 去重;首用仍消耗解码 CPU、读取带宽与一份源大小的存储,之后修订为 JSON。
- 固定源不随旧修订目录清理;目前没有自动 GC,以免删除在途审核及已交付索引引用。启用前评估容量,后续需基于引用和保留期设计 GC。
- Worker 识别源索引输入时必须使用 `ANNOTATION_SOURCE_PACKAGE_IMAGE` 指定的不可变 package 镜像摘要;旧 package 镜像不支持该协议,缺配置即启动任务前拒绝。

View File

@ -0,0 +1,36 @@
# 标注确认、时间映射与存量兼容
## 2026-10-22 修复契约
合法的源帧边界不等于用户确认了该边界。页面恢复自动保存草稿,不能使失败重试静默采用另一份已确认工作区。切片、动作、字典版本与标签、舍弃理由必须属于同一次确认。
普通失败重试只接受与原生成确认相同的业务内容。服务端比较规范化的切片、动作和状态,不将保存时间及压缩后的操作历史作为业务差异。有较新的不同草稿时返回 `409 slice_retry_snapshot_conflict`,不入队、不清空或替换两份保存内容。
操作者可显式选择“按当前保存内容重新确认生成”。该请求仅传任务版本和 `reconfirmSaved`,服务端读取已保存草稿,核对操作者、任务版本、保存先后、字典及统一帧边界,建立新的持久作业;客户端提供的重试快照与内容哈希不被信任。它不沿用初次切片确认清空动作的语义。
每次确认记录业务内容指纹。后台生成前及完成发布前分别核对指纹与当前可恢复内容;不一致时生成失败,保留内容及失败作业输入,不能通过删除新草稿发布旧结果。部署前已入队、没有新指纹的作业也执行内容比较。旧租约和任务版本机制继续阻止过期进程写入。
重新确认前,旧工作区和可获得的失败作业快照以内部准备事件保存;准备事件不是已经受理或生成成功的证据。若事件保存失败或超过完整保存上限,请求停止。生成结果及内部事件关联确认指纹;这些过程字段不进入客户总 JSON。
## 时间与来源检查
原时间网格和已保存秒数不在此次修复中重算。硬件 PTS、左右相机各自帧范围及半开区间语义保持不变。新 v4 要求可靠完整的秒数与硬件 PTS 映射;仅有首尾连续性不能证明两种坐标一致,缺证据时拒绝新交付,而不是按固定帧率或时长插值猜补。
接入时的原始 Run 已保存所选对象键及由 `key|ETag|size` 生成的指纹。v4 在打包执行前、写交付目录归属标记前,以及批次预览/发布时重新 HEAD 同一选择并核对该指纹,包括原选择中的可选音频。不能从当前列表、Capture 的新状态或当前文件重新生成“期望值”。读取键有界,不重新下载或哈希整段视频。
ETag/大小指纹用于检测对象身份变化,不等同于逐对象 SHA256 或不可变对象版本;无法证明外部对象从未被覆盖后恢复。保留内部固定源的现有 SHA256 校验。原始选择缺失、范围不一致或身份发生变化时,只拒绝该次 v4 发布,不自动修复历史。
所有 v4 打包节点和源索引输入使用 Worker 已有的 `ANNOTATION_SOURCE_PACKAGE_IMAGE` 不可变镜像配置。物理 v1–v3 输入继续使用原工作流镜像,历史读者与已交付文件保持兼容;此次发布不原地改写工作流定义。
## 旧数据处理
- 编辑中或生成失败的冲突内容:两份保留,操作者核对后显式重新确认;不按保存时间自动择新。
- 已生成、待质检或待验收:被审修订固定。发现确认/生成不一致时走正式返修,不以新草稿替换被审结果。
- 已交付:不覆盖原文件。确认错误后重新验收并建立新不可变批次;证据一致的历史交付无需重生成。
- 缺历史来源或映射:报告缺口,不猜补、插值或把当前证据冒充过去证据。
本次应用升级没有启动历史盘点、回填、时间重算或批量重生成。任何生产修复脚本需要另列具体范围、预览及授权。
## 验证要求
回归覆盖保存版本冲突、显式重新确认、原标签与动作保留、任务归属及版本拒绝、失败/进程重启恢复、旧作业与后来的草稿竞争,以及物理/源索引两种 v4 独立消费路径。构造“秒数连续但 PTS 错误”及“同大小、不同 ETag 的原始文件”,必须在发布前拒绝。

View File

@ -34,6 +34,12 @@
平台只输出分类索引,不按标签执行 BOS 数据切割、搬运或集中打包。内部审核文件不在客户目录内,也不从主 JSON 引用。
## 2026-10-22 时间与来源门禁补充
新 v4 不再接受缺少秒数↔PTS 映射、仅靠动作首尾连续的旧证据;必须与固定同步窗口一致,不能从帧率或时长猜补。保留原映射与秒数,缺证据的历史结果需另行核对后生成新修订,不能自动迁移。
原始来源检查补充核对 Run 接入时保存的选择指纹:对完全相同的对象键 HEAD 后,按既有 `key|ETag|size` 算法比较,包含当时选择的可选音频。打包前、Worker 发布前和批次预览/发布均校验;不再仅证明对象存在。该指纹不是逐文件 SHA256 或不可变对象版本保证,也不按当前目录重新生成期望指纹。历史 v1–v3 和已交付字节不变,详见 [标注时间完整性](annotation-time-integrity.zh-CN.md)。
本地可用 `python -m tests.backend.export_delivery_v4_fixture <全新输出目录>` 生成明确标记的合成样例。目录中 `partNNNN/part.json` 是客户主文件;`internal/partNNNN/review-history.json` 是分离的内部示例,不属于客户包。不要把整个验证输出目录直接交付给客户。
独立消费验证:设置 `FANYU_DELIVERY_CONSUMER_FIXTURE_DIR` 为全新绝对目录,先运行 Go `TestDeliveryV4ConsumerReplicaFixture`,再以同一环境变量运行 `tests/backend/test_delivery_v4_consumer.py`。合成真实双目视频分别经过物理切片、源索引和元数据复用;独立进程仅读取导出 JSON 与原始文件,核对每帧覆盖、非零 part 起点、标签继承、无效与舍弃。独立读取器 `tests/backend/consume_delivery_v4.py` 只生成处理计划,不执行切割或写源数据。

View File

@ -8,9 +8,10 @@ import (
)
type annotationEditorState struct {
Phase string `json:"phase"`
CanConfirmDescriptions bool `json:"canConfirmDescriptions"`
CanRecoverGeneration bool `json:"canRecoverGeneration"`
Phase string `json:"phase"`
CanConfirmDescriptions bool `json:"canConfirmDescriptions"`
CanRecoverGeneration bool `json:"canRecoverGeneration"`
CanReconfirmSavedGeneration bool `json:"canReconfirmSavedGeneration"`
}
// Use the same persisted evidence as the write gate, including legacy tasks
@ -34,6 +35,7 @@ func (server *server) annotationEditorStateForTask(ctx context.Context, task dom
jobStatus = item.ProcessingJob.Status
}
if status == "failed" || jobStatus == "failed" {
state.CanReconfirmSavedGeneration = failedSliceReconfirmationAvailable(item)
return state, nil
}
if status == "processing" || status == "queued" || status == "running" || status == "materializing" || jobStatus == "queued" || jobStatus == "running" || jobStatus == "processing" || item.PendingCombinedSubmissionID != "" {

View File

@ -6,6 +6,7 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"math"
@ -1420,13 +1421,15 @@ type combinedAnnotationDraftInput struct {
}
type sliceConfirmationInput struct {
RecoverExisting bool `json:"recoverExisting,omitempty"`
RetryFailed bool `json:"retryFailed,omitempty"`
RetryWorkspace map[string]any `json:"retryWorkspace,omitempty"`
Version int `json:"version"`
SliceSegments []domain.SliceSegment `json:"sliceSegments"`
SliceOperations []domain.SliceOperation `json:"sliceOperations,omitempty"`
ChangeReason string `json:"changeReason,omitempty"`
ReconfirmSaved bool `json:"reconfirmSaved,omitempty"`
SnapshotContentSHA string `json:"snapshotContentSha256,omitempty"`
RecoverExisting bool `json:"recoverExisting,omitempty"`
RetryFailed bool `json:"retryFailed,omitempty"`
RetryWorkspace map[string]any `json:"retryWorkspace,omitempty"`
Version int `json:"version"`
SliceSegments []domain.SliceSegment `json:"sliceSegments"`
SliceOperations []domain.SliceOperation `json:"sliceOperations,omitempty"`
ChangeReason string `json:"changeReason,omitempty"`
}
type sliceResetInput struct {
@ -2039,7 +2042,11 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
Revision: revision, Segments: slices, Operations: payload.SliceOperations, AuthorUserID: actorUserID,
ChangeReason: strings.TrimSpace(payload.ChangeReason), CreatedAt: nowISO(),
}
materialization, processErr := server.materializeCombinedSliceSubmission(ctx, *task, *item, combinedAnnotationSubmissionInput{sliceOnly: true, SliceSegments: slices, Segments: []domain.AnnotationSegment{}}, revision)
processErr := validateSliceCompletionSnapshot(item, payload)
var materialization *humanSliceMaterialization
if processErr == nil {
materialization, processErr = server.materializeCombinedSliceSubmission(ctx, *task, *item, combinedAnnotationSubmissionInput{sliceOnly: true, SliceSegments: slices, Segments: []domain.AnnotationSegment{}}, revision)
}
close(done)
<-heartbeatStopped
if _, durable := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease); durable && (ctx.Err() != nil || server.assertAnnotationLease(ctx) != nil) {
@ -2059,6 +2066,9 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
if strings.TrimSpace(fmt.Sprint(workflow["requestId"])) != requestID || stringValue(workflow["status"]) != "processing" {
return
}
if processErr == nil {
processErr = validateSliceCompletionSnapshot(current, payload)
}
if processErr == nil {
processErr = server.store.UpsertSliceRevision(ctx, sliceRevision)
}
@ -2070,11 +2080,13 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
workflow["updatedAt"] = now
} else {
log.Printf("[api] finishSliceConfirmation succeeded for task=%s item=%s requestID=%s revision=%d slices=%d", taskID, itemID, requestID, revision, len(materialization.Slices))
materialization.Payload["confirmationSnapshotSha256"] = payload.SnapshotContentSHA
workflow = map[string]any{
"status": "completed", "phase": "annotate", "requestId": requestID, "revision": revision,
"boundaryContract": "hardware-frame-v1",
"sliceCount": len(materialization.Slices), "materialization": materialization.Payload, "updatedAt": now,
}
workflow["snapshotContentSha256"] = payload.SnapshotContentSHA
if payload.RetryFailed {
workflow["retryOf"] = w["retryOf"]
}
@ -2082,6 +2094,7 @@ func (server *server) finishSliceConfirmation(ctx context.Context, taskID string
"sliceSegments": materialization.Slices, "operations": payload.SliceOperations,
"segments": []domain.AnnotationSegment{}, "statuses": pendingSliceStatuses(materialization.Slices),
"userId": actorUserID, "savedAt": now,
"confirmationSnapshotSha256": payload.SnapshotContentSHA,
}
if payload.RetryFailed {
// Retry only regenerated physical media; retained semantic work and
@ -2142,8 +2155,32 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
return
}
payload.RetryWorkspace = nil // Private durable snapshot is always server-owned.
payload.SnapshotContentSHA = ""
if payload.Version != item.Version {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"})
return
}
if payload.ReconfirmSaved && (payload.RetryFailed || payload.RecoverExisting) {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "请选择一种切片生成操作", "code": "slice_confirmation_mode_invalid"})
return
}
var retained combinedAnnotationDraftInput
if payload.RecoverExisting {
if payload.ReconfirmSaved {
workflow := mapValue(item.Subject.Payload["sliceWorkflow"])
job := item.ProcessingJob
if stringValue(workflow["status"]) != "failed" || (job != nil && (job.Status == "queued" || job.Status == "running" || job.Kind != "slice")) {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "只有生成失败且未在后台处理的任务可以重新确认", "code": "slice_retry_unavailable"})
return
}
var recoveryErr error
payload, retained, recoveryErr = retainedSliceRecoveryInput(item, payload.Version)
if recoveryErr != nil {
renderJSON(writer, http.StatusUnprocessableEntity, map[string]string{"error": recoveryErr.Error(), "code": "slice_retry_workspace_invalid"})
return
}
payload.ReconfirmSaved = true
payload.ChangeReason = "按当前已保存内容重新确认切片生成"
} else if payload.RecoverExisting {
ready, err := server.legacyAnnotationConfirmationReady(request.Context(), *task, item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
@ -2170,17 +2207,21 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
}
var retryErr error
recoverExisting := boolValue(workflow["recoverExisting"])
if recoverExisting {
payload, retained, retryErr = retainedSliceRecoveryInput(item, payload.Version)
payload.RecoverExisting = true
} else {
payload, retained, retryErr = retainedSliceRetryInput(item, payload.Version)
}
payload, retained, retryErr = retainedSliceRetryInput(item, payload.Version)
payload.RecoverExisting = recoverExisting
if retryErr != nil {
if errors.Is(retryErr, errSliceSnapshotConflict) {
renderJSON(writer, http.StatusConflict, map[string]string{"error": retryErr.Error(), "code": "slice_retry_snapshot_conflict"})
return
}
renderJSON(writer, http.StatusUnprocessableEntity, map[string]string{"error": retryErr.Error(), "code": "slice_retry_workspace_invalid"})
return
}
}
if !payload.RetryFailed && stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["status"]) == "failed" {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "请按已保存内容重新确认生成,已有标注仍保留", "code": "slice_retry_snapshot_conflict"})
return
}
payload.SliceSegments = descriptionOnlySlices(payload.SliceSegments)
if payload.Version != item.Version {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"})
@ -2250,6 +2291,39 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
}
probe.validated()
requestID := domain.NewID("slice-materialization")
contentSHA, hashErr := sliceWorkspaceContentSHA(sliceConfirmationWorkspace(payload))
if hashErr != nil {
renderError(writer, http.StatusUnprocessableEntity, hashErr)
return
}
payload.SnapshotContentSHA = contentSHA
if payload.ReconfirmSaved {
prior, archiveErr := compactSliceWorkspaceForArchive(annotationWorkspacePayload(item))
if archiveErr != nil {
renderError(writer, http.StatusUnprocessableEntity, archiveErr)
return
}
archive := map[string]any{"requestId": requestID, "selectedContentSha256": contentSHA, "previousWorkspace": prior}
if job := item.ProcessingJob; job != nil && job.Input != nil {
if raw := mapValue(job.Input["retryWorkspace"]); len(raw) > 0 {
previousInput, err := compactSliceWorkspaceForArchive(raw)
if err != nil {
renderError(writer, http.StatusUnprocessableEntity, err)
return
}
archive["previousFailedSnapshot"] = previousInput
}
}
encoded, err := json.Marshal(archive)
if err != nil || len(encoded) > 8<<20 {
renderJSON(writer, http.StatusRequestEntityTooLarge, map[string]string{"error": "历史快照过大,无法完整保留,未开始重新生成", "code": "slice_snapshot_archive_too_large"})
return
}
if err := server.appendHumanEvent(request.Context(), *item, "slice_reconfirmation_snapshot_prepared", principal.User.ID, archive); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
now := nowISO()
if item.Subject.Payload == nil {
item.Subject.Payload = map[string]any{}
@ -2257,7 +2331,8 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter,
item.Subject.Payload["sliceWorkflow"] = map[string]any{
"status": "processing", "phase": "materializing", "requestId": requestID, "revision": revision,
"sliceCount": len(payload.SliceSegments), "startedAt": now,
"leaseExpiresAt": time.Now().Add(sliceLeaseDuration).Format(time.RFC3339Nano),
"snapshotContentSha256": contentSHA,
"leaseExpiresAt": time.Now().Add(sliceLeaseDuration).Format(time.RFC3339Nano),
}
if payload.RetryFailed {
// Keep the original snapshot intact while queued/running; all workbench

View File

@ -309,7 +309,6 @@ func (s *server) executeAnnotationJob(parent context.Context, claimed domain.Hum
}
j.Status = "completed"
j.Stage = "completed"
j.Input = nil
j.CompletedSlices = j.TotalSlices
j.UpdatedAt = nowISO()
if e != nil {
@ -317,6 +316,8 @@ func (s *server) executeAnnotationJob(parent context.Context, claimed domain.Hum
j.Stage = "failed"
j.Error = e.Error()
log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e)
} else {
j.Input = nil
}
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
if e != nil {

View File

@ -16,6 +16,7 @@ import (
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/sourceidentity"
"droboticflow/internal/workspacescope"
)
@ -180,22 +181,26 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla
}
expected, meta, e := indexBatchSourceEvidence(r.Context(), sourceBrowser, record.Bucket, part.Capture.Prefix, part.Capture.ID)
if e == nil && part.Schema == deliveryindex.ClassificationPartSchema {
// HEAD only: prove the frozen references still resolve without reading
// video bodies, recutting media or expanding the original selection.
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
rel := part.OriginalSource.Files[role].Path
media := role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu"
if rel == "" || path.Clean(rel) != rel || strings.ContainsAny(rel, "\\\x00") || strings.HasPrefix(rel, "/") || strings.HasPrefix(rel, "../") || rel == ".." || media && !strings.HasPrefix(rel, part.PartID+"/") || !media && path.Base(rel) != rel {
e = fmt.Errorf("分类 JSON 原始文件路径无效:%s", role)
break
originalRun, runErr := s.store.GetRun(r.Context(), part.Lineage.RunID)
if runErr != nil {
e = runErr
} else {
var selection *sourceidentity.Selection
selection, e = sourceidentity.FromRun(originalRun, record)
if e == nil {
paths := map[string]string{}
for role, file := range part.OriginalSource.Files {
paths[role] = file.Path
}
e = selection.MatchFiles(part.OriginalSource.Provider, part.OriginalSource.Bucket, part.OriginalSource.Prefix, part.OriginalSource.PartPrefix, paths)
}
key := part.Capture.Prefix + "/" + rel
info, err := sourceBrowser.GetObjectSummary(r.Context(), record.Bucket, key)
if err != nil || info.Key != key || info.Kind != "file" || info.SizeBytes <= 0 {
e = fmt.Errorf("分类 JSON 原始文件不存在或不可读:%s", role)
break
if e == nil {
e = selection.Recheck(r.Context(), sourceBrowser.GetObjectSummary)
}
}
if e != nil {
e = fmt.Errorf("原始数据身份核验失败,停止发布:%w", e)
}
}
if closeSource != nil {
closeSource()

View File

@ -15,6 +15,7 @@ import (
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/sourceidentity"
"droboticflow/internal/store"
)
@ -30,7 +31,7 @@ func (b *indexTestBrowser) GetObjectSummary(_ context.Context, bucket, key strin
if !ok {
return domain.ObjectStorageObjectSummary{}, os.ErrNotExist
}
return domain.ObjectStorageObjectSummary{Bucket: bucket, Key: key, Kind: "file", SizeBytes: int64(len(body))}, nil
return domain.ObjectStorageObjectSummary{Bucket: bucket, Key: key, Kind: "file", SizeBytes: int64(len(body)), ETag: deliveryindex.Hash(body)}, nil
}
func (b *indexTestBrowser) PutObjectConditional(_ context.Context, _ string, key string, body []byte, _ string, match, none string, _ map[string]string) (domain.ObjectStorageObjectSummary, error) {
@ -185,6 +186,26 @@ func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authent
must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, OrganizationID: org, ProjectID: "project-A", RunID: sourceRun, CaptureID: "capture-A/" + part, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A/" + part, SourcePrefix: "incoming/capture-A", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix}}))
}
b.objects["incoming/capture-A/meta_capture-A.json"] = []byte(`{"synthetic":true}`)
if v4 {
for _, part := range []string{"part0001", "part0002"} {
var doc struct {
Original struct {
Files map[string]struct {
Path string `json:"path"`
} `json:"files"`
} `json:"originalSource"`
}
must(json.Unmarshal(b.objects[root+"/capture-A/"+part+"/part.json"], &doc))
keys := []string{}
objects := []sourceidentity.Object{}
for _, file := range doc.Original.Files {
key := "incoming/capture-A/" + file.Path
keys = append(keys, key)
objects = append(objects, sourceidentity.Object{Key: key, ETag: deliveryindex.Hash(b.objects[key]), Size: int64(len(b.objects[key]))})
}
must(ds.UpsertRun(ctx, domain.RunRecord{ID: "source-run-" + part, OrganizationID: org, ProjectID: "project-A", Inputs: map[string]domain.ArtifactPayload{"source": {Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, Kind: domain.ArtifactRefKindDirectory, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A", CaptureUnit: "capture-A/" + part, ObjectKeys: keys, VersionID: sourceidentity.Fingerprint(objects)}}}}))
}
}
old := storageConnectionBrowserResolver
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
if c.OrganizationID != org {
@ -382,7 +403,7 @@ func TestClassificationBatchRejectsMissingOrChangedOriginalSource(t *testing.T)
t.Fatal(err)
}
} else {
delete(b.objects, "incoming/capture-A/part0001/leftVideo")
delete(b.objects, "incoming/capture-A/part0001/capture-A_camera_left_part0001.mp4")
}
w := indexCall(NewHandler(ds, nil), p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
if w.Code != 409 || len(b.puts) != 0 || !strings.Contains(w.Body.String(), "原始") {
@ -401,8 +422,23 @@ func enrichConsumerFixture(p map[string]any) {
sources := map[string]any{}
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
name := part + "/" + role
switch role {
case "leftVideo":
name = part + "/capture-A_camera_left_" + part + ".mp4"
case "rightVideo":
name = part + "/capture-A_camera_right_" + part + ".mp4"
case "leftPts":
name = part + "/capture-A_camera_left_" + part + "_pts.csv"
case "rightPts":
name = part + "/capture-A_camera_right_" + part + "_pts.csv"
case "imu":
name = part + "/capture-A_imu_" + part + ".csv"
}
if role == "cameraCalibration" || role == "imuCalibration" {
name = role + ".yaml"
name = "capture-A_calibration_camera.yaml"
if role == "imuCalibration" {
name = "capture-A_calibration_imu.yaml"
}
}
if role == "metadata" {
name = "meta_capture-A.json"

View File

@ -0,0 +1,166 @@
package api
import (
"bytes"
"context"
"encoding/json"
"net/http"
"strings"
"testing"
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/sourceidentity"
"droboticflow/internal/store"
)
type identityIndexBrowser struct {
*indexTestBrowser
heads int
}
func (b *identityIndexBrowser) GetObjectSummary(ctx context.Context, bucket, key string) (domain.ObjectStorageObjectSummary, error) {
info, err := b.indexTestBrowser.GetObjectSummary(ctx, bucket, key)
if err == nil {
info.ETag = deliveryindex.Hash(b.objects[key])
b.heads++
}
return info, err
}
func seedOriginalIdentityIndex(t *testing.T, ds store.Store) (*identityIndexBrowser, *authenticatedPrincipal) {
t.Helper()
t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1")
t.Setenv("DELIVERY_V4_FIXTURE_DIR", "")
base, p, root := seedIndexFixture(t, ds)
b := &identityIndexBrowser{indexTestBrowser: base}
for _, part := range []string{"part0001", "part0002"} {
prefix := root + "/capture-A/" + part
body := base.objects[prefix+"/part.json"]
replacements := map[string]string{"leftVideo": part + "/capture-A_camera_left_" + part + ".mp4", "rightVideo": part + "/capture-A_camera_right_" + part + ".mp4", "leftPts": part + "/capture-A_camera_left_" + part + "_pts.csv", "rightPts": part + "/capture-A_camera_right_" + part + "_pts.csv", "imu": part + "/capture-A_imu_" + part + ".csv", "cameraCalibration": "capture-A_calibration_camera.yaml", "imuCalibration": "capture-A_calibration_imu.yaml", "metadata": "meta_capture-A.json"}
keys := []string{}
objects := []sourceidentity.Object{}
for role, rel := range replacements {
oldRel := part + "/" + role
if role == "cameraCalibration" || role == "imuCalibration" {
oldRel = role + ".yaml"
}
if role == "metadata" {
oldRel = "meta_capture-A.json"
}
body = bytes.ReplaceAll(body, []byte(`"`+oldRel+`"`), []byte(`"`+rel+`"`))
key := "incoming/capture-A/" + rel
if role != "metadata" {
base.objects[key] = []byte("original synthetic object")
}
keys = append(keys, key)
objects = append(objects, sourceidentity.Object{Key: key, ETag: deliveryindex.Hash(base.objects[key]), Size: int64(len(base.objects[key]))})
}
base.objects[prefix+"/part.json"] = body
entries := []deliveryindex.Entry{{Path: "part.json", Key: prefix + "/part.json", SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)}}
base.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 1, "totalBytes": len(body), "entries": entries})
child, err := ds.GetRun(t.Context(), "package-run-"+part)
if err != nil {
t.Fatal(err)
}
for i := range child.Artifacts {
if domain.IsInternalDeliveryAudit(child.Artifacts[i]) {
child.Artifacts[i].Data["partSha256"] = deliveryindex.Hash(body)
}
}
child.NodeExecutions[0].Artifacts = child.Artifacts
if err = ds.UpsertRun(t.Context(), *child); err != nil {
t.Fatal(err)
}
capture, err := ds.GetCaptureRecordByRunID(t.Context(), "source-run-"+part)
if err != nil {
t.Fatal(err)
}
original := domain.RunRecord{ID: capture.RunID, OrganizationID: capture.OrganizationID, ProjectID: capture.ProjectID, WorkflowID: capture.WorkflowID, Inputs: map[string]domain.ArtifactPayload{"source": {Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, Kind: domain.ArtifactRefKindDirectory, ConnectionID: capture.ConnectionID, Bucket: capture.Bucket, Prefix: capture.SourcePrefix, CaptureUnit: capture.CaptureID, ObjectKeys: keys, VersionID: sourceidentity.Fingerprint(objects)}}}}
if err = ds.UpsertRun(t.Context(), original); err != nil {
t.Fatal(err)
}
}
storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
return b, nil, nil
}
return b, p
}
func TestClassificationBatchOriginalIdentityHTTP(t *testing.T) {
for _, scenario := range []string{"correct", "same-size-etag", "changed-size", "missing", "missing-frozen-version", "wrong-tenant", "extra-selection"} {
t.Run(scenario, func(t *testing.T) {
ds := store.NewMemoryStore()
b, p := seedOriginalIdentityIndex(t, ds)
h := NewHandler(ds, nil)
query := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001"
preview := indexCall(h, p, http.MethodGet, query, nil)
if preview.Code != 200 {
t.Fatalf("valid preview: %d %s", preview.Code, preview.Body.String())
}
var plan struct {
ID string `json:"snapshotId"`
}
json.Unmarshal(preview.Body.Bytes(), &plan)
key := "incoming/capture-A/part0001/capture-A_camera_left_part0001.mp4"
switch scenario {
case "same-size-etag":
b.objects[key] = bytes.Repeat([]byte("x"), len(b.objects[key]))
case "changed-size":
b.objects[key] = append(b.objects[key], 'x')
case "missing":
delete(b.objects, key)
case "missing-frozen-version", "wrong-tenant", "extra-selection":
run, _ := ds.GetRun(t.Context(), "source-run-part0001")
switch scenario {
case "missing-frozen-version":
run.Inputs["source"].Ref.VersionID = ""
case "wrong-tenant":
run.OrganizationID = "other"
case "extra-selection":
run.Inputs["source"].Ref.ObjectKeys = append(run.Inputs["source"].Ref.ObjectKeys, "incoming/capture-A/extra.json")
}
if err := ds.UpsertRun(t.Context(), *run); err != nil {
t.Fatal(err)
}
}
result := indexCall(h, p, http.MethodPost, "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: plan.ID})
if scenario == "correct" {
if result.Code != 200 {
t.Fatalf("publish: %d %s", result.Code, result.Body.String())
}
if b.heads < 16 {
t.Fatalf("POST did not recheck: %d", b.heads)
}
} else if result.Code != 409 || len(b.puts) != 0 {
t.Fatalf("changed source published: %d %s writes=%d", result.Code, result.Body.String(), len(b.puts))
}
})
}
}
func TestClassificationBatchRejectsDeliveryFilesOutsideFrozenSelection(t *testing.T) {
ds := store.NewMemoryStore()
b, p := seedOriginalIdentityIndex(t, ds)
// Change the immutable run selection while preserving a valid selection shape.
run, _ := ds.GetRun(t.Context(), "source-run-part0001")
ref := run.Inputs["source"].Ref
for i, key := range ref.ObjectKeys {
if strings.HasSuffix(key, "_left_part0001.mp4") {
ref.ObjectKeys[i] = strings.TrimSuffix(key, ".mp4") + "_alternate.mp4"
b.objects[ref.ObjectKeys[i]] = b.objects[key]
}
}
objects := []sourceidentity.Object{}
for _, key := range ref.ObjectKeys {
objects = append(objects, sourceidentity.Object{Key: key, ETag: deliveryindex.Hash(b.objects[key]), Size: int64(len(b.objects[key]))})
}
ref.VersionID = sourceidentity.Fingerprint(objects)
if err := ds.UpsertRun(t.Context(), *run); err != nil {
t.Fatal(err)
}
result := indexCall(NewHandler(ds, nil), p, http.MethodGet, "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
if result.Code != 409 || len(b.puts) > 0 {
t.Fatalf("files outside frozen selection accepted: %d %s", result.Code, result.Body.String())
}
}

View File

@ -11,26 +11,67 @@ import (
// A retry reads the retained server workspace. Client-supplied slices must never
// replace it, and malformed historic containers must not become empty content.
func retainedSliceRetryInput(item *domain.HumanItemRecord, version int) (sliceConfirmationInput, combinedAnnotationDraftInput, error) {
return retainedSliceInput(annotationWorkspacePayload(item), version)
selected, saved, err := retainedSliceRecoveryInput(item, version)
if err != nil {
return selected, saved, err
}
expected := stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["snapshotContentSha256"])
if expected == "" {
var frozen map[string]any
if job := item.ProcessingJob; job != nil && job.Kind == "slice" && len(job.Input) > 0 {
encoded, marshalErr := json.Marshal(job.Input)
var prior sliceConfirmationInput
if marshalErr != nil {
return selected, saved, marshalErr
}
if decodeErr := json.Unmarshal(encoded, &prior); decodeErr != nil {
return selected, saved, decodeErr
}
frozen = sliceConfirmationWorkspace(prior)
} else {
frozen = mapValue(item.Subject.Payload["annotationWorkspace"])
}
if len(frozen) == 0 {
return selected, saved, errSliceSnapshotConflict
}
expected, err = sliceWorkspaceContentSHA(frozen)
if err != nil {
return selected, saved, err
}
}
actual, err := sliceWorkspaceContentSHA(selected.RetryWorkspace)
if err != nil {
return selected, saved, err
}
if actual != expected {
return selected, saved, errSliceSnapshotConflict
}
return selected, saved, nil
}
// Legacy recovery must retain the same newer, owned autosave restored by the
// editor. Failed-job retries keep their separately frozen confirmation snapshot.
// editor. Ordinary retries must additionally match their frozen content hash.
func retainedSliceRecoveryInput(item *domain.HumanItemRecord, version int) (sliceConfirmationInput, combinedAnnotationDraftInput, error) {
workspace := annotationWorkspacePayload(item)
baseline := mapValue(item.Subject.Payload["annotationWorkspace"])
if raw := item.Subject.Payload["annotationDraft"]; raw != nil && len(baseline) > 0 {
if raw := item.Subject.Payload["annotationDraft"]; raw != nil {
draft := mapValue(raw)
if len(draft) == 0 || item.AssignedUserID == "" || stringValue(draft["userId"]) != item.AssignedUserID || intValue(draft["savedVersion"]) > item.Version {
return sliceConfirmationInput{}, combinedAnnotationDraftInput{}, fmt.Errorf("自动保存草稿的归属或版本无法核对,请先保存当前进度;原内容仍保留")
}
savedAt, savedErr := time.Parse(time.RFC3339Nano, stringValue(draft["savedAt"]))
if savedErr != nil {
return sliceConfirmationInput{}, combinedAnnotationDraftInput{}, fmt.Errorf("自动保存草稿缺少可靠保存时间,请先保存当前进度;原内容仍保留")
}
if len(baseline) == 0 {
return retainedSliceInput(draft, version)
}
baselineTime := stringValue(baseline["confirmedAt"])
if baselineTime == "" {
baselineTime = stringValue(baseline["savedAt"])
}
confirmedAt, confirmedErr := time.Parse(time.RFC3339Nano, baselineTime)
savedAt, savedErr := time.Parse(time.RFC3339Nano, stringValue(draft["savedAt"]))
if confirmedErr != nil || savedErr != nil {
if confirmedErr != nil {
return sliceConfirmationInput{}, combinedAnnotationDraftInput{}, fmt.Errorf("无法核对自动保存草稿与已确认内容的先后,请先核对保存记录;原内容仍保留")
}
if !savedAt.Before(confirmedAt) {

View File

@ -274,7 +274,7 @@ func TestFailedSliceRetryRejectsUnsafeSnapshotsWithoutMutation(t *testing.T) {
}
}
func TestFailedSliceRetryFreezesLegacyDraftAgainstLateWriter(t *testing.T) {
func TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
h := seedFailedSliceRetry(t, f)
@ -282,7 +282,7 @@ func TestFailedSliceRetryFreezesLegacyDraftAgainstLateWriter(t *testing.T) {
delete(h.Subject.Payload, "annotationWorkspace")
f.store.UpsertHumanItem(f.ctx, h)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
map[string]any{"version": h.Version, "retryFailed": true, "retryWorkspace": map[string]any{"segments": []any{}}})
map[string]any{"version": h.Version, "reconfirmSaved": true, "retryWorkspace": map[string]any{"segments": []any{}}})
if r.Code != 202 {
t.Fatal(r.Code, r.Body.String())
}
@ -304,7 +304,9 @@ func TestFailedSliceRetryFreezesLegacyDraftAgainstLateWriter(t *testing.T) {
s.executeAnnotationJob(context.Background(), *claimed)
saved, _ := f.store.GetHumanItem(f.ctx, h.ID)
actions := decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(saved)["segments"])
if saved.ProcessingJob.Status != "completed" || len(actions) != 1 || actions[0].ID != "retained-action" {
t.Fatal("retry did not restore the server snapshot", saved.ProcessingJob, actions)
retained := mapValue(saved.ProcessingJob.Input["retryWorkspace"])
retainedActions := decodeMaterializationValue[domain.AnnotationSegment](retained["segments"])
if saved.ProcessingJob.Status != "failed" || len(actions) != 0 || len(retainedActions) != 1 || retainedActions[0].ID != "retained-action" {
t.Fatal("late edit or original job snapshot was lost", saved.ProcessingJob, actions)
}
}

View File

@ -0,0 +1,84 @@
package api
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"droboticflow/internal/domain"
)
var errSliceSnapshotConflict = errors.New("当前保存内容与原失败快照不同,请核对后按当前保存内容重新确认生成;两份内容仍保留")
// Compare annotation content, not autosave timestamps or compacted operation
// history. Frame identities, dictionaries, disposition and actions all matter.
func sliceWorkspaceContentSHA(workspace map[string]any) (string, error) {
_, saved, err := retainedSliceInput(workspace, 0)
if err != nil {
return "", err
}
content := struct {
Slices []domain.SliceSegment `json:"slices"`
Segments []domain.AnnotationSegment `json:"segments"`
Statuses map[string]domain.SliceAnnotationStatus `json:"statuses"`
}{saved.normalizedSliceSegments(), saved.normalizedSegments(), saved.normalizedStatuses()}
body, err := json.Marshal(content)
if err != nil {
return "", err
}
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:]), nil
}
func sliceConfirmationWorkspace(payload sliceConfirmationInput) map[string]any {
if payload.RetryFailed {
return payload.RetryWorkspace
}
return map[string]any{"sliceSegments": payload.SliceSegments, "segments": []domain.AnnotationSegment{}, "statuses": pendingSliceStatuses(payload.SliceSegments)}
}
// This gate also applies to pre-upgrade queued jobs lacking a content hash.
// A valid frame in a different saved snapshot is still the wrong annotation.
func validateSliceCompletionSnapshot(item *domain.HumanItemRecord, payload sliceConfirmationInput) error {
selectedSHA, err := sliceWorkspaceContentSHA(sliceConfirmationWorkspace(payload))
if err != nil {
return err
}
if payload.SnapshotContentSHA != "" && payload.SnapshotContentSHA != selectedSHA {
return fmt.Errorf("切片生成快照校验失败,原内容仍保留")
}
flowSHA := stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["snapshotContentSha256"])
if flowSHA != "" && flowSHA != selectedSHA {
return fmt.Errorf("切片生成确认版本已变化,原内容仍保留")
}
current, _, err := retainedSliceRecoveryInput(item, item.Version)
if err != nil {
return err
}
currentSHA, err := sliceWorkspaceContentSHA(current.RetryWorkspace)
if err != nil {
return err
}
if currentSHA != selectedSHA {
return errSliceSnapshotConflict
}
return nil
}
func failedSliceReconfirmationAvailable(item *domain.HumanItemRecord) bool {
if item == nil || stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["status"]) != "failed" {
return false
}
_, _, err := retainedSliceRetryInput(item, item.Version)
return errors.Is(err, errSliceSnapshotConflict)
}
func compactSliceWorkspaceForArchive(workspace map[string]any) (map[string]any, error) {
_, saved, err := retainedSliceInput(workspace, 0)
if err != nil {
return nil, err
}
return map[string]any{"sliceSegments": saved.normalizedSliceSegments(), "segments": saved.normalizedSegments(), "statuses": saved.normalizedStatuses(), "operations": saved.normalizedOperations(), "userId": workspace["userId"], "savedAt": workspace["savedAt"], "confirmedAt": workspace["confirmedAt"]}, nil
}

View File

@ -0,0 +1,246 @@
package api
import (
"context"
"net/http"
"reflect"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
)
func seedNewerSliceSnapshot(t *testing.T, f combinedDraftFixture) domain.HumanItemRecord {
t.Helper()
h := seedFailedSliceRetry(t, f)
draft := map[string]any{}
for k, v := range annotationWorkspacePayload(&h) {
draft[k] = v
}
draft["sliceSegments"] = []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 0, End: 3}, {ID: "b", Name: "测试切片", Start: 3, End: 4}}
draft["segments"] = []domain.AnnotationSegment{{ID: "new-action", Start: 0, End: 3, Status: "pending", Dictionary: &domain.DictionarySelection{ID: "historical-action", Kind: "action", Version: 2, Name: "历史动作", Tags: []string{"原标签"}}}}
draft["savedAt"], draft["savedVersion"] = nowISO(), h.Version
h.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
stored, err := f.store.GetHumanItem(f.ctx, h.ID)
if err != nil || stored == nil {
t.Fatal("saved fixture unavailable", err)
}
return *stored
}
func TestFailedSliceRetryRejectsDifferentSavedSnapshot(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, _ := annotationProcessorFixture(t)
h := seedNewerSliceSnapshot(t, f)
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "retryFailed": true})
if r.Code != http.StatusConflict || !strings.Contains(r.Body.String(), "slice_retry_snapshot_conflict") {
t.Fatalf("stale snapshot accepted: %d %s", r.Code, r.Body.String())
}
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(before, after) {
t.Fatal("rejected retry changed either retained snapshot")
}
state, err := (&server{store: f.store}).annotationEditorStateForTask(f.ctx, f.task, after)
if err != nil || !state.CanReconfirmSavedGeneration {
t.Fatalf("explicit saved confirmation not offered: %+v %v", state, err)
}
}
func TestReconfirmSavedSlicesPreservesSelectedBoundariesAcrossRestart(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
h := seedNewerSliceSnapshot(t, f)
draft := mapValue(h.Subject.Payload["annotationDraft"])
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "reconfirmSaved": true, "retryWorkspace": map[string]any{"segments": []any{}}, "sliceSegments": []any{}})
if r.Code != http.StatusAccepted {
t.Fatalf("explicit reconfirmation: %d %s", r.Code, r.Body.String())
}
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(mapValue(queued.Subject.Payload["annotationDraft"]), draft) {
t.Fatal("queueing changed saved draft")
}
now := time.Now().UTC()
old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "snapshot-old", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano))
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "snapshot-new", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 {
t.Fatal("reconfirmation not recoverable after restart")
}
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
s.executeAnnotationJob(context.Background(), *claimed)
done, _ := f.store.GetHumanItem(f.ctx, h.ID)
if done.ProcessingJob.Status != "completed" {
t.Fatalf("failed reconfirmation: %+v", done.ProcessingJob)
}
workspace := annotationWorkspacePayload(done)
slices := decodeMaterializationValue[domain.SliceSegment](workspace["sliceSegments"])
if len(slices) != 2 || slices[0].End != 3 || slices[1].Start != 3 {
t.Fatalf("selected boundary changed: %+v", slices)
}
if !reflect.DeepEqual(decodeMaterializationValue[domain.AnnotationSegment](workspace["segments"]), decodeMaterializationValue[domain.AnnotationSegment](draft["segments"])) {
t.Fatal("reconfirmation erased actions")
}
if stringValue(mapValue(done.Subject.Payload["sliceWorkflow"])["snapshotContentSha256"]) == "" || workspace["confirmationSnapshotSha256"] == nil {
t.Fatal("confirmation identity missing from generated result")
}
}
func TestFailedSliceRetryIgnoresAutosaveMetadataOnly(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, _ := annotationProcessorFixture(t)
h := seedFailedSliceRetry(t, f)
draft := map[string]any{}
for k, v := range annotationWorkspacePayload(&h) {
draft[k] = v
}
draft["savedAt"], draft["savedVersion"] = nowISO(), h.Version
h.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "retryFailed": true})
if r.Code != http.StatusAccepted {
t.Fatalf("metadata-only autosave blocked unchanged retry: %d %s", r.Code, r.Body.String())
}
}
func TestReconfirmSavedSlicesRejectsAmbiguousEvidence(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
for _, reason := range []string{"wrong-owner", "future-version", "missing-time", "stale-client"} {
t.Run(reason, func(t *testing.T) {
f, _ := annotationProcessorFixture(t)
h := seedNewerSliceSnapshot(t, f)
draft := mapValue(h.Subject.Payload["annotationDraft"])
version := h.Version
switch reason {
case "wrong-owner":
draft["userId"] = f.reviewer.ID
case "future-version":
draft["savedVersion"] = h.Version + 1
case "missing-time":
delete(draft, "savedAt")
case "stale-client":
version--
}
h.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": version, "reconfirmSaved": true, "snapshotContentSha256": "forged"})
if r.Code != http.StatusConflict && r.Code != http.StatusUnprocessableEntity {
t.Fatalf("unsafe reconfirmation: %d %s", r.Code, r.Body.String())
}
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(before, after) {
t.Fatal("rejected reconfirmation changed saved data")
}
})
}
}
func TestLegacyFailedSliceRetryUsesFrozenJobRatherThanCurrentDraft(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, _ := annotationProcessorFixture(t)
h := seedNewerSliceSnapshot(t, f)
original := annotationWorkspacePayload(&h)
delete(h.Subject.Payload, "annotationWorkspace")
h.ProcessingJob = &domain.AnnotationProcessingJob{ID: "old-job", Kind: "slice", Status: "failed", Input: map[string]any{"retryFailed": true, "retryWorkspace": original, "sliceSegments": original["sliceSegments"]}}
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "retryFailed": true})
if r.Code != http.StatusConflict || !strings.Contains(r.Body.String(), "slice_retry_snapshot_conflict") {
t.Fatalf("old frozen job input ignored: %d %s", r.Code, r.Body.String())
}
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(before, after) {
t.Fatal("rejection changed either old-job or draft evidence")
}
}
func TestDraftOnlyReconfirmationRequiresOwnedDatedSavedEvidence(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
for _, reason := range []string{"wrong-owner", "future-version", "missing-time"} {
t.Run(reason, func(t *testing.T) {
f, _ := annotationProcessorFixture(t)
h := seedNewerSliceSnapshot(t, f)
delete(h.Subject.Payload, "annotationWorkspace")
draft := mapValue(h.Subject.Payload["annotationDraft"])
switch reason {
case "wrong-owner":
draft["userId"] = f.reviewer.ID
case "future-version":
draft["savedVersion"] = h.Version + 1
case "missing-time":
delete(draft, "savedAt")
}
h.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "reconfirmSaved": true})
if r.Code != http.StatusUnprocessableEntity {
t.Fatalf("unverified draft accepted without baseline: %d %s", r.Code, r.Body.String())
}
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(before, after) {
t.Fatal("invalid draft-only confirmation mutated state")
}
})
}
}
func TestPreUpgradeSliceJobCannotReplaceNewerSavedSnapshot(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
h := seedFailedSliceRetry(t, f)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "retryFailed": true})
if r.Code != http.StatusAccepted {
t.Fatal(r.Code, r.Body.String())
}
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
// Simulate an already queued job from an old API, without new proof fields.
delete(queued.ProcessingJob.Input, "snapshotContentSha256")
flow := mapValue(queued.Subject.Payload["sliceWorkflow"])
delete(flow, "snapshotContentSha256")
queued.Subject.Payload["sliceWorkflow"] = flow
draft := map[string]any{}
for k, v := range annotationWorkspacePayload(queued) {
draft[k] = v
}
draft["sliceSegments"] = []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 0, End: 3}, {ID: "b", Name: "测试切片", Start: 3, End: 4}}
draft["segments"] = []domain.AnnotationSegment{{ID: "new-action", Start: 0, End: 3, Status: "pending"}}
draft["savedAt"], draft["savedVersion"] = nowISO(), queued.Version
queued.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, *queued); err != nil {
t.Fatal(err)
}
stored, _ := f.store.GetHumanItem(f.ctx, h.ID)
before := annotationWorkspacePayload(stored)
draft = mapValue(stored.Subject.Payload["annotationDraft"])
now := time.Now().UTC()
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "snapshot-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if claimed == nil {
t.Fatal("job unavailable")
}
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
s.executeAnnotationJob(context.Background(), *claimed)
done, _ := f.store.GetHumanItem(f.ctx, h.ID)
if done.ProcessingJob.Status != "failed" {
t.Fatalf("old snapshot published: %+v", done.ProcessingJob)
}
if !reflect.DeepEqual(annotationWorkspacePayload(done), before) || !reflect.DeepEqual(mapValue(done.Subject.Payload["annotationDraft"]), draft) {
t.Fatal("obsolete completion replaced retained content")
}
revisions, err := f.store.ListSliceRevisions(f.ctx, f.task.OrganizationID, f.task.ID)
if err != nil || len(revisions) != 0 {
t.Fatalf("obsolete revision published: %d %v", len(revisions), err)
}
}

View File

@ -0,0 +1,139 @@
// Package sourceidentity verifies frozen BOS selections using object metadata.
// ETag plus size detects identity changes; it is not a cryptographic byte proof.
package sourceidentity
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"path"
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
)
type Object struct {
Key, ETag string
Size int64
}
// Fingerprint preserves the capture discovery fingerprint byte for byte.
func Fingerprint(objects []Object) string {
lines := make([]string, 0, len(objects))
for _, o := range objects {
lines = append(lines, o.Key+"|"+strings.Trim(o.ETag, `"`)+"|"+strconv.FormatInt(o.Size, 10))
}
sort.Strings(lines)
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return hex.EncodeToString(sum[:])
}
type Selection struct {
Ref domain.ArtifactReference
Roles map[string]string
PartPrefix string
}
// FromRun uses only immutable run input evidence. The caller must resolve the
// exact original run (not a release child) through its persisted lineage.
func FromRun(run *domain.RunRecord, capture *domain.CaptureRecord) (*Selection, error) {
if run == nil || capture == nil || run.OrganizationID == "" || run.OrganizationID != capture.OrganizationID || run.ProjectID != capture.ProjectID || run.WorkflowID != capture.WorkflowID {
return nil, fmt.Errorf("original source tenant/project/workflow mismatch")
}
root, part := capture.SourcePrefix, path.Base(capture.Prefix)
if capture.ConnectionID == "" || capture.Bucket == "" || root == "." || root == "" || path.Clean(root) != root || strings.HasPrefix(root, "/") || strings.ContainsAny(root, "\\\x00\n\r|") || root == ".." || strings.HasPrefix(root, "../") || capture.Prefix != root+"/"+part || !regexp.MustCompile(`^part[0-9]{4,}$`).MatchString(part) || capture.CaptureID != path.Base(root)+"/"+part {
return nil, fmt.Errorf("invalid original capture/part scope")
}
var selected *Selection
for _, input := range run.Inputs {
ref := input.Ref
if ref == nil || ref.Provider != domain.StorageProviderBOS || ref.Kind != domain.ArtifactRefKindDirectory || ref.ConnectionID != capture.ConnectionID || ref.Bucket != capture.Bucket || ref.Prefix != root || ref.CaptureUnit != "" && ref.CaptureUnit != capture.CaptureID {
continue
}
digest, err := hex.DecodeString(ref.VersionID)
if err != nil || len(digest) != 32 || len(ref.ObjectKeys) < 8 || len(ref.ObjectKeys) > 10 {
return nil, fmt.Errorf("missing frozen original identity; explicit recovery required")
}
copyRef := *ref
copyRef.ObjectKeys = append([]string(nil), ref.ObjectKeys...)
sort.Strings(copyRef.ObjectKeys)
roles := map[string]string{}
capRE, partRE := regexp.QuoteMeta(path.Base(root)), regexp.QuoteMeta(part)
patterns := map[string]string{
"leftVideo": partRE + `/` + capRE + `_camera_left_` + partRE + `(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)`,
"rightVideo": partRE + `/` + capRE + `_camera_right_` + partRE + `(?:_[a-z0-9_]+)?\.(?:mp4|mov|mkv)`,
"leftPts": partRE + `/` + capRE + `_camera_left_` + partRE + `(?:_[a-z0-9_]+)?_pts\.csv`,
"rightPts": partRE + `/` + capRE + `_camera_right_` + partRE + `(?:_[a-z0-9_]+)?_pts\.csv`,
"imu": partRE + `/` + capRE + `_imu_` + partRE + `\.csv`,
"cameraCalibration": capRE + `_calibration_camera\.yaml`,
"imuCalibration": capRE + `_calibration_imu\.yaml`,
"metadata": `meta_` + capRE + `\.json`,
}
for i, key := range copyRef.ObjectKeys {
if !strings.HasPrefix(key, root+"/") || path.Clean(key) != key || strings.ContainsAny(key, "\\\x00\n\r|") || i > 0 && key == copyRef.ObjectKeys[i-1] {
return nil, fmt.Errorf("unsafe/duplicate frozen source key")
}
rel := strings.TrimPrefix(key, root+"/")
matched := false
for role, pattern := range patterns {
if regexp.MustCompile(`(?i)^` + pattern + `$`).MatchString(rel) {
if roles[role] != "" {
return nil, fmt.Errorf("ambiguous frozen source role")
}
roles[role] = rel
matched = true
}
}
if !matched && !regexp.MustCompile(`(?i)^`+partRE+`/`+capRE+`_audio_`+partRE+`(?:_pts\.csv|\.wav)$`).MatchString(rel) {
return nil, fmt.Errorf("extra frozen source selection")
}
}
if len(roles) != 8 {
return nil, fmt.Errorf("incomplete frozen original source selection")
}
next := &Selection{Ref: copyRef, Roles: roles, PartPrefix: capture.Prefix}
if selected != nil && !reflect.DeepEqual(selected, next) {
return nil, fmt.Errorf("ambiguous frozen original inputs")
}
selected = next
}
if selected == nil {
return nil, fmt.Errorf("missing original run selection; explicit recovery required")
}
return selected, nil
}
func (s *Selection) MatchFiles(provider, bucket, prefix, partPrefix string, files map[string]string) error {
if provider != "BOS" || bucket != s.Ref.Bucket || prefix != s.Ref.Prefix || partPrefix != s.PartPrefix {
return fmt.Errorf("delivery original source scope mismatch")
}
if !reflect.DeepEqual(files, s.Roles) {
return fmt.Errorf("delivery original files differ from frozen run selection")
}
return nil
}
// Recheck performs at most ten sequential HEAD calls, including optional audio.
// It never lists objects or derives an expected fingerprint from current state.
func (s *Selection) Recheck(ctx context.Context, head func(context.Context, string, string) (domain.ObjectStorageObjectSummary, error)) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
objects := make([]Object, 0, len(s.Ref.ObjectKeys))
for _, key := range s.Ref.ObjectKeys {
info, err := head(ctx, s.Ref.Bucket, key)
if err != nil || info.Key != key || info.Bucket != "" && info.Bucket != s.Ref.Bucket || info.Kind != "file" || info.SizeBytes <= 0 || strings.Trim(info.ETag, `"`) == "" {
return fmt.Errorf("original source HEAD missing/invalid: %s", key)
}
objects = append(objects, Object{Key: key, ETag: info.ETag, Size: info.SizeBytes})
}
if Fingerprint(objects) != s.Ref.VersionID {
return fmt.Errorf("original source identity changed; explicit recovery required")
}
return nil
}

View File

@ -0,0 +1,172 @@
package sourceidentity
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"testing"
"droboticflow/internal/domain"
)
func fixture(t *testing.T) (*domain.RunRecord, *domain.CaptureRecord, map[string]domain.ObjectStorageObjectSummary) {
t.Helper()
root := "incoming/capture-A"
part := "part0001"
keys := []string{root + "/meta_capture-A.json", root + "/capture-A_calibration_camera.yaml", root + "/capture-A_calibration_imu.yaml", root + "/" + part + "/capture-A_imu_" + part + ".csv"}
for _, side := range []string{"left", "right"} {
keys = append(keys, root+"/"+part+"/capture-A_camera_"+side+"_"+part+".mp4", root+"/"+part+"/capture-A_camera_"+side+"_"+part+"_pts.csv")
}
keys = append(keys, root+"/"+part+"/capture-A_audio_"+part+".wav", root+"/"+part+"/capture-A_audio_"+part+"_pts.csv")
objects := []Object{}
heads := map[string]domain.ObjectStorageObjectSummary{}
for _, key := range keys {
objects = append(objects, Object{key, `"original-etag"`, 100})
heads[key] = domain.ObjectStorageObjectSummary{Bucket: "b", Key: key, Kind: "file", ETag: `"original-etag"`, SizeBytes: 100}
}
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, Kind: domain.ArtifactRefKindDirectory, ConnectionID: "c", Bucket: "b", Prefix: root, CaptureUnit: "capture-A/" + part, ObjectKeys: keys, VersionID: Fingerprint(objects)}
run := &domain.RunRecord{ID: "original", OrganizationID: "org", ProjectID: "p", WorkflowID: "w", Inputs: map[string]domain.ArtifactPayload{"source": {Ref: ref}}}
// Current capture state is deliberately different from immutable run evidence.
capture := &domain.CaptureRecord{OrganizationID: "org", ProjectID: "p", WorkflowID: "w", ConnectionID: "c", Bucket: "b", SourcePrefix: root, Prefix: root + "/" + part, CaptureID: ref.CaptureUnit, Fingerprint: "later-discovery", SourceKeys: []string{"later-key"}}
return run, capture, heads
}
func TestOriginalIdentityRecheck(t *testing.T) {
for _, scenario := range []string{"correct", "same-size-etag-tamper", "size-change", "missing-key", "empty-etag", "head-error", "optional-audio-tamper"} {
t.Run(scenario, func(t *testing.T) {
run, capture, heads := fixture(t)
selection, err := FromRun(run, capture)
if err != nil {
t.Fatal(err)
}
key := selection.Ref.ObjectKeys[0]
if scenario == "optional-audio-tamper" {
for _, candidate := range selection.Ref.ObjectKeys {
if strings.HasSuffix(candidate, ".wav") {
key = candidate
}
}
}
info := heads[key]
switch scenario {
case "same-size-etag-tamper", "optional-audio-tamper":
info.ETag = "changed"
heads[key] = info
case "size-change":
info.SizeBytes++
heads[key] = info
case "missing-key":
delete(heads, key)
case "empty-etag":
info.ETag = ""
heads[key] = info
}
calls := 0
err = selection.Recheck(context.Background(), func(ctx context.Context, bucket, key string) (domain.ObjectStorageObjectSummary, error) {
calls++
if _, ok := ctx.Deadline(); !ok {
t.Fatal("unbounded HEAD")
}
if scenario == "head-error" {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("head unavailable")
}
return heads[key], nil
})
if (err == nil) != (scenario == "correct") {
t.Fatalf("err=%v", err)
}
if calls > 10 || scenario == "correct" && calls != 10 {
t.Fatalf("HEAD count=%d", calls)
}
})
}
}
func TestOriginalSelectionScopeAndEvidence(t *testing.T) {
for _, scenario := range []string{"tenant", "project", "workflow", "connection", "bucket", "prefix", "capture-unit", "extra-selection", "foreign-part", "duplicate", "missing-version", "fake-version", "missing-inputs", "ambiguous-inputs"} {
t.Run(scenario, func(t *testing.T) {
run, capture, _ := fixture(t)
ref := run.Inputs["source"].Ref
switch scenario {
case "tenant":
run.OrganizationID = "other"
case "project":
run.ProjectID = "other"
case "workflow":
run.WorkflowID = "other"
case "connection":
ref.ConnectionID = "other"
case "bucket":
ref.Bucket = "other"
case "prefix":
ref.Prefix = "other"
case "capture-unit":
ref.CaptureUnit = "other"
case "extra-selection":
ref.ObjectKeys[0] = ref.Prefix + "/extra.yaml"
case "foreign-part":
ref.ObjectKeys[0] = ref.Prefix + "/part0002/foreign.mp4"
case "duplicate":
ref.ObjectKeys[0] = ref.ObjectKeys[1]
case "missing-version":
ref.VersionID = ""
case "fake-version":
ref.VersionID = "selectionFP"
case "missing-inputs":
run.Inputs = nil
case "ambiguous-inputs":
other := *ref
other.VersionID = strings.Repeat("a", 64)
run.Inputs["other"] = domain.ArtifactPayload{Ref: &other}
}
if _, err := FromRun(run, capture); err == nil {
t.Fatal("invalid selection accepted")
}
})
}
}
func TestOriginalDeliveryFilesMatchFrozenSelection(t *testing.T) {
run, capture, _ := fixture(t)
selection, err := FromRun(run, capture)
if err != nil {
t.Fatal(err)
}
if err = selection.MatchFiles("BOS", "b", capture.SourcePrefix, capture.Prefix, selection.Roles); err != nil {
t.Fatal(err)
}
for _, scenario := range []string{"extra", "changed", "missing", "wrong-bucket", "wrong-part"} {
t.Run(scenario, func(t *testing.T) {
files := map[string]string{}
for k, v := range selection.Roles {
files[k] = v
}
bucket, part := "b", capture.Prefix
switch scenario {
case "extra":
files["audio"] = "extra.wav"
case "changed":
files["leftVideo"] = "part0001/other.mp4"
case "missing":
delete(files, "imu")
case "wrong-bucket":
bucket = "other"
case "wrong-part":
part += "2"
}
if selection.MatchFiles("BOS", bucket, capture.SourcePrefix, part, files) == nil {
t.Fatal("mismatched delivery accepted")
}
})
}
}
func TestCaptureFingerprintHistoricalBytes(t *testing.T) {
// Independently pinned old key|trim(etag)|size serialization, sorted as strings.
sum := sha256.Sum256([]byte("a|abc|12\nz|def|34"))
if got := Fingerprint([]Object{{"z", `"def"`, 34}, {"a", `"abc"`, 12}}); got != hex.EncodeToString(sum[:]) {
t.Fatalf("old identity changed: %s", got)
}
}

View File

@ -9,11 +9,11 @@ import (
"io"
"regexp"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/sourceidentity"
"github.com/baidubce/bce-sdk-go/services/bos"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
@ -23,11 +23,7 @@ const captureAutomationNodeID = "__capture_automation__"
const captureRequeueStaleAfter = 2 * time.Minute
type captureObject struct {
Key string
ETag string
Size int64
}
type captureObject = sourceidentity.Object
type captureGroup struct {
SourcePrefix string
@ -104,15 +100,7 @@ func captureRequirements(config domain.CaptureAutomationConfig) []captureRequire
return items
}
func captureFingerprint(objects []captureObject) string {
parts := make([]string, 0, len(objects))
for _, object := range objects {
parts = append(parts, object.Key+"|"+strings.Trim(object.ETag, `"`)+"|"+strconv.FormatInt(object.Size, 10))
}
sort.Strings(parts)
sum := sha256.Sum256([]byte(strings.Join(parts, "\n")))
return hex.EncodeToString(sum[:])
}
func captureFingerprint(objects []captureObject) string { return sourceidentity.Fingerprint(objects) }
func captureRecordID(workflowID, prefix string) string {
sum := sha256.Sum256([]byte(strings.TrimSpace(workflowID) + "|" + strings.Trim(strings.TrimSpace(prefix), "/")))

View File

@ -165,6 +165,9 @@ func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.R
}
return publisher.Delete(ctx, *connection, target.Bucket, stage)
}
if err := runtime.recheckV4DeliveryArtifact(ctx, run, artifact); err != nil {
return nil, err
}
known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker)
if err != nil {
return nil, err

View File

@ -1075,6 +1075,9 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord,
node = applyNodeOverridesToNode(node, override)
}
}
if err := runtime.preflightV4OriginalSource(ctx, run, node); err != nil {
return executionResult{}, nil, err
}
if err := validateRequiredNodeInputs(node, inputBindings); err != nil {
return executionResult{}, nil, err
}

View File

@ -0,0 +1,156 @@
package worker
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/sourceidentity"
"droboticflow/internal/workspacescope"
)
// Release children carry accepted artifacts but no original Inputs. Resolve
// only the persisted human-release edge; retries are independent selections.
func (r *Runtime) originalDeliverySelection(ctx context.Context, run *domain.RunRecord) (*sourceidentity.Selection, error) {
if run == nil {
return nil, fmt.Errorf("missing original delivery run")
}
original := run
if run.Trigger == domain.RunTriggerHumanRelease {
parent, err := r.store.GetRun(ctx, run.ParentRunID)
if err != nil {
return nil, err
}
if parent == nil || parent.ID == run.ID || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != run.TaskID {
return nil, fmt.Errorf("original delivery parent scope mismatch")
}
original = parent
}
capture, err := r.store.GetCaptureRecordByRunID(ctx, original.ID)
if err != nil {
return nil, err
}
if capture == nil || capture.ID != original.TaskID || capture.RunID != original.ID && capture.RunID != run.ID {
return nil, fmt.Errorf("original delivery capture lineage mismatch")
}
return sourceidentity.FromRun(original, capture)
}
func (r *Runtime) recheckOriginalDeliverySelection(ctx context.Context, run *domain.RunRecord, selection *sourceidentity.Selection) error {
ref := selection.Ref
c, err := r.store.GetStorageConnection(ctx, ref.ConnectionID)
if err != nil {
return err
}
if c == nil || c.Provider != domain.StorageProviderBOS {
return fmt.Errorf("original BOS connection missing")
}
m, err := workspacescope.ForOrganization(run.OrganizationID)
if err != nil {
return err
}
if c.OrganizationID != run.OrganizationID {
if m == nil || m.ConnectionID != c.ID || m.ReadConnectionOrganizationID != c.OrganizationID || m.Bucket != ref.Bucket || !workspacescope.PrefixAllows(ref.Prefix, m.ReadPrefixes) {
return fmt.Errorf("original BOS connection outside delegated read scope")
}
}
if m != nil && m.ConnectionID == c.ID && (m.Bucket != ref.Bucket || !workspacescope.PrefixAllows(ref.Prefix, m.ReadPrefixes)) {
return fmt.Errorf("original BOS prefix outside read scope")
}
connection := *c
if connection.Endpoint == "" {
connection.Endpoint = r.globalBOSEndpoint
}
return selection.Recheck(ctx, func(ctx context.Context, bucket, key string) (domain.ObjectStorageObjectSummary, error) {
meta, err := r.outputArtifactPublisher().Head(ctx, connection, bucket, key)
kind := ""
if meta.Exists {
kind = "file"
}
return domain.ObjectStorageObjectSummary{Bucket: bucket, Key: key, Kind: kind, ETag: meta.ETag, SizeBytes: meta.SizeBytes}, err
})
}
func (r *Runtime) preflightV4OriginalSource(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) error {
if !workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{node}}) {
return nil
}
selection, err := r.originalDeliverySelection(ctx, run)
if err != nil {
return err
}
return r.recheckOriginalDeliverySelection(ctx, run, selection)
}
// Detect the actual output schema, independent of mutable workflow switches.
// This guard runs before prepareDeliveryTarget writes even its owner marker.
func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.RunRecord, artifact domain.ExecutionArtifact) error {
name := artifact.Path
info, err := os.Stat(name)
if err != nil {
return err
}
if info.IsDir() {
name = filepath.Join(name, "part.json")
} else if filepath.Base(name) != "part.json" {
return nil
}
f, err := os.Open(name)
if os.IsNotExist(err) {
return nil
} // Legacy directory has no part.json.
if err != nil {
return err
}
defer f.Close()
raw, err := io.ReadAll(io.LimitReader(f, partIndexMaxPeekBytes+1))
if err != nil {
return err
}
var part struct {
Schema string `json:"schemaVersion"`
Capture struct {
ID string `json:"captureId"`
Prefix string `json:"prefix"`
} `json:"capture"`
PartID string `json:"partId"`
OriginalSource struct {
Provider string `json:"provider"`
Bucket string `json:"bucket"`
Prefix string `json:"prefix"`
PartPrefix string `json:"partPrefix"`
Files map[string]struct {
Path string `json:"path"`
} `json:"files"`
} `json:"originalSource"`
}
if json.Unmarshal(raw, &part) != nil {
return fmt.Errorf("invalid delivery part JSON")
}
if part.Schema != deliveryindex.ClassificationPartSchema {
return nil
}
if len(raw) > 8<<20 {
return fmt.Errorf("classification part exceeds 8 MiB")
}
selection, err := r.originalDeliverySelection(ctx, run)
if err != nil {
return err
}
if part.Capture.ID != filepath.Base(selection.Ref.Prefix) || part.Capture.Prefix != selection.Ref.Prefix || part.PartID != filepath.Base(selection.PartPrefix) {
return fmt.Errorf("classification capture/part differs from original run")
}
paths := map[string]string{}
for role, file := range part.OriginalSource.Files {
paths[role] = file.Path
}
if err = selection.MatchFiles(part.OriginalSource.Provider, part.OriginalSource.Bucket, part.OriginalSource.Prefix, part.OriginalSource.PartPrefix, paths); err != nil {
return err
}
return r.recheckOriginalDeliverySelection(ctx, run, selection)
}

View File

@ -0,0 +1,170 @@
package worker
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"droboticflow/internal/deliveryindex"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"droboticflow/internal/sourceidentity"
"droboticflow/internal/store"
)
type originalIdentityPublisher struct {
deliveryPublisher
heads map[string]objectsync.ObjectMetadata
calls int
}
func (p *originalIdentityPublisher) Head(ctx context.Context, c domain.StorageConnectionRecord, b, k string) (objectsync.ObjectMetadata, error) {
if m, ok := p.heads[k]; ok {
p.calls++
return m, nil
}
return p.deliveryPublisher.Head(ctx, c, b, k)
}
func originalWorkerFixture(t *testing.T) (*Runtime, *originalIdentityPublisher, domain.RunRecord, domain.WorkflowNode, domain.ExecutionArtifact) {
t.Helper()
ctx := t.Context()
db := store.NewMemoryStore()
pub := &originalIdentityPublisher{deliveryPublisher: deliveryPublisher{memoryArtifactPublisher: memoryArtifactPublisher{objects: map[string][]byte{}}}, heads: map[string]objectsync.ObjectMetadata{}}
root, part := "incoming/capture-A", "part0001"
roles := map[string]string{"leftVideo": part + "/capture-A_camera_left_" + part + ".mp4", "rightVideo": part + "/capture-A_camera_right_" + part + ".mp4", "leftPts": part + "/capture-A_camera_left_" + part + "_pts.csv", "rightPts": part + "/capture-A_camera_right_" + part + "_pts.csv", "imu": part + "/capture-A_imu_" + part + ".csv", "cameraCalibration": "capture-A_calibration_camera.yaml", "imuCalibration": "capture-A_calibration_imu.yaml", "metadata": "meta_capture-A.json"}
keys := []string{}
objects := []sourceidentity.Object{}
files := map[string]any{}
for role, rel := range roles {
key := root + "/" + rel
keys = append(keys, key)
objects = append(objects, sourceidentity.Object{Key: key, ETag: "original", Size: 100})
pub.heads[key] = objectsync.ObjectMetadata{Exists: true, ETag: "original", SizeBytes: 100}
files[role] = map[string]string{"path": rel}
}
ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, Kind: domain.ArtifactRefKindDirectory, ConnectionID: "c", Bucket: "b", Prefix: root, CaptureUnit: "capture-A/" + part, ObjectKeys: keys, VersionID: sourceidentity.Fingerprint(objects)}
parent := domain.RunRecord{ID: "original", OrganizationID: "org", ProjectID: "p", WorkflowID: "w", TaskID: "capture", Inputs: map[string]domain.ArtifactPayload{"source": {Ref: ref}}}
child := domain.RunRecord{ID: "release-child", OrganizationID: "org", ProjectID: "p", WorkflowID: "w", TaskID: "capture", Trigger: domain.RunTriggerHumanRelease, ParentRunID: parent.ID}
for _, run := range []domain.RunRecord{parent, child} {
if err := db.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
}
if err := db.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "c", OrganizationID: "org", Provider: domain.StorageProviderBOS}); err != nil {
t.Fatal(err)
}
if err := db.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture", OrganizationID: "org", ProjectID: "p", WorkflowID: "w", RunID: child.ID, CaptureID: ref.CaptureUnit, ConnectionID: "c", Bucket: "b", SourcePrefix: root, Prefix: root + "/" + part, Fingerprint: "changed-current-discovery", SourceKeys: []string{"do-not-use"}, Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: "deliveries/project__p/capture-A/" + part}}); err != nil {
t.Fatal(err)
}
dir := t.TempDir()
body, _ := json.Marshal(map[string]any{"schemaVersion": deliveryindex.ClassificationPartSchema, "capture": map[string]string{"captureId": "capture-A", "prefix": root}, "partId": part, "originalSource": map[string]any{"provider": "BOS", "bucket": "b", "prefix": root, "partPrefix": root + "/" + part, "files": files}})
if err := os.WriteFile(filepath.Join(dir, "part.json"), body, 0600); err != nil {
t.Fatal(err)
}
node := domain.WorkflowNode{ID: "package", Data: domain.WorkflowNodeData{OperatorID: "operator-builtin-package", Hyperparams: []domain.HyperParam{{Key: "DELIVERY_JSON_V4", Value: "true"}}}}
return &Runtime{store: db, artifactPublisher: pub}, pub, child, node, domain.ExecutionArtifact{Name: "delivery.directory.index", Path: dir, Type: domain.PortTypeFile}
}
func TestV4WorkerOriginalIdentityGuards(t *testing.T) {
for _, scenario := range []string{"correct", "same-size-etag", "changed-size", "missing", "wrong-tenant", "wrong-parent", "missing-input", "wrong-files"} {
t.Run(scenario, func(t *testing.T) {
r, pub, run, node, artifact := originalWorkerFixture(t)
key := "incoming/capture-A/part0001/capture-A_camera_left_part0001.mp4"
m := pub.heads[key]
switch scenario {
case "same-size-etag":
m.ETag = "tampered"
pub.heads[key] = m
case "changed-size":
m.SizeBytes++
pub.heads[key] = m
case "missing":
m.Exists = false
pub.heads[key] = m
case "wrong-tenant":
run.OrganizationID = "foreign"
case "wrong-parent":
run.ParentRunID = "nonexistent"
case "missing-input":
parent, _ := r.store.GetRun(t.Context(), run.ParentRunID)
parent.Inputs = nil
r.store.UpsertRun(t.Context(), *parent)
case "wrong-files":
name := filepath.Join(artifact.Path, "part.json")
raw, _ := os.ReadFile(name)
var doc map[string]any
json.Unmarshal(raw, &doc)
doc["originalSource"].(map[string]any)["files"].(map[string]any)["leftVideo"] = map[string]string{"path": "part0001/other.mp4"}
raw, _ = json.Marshal(doc)
os.WriteFile(name, raw, 0600)
}
err := r.preflightV4OriginalSource(t.Context(), &run, node)
if (err == nil) != (scenario == "correct" || scenario == "wrong-files") {
t.Fatalf("preflight err=%v", err)
}
err = r.recheckV4DeliveryArtifact(t.Context(), &run, artifact)
if (err == nil) != (scenario == "correct") {
t.Fatalf("artifact guard err=%v", err)
}
if scenario == "correct" {
if _, err = r.publishCaptureDelivery(t.Context(), &run, "package", artifact); err != nil {
t.Fatal(err)
}
if _, ok := pub.objects["b/deliveries/project__p/capture-A/part0001/manifest.json"]; !ok {
t.Fatal("valid delivery did not complete")
}
}
if scenario != "correct" {
if _, err = r.publishCaptureDelivery(t.Context(), &run, "package", artifact); err == nil {
t.Fatal("guard allowed publication")
}
if len(pub.puts) > 0 || len(pub.published) > 0 {
t.Fatal("guard wrote destination/staging before rejection")
}
}
})
}
}
func TestV4ExecutionGuardRunsBeforePackageLaunch(t *testing.T) {
r, pub, run, node, _ := originalWorkerFixture(t)
parent, _ := r.store.GetRun(t.Context(), run.ParentRunID)
parent.Inputs = nil
r.store.UpsertRun(t.Context(), *parent)
_, job, err := r.executeNode(t.Context(), &run, node, nil, nil)
if err == nil || job != nil || pub.calls != 0 {
t.Fatalf("missing frozen identity reached execution: err=%v job=%v heads=%d", err, job, pub.calls)
}
}
func TestOriginalSourceGuardLeavesLegacyPackageAlone(t *testing.T) {
r, _, run, node, artifact := originalWorkerFixture(t)
run.ParentRunID = "missing"
node.Data.Hyperparams = nil
if err := r.preflightV4OriginalSource(t.Context(), &run, node); err != nil {
t.Fatal(err)
}
for _, schema := range []string{"droboticflow.delivery.part-index/v1", "droboticflow.delivery.part-index/v2", "droboticflow.delivery.part-index/v3"} {
body, _ := json.Marshal(map[string]string{"schemaVersion": schema})
os.WriteFile(filepath.Join(artifact.Path, "part.json"), body, 0600)
if err := r.recheckV4DeliveryArtifact(t.Context(), &run, artifact); err != nil {
t.Fatal(err)
}
}
}
func TestOriginalSourceGuardPreservesLargeLegacyPart(t *testing.T) {
r, _, run, _, artifact := originalWorkerFixture(t)
run.ParentRunID = "missing"
body := []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","filler":"` + strings.Repeat("x", 9<<20) + `"}`)
if err := os.WriteFile(filepath.Join(artifact.Path, "part.json"), body, 0600); err != nil {
t.Fatal(err)
}
if err := r.recheckV4DeliveryArtifact(t.Context(), &run, artifact); err != nil {
t.Fatal(err)
}
}

View File

@ -1,30 +1,34 @@
package worker
import (
"droboticflow/internal/domain"
"fmt"
"os"
"regexp"
"strings"
"droboticflow/internal/domain"
)
var sourcePackageDigestImage = regexp.MustCompile(`^[^\s@]+@sha256:[a-f0-9]{64}$`)
// Only new source-backed acceptance inputs select the new package image. Old
// physical revisions keep their workflow image and independent final-image
// contract. A missing override must fail before launching an old incompatible
// package operator, rather than failing after acceptance in the container.
// Source-index inputs and every v4 package require the deployment's compatible
// digest-pinned package image. Physical v1-v3 revisions keep their workflow image.
// A missing override fails before launching an incompatible package operator.
func sourceIndexPackageImage(node domain.WorkflowNode, inputs []domain.ExecutionArtifact, fallback string) (string, error) {
if node.Data.OperatorID != "operator-builtin-package" {
return chooseImage(node, fallback), nil
}
requiresOverride := workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{node}})
for _, input := range inputs {
if input.Data == nil || input.Data["sourceIndexSchema"] != "droboticflow.source-slice-index/v1" {
continue
if input.Data != nil && input.Data["sourceIndexSchema"] == "droboticflow.source-slice-index/v1" {
requiresOverride = true
break
}
}
if requiresOverride {
image := strings.TrimSpace(os.Getenv("ANNOTATION_SOURCE_PACKAGE_IMAGE"))
if !sourcePackageDigestImage.MatchString(image) {
return "", fmt.Errorf("source-index delivery requires ANNOTATION_SOURCE_PACKAGE_IMAGE pinned by digest")
return "", fmt.Errorf("v4 or source-index delivery requires ANNOTATION_SOURCE_PACKAGE_IMAGE pinned by digest")
}
return image, nil
}

View File

@ -26,3 +26,70 @@ func TestSourceIndexPackageRequiresCompatibleImmutableImageWithoutChangingLegacy
t.Fatal("legacy workflow image changed", got, e)
}
}
func TestV4PhysicalPackageRequiresExistingImmutableOverride(t *testing.T) {
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{
OperatorID: "operator-builtin-package",
Image: "registry.test/package@sha256:" + strings.Repeat("d", 64),
Hyperparams: []domain.HyperParam{{Key: "DELIVERY_JSON_V4", Value: "true"}},
}}
physical := []domain.ExecutionArtifact{{Data: map[string]any{"schemaVersion": "physical-revision"}}}
for _, override := range []string{"", "registry.test/package:latest", "registry.test/package@sha256:short"} {
t.Run("reject-"+override, func(t *testing.T) {
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", override)
if got, err := sourceIndexPackageImage(node, physical, "runner"); err == nil || got != "" {
t.Fatalf("v4 physical input fell back to old workflow image: image=%q err=%v", got, err)
}
})
}
strict := "registry.test/package@sha256:" + strings.Repeat("e", 64)
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", strict)
for _, enabled := range []string{"true", "1", "yes", "on", " TRUE "} {
node.Data.Hyperparams[0].Value = enabled
if got, err := sourceIndexPackageImage(node, physical, "runner"); err != nil || got != strict {
t.Fatalf("v4=%q: image=%q err=%v", enabled, got, err)
}
}
}
func TestPackageProtocolsPreservePhysicalFallbackAndSourceIndexOverride(t *testing.T) {
strict := "registry.test/package@sha256:" + strings.Repeat("f", 64)
for _, protocol := range []string{"v1", "v2", "v3", "v4"} {
t.Run(protocol, func(t *testing.T) {
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{OperatorID: "operator-builtin-package", Image: "old-package:approved"}}
if protocol != "v1" {
flag := map[string]string{"v2": "SOURCE_INDEX_DELIVERY", "v3": "DELIVERY_JSON_V3", "v4": "DELIVERY_JSON_V4"}[protocol]
node.Data.Hyperparams = []domain.HyperParam{{Key: flag, Value: "true"}}
}
source := []domain.ExecutionArtifact{{Data: map[string]any{"sourceIndexSchema": "droboticflow.source-slice-index/v1"}}}
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", strict)
if got, err := sourceIndexPackageImage(node, source, "runner"); err != nil || got != strict {
t.Fatalf("source-index override changed: image=%q err=%v", got, err)
}
expected := "old-package:approved"
if protocol == "v4" {
expected = strict
}
if got, err := sourceIndexPackageImage(node, nil, "runner"); err != nil || got != expected {
t.Fatalf("physical selection: image=%q err=%v", got, err)
}
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "")
if _, err := sourceIndexPackageImage(node, source, "runner"); err == nil {
t.Fatal("source-index input accepted missing override")
}
if protocol != "v4" {
if got, err := sourceIndexPackageImage(node, nil, "runner"); err != nil || got != "old-package:approved" {
t.Fatalf("legacy physical input now requires override: image=%q err=%v", got, err)
}
}
})
}
}
func TestV4ImageOverrideDoesNotAffectOtherOperators(t *testing.T) {
t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "")
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{OperatorID: "other-operator", Image: "other:approved", Hyperparams: []domain.HyperParam{{Key: "DELIVERY_JSON_V4", Value: "true"}}}}
if got, err := sourceIndexPackageImage(node, nil, "runner"); err != nil || got != "other:approved" {
t.Fatal(got, err)
}
}

View File

@ -950,6 +950,26 @@ def _v4_consumer_payload(payload, manifest, source_root):
start, end = (_v4_number(split[k], k) for k in ("startSecond", "endSecond"))
first, last = (_v4_timestamp(split[k], k) for k in ("startTimestampUs", "endTimestampUs"))
if start < 0 or end <= start or last <= first: raise RuntimeError("v4 invalid accepted slice range")
row = rows[item["sliceId"]]
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
seconds, stamps = sync.get("boundarySeconds"), sync.get("boundaryTimestampUs")
# A continuous annotation is not proof of its relation to media seconds.
# Require the accepted revision's mapping for kept and discarded ranges;
# never infer hardware PTS from duration, cadence, or action endpoints.
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
raise RuntimeError("v4 missing or invalid hardware boundary mapping")
seconds = [_v4_number(value, "boundary") for value in seconds]
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
stamps[0] != first or stamps[-1] != last or
any(a >= b for a, b in zip(seconds, seconds[1:])) or
any(a >= b for a, b in zip(stamps, stamps[1:]))):
raise RuntimeError("v4 invalid hardware boundary mapping")
window = sync.get("window")
if (not isinstance(window, dict) or window.get("semantics") != "half-open" or
_v4_timestamp(window.get("startTimestampUs"), "synchronization window start") != first or
_v4_timestamp(window.get("endTimestampUs"), "synchronization window end") != last):
raise RuntimeError("v4 boundary mapping disagrees with frozen synchronization window")
if item["classification"]["deliveryDisposition"] == "discard":
item["sliceName"] = ""
evidence = item["classification"]["discardEvidence"]
@ -966,25 +986,10 @@ def _v4_consumer_payload(payload, manifest, source_root):
continue
parent_tags = _v4_tags(item["classification"].get("dictionary"), "slice")
item["classification"]["tags"] = parent_tags
row = rows[item["sliceId"]]
sync = json.loads((source_root / row["directory"] / "synchronized-slice.json").read_text())
seconds, stamps = sync.get("boundarySeconds", []), sync.get("boundaryTimestampUs", [])
if seconds or stamps:
if not isinstance(seconds, list) or not isinstance(stamps, list) or len(seconds) < 2 or len(seconds) != len(stamps):
raise RuntimeError("v4 invalid hardware boundary mapping")
seconds = [_v4_number(value, "boundary") for value in seconds]
stamps = [_v4_timestamp(value, "boundary timestamp") for value in stamps]
if (abs(seconds[0]) > 1e-7 or abs(seconds[-1] - (end-start)) > 1e-7 or
stamps[0] != first or stamps[-1] != last or
any(a >= b for a, b in zip(seconds, seconds[1:])) or
any(a >= b for a, b in zip(stamps, stamps[1:]))):
raise RuntimeError("v4 invalid hardware boundary mapping")
def timestamp(second):
if abs(second) < 1e-7: return first
if abs(second - (end-start)) < 1e-7: return last
hits = [stamps[i] for i, value in enumerate(seconds) if abs(_v4_number(value,"boundary") - second) < 1e-7]
hits = [stamps[i] for i, value in enumerate(seconds) if abs(value - second) < 1e-7]
if len(hits) != 1: raise RuntimeError("v4 action boundary has no original PTS mapping")
return _v4_timestamp(hits[0], "boundary timestamp")
return hits[0]
actions = []
for action in item["segments"]:
a = _v4_number(action.get("clipStartMs"), "action clipStartMs") / 1000
@ -992,7 +997,7 @@ def _v4_consumer_payload(payload, manifest, source_root):
if a < -1e-7 or b <= a or b > end-start+1e-7: raise RuntimeError("v4 action escapes its parent slice")
ta, tb = (_v4_timestamp(action.get(k), k) for k in ("startTimestampUs", "endTimestampUs"))
if not first <= ta < tb <= last: raise RuntimeError("v4 action timestamps escape its parent slice")
if seconds and (timestamp(a) != ta or timestamp(b) != tb): raise RuntimeError("v4 action seconds disagree with original PTS")
if timestamp(a) != ta or timestamp(b) != tb: raise RuntimeError("v4 action seconds disagree with original PTS")
action.update(startSecond=a, endSecond=b, partStartSecond=start+a, partEndSecond=start+b)
own = action.get("dictionary")
action["tags"] = _v4_tags(own, "action") if own is not None else list(parent_tags)

View File

@ -1,6 +1,8 @@
type Props = { onRetry: () => void; busy: boolean; blockedReason?: string };
import { RECONFIRM_SAVED_GENERATION_LABEL } from "../lib/annotation-generation-retry.js";
export function FailedSliceRecoveryNotice({ onRetry, busy, blockedReason = "" }: Props) {
type Props = { onRetry: () => void; onReconfirmSaved?: () => void; busy: boolean; blockedReason?: string };
export function FailedSliceRecoveryNotice({ onRetry, onReconfirmSaved, busy, blockedReason = "" }: Props) {
return (
<div
role="status"
@ -8,15 +10,20 @@ export function FailedSliceRecoveryNotice({ onRetry, busy, blockedReason = "" }:
>
<div>
<p>切片生成失败,已保存的切片、标注和任务归属仍保留。</p>
<p className="mt-1 text-xs">{blockedReason || "可按已保存内容重试,无需清空或重新切分。"}</p>
<p className="mt-1 text-xs">
{onReconfirmSaved
? "保存内容已更新,请按当前保存内容重新确认生成。"
: "将重试上次确认的内容,已保存的编辑仍保留。"}
</p>
{blockedReason ? <p className="mt-1 text-xs">{blockedReason}</p> : null}
</div>
<button
type="button"
onClick={onRetry}
onClick={onReconfirmSaved || onRetry}
disabled={busy || Boolean(blockedReason)}
className="rounded-lg border px-3 py-2 disabled:opacity-50"
>
{busy ? "正在核对重试…" : "保留已保存切片重试"}
{onReconfirmSaved ? RECONFIRM_SAVED_GENERATION_LABEL : busy ? "正在核对重试…" : "重试生成"}
</button>
</div>
);

View File

@ -0,0 +1,48 @@
export const RECONFIRM_SAVED_GENERATION_LABEL = "按当前保存内容重新确认生成";
export const SLICE_RETRY_CONFLICT_MESSAGE = "保存内容已更新,请按当前保存内容重新确认生成。当前编辑和历史已保留。";
/** Render only server-owned eligibility; legacy recovery keeps its separate meaning. */
export function annotationGenerationConfirmationMode(editorState) {
if (editorState?.canRecoverGeneration) return "recoverExisting";
if (editorState?.canReconfirmSavedGeneration) return "reconfirmSaved";
return "retryFailed";
}
/** Shared by the failed notice and the editor confirmation button. Never fall
* back to another snapshot after rejection. Persistence supplies the saved CAS
* version and fences every completion to the current task/account lease. */
export async function retryAnnotationGeneration({
taskId,
mode,
canStart,
persistence,
owns,
api,
accepted,
conflict,
reconcile,
}) {
if (!canStart()) return;
const token = await persistence.begin(taskId);
if (!token) return;
try {
if (!owns(token)) return;
const response =
mode === "recoverExisting"
? await api.recoverExistingSlices(taskId, token.version)
: mode === "reconfirmSaved"
? await api.reconfirmSavedSlices(taskId, token.version)
: await api.retryFailedSlices(taskId, token.version);
if (!owns(token)) return;
accepted(response);
} catch (failure) {
if (!owns(token)) return;
if (failure?.status === 409 && failure?.code === "slice_retry_snapshot_conflict") {
await conflict(SLICE_RETRY_CONFLICT_MESSAGE, token);
return;
}
await reconcile(failure, token);
} finally {
persistence.end(token);
}
}

View File

@ -1,3 +1,8 @@
import {
annotationGenerationConfirmationMode,
retryAnnotationGeneration,
RECONFIRM_SAVED_GENERATION_LABEL,
} from "../lib/annotation-generation-retry.js";
import { createReviewDecisionCompletion } from "../lib/review-decision-completion.js";
import { createSliceEditSession, freezeSliceEditSession } from "../lib/slice-edit-session.js";
import { createAnnotationPersistence } from "../lib/annotation-persistence.js";
@ -4203,40 +4208,61 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
}
};
const retryFailedSlices = async (recoverExisting = false) => {
if (!detail?.humanItem || !selectedClaimedByMe || annotationDirty || combinedSubmissionLocked || saving) return;
const retryFailedSlices = async (mode = "retryFailed") => {
if (!detail?.humanItem) return;
const taskId = detail.task.id;
const token = await beginAnnotationMutation(taskId);
if (!token) return;
try {
const response = recoverExisting
? await annotationOperationsApi.recoverExistingSlices(taskId, token.version)
: await annotationOperationsApi.retryFailedSlices(taskId, token.version);
if (!annotationMutationScopeRef.current.owns(token, currentAnnotationMutationContext())) return;
setDetail((current) => (current?.task.id === taskId ? { ...current, humanItem: response.item } : current));
clearEditHistory();
setError("");
setCombinedEditorPhase("materializing");
setTaskDetailRefreshVersion((value) => value + 1);
setRandomClaimNotice("已受理重试,已保存的切片和标注仍保留;后台完成后可继续处理。");
} catch (failure) {
if (annotationMutationScopeRef.current.owns(token, currentAnnotationMutationContext()))
await reconcileStageOperation(
taskId,
"slice",
token,
getActionErrorMessage(failure, "切片重试未完成,原内容仍保留"),
await retryAnnotationGeneration({
taskId,
mode,
canStart: () => selectedClaimedByMe && !annotationDirty && !combinedSubmissionLocked && !saving,
persistence: annotationPersistence,
owns: (token: AnnotationMutationToken) =>
annotationMutationScopeRef.current.owns(token, currentAnnotationMutationContext()),
api: annotationOperationsApi,
accepted: (response: { item: HumanItem }) => {
setDetail((current) => (current?.task.id === taskId ? { ...current, humanItem: response.item } : current));
clearEditHistory();
setError("");
setCombinedEditorPhase("materializing");
setTaskDetailRefreshVersion((value) => value + 1);
setRandomClaimNotice(
mode === "reconfirmSaved"
? "已按当前保存内容受理重新生成,切片、标注和分类将保留。"
: mode === "recoverExisting"
? "已受理恢复生成,已保存的切片和标注仍保留。"
: "已受理原失败快照重试;后台完成后可继续处理。",
);
} finally {
endAnnotationMutation(token);
}
},
conflict: async (message: string, token: AnnotationMutationToken) => {
setError(message);
// Refresh only server-owned action eligibility. Do not replace local
// editing, the saved version, or history on this rejected command.
try {
const next = await annotationOperationsApi.getTask(taskId);
if (annotationMutationScopeRef.current.owns(token, currentAnnotationMutationContext()))
setDetail((current) =>
current?.task.id === taskId ? { ...current, editorState: next.editorState } : current,
);
} catch {
// The rejection is definitive; a failed eligibility refresh does not
// turn it into an unknown mutation or discard the editor state.
}
},
reconcile: (failure: unknown, token: AnnotationMutationToken) =>
reconcileStageOperation(taskId, "slice", token, getActionErrorMessage(failure, "切片重试未完成,原内容仍保留")),
});
};
const confirmSlicesAndMaterialize = async () => {
if (!annotationAuthoringRef.current) return;
if (detail?.editorState?.canRecoverGeneration) {
if (annotationDirty) setError("请先保存当前进度,再恢复切片生成;已有标注将保留。");
else await retryFailedSlices(true);
else await retryFailedSlices("recoverExisting");
return;
}
if (annotationGenerationConfirmationMode(detail?.editorState) === "reconfirmSaved") {
if (annotationDirty) setError("请先保存当前编辑,再按当前保存内容重新确认生成。");
else await retryFailedSlices("reconfirmSaved");
return;
}
if ((detail?.humanItem?.subject.payload?.sliceWorkflow as { status?: string } | undefined)?.status === "failed") {
@ -4274,7 +4300,6 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
}));
const token = await beginAnnotationMutation(detail.task.id);
if (!token) return;
clearEditHistory();
try {
const response = await annotationOperationsApi.confirmSlices(detail.task.id, {
version: token.version,
@ -4283,6 +4308,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
changeReason: detail.sliceRevisions.length ? "重新确认切片" : "首次确认切片",
});
if (!annotationMutationScopeRef.current.owns(token, currentAnnotationMutationContext())) return;
clearEditHistory();
setDetail((current) => (current ? { ...current, humanItem: response.item } : current));
if (response.item) {
setHumanPools((current) => ({
@ -6540,7 +6566,12 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
"failed" ? (
<FailedSliceRecoveryNotice
onRetry={() => void retryFailedSlices()}
busy={saving}
onReconfirmSaved={
detail?.editorState?.canReconfirmSavedGeneration === true
? () => void retryFailedSlices("reconfirmSaved")
: undefined
}
busy={saving || combinedSubmissionLocked}
blockedReason={
stageResultUnknown
? "请先核对上一次操作结果。"
@ -7792,7 +7823,13 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
}}
onDeleteSlice={deleteSlice}
onConfirmSlices={() => void confirmSlicesAndMaterialize()}
confirmSlicesLabel={detail.editorState?.canRecoverGeneration ? "恢复生成并保留标注" : undefined}
confirmSlicesLabel={
annotationGenerationConfirmationMode(detail.editorState) === "recoverExisting"
? "恢复生成并保留标注"
: annotationGenerationConfirmationMode(detail.editorState) === "reconfirmSaved"
? RECONFIRM_SAVED_GENERATION_LABEL
: undefined
}
canConfirmSlices={
!sliceAlignmentIssue &&
sliceCoverage.complete &&

View File

@ -344,6 +344,12 @@ export const annotationOperationsApi = {
{ method: "POST" },
{ version, retryFailed: true },
),
reconfirmSavedSlices: (taskId: string, version: number) =>
apiRequest<{ item: HumanItem; sliceWorkflow: Record<string, unknown> }>(
`/api/plugins/annotation-qa/tasks/${taskId}/slice-confirmations`,
{ method: "POST" },
{ version, reconfirmSaved: true },
),
recoverExistingSlices: (taskId: string, version: number) =>
apiRequest<{ item: HumanItem; sliceWorkflow: Record<string, unknown> }>(
`/api/plugins/annotation-qa/tasks/${taskId}/slice-confirmations`,

View File

@ -2159,6 +2159,7 @@ export interface AnnotationTaskDetail {
phase: "slice" | "materializing" | "annotate";
canConfirmDescriptions: boolean;
canRecoverGeneration?: boolean;
canReconfirmSavedGeneration?: boolean;
};
sliceRevisions: SliceRevision[];
revisions: AnnotationRevision[];

View File

@ -29,7 +29,7 @@ def test_tags_inherit_by_dictionary_selection_not_description_text(tmp_path):
assert "workspaceKey" not in json.dumps(doc)
@pytest.mark.parametrize("failure,match",[("parent-tags","slice tags"),("action-tags","action tags"),("source","original BOS"),("timestamps","startTimestampUs"),("overlap","overlapping"),("clock","disagree"),("missing-file","missing original files"),("unsafe-key","unsafe"),("boundary","boundary mapping"),("pts-gap","discontinuous"),("discard-dictionary","discard reason disagrees")])
@pytest.mark.parametrize("failure,match",[("parent-tags","slice tags"),("action-tags","action tags"),("source","original BOS"),("timestamps","startTimestampUs"),("overlap","overlapping"),("clock","disagree"),("missing-file","missing original files"),("unsafe-key","unsafe"),("boundary","boundary mapping"),("pts-gap","boundary mapping"),("discard-dictionary","discard reason disagrees")])
def test_v4_refuses_incomplete_consumer_instructions(tmp_path,failure,match):
inputs=prepare(tmp_path,[{"description":"one"},{"description":"two","dictionary":dict(ACTION)}])
root=Path(inputs[0]["path"])
@ -57,6 +57,126 @@ def test_v4_refuses_incomplete_consumer_instructions(tmp_path,failure,match):
with pytest.raises(RuntimeError,match=match):read_outputs(tmp_path,inputs)
@pytest.mark.parametrize("grid", ["absent", "empty", "seconds-only", "timestamps-only", "null", "object"])
@pytest.mark.parametrize("shift", [False, True], ids=["correct-pts", "coherent-wrong-pts"])
def test_v4_requires_complete_time_mapping_at_package_seam(tmp_path, grid, shift):
inputs = prepare(tmp_path, [{"description": "one"}, {"description": "two"}])
root = Path(inputs[0]["path"]) / "slices/slice-000001"
sync_path, ann_path = root / "synchronized-slice.json", root / "annotation.json"
sync, ann = json.loads(sync_path.read_text()), json.loads(ann_path.read_text())
assert ann["actions"][0]["clipEndMs"] == ann["actions"][1]["clipStartMs"] == 500
if shift:
# Preserve continuity and parent endpoints: only the sec<->PTS relation is wrong.
ann["actions"][0]["endTimestampUs"] = ann["actions"][1]["startTimestampUs"] = 1400000
if grid == "absent":
sync.pop("boundarySeconds"); sync.pop("boundaryTimestampUs")
elif grid == "empty":
sync.update(boundarySeconds=[], boundaryTimestampUs=[])
elif grid == "seconds-only":
sync.pop("boundaryTimestampUs")
elif grid == "timestamps-only":
sync.pop("boundarySeconds")
elif grid == "null":
sync.update(boundarySeconds=None, boundaryTimestampUs=None)
else:
sync.update(boundarySeconds={}, boundaryTimestampUs={})
sync_path.write_text(json.dumps(sync)); ann_path.write_text(json.dumps(ann))
with pytest.raises(RuntimeError, match="boundary mapping"):
read_outputs(tmp_path, inputs)
@pytest.mark.parametrize("failure", ["missing", "start", "end", "semantics"])
def test_v4_mapping_must_agree_with_frozen_sync_window(tmp_path, failure):
inputs = prepare(tmp_path, [{"description": "one"}])
path = Path(inputs[0]["path"]) / "slices/slice-000001/synchronized-slice.json"
sync = json.loads(path.read_text())
sync["window"] = {"startTimestampUs": 1000000, "endTimestampUs": 2000000, "semantics": "half-open"}
if failure == "missing": sync.pop("window")
elif failure == "semantics": sync["window"]["semantics"] = "closed"
else: sync["window"][failure + "TimestampUs"] += 1
path.write_text(json.dumps(sync))
with pytest.raises(RuntimeError, match="synchronization window"):
read_outputs(tmp_path, inputs)
@pytest.mark.parametrize("failure", ["missing-blank", "length", "seconds-order", "pts-order", "missing-interior", "ambiguous-interior"])
def test_v4_rejects_unusable_mapping_without_inventing_boundaries(tmp_path, failure):
actions = [] if failure == "missing-blank" else [{"description": "one"}, {"description": "two"}]
inputs = prepare(tmp_path, actions)
path = Path(inputs[0]["path"]) / "slices/slice-000001/synchronized-slice.json"
sync = json.loads(path.read_text())
if failure == "missing-blank":
sync.pop("boundarySeconds"); sync.pop("boundaryTimestampUs")
elif failure == "length": sync["boundarySeconds"].pop()
elif failure == "seconds-order": sync["boundarySeconds"][1] = 0
elif failure == "pts-order": sync["boundaryTimestampUs"][1] = 2000000
elif failure == "missing-interior":
sync["boundarySeconds"].pop(1); sync["boundaryTimestampUs"].pop(1)
else:
sync["boundarySeconds"].insert(2, .50000001)
sync["boundaryTimestampUs"].insert(2, 1500001)
path.write_text(json.dumps(sync))
with pytest.raises(RuntimeError, match="boundary mapping|no original PTS mapping"):
read_outputs(tmp_path, inputs)
@pytest.mark.parametrize("all_discard", [False, True], ids=["mixed", "all-discard"])
@pytest.mark.parametrize("failure", ["missing", "empty", "seconds-only", "timestamps-only", "range", "window", "coherent-range"])
def test_v4_validates_discard_time_evidence_at_package_seam(tmp_path, all_discard, failure):
inputs = prepare(tmp_path, [])
root = Path(inputs[0]["path"])
manifest_path = root / "revision-manifest.json"
manifest = json.loads(manifest_path.read_text())
if all_discard:
manifest["items"][0].update(deliveryDisposition="discard", discardEvidence=manifest["items"][1]["discardEvidence"])
row = manifest["items"][1]
path = root / row["directory"] / "synchronized-slice.json"
sync = json.loads(path.read_text())
assert sync["boundaryTimestampUs"] == [2000000, 2500000, 3000000]
assert sync["boundarySeconds"] == [0, .5, 1]
if failure == "missing":
sync.pop("boundarySeconds"); sync.pop("boundaryTimestampUs")
elif failure == "empty": sync.update(boundarySeconds=[], boundaryTimestampUs=[])
elif failure == "seconds-only": sync.pop("boundaryTimestampUs")
elif failure == "timestamps-only": sync.pop("boundarySeconds")
elif failure == "range": sync["boundaryTimestampUs"][-1] -= 1
elif failure == "window": sync["window"]["endTimestampUs"] -= 1
else:
# Parent and grid agree with each other, but contradict the frozen window.
row["endTimestampUs"] -= 1
sync["boundaryTimestampUs"][-1] -= 1
path.write_text(json.dumps(sync))
manifest_path.write_text(json.dumps(manifest))
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
with pytest.raises(RuntimeError, match="boundary mapping|synchronization window"):
read_outputs(tmp_path, inputs)
@pytest.mark.parametrize("all_discard", [False, True], ids=["mixed", "all-discard"])
def test_v4_valid_discard_time_evidence_needs_no_slice_tags_or_actions(tmp_path, all_discard):
inputs = prepare(tmp_path, [])
root = Path(inputs[0]["path"])
manifest_path = root / "revision-manifest.json"
manifest = json.loads(manifest_path.read_text())
if all_discard:
manifest["items"][0].update(deliveryDisposition="discard", discardEvidence=manifest["items"][1]["discardEvidence"])
for row in manifest["items"]:
if row["deliveryDisposition"] != "discard": continue
folder = root / row["directory"]
sync_path = folder / "synchronized-slice.json"
sync = json.loads(sync_path.read_text())
sync.pop("dictionary", None)
sync_path.write_text(json.dumps(sync))
(folder / "annotation.json").unlink()
manifest_path.write_text(json.dumps(manifest))
inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
doc, _, _ = read_outputs(tmp_path, inputs)
assert len(doc["discardedSlices"]) == (2 if all_discard else 1)
for item in doc["discardedSlices"]:
assert item["segments"] == []
assert "tags" not in item["classification"]
def test_legacy_partial_annotations_keep_inherited_gap_coverage(tmp_path):
inputs=prepare(tmp_path,[{"description":"拿起","dictionary":ACTION,"clipStartMs":250,"clipEndMs":750}])
doc,_,_=read_outputs(tmp_path,inputs)

View File

@ -1,4 +1,5 @@
"""Final classifications and private review history are different outputs."""
import csv
import hashlib
import json
from pathlib import Path
@ -26,6 +27,31 @@ def original_source(capture="capture-A", part="part0001"):
return {"provider":"BOS","bucket":"test-bucket","prefix":root,"partPrefix":root+"/"+part,"captureUnit":capture+"/"+part,"objectKeys":[root+"/"+p for p in paths],"snapshotFingerprint":"synthetic-source-version"}
def add_synthetic_discard_sync(root, manifest):
"""Complete only the v4 synthetic fixture from its existing PTS evidence."""
for row in manifest["items"]:
if row["deliveryDisposition"] != "discard":
continue
folder = root / row["directory"]
with (folder / "left_pts.csv").open() as source:
points = list(csv.DictReader(source))
seconds = [float(point["clip_time_ms"]) / 1000 for point in points]
stamps = [int(float(point["timestamp_ms"]) * 1000) for point in points]
assert seconds[0] == 0 and stamps[0] == row["startTimestampUs"]
assert seconds[-1] < row["endSecond"] - row["startSecond"]
assert stamps[-1] < row["endTimestampUs"]
path = folder / "synchronized-slice.json"
sync = json.loads(path.read_text())
sync.update(
boundarySeconds=seconds + [row["endSecond"] - row["startSecond"]],
boundaryTimestampUs=stamps + [row["endTimestampUs"]],
rightBoundarySeconds=seconds + [row["endSecond"] - row["startSecond"]],
window={"startTimestampUs": row["startTimestampUs"],
"endTimestampUs": row["endTimestampUs"], "semantics": "half-open"},
)
path.write_text(json.dumps(sync))
def prepare(tmp_path, actions):
inputs = inputs_for(tmp_path, actions)
root = Path(inputs[0]["path"])
@ -41,10 +67,12 @@ def prepare(tmp_path, actions):
action.setdefault("endTimestampUs",1000000+round(action["clipEndMs"]*1000))
(root/"slices/slice-000001/annotation.json").write_text(json.dumps({"actions":actions}))
points=sorted({0.0,1.0,*[a[k]/1000 for a in actions for k in ("clipStartMs","clipEndMs")]})
doc["window"] = {"startTimestampUs":1000000,"endTimestampUs":2000000,"semantics":"half-open"}
doc.update(boundarySeconds=points,boundaryTimestampUs=[1000000+round(p*1000000) for p in points],rightBoundarySeconds=points)
sync.write_text(json.dumps(doc))
manifest_path = root / "revision-manifest.json"
m = json.loads(manifest_path.read_text())
add_synthetic_discard_sync(root, m)
m["runId"] = "source-run"
m["originalSource"] = original_source()
m["items"][0]["sliceName"] = BASE["name"]
@ -105,6 +133,7 @@ def test_v4_all_discard_outputs_classification_and_legacy_stays_unchanged(tmp_pa
inputs[1]["data"].update(capture={"captureId":"capture-A/part0001","prefix":"incoming/capture-A/part0001"},trace=[])
inputs[1]["data"]["finalAcceptance"]["acceptedAt"] = "2026-10-20T01:00:00Z"
root=Path(inputs[0]["path"]);p=root/"revision-manifest.json";m=json.loads(p.read_text());m["originalSource"]=original_source();p.write_text(json.dumps(m));inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(p.read_bytes()).hexdigest()
add_synthetic_discard_sync(root, m)
doc, _, _ = read_outputs(tmp_path, inputs)
assert doc["slices"] == [] and len(doc["discardedSlices"]) == 2
assert doc["deliveryClassification"]["discardedSliceCount"] == 2

View File

@ -0,0 +1,175 @@
import test from "node:test";
import assert from "node:assert/strict";
import {
annotationGenerationConfirmationMode,
retryAnnotationGeneration,
SLICE_RETRY_CONFLICT_MESSAGE,
} from "../../src/lib/annotation-generation-retry.js";
import { createAnnotationPersistence } from "../../src/lib/annotation-persistence.js";
import {
createAuthoritativeMutationContext,
createAuthoritativeMutationScope,
} from "../../src/lib/annotation-authoritative-mutation.js";
function fixture() {
const scope = createAuthoritativeMutationScope();
const context = createAuthoritativeMutationContext(scope, { taskId: "task", userId: "operator" });
const state = {
version: 4,
dirty: false,
saving: false,
locked: false,
owned: true,
history: ["edit"],
slices: [{ id: "saved", start: 0, end: 0.366728 }],
wait: async () => true,
request: async () => ({ item: { version: 6 } }),
};
const calls = [],
events = [];
const persistence = createAnnotationPersistence({
scope,
context: context.current,
adapter: {
waitForPending: () => state.wait(),
version: () => state.version,
},
view: {
canWrite: () => state.owned,
begin() {},
idle() {},
blocked() {},
},
});
const api = Object.fromEntries(
["retryFailedSlices", "reconfirmSavedSlices", "recoverExistingSlices"].map((method) => [
method,
async (taskId, version) => {
calls.push({ method, taskId, version });
return state.request();
},
]),
);
const run = (mode = "retryFailed") =>
retryAnnotationGeneration({
taskId: "task",
mode,
canStart: () => !state.dirty && !state.saving && !state.locked && state.owned,
persistence,
owns: (token) => scope.owns(token, context.current()),
api,
accepted: () => {
state.history = [];
events.push("accepted");
},
conflict: (message) => events.push(message),
reconcile: () => events.push("reconcile"),
});
return { state, calls, events, run, scope, context };
}
test("confirmation uses server eligibility, including legacy recovery precedence", () => {
assert.equal(annotationGenerationConfirmationMode(undefined), "retryFailed");
assert.equal(annotationGenerationConfirmationMode({ canReconfirmSavedGeneration: false }), "retryFailed");
assert.equal(annotationGenerationConfirmationMode({ canReconfirmSavedGeneration: true }), "reconfirmSaved");
assert.equal(
annotationGenerationConfirmationMode({ canRecoverGeneration: true, canReconfirmSavedGeneration: true }),
"recoverExisting",
);
});
test("each explicit generation action uses the post-autosave version and clears history only on acceptance", async () => {
for (const [mode, method] of [
["retryFailed", "retryFailedSlices"],
["reconfirmSaved", "reconfirmSavedSlices"],
["recoverExisting", "recoverExistingSlices"],
]) {
const f = fixture();
f.state.wait = async () => {
f.state.version = 5;
return true;
};
f.state.request = async () => {
assert.deepEqual(f.state.history, ["edit"]);
return { item: { version: 6 } };
};
await f.run(mode);
assert.deepEqual(f.calls, [{ method, taskId: "task", version: 5 }]);
assert.deepEqual(f.events, ["accepted"]);
assert.deepEqual(f.state.history, []);
assert.deepEqual(f.state.slices, [{ id: "saved", start: 0, end: 0.366728 }]);
assert.equal(f.scope.busy(), false);
}
});
test("retry snapshot conflict never switches to saved content or resets editor/history", async () => {
const f = fixture();
f.state.request = async () => {
throw { status: 409, code: "slice_retry_snapshot_conflict", message: "internal details" };
};
const before = structuredClone(f.state.slices);
await f.run();
assert.deepEqual(
f.calls.map((c) => c.method),
["retryFailedSlices"],
);
assert.deepEqual(f.events, [SLICE_RETRY_CONFLICT_MESSAGE]);
assert.deepEqual(f.state.history, ["edit"]);
assert.deepEqual(f.state.slices, before);
assert.equal(f.scope.busy(), false);
// A later explicit button click is the only way to choose current saved data.
f.state.request = async () => ({ item: { version: 6 } });
await f.run("reconfirmSaved");
assert.deepEqual(
f.calls.map((c) => c.method),
["retryFailedSlices", "reconfirmSavedSlices"],
);
});
test("other rejection or response loss reconciles without retrying another snapshot", async () => {
for (const mode of ["retryFailed", "reconfirmSaved", "recoverExisting"]) {
const f = fixture();
f.state.request = async () => {
throw new Error("response lost");
};
await f.run(mode);
assert.equal(f.calls.length, 1);
assert.deepEqual(f.events, ["reconcile"]);
assert.deepEqual(f.state.history, ["edit"]);
assert.equal(f.scope.busy(), false);
}
});
test("dirty, saving, locked, unowned and unsettled autosave block all generation commands", async () => {
for (const mode of ["retryFailed", "reconfirmSaved", "recoverExisting"]) {
for (const field of ["dirty", "saving", "locked", "owned", "wait"]) {
const f = fixture();
f.state[field] = field === "wait" ? async () => false : field !== "owned";
await f.run(mode);
assert.deepEqual(f.calls, []);
assert.deepEqual(f.state.history, ["edit"]);
assert.equal(f.scope.busy(), false);
}
}
});
test("task/account switches during autosave or a request fence stale completions", async () => {
for (const userId of ["operator", "other"]) {
for (const stage of ["wait", "request"]) {
const f = fixture();
let finish;
f.state[stage] = () =>
new Promise((resolve) => {
finish = resolve;
});
const pending = f.run("reconfirmSaved");
while (!finish) await Promise.resolve();
f.context.switchTo({ taskId: userId === "operator" ? "other-task" : "task", userId });
finish(stage === "wait" ? true : { item: { version: 6 } });
await pending;
assert.equal(f.calls.length, stage === "wait" ? 0 : 1);
assert.deepEqual(f.events, []);
assert.deepEqual(f.state.history, ["edit"]);
}
}
});

View File

@ -0,0 +1,71 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import { createRequire } from "node:module";
import ts from "typescript";
import { createElement } from "react";
import { renderToStaticMarkup } from "react-dom/server";
import * as generation from "../../src/lib/annotation-generation-retry.js";
const require = createRequire(import.meta.url);
const code = ts.transpileModule(
fs.readFileSync(new URL("../../src/components/FailedSliceRecoveryNotice.tsx", import.meta.url), "utf8"),
{ compilerOptions: { module: ts.ModuleKind.CommonJS, jsx: ts.JsxEmit.ReactJSX } },
).outputText;
const component = {};
new Function("require", "exports", code)(
(path) => (path === "../lib/annotation-generation-retry.js" ? generation : require(path)),
component,
);
const { FailedSliceRecoveryNotice } = component;
const buttons = (element) => {
if (!element || typeof element !== "object") return [];
if (element.type === "button") return [element];
return [element.props?.children].flat(Infinity).flatMap(buttons);
};
test("updated saved content shows only the explicit reconfirm button and dispatches that action", () => {
let retries = 0,
reconfirms = 0;
const props = {
busy: false,
onRetry: () => retries++,
onReconfirmSaved: () => reconfirms++,
};
const markup = renderToStaticMarkup(createElement(FailedSliceRecoveryNotice, props));
assert.match(markup, /保存内容已更新/);
assert.match(markup, /按当前保存内容重新确认生成/);
assert.doesNotMatch(markup, /快照|新草稿|重试生成/);
const actions = buttons(FailedSliceRecoveryNotice(props));
assert.equal(actions.length, 1);
assert.equal(actions[0].props.disabled, false);
actions[0].props.onClick();
assert.equal(retries, 0);
assert.equal(reconfirms, 1);
});
test("without reconfirm eligibility the sole action remains a normal retry", () => {
let retries = 0;
const props = { busy: false, onRetry: () => retries++ };
const markup = renderToStaticMarkup(createElement(FailedSliceRecoveryNotice, props));
assert.doesNotMatch(markup, /保存内容已更新|按当前保存内容重新确认生成/);
const actions = buttons(FailedSliceRecoveryNotice(props));
assert.equal(actions.length, 1);
assert.equal(actions[0].props.children, "重试生成");
actions[0].props.onClick();
assert.equal(retries, 1);
});
test("saving or a blocked editor disables the only visible action in both modes", () => {
for (const onReconfirmSaved of [undefined, () => {}]) {
for (const guard of [{ busy: true }, { busy: false, blockedReason: "请先保存当前编辑。" }]) {
const props = { onRetry() {}, onReconfirmSaved, ...guard };
const actions = buttons(FailedSliceRecoveryNotice(props));
assert.equal(actions.length, 1);
assert.equal(actions[0].props.disabled, true);
const markup = renderToStaticMarkup(createElement(FailedSliceRecoveryNotice, props));
assert.match(markup, /disabled=""/);
if (guard.blockedReason) assert.match(markup, /请先保存当前编辑。/);
}
}
});