From 65c19de0bf402cd8202714ebbfd8a05551cd5a25 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Tue, 20 Oct 2026 21:17:58 +0800 Subject: [PATCH 01/54] =?UTF-8?q?=E2=9C=A8=20Add=20reconciled=20annotation?= =?UTF-8?q?=20output=20dashboards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/customer-dashboard-20261020/spec.md | 28 +- .../customer-dashboard-20261020/report.md | 49 ++ internal/api/annotation_output.go | 250 ++++++ internal/api/annotation_output_test.go | 98 +++ internal/api/routes_dashboard.go | 1 + internal/api/testdata/routes.golden | 3 +- internal/store/annotation_output.go | 192 +++++ internal/store/annotation_output_test.go | 86 +++ internal/store/store.go | 1 + internal/workforcestats/output.go | 670 ++++++++++++++++ internal/workforcestats/output_test.go | 224 ++++++ internal/workforcestats/output_types.go | 105 +++ package.json | 3 +- src/components/AnnotationOutputDashboard.tsx | 590 ++++++++++++++ src/pages/Dashboard.tsx | 731 +++++++++--------- src/services/api/annotation-output.ts | 95 +++ src/styles/annotation-output.css | 312 ++++++++ tests/browser/annotation-output-replica.mjs | 113 +++ .../support/annotation-output-replica/main.go | 109 +++ 19 files changed, 3304 insertions(+), 356 deletions(-) create mode 100644 docs/verification/customer-dashboard-20261020/report.md create mode 100644 internal/api/annotation_output.go create mode 100644 internal/api/annotation_output_test.go create mode 100644 internal/store/annotation_output.go create mode 100644 internal/store/annotation_output_test.go create mode 100644 internal/workforcestats/output.go create mode 100644 internal/workforcestats/output_test.go create mode 100644 internal/workforcestats/output_types.go create mode 100644 src/components/AnnotationOutputDashboard.tsx create mode 100644 src/services/api/annotation-output.ts create mode 100644 src/styles/annotation-output.css create mode 100644 tests/browser/annotation-output-replica.mjs create mode 100644 tests/browser/support/annotation-output-replica/main.go diff --git a/.scratch/customer-dashboard-20261020/spec.md b/.scratch/customer-dashboard-20261020/spec.md index 4466cada..f54fe679 100644 --- a/.scratch/customer-dashboard-20261020/spec.md +++ b/.scratch/customer-dashboard-20261020/spec.md @@ -1,6 +1,6 @@ # 客户可对账看板 -Status: ready-for-agent +Status: resolved ## 已确认 @@ -12,10 +12,10 @@ Status: ready-for-agent - 提交:首次成功送检,按该日期入账;反复保存/返工重提不重复累计。明确重跑或撤销完成证据的规则需沿用正式版本链,不按 revision 大小猜测。 - 质检通过、验收通过:各按有效正式决定实际完成日期入账,重复决定/重试不重复;同一 part 可以在不同日期分别进入各阶段。撤销/打回后的处理保留事件历史及当前状态,明细显示证据,不能直接把当前最新状态当历史结论。 -- 返工中:当前状态快照,不是期间返工次数;范围为当前筛选归属、在查询截止时刻仍需返工的 part。日期范围不能把历史已解决返工再次算成当前返工。若要历史截止日快照,必须有可靠状态事件重放;否则仅显示当前时刻并注明。 +- 返工中:当前状态快照,不是期间返工次数;范围为当前筛选归属、在查询截止时刻仍需返工的 part。日期范围不能把历史已解决返工再次算成当前返工。若要历史截止日快照,必须有可靠状态事件重放;本版使用当前时刻快照,接口返回 asOf;不添加页面规则说明。 - 时区:显示当前账号配置的统计时区;按本地日界转 UTC 半开区间,查询与导出一致。 - 可结算参考:本期有效验收通过整段时长,不表示已付款或新增结算凭证;本期去重与跨期不重复结算需将来另立实际结算台账,原型不冒充结算系统。 -- 归属:使用提交/被审版本冻结的作者、供应商;不能按当前任务分配人或现任角色重写历史。缺证据显示待核对,并显示已核实/待核对覆盖范围。 +- 归属:使用提交/被审版本冻结的作者、供应商;不能按当前任务分配人或现任角色重写历史。缺少可靠证据的字段显示“—”;不在产品界面显示“待核对”、覆盖率或调试说明。 ## 个人版式 @@ -27,10 +27,26 @@ Status: ready-for-agent ## Mongo 补录边界 -先读已有证据生成缺口清单。可恢复的字段逐项列来源与冲突,须经用户确认才补录;不能恢复的留空待核对,不为凑齐看板造记录。报表计算、可选内部统计快照与客户交付 JSON 分开。 +先读已有证据生成缺口清单。可恢复的字段逐项列来源与冲突,须经用户确认才补录;不能恢复的留空,不为凑齐看板造记录。报表计算、可选内部统计快照与客户交付 JSON 分开。 + +## 页面与分页 + +- 数量直接显示数字,不附加“个 part”。缺失值显示“—”,不添加长口径解释、黄色提醒、“待核对”或演示日期。 +- 任务、供应商、人员每日表分别分页,默认 5 条,可选 5/10/20/50;展示总条数、页码和上一页/下一页。筛选变更回到第一页,汇总始终覆盖完整筛选结果。 +- 导出完整筛选任务明细,不受当前页影响;详情弹层只读,支持关闭和 Esc,返回保留位置。 ## 本次交付状态 -角色筛选的 role 投影缺陷已修复并完成本地 Mongo/真实 router 回归,待合入和另行发布。本地看板原型仅供版式/交互评审,使用明确标识的合成数据。真实指标接口、完整导出和生产历史覆盖率仍待实现/核实。 +2026-10-20 已按用户批准的原型完成正式实现,保存在 `codex/customer-dashboard-20261020` 分支: -原型源码保存在独立 `codex/dashboard-reconciliation-prototype-20261020` 分支,不进入生产代码;入口为 tests/browser/fixtures/dashboard-reconciliation-prototype.html。 +- 新增只读 `GET /api/dashboard/annotation-output`,支持阶段归属、日期/项目/供应商/人员筛选、独立分页和完整 CSV 导出。组织管理员看组织内正式纳入的项目,供应商管理员看本供应商,标注员看本人;客户端参数不能扩大服务端权限。 +- Mongo 仅投影读取已有事实;统计源使用正式生产范围配置,不添加客户数据、不回填历史、不改变交付 JSON。时长只使用明确的整段视频源时长,不使用保留切片时长或按 15 分钟估算。 +- 个人与管理员/供应商管理员接入真实 React 页面;旧的源采集生产进度/运行概览保留在管理员折叠区;质检与验收角色原有工作看板保留。 +- 已验证:真实 router 与隔离合成 Mongo 的三角色浏览器流程;权限、返工/重试去重、跨期、冻结归属、缺失时长、207 条分页与完整导出;前端/Go 既有失败基线无新增;生产前端启动与 Go 构建通过。 +- 当前本地预览使用独立合成数据。生产历史完整性和数据规模下的性能尚未线上核实;历史补录与公司集群发布均未执行。 + +验证记录:[`docs/verification/customer-dashboard-20261020/report.md`](../../docs/verification/customer-dashboard-20261020/report.md)。 + +## 原型历史 + +原型源码保存在独立 `codex/dashboard-reconciliation-prototype-20261020` 分支,不进入生产代码;入口为 `tests/browser/fixtures/dashboard-reconciliation-prototype.html`。用户确认版式后,正式页面重新接入真实接口,原型数据没有进入运行服务。 diff --git a/docs/verification/customer-dashboard-20261020/report.md b/docs/verification/customer-dashboard-20261020/report.md new file mode 100644 index 00000000..dcb42f99 --- /dev/null +++ b/docs/verification/customer-dashboard-20261020/report.md @@ -0,0 +1,49 @@ +# 客户可对账看板:本地实现验证 + +日期:2026-10-20。实现分支:`codex/customer-dashboard-20261020`,基于 `d6fe5e9be3f9e0002694cffdf776ae7c886235e5`。本次为本地修改,不含公司集群发布或客户历史数据补录。 + +## 最终行为 + +- 标注员看本人产出与任务;供应商管理员看本供应商;组织管理员看正式生产统计范围内的项目并按供应商、人员下钻。项目、账号现任角色和当前分配变化不重写历史归属。 +- 单位为原始 part,整段源视频只计一次。提交取首次有效成功送检日期;质检、最终验收取有效正式决定完成日期;重提、请求重试不重复加产出。完成计划在 applying/未知状态时不计成功。返工按当前状态快照;累计截至所选结束日。 +- 时长只取明确的整段源时长;工作日志的 `videoSeconds` 是切片时长,不能用来补数。无法证实的时长为 null,界面和明细显示“—”;含未知时长的汇总也不展示伪装成完整值的部分和。 +- 每张表独立分页,默认 5,可选 5/10/20/50;换筛选/页大小回到第一页;汇总与全量 CSV 导出不受当前页影响。 +- 保留用户批准版式:不展示“个 part”“待核对”、长规则说明或调试提示。详情只读,支持 Esc 和焦点返回。管理员旧生产进度与运行概览保留在折叠区;审核角色原有看板保留。 + +## 接口与兼容边界 + +新增 `GET /api/dashboard/annotation-output`;不改变旧 statistics 接口、Mongo 文档、工作流、任务状态或交付 JSON。 + +查询参数:`from`/`to` 为包含端点的本地日期,`timeZone` 为 IANA 时区(默认 Asia/Shanghai);服务端转换为 UTC 半开区间,最多 366 个日历日。`projectId`、`supplierId`、`userId` 为筛选。`tasksPage`/`tasksPageSize`、`suppliersPage`/`suppliersPageSize`、`daysPage`/`daysPageSize` 分别分页;每页最大 100,超末页归并到末页。`dailySupplierId`/`dailyUserId` 只收窄每日表。`format=csv` 导出全部筛选明细,超过 100000 行明确拒绝。 + +组织、本人或供应商边界由服务端主体验证,查询参数不能扩大权限。只读投影不加载草稿、描述正文、媒体或凭证;个人和供应商结果不返回原始审核证据列表。CSV 保留阶段归属、防公式注入并使用 UTF-8 BOM。 + +## 验证结果 + +| 范围 | 结果与证据 | +| --- | --- | +| TypeScript、Prettier | `npm run lint:web`、`npm run format:check` 通过。 | +| 定向 Go + 本机 Mongo | `TEST_WORKFORCE_MONGO_URI=mongodb://127.0.0.1:27018 go test ./internal/api ./internal/store ./internal/workforcestats -run TestAnnotationOutput -count=1` 通过。覆盖权限、冻结归属、跨期、重试、返工撤销、未完成计划、多任务合并、来源标识、缺少整段时长、207 条分页及完整导出、时区与夏令时、CSV 公式转义、Memory/Mongo 一致性。 | +| 全量前端基线门禁 | 13 项失败均属于 14 项既有基线;1 项既有失败已不再出现,未擅改基线。 | +| 全量 Go 基线门禁 | 70 项失败均属于 70 项既有基线,无新增失败。最后时长收紧后另跑定向 Go 回归及 API 构建。 | +| 路由快照 | 更新后 369 条,仅新增本接口;处理器权限测试另验未授权角色 403。 | +| 三角色真实浏览器 | 管理员、标注员、供应商管理员,真实 HTTP router + 本机隔离合成 Mongo。验证分页汇总、从第 2 页全量导出、页大小重置、详情与 Esc、供应商/人员/每日下钻、项目切换;无页面异常,无非认证写请求。 | +| 构建与生产入口启动 | `npm run build:web`、Go API 构建通过;生产静态包真实浏览器启动成功,登录表可见,无 React 启动错误。仍有既有大 bundle 提醒。 | + +原始日志、浏览器 JSON 报告和截图保存在当前 worktree 的未跟踪目录 `local-data/verification/customer-dashboard-20261020/`。临时测试环境启动与浏览器代理配置故障保留在带 failure 后缀的报告中,最终结果见 `browser-report.json`、`final-targeted-tests.txt`、`production-boot.txt`。 + +## 本地复现与预览 + +已存在本机 Mongo `127.0.0.1:27018` 时,在本分支依次启动(各占一个终端): + +```sh +go run ./tests/browser/support/annotation-output-replica +VITE_DEV_API_BASE_URL=http://127.0.0.1:3026 DISABLE_HMR=true npx vite --host 127.0.0.1 --port 5196 --strictPort +npm run test:browser:annotation-output +``` + +入口:`http://127.0.0.1:5196/?view=dashboard`。测试服务仅监听 127.0.0.1:3026,每次创建独立 `annotation_output_browser_*` 合成数据库,账号密码随机生成至本地权限 0600 的 `accounts.json`,不写入仓库。浏览器默认使用本机 Edge;可用 `BROWSER_EXECUTABLE` 指定已安装浏览器。 + +## 尚未验证的范围 + +本次没有连接生产环境核实全量历史完整性或真实规模下的查询性能。读取器按组织/正式生产项目投影加载已有事件历史,未增加持久化统计快照或索引迁移。不能关联来源的旧任务暂按独立源任务保留,缺少冻结归属的记录不猜测作者;正式发布前需只读抽样评估这些历史覆盖范围。历史补录必须另列来源与冲突,经用户确认后执行。 diff --git a/internal/api/annotation_output.go b/internal/api/annotation_output.go new file mode 100644 index 00000000..caa40dcf --- /dev/null +++ b/internal/api/annotation_output.go @@ -0,0 +1,250 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "encoding/csv" + "fmt" + "net/http" + "strconv" + "strings" + "time" + _ "time/tzdata" + "unicode" +) + +type outputPage[T any] struct { + Items []T `json:"items"` + Total int `json:"total"` + Page int `json:"page"` + PageSize int `json:"pageSize"` +} + +func paginateOutput[T any](items []T, page, size int) outputPage[T] { + pages := (len(items) + size - 1) / size + if pages < 1 { + pages = 1 + } + if page > pages { + page = pages + } + start := (page - 1) * size + end := start + size + if end > len(items) { + end = len(items) + } + return outputPage[T]{items[start:end], len(items), page, size} +} +func outputPagination(r *http.Request, key string) (int, int, error) { + parse := func(name string, fallback, max int) (int, error) { + v := r.URL.Query().Get(name) + if v == "" { + return fallback, nil + } + n, e := strconv.Atoi(v) + if e != nil || n < 1 || n > max { + return 0, fmt.Errorf("invalid %s", name) + } + return n, nil + } + page, e := parse(key+"Page", 1, 10000000) + if e != nil { + return 0, 0, e + } + size, e := parse(key+"PageSize", 5, 100) + return page, size, e +} +func outputDateRange(r *http.Request, now time.Time) (time.Time, time.Time, *time.Location, error) { + zone := r.URL.Query().Get("timeZone") + if zone == "" { + zone = "Asia/Shanghai" + } + loc, e := time.LoadLocation(zone) + if e != nil { + return time.Time{}, time.Time{}, nil, fmt.Errorf("invalid time zone") + } + today := now.In(loc) + to := time.Date(today.Year(), today.Month(), today.Day()+1, 0, 0, 0, 0, loc) + from := to.AddDate(0, 0, -30) + if s := r.URL.Query().Get("from"); s != "" { + from, e = time.ParseInLocation("2006-01-02", s, loc) + if e != nil { + return time.Time{}, time.Time{}, nil, fmt.Errorf("invalid start date") + } + } + if s := r.URL.Query().Get("to"); s != "" { + var last time.Time + last, e = time.ParseInLocation("2006-01-02", s, loc) + if e != nil { + return time.Time{}, time.Time{}, nil, fmt.Errorf("invalid end date") + } + to = last.AddDate(0, 0, 1) + } + if !from.Before(to) || to.After(from.AddDate(0, 0, 366)) { + return time.Time{}, time.Time{}, nil, fmt.Errorf("date range must be between 1 and 366 days") + } + return from.UTC(), to.UTC(), loc, nil +} +func (s *server) handleAnnotationOutput(w http.ResponseWriter, r *http.Request) { + p, ok := currentPrincipal(r) + if !ok { + renderJSON(w, 401, map[string]string{"error": "authentication required"}) + return + } + admin := isOrganizationAdmin(p) + vendor := p.User.Role == domain.UserRoleSupplierManager + if !admin && !vendor && p.User.Role != domain.UserRoleAnnotator { + renderJSON(w, 403, map[string]string{"error": "annotation output access required"}) + return + } + now := time.Now().UTC() + from, to, loc, err := outputDateRange(r, now) + if err != nil { + renderError(w, 400, err) + return + } + tasksPage, tasksSize, err := outputPagination(r, "tasks") + if err != nil { + renderError(w, 400, err) + return + } + suppliersPage, suppliersSize, err := outputPagination(r, "suppliers") + if err != nil { + renderError(w, 400, err) + return + } + daysPage, daysSize, err := outputPagination(r, "days") + if err != nil { + renderError(w, 400, err) + return + } + format := r.URL.Query().Get("format") + if format != "" && format != "csv" { + renderError(w, 400, fmt.Errorf("invalid export format")) + return + } + q := workforcestats.OutputQuery{From: from, To: to, Now: now, Location: loc, ProjectID: r.URL.Query().Get("projectId"), SupplierID: r.URL.Query().Get("supplierId"), UserID: r.URL.Query().Get("userId")} + audience := "platform" + if vendor { + audience = "vendor" + q.MatchSupplier = true + q.ScopeSupplierID = p.User.SupplierID + } else if !admin { + audience = "self" + q.MatchUser = true + q.ScopeUserID = p.User.ID + } + ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second) + defer cancel() + org := s.requestOrganizationID(r) + scope, err := s.store.GetProductionStatsScope(ctx, org) + if err != nil { + renderError(w, 500, err) + return + } + projects, err := s.store.ListProjectsByOrganization(ctx, org) + if err != nil { + renderError(w, 500, err) + return + } + admitted := map[string]bool{} + for _, id := range scope.ProjectIDs { + admitted[id] = true + } + ids := []string{} + for _, project := range projects { + if admitted[project.ID] { + ids = append(ids, project.ID) + } + } + // A supplier manager without a supplier never gains the organization scope. + if vendor && p.User.SupplierID == "" { + ids = []string{} + } + // Do not narrow the load to a requested person: a review may address sources + // attributed to several authors. The read model applies the principal fence. + snapshot, err := s.store.LoadAnnotationOutput(ctx, org, ids) + if err != nil { + renderError(w, 500, err) + return + } + report := workforcestats.BuildOutput(snapshot, q) + if format == "csv" { + if len(report.Tasks) > 100000 { + renderError(w, 400, fmt.Errorf("export exceeds 100000 rows; narrow the filters")) + return + } + writeOutputCSV(w, report.Tasks, loc) + return + } + days := report.Days + if selected := r.URL.Query().Get("dailySupplierId"); selected != "" { + days = []workforcestats.OutputDay{} + for _, day := range report.Days { + if day.SupplierID == selected { + days = append(days, day) + } + } + } + if selected := r.URL.Query().Get("dailyUserId"); selected != "" { + filtered := []workforcestats.OutputDay{} + for _, day := range days { + if day.UserID == selected { + filtered = append(filtered, day) + } + } + days = filtered + } + w.Header().Set("Cache-Control", "no-store") + renderJSON(w, 200, map[string]any{"asOf": now.Format(time.RFC3339Nano), "audienceScope": audience, "from": from.In(loc).Format("2006-01-02"), "to": to.In(loc).AddDate(0, 0, -1).Format("2006-01-02"), "timeZone": loc.String(), "totals": report.Totals, "projects": report.Projects, "suppliers": report.SupplierOptions, "users": report.Users, "tasks": paginateOutput(report.Tasks, tasksPage, tasksSize), "supplierOutput": paginateOutput(report.Suppliers, suppliersPage, suppliersSize), "dailyOutput": paginateOutput(days, daysPage, daysSize)}) +} +func outputCSVCell(s string) string { + v := strings.TrimLeftFunc(s, unicode.IsSpace) + if v != "" && strings.ContainsRune("=+-@", rune(v[0])) { + return "'" + s + } + return s +} +func writeOutputCSV(w http.ResponseWriter, tasks []workforcestats.OutputTask, loc *time.Location) { + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", `attachment; filename="annotation-output.csv"`) + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write([]byte{0xef, 0xbb, 0xbf}) + writer := csv.NewWriter(w) + defer writer.Flush() + _ = writer.Write([]string{"视频编号", "项目", "供应商", "标注员", "整段视频秒数", "首次提交时间", "质检通过时间", "最终验收通过时间", "当前状态", "首次提交标注员", "首次提交供应商", "验收产出标注员", "验收产出供应商"}) + date := func(s string) string { + t, e := time.Parse(time.RFC3339Nano, s) + if e != nil { + return "" + } + return t.In(loc).Format("2006-01-02 15:04:05") + } + for _, t := range tasks { + seconds := "" + if t.Seconds != nil { + seconds = strconv.FormatFloat(*t.Seconds, 'f', 3, 64) + } + row := []string{t.VideoID, t.ProjectName, t.SupplierName, t.UserName, seconds, date(t.SubmittedAt), date(t.QualityAt), date(t.AcceptedAt), outputStatusLabel(t.Status), t.Attribution["submitted"].UserName, t.Attribution["submitted"].SupplierName, t.Attribution["acceptance"].UserName, t.Attribution["acceptance"].SupplierName} + for i := range row { + row[i] = outputCSVCell(row[i]) + } + if writer.Write(row) != nil { + return + } + } +} +func outputStatusLabel(s string) string { + switch s { + case "accepted": + return "已验收" + case "pending_acceptance": + return "待最终验收" + case "pending_quality": + return "待质检" + case "rework": + return "返工中" + } + return "" +} diff --git a/internal/api/annotation_output_test.go b/internal/api/annotation_output_test.go new file mode 100644 index 00000000..88c6652d --- /dev/null +++ b/internal/api/annotation_output_test.go @@ -0,0 +1,98 @@ +package api + +import ( + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/workforcestats" + "encoding/csv" + "encoding/json" + "fmt" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "testing" + "time" +) + +func TestAnnotationOutputAPIAndCSVScope(t *testing.T) { + f := newCombinedDraftFixture(t, "output-vendor") + check := func(e error) { + t.Helper() + if e != nil { + t.Fatal(e) + } + } + _, e := f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{f.task.ProjectID}, Revision: 1}, 0) + check(e) + at := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano) + for i := 0; i < 207; i++ { + id := fmt.Sprintf("output-task-%d", i) + check(f.store.UpsertAnnotationTask(f.ctx, domain.AnnotationTaskRecord{ID: id, ProjectID: f.task.ProjectID, OrganizationID: domain.DefaultOrganizationID, SourceIsFullVideo: true, SourceDurationSeconds: 900, CreatedAt: at})) + check(f.store.UpsertAnnotationRevision(f.ctx, domain.AnnotationRevisionRecord{ID: id + "-rev", OrganizationID: domain.DefaultOrganizationID, TaskID: id, Revision: 1, AuthorUserID: f.annotator.ID, CountsAsOutput: true})) + check(f.store.UpsertAnnotationWorklogEvent(f.ctx, domain.AnnotationWorklogEventRecord{ID: id + "-event", ProjectID: f.task.ProjectID, OrganizationID: domain.DefaultOrganizationID, TaskID: id, Revision: 1, UserID: f.annotator.ID, SupplierID: f.annotator.SupplierID, EventType: "combined_submitted", CreatedAt: at})) + } + before, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + for _, user := range []domain.UserRecord{f.admin, f.annotator} { + r := f.request("GET", "/api/dashboard/annotation-output?tasksPage=42&tasksPageSize=5", user, nil) + if r.Code != 200 { + t.Fatalf("read %d %s", r.Code, r.Body.String()) + } + var d struct { + Tasks outputPage[workforcestats.OutputTask] `json:"tasks"` + Totals workforcestats.OutputTotals `json:"totals"` + } + check(json.Unmarshal(r.Body.Bytes(), &d)) + if d.Tasks.Total != 207 || len(d.Tasks.Items) != 2 || d.Totals.Submitted.Count != 207 { + t.Fatalf("pagination changed totals %s", r.Body.String()) + } + exported := f.request("GET", "/api/dashboard/annotation-output?format=csv&tasksPage=42&tasksPageSize=5", user, nil) + if exported.Code != 200 { + t.Fatalf("export %s", exported.Body.String()) + } + rows, e := csv.NewReader(strings.NewReader(strings.TrimPrefix(exported.Body.String(), "\ufeff"))).ReadAll() + check(e) + if len(rows) != 208 { + t.Fatalf("export only current page: %d", len(rows)) + } + } + for _, query := range []string{"userId=" + f.reviewer.ID, "supplierId=foreign"} { + r := f.request("GET", "/api/dashboard/annotation-output?"+query, f.annotator, nil) + var body struct { + Tasks outputPage[workforcestats.OutputTask] `json:"tasks"` + } + check(json.Unmarshal(r.Body.Bytes(), &body)) + if r.Code != 200 || body.Tasks.Total != 0 { + t.Fatalf("scope enlarged %s", r.Body.String()) + } + } + if r := f.request("GET", "/api/dashboard/annotation-output", f.reviewer, nil); r.Code != 403 { + t.Fatal("reviewer gained author report") + } + for _, query := range []string{"tasksPage=-1", "tasksPageSize=1000", "timeZone=invalid", "from=2026-10-20&to=2026-10-01", "format=unknown"} { + if r := f.request("GET", "/api/dashboard/annotation-output?"+query, f.admin, nil); r.Code != 400 { + t.Fatalf("invalid query accepted %s: %d", query, r.Code) + } + } + after, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + if !reflect.DeepEqual(before, after) { + t.Fatal("report mutated customer task") + } +} +func TestAnnotationOutputDateBoundariesAndCSVCells(t *testing.T) { + for _, tc := range []struct { + zone, from, to string + hours time.Duration + }{{"Asia/Shanghai", "2026-10-01", "2026-10-01", 24}, {"America/New_York", "2026-03-08", "2026-03-08", 23}, {"America/New_York", "2026-11-01", "2026-11-01", 25}} { + req := httptest.NewRequest("GET", "/api/dashboard/annotation-output?"+url.Values{"timeZone": {tc.zone}, "from": {tc.from}, "to": {tc.to}}.Encode(), nil) + from, to, _, e := outputDateRange(req, time.Now()) + if e != nil || to.Sub(from) != tc.hours*time.Hour { + t.Fatalf("DST range %+v: %v %v", tc, e, to.Sub(from)) + } + } + for _, s := range []string{"=SUM(A1)", " +123", "@unsafe", "\t-formula"} { + if !strings.HasPrefix(outputCSVCell(s), "'") { + t.Fatal("formula passed to spreadsheet") + } + } +} diff --git a/internal/api/routes_dashboard.go b/internal/api/routes_dashboard.go index 0a841dec..5bb26489 100644 --- a/internal/api/routes_dashboard.go +++ b/internal/api/routes_dashboard.go @@ -8,6 +8,7 @@ func (server *server) mountDashboardRoutes(router chi.Router) { router.Get("/api/dashboard", server.withDashboardResponseCache("dashboard", server.handleDashboard)) router.Get("/api/workforce-returns", server.handleWorkforceReturns) router.Get("/api/dashboard/workforce", server.handleDashboardWorkforce) + router.Get("/api/dashboard/annotation-output", server.handleAnnotationOutput) router.Get("/api/dashboard/production", server.withDashboardResponseCache("production", server.handleProductionDashboard)) router.Get("/api/dashboard/production/scope", server.handleProductionScope) router.Put("/api/dashboard/production/scope", server.handleProductionScope) diff --git a/internal/api/testdata/routes.golden b/internal/api/testdata/routes.golden index 36c4cae3..04628c0a 100644 --- a/internal/api/testdata/routes.golden +++ b/internal/api/testdata/routes.golden @@ -1,7 +1,7 @@ # Generated by TestRouteSnapshot; see routes_snapshot_test.go. # access: reach = endpoint reached; otherwise the status the middleware chain returned. global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache] -routes=368 +routes=369 DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403 @@ -37,6 +37,7 @@ GET /api/audit-tasks | mw=[(*server).requireBusinessRole.1] | (*server).handleAu GET /api/auth/bootstrap-status | mw=[] | (*server).handleAuthBootstrapStatus-fm | anon=reach token=reach super=reach admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/auth/me | mw=[] | (*server).handleAuthMe-fm | anon=reach token=reach super=reach admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/dashboard | mw=[] | (*server).withDashboardResponseCache.1 | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +GET /api/dashboard/annotation-output | mw=[] | (*server).handleAnnotationOutput-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/dashboard/production | mw=[] | (*server).withDashboardResponseCache.1 | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/dashboard/production/scope | mw=[] | (*server).handleProductionScope-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/dashboard/workforce | mw=[] | (*server).handleDashboardWorkforce-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach diff --git a/internal/store/annotation_output.go b/internal/store/annotation_output.go new file mode 100644 index 00000000..ef56af71 --- /dev/null +++ b/internal/store/annotation_output.go @@ -0,0 +1,192 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/sync/errgroup" + "time" +) + +func (s *MemoryStore) LoadAnnotationOutput(_ context.Context, org string, projects []string) (workforcestats.OutputSnapshot, error) { + s.mu.Lock() + defer s.mu.Unlock() + d := workforcestats.OutputSnapshot{} + allowed := map[string]bool{} + for _, id := range projects { + allowed[id] = true + } + belongs := func(o, p string) bool { return organizationMatches(o, org) && allowed[p] } + tasks := map[string]bool{} + items := map[string]bool{} + for _, v := range s.annotationTasks { + if belongs(v.OrganizationID, v.ProjectID) { + d.Tasks = append(d.Tasks, clone(v)) + tasks[v.ID] = true + } + } + for _, v := range s.humanItems { + if belongs(v.OrganizationID, v.ProjectID) { + v.Subject.Payload = nil + d.Items = append(d.Items, clone(v)) + items[v.ID] = true + } + } + for _, v := range s.captureRecords { + if belongs(v.OrganizationID, v.ProjectID) { + d.Captures = append(d.Captures, clone(v)) + } + } + runIDs, captureIDs := map[string]bool{}, map[string]bool{} + for _, t := range d.Tasks { + runIDs[t.RunID] = true + } + for _, c := range d.Captures { + runIDs[c.RunID] = true + captureIDs[c.ID] = true + } + for _, v := range s.runs { + if organizationMatches(v.OrganizationID, org) && (allowed[v.ProjectID] || runIDs[v.ID] || captureIDs[v.TaskID]) { + d.Runs = append(d.Runs, domain.RunRecord{ID: v.ID, TaskID: v.TaskID, ParentRunID: v.ParentRunID, Mode: v.Mode, CreatedAt: v.CreatedAt}) + } + } + for _, v := range s.annotationWorklogEvents { + if belongs(v.OrganizationID, v.ProjectID) { + d.Events = append(d.Events, clone(v)) + } + } + for _, v := range s.reviewOperations { + if belongs(v.OrganizationID, v.ProjectID) { + d.Operations = append(d.Operations, cloneReviewOperation(v)) + } + } + for _, v := range s.annotationReviews { + if organizationMatches(v.OrganizationID, org) && tasks[v.TaskID] { + d.Reviews = append(d.Reviews, clone(v)) + } + } + for _, v := range s.annotationRevisions { + if organizationMatches(v.OrganizationID, org) && tasks[v.TaskID] { + v.Segments = nil + v.ObjectHints = nil + d.Revisions = append(d.Revisions, v) + } + } + for _, v := range s.humanEvents { + if organizationMatches(v.OrganizationID, org) && items[v.ItemID] { + d.HumanEvents = append(d.HumanEvents, clone(v)) + } + } + for _, v := range s.deliveryPackages { + if belongs(v.OrganizationID, v.ProjectID) { + d.Packages = append(d.Packages, clone(v)) + } + } + for _, v := range s.users { + if organizationMatches(v.OrganizationID, org) { + d.Users = append(d.Users, domain.UserRecord{ID: v.ID, Name: v.Name, Role: v.Role, SupplierID: v.SupplierID}) + } + } + for _, v := range s.projects { + if organizationMatches(v.OrganizationID, org) && allowed[v.ID] { + d.Projects = append(d.Projects, domain.ProjectRecord{ID: v.ID, Name: v.Name}) + } + } + for _, v := range s.suppliers { + if organizationMatches(v.OrganizationID, org) { + d.Suppliers = append(d.Suppliers, domain.SupplierRecord{ID: v.ID, Name: v.Name}) + } + } + return d, nil +} + +// Project/organization-scoped projections keep history sufficient for cumulative +// output while excluding descriptions, secrets, drafts and media. No UI list cap. +func (s *MongoStore) LoadAnnotationOutput(ctx context.Context, org string, projects []string) (workforcestats.OutputSnapshot, error) { + d := workforcestats.OutputSnapshot{} + if len(projects) == 0 { + return d, nil + } + f := organizationScopedFilter(org, bson.M{"projectId": bson.M{"$in": projects}}) + fields := func(names ...string) bson.M { + m := bson.M{"_id": 0} + for _, n := range names { + m[n] = 1 + } + return m + } + read := func(c *mongo.Collection, filter, projection bson.M, out any) error { + cursor, err := c.Find(ctx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second)) + if err != nil { + return err + } + defer cursor.Close(ctx) + return cursor.All(ctx, out) + } + var g errgroup.Group + g.Go(func() error { + return read(s.annotationTasks, f, fields("id", "organizationId", "projectId", "dataTaskId", "runId", "sourceIsFullVideo", "sourceDurationSeconds", "clipStart", "clipEnd", "supplierId", "status", "currentRevision", "createdAt", "updatedAt"), &d.Tasks) + }) + g.Go(func() error { + return read(s.humanItems, f, fields("id", "projectId", "runId", "legacyTaskId", "subject.itemKey", "currentStageId", "status", "currentRevision", "createdAt", "updatedAt"), &d.Items) + }) + g.Go(func() error { + return read(s.captureRecords, f, fields("id", "projectId", "connectionId", "bucket", "prefix", "captureId", "runId"), &d.Captures) + }) + + g.Go(func() error { + return read(s.annotationWorklogEvents, f, fields("id", "taskId", "userId", "supplierId", "eventType", "submissionId", "submissionKind", "videoSeconds", "outputDelta", "revision", "createdAt", "mode", "temporary"), &d.Events) + }) + g.Go(func() error { + return read(s.db.Collection("reviewOperations"), f, fields("_id", "operationId", "projectId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"), &d.Operations) + }) + g.Go(func() error { + return read(s.deliveryPackages, f, fields("id", "itemIds", "parentRunId", "mode", "temporary", "qualitySummary.prePackageAcceptance", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.createdAt"), &d.Packages) + }) + g.Go(func() error { + return read(s.users, organizationScopedFilter(org, bson.M{}), fields("id", "name", "role", "supplierId"), &d.Users) + }) + g.Go(func() error { + return read(s.suppliers, organizationScopedFilter(org, bson.M{}), fields("id", "name"), &d.Suppliers) + }) + g.Go(func() error { + return read(s.projects, organizationScopedFilter(org, bson.M{"id": bson.M{"$in": projects}}), fields("id", "name"), &d.Projects) + }) + if err := g.Wait(); err != nil { + return d, err + } + runIDs, captureIDs := []string{}, []string{} + for _, t := range d.Tasks { + runIDs = append(runIDs, t.RunID) + } + for _, c := range d.Captures { + runIDs = append(runIDs, c.RunID) + captureIDs = append(captureIDs, c.ID) + } + g.Go(func() error { + return read(s.runs, organizationScopedFilter(org, bson.M{"$or": bson.A{bson.M{"projectId": bson.M{"$in": projects}}, bson.M{"id": bson.M{"$in": runIDs}}, bson.M{"taskId": bson.M{"$in": captureIDs}}}}), fields("id", "taskId", "parentRunId", "mode", "createdAt"), &d.Runs) + }) + ids := []string{} + for _, t := range d.Tasks { + ids = append(ids, t.ID) + } + related := organizationScopedFilter(org, bson.M{"taskId": bson.M{"$in": ids}}) + g.Go(func() error { + return read(s.annotationReviews, related, fields("id", "taskId", "revision", "stage", "decision", "createdAt"), &d.Reviews) + }) + g.Go(func() error { + return read(s.annotationRevisions, related, fields("id", "taskId", "revision", "authorUserId", "isRework", "countsAsOutput", "createdAt"), &d.Revisions) + }) + itemIDs := []string{} + for _, i := range d.Items { + itemIDs = append(itemIDs, i.ID) + } + g.Go(func() error { + return read(s.humanEvents, organizationScopedFilter(org, bson.M{"itemId": bson.M{"$in": itemIDs}}), fields("id", "itemId", "stageId", "eventType", "createdAt", "payload.operationId", "payload.sourceItemIds", "payload.deliveryPackageId", "payload.reviewedRevision"), &d.HumanEvents) + }) + err := g.Wait() + return d, err +} diff --git a/internal/store/annotation_output_test.go b/internal/store/annotation_output_test.go new file mode 100644 index 00000000..354c75c9 --- /dev/null +++ b/internal/store/annotation_output_test.go @@ -0,0 +1,86 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "reflect" + "testing" + "time" +) + +func seedAnnotationOutput(t *testing.T, s Store) { + t.Helper() + ctx := t.Context() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + check(s.UpsertProject(ctx, domain.ProjectRecord{ID: "p", OrganizationID: "out", Name: "项目"})) + check(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: "v", OrganizationID: "out", Name: "供应商"})) + check(s.UpsertUser(ctx, domain.UserRecord{ID: "u", OrganizationID: "out", Name: "标注员", Role: domain.UserRoleAnnotator})) + check(s.UpsertRun(ctx, domain.RunRecord{ID: "r", ProjectID: "p", OrganizationID: "out", TaskID: "c", CreatedAt: "2026-10-01T00:00:00Z"})) + check(s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "c", ProjectID: "p", OrganizationID: "out", RunID: "r", Prefix: "source/c/part0001", CaptureID: "c/part0001"})) + check(s.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: "t", ProjectID: "p", OrganizationID: "out", RunID: "r", SourceIsFullVideo: true, SourceDurationSeconds: 900, CurrentRevision: 1})) + check(s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "i", ProjectID: "p", OrganizationID: "out", LegacyTaskID: "t", Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": "never export"}}})) + check(s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "rev", OrganizationID: "out", TaskID: "t", Revision: 1, AuthorUserID: "u", CountsAsOutput: true})) + check(s.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "s", OrganizationID: "out", ProjectID: "p", TaskID: "t", UserID: "u", SupplierID: "v", EventType: "combined_submitted", Revision: 1, CreatedAt: "2026-10-02T00:00:00Z"})) + check(s.UpsertAnnotationReview(ctx, domain.AnnotationReviewRecord{ID: "q", OrganizationID: "out", TaskID: "t", Revision: 1, Stage: domain.AnnotationReviewStageQuality, Decision: domain.AnnotationReviewApproved, CreatedAt: "2026-10-03T00:00:00Z"})) + ok, err := s.CompareAndSwapReviewOperation(ctx, domain.ReviewOperation{ID: "ac", OrganizationID: "out", ProjectID: "p", StageID: "prepackage_acceptance", Decision: "approved", Status: "completed", SourceItemIDs: []string{"i"}, CompletedAt: "2026-10-04T00:00:00Z"}, 0) + check(err) + if !ok { + t.Fatal("seed operation") + } +} +func outputStoreReport(t *testing.T, s Store) workforcestats.OutputReport { + t.Helper() + d, e := s.LoadAnnotationOutput(t.Context(), "out", []string{"p"}) + if e != nil { + t.Fatal(e) + } + q := workforcestats.OutputQuery{From: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC), Now: time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)} + return workforcestats.BuildOutput(d, q) +} +func TestAnnotationOutputStoreScope(t *testing.T) { + s := NewMemoryStore() + seedAnnotationOutput(t, s) + r := outputStoreReport(t, s) + if r.Totals.Acceptance.Count != 1 { + t.Fatalf("report %+v", r) + } + for _, ids := range [][]string{nil, {"foreign-project"}} { + d, e := s.LoadAnnotationOutput(t.Context(), "out", ids) + if e != nil || len(d.Tasks) != 0 { + t.Fatalf("scope %v %v", ids, e) + } + } +} +func TestAnnotationOutputMongoReplicaParity(t *testing.T) { + if os.Getenv("TEST_WORKFORCE_MONGO_URI") != "mongodb://127.0.0.1:27018" { + t.Skip("local isolated Mongo required") + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018")) + if e != nil { + t.Fatal(e) + } + defer client.Disconnect(context.Background()) + db := client.Database(fmt.Sprintf("annotation_output_test_%d", time.Now().UnixNano())) + defer db.Drop(context.Background()) + s := NewMongoStore(db) + seedAnnotationOutput(t, s) + actual := outputStoreReport(t, s) + m := NewMemoryStore() + seedAnnotationOutput(t, m) + expected := outputStoreReport(t, m) + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("projection mismatch\nMongo: %+v\nMemory: %+v", actual, expected) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index e72a0c86..a94665cb 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -36,6 +36,7 @@ type Store interface { InsertClientMoviStat(context.Context, domain.ClientMoviStatRecord) error ListClientMoviStats(context.Context, string, int) ([]domain.ClientMoviStatRecord, error) LoadWorkforceStats(context.Context, string, workforcestats.Query) (workforcestats.Snapshot, error) + LoadAnnotationOutput(context.Context, string, []string) (workforcestats.OutputSnapshot, error) GetProductionStatsScope(context.Context, string) (productionstats.Scope, error) SaveProductionStatsScope(context.Context, string, productionstats.Scope, int) (bool, error) LoadProductionStats(context.Context, string, []string) (productionstats.Snapshot, error) diff --git a/internal/workforcestats/output.go b/internal/workforcestats/output.go new file mode 100644 index 00000000..e66f7434 --- /dev/null +++ b/internal/workforcestats/output.go @@ -0,0 +1,670 @@ +package workforcestats + +import ( + "crypto/sha256" + "droboticflow/internal/domain" + "fmt" + "math" + "path" + "reflect" + "sort" + "strings" + "time" +) + +type outputOwner struct{ user, supplier string } +type outputFact struct { + at string + owner outputOwner + seconds *float64 +} +type outputEvent struct { + OutputEvidence + owner outputOwner + first bool +} +type outputUnit struct { + task domain.AnnotationTaskRecord + part, video, attempt string + events []outputEvent + first, submitted, quality, acceptance *outputFact + qualityStatus, acceptanceStatus, status string + evidence []OutputEvidence + seconds *float64 +} +type outputPart struct { + row OutputTask + submitted, quality, acceptance, rework *outputFact +} + +func outputTime(s string) time.Time { t, _ := time.Parse(time.RFC3339Nano, s); return t } +func outputEarlier(a, b string) bool { + return !outputTime(a).IsZero() && (outputTime(b).IsZero() || outputTime(a).Before(outputTime(b))) +} +func outputSeconds(v float64) *float64 { + if math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 { + return nil + } + return &v +} +func outputPartKey(c domain.CaptureRecord) string { + return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join([]string{c.ProjectID, c.ConnectionID, c.Bucket, strings.TrimRight(c.Prefix, "/")}, "\x00")))) +} +func outputIDs(v any) []string { + r := reflect.ValueOf(v) + out := []string{} + if !r.IsValid() || (r.Kind() != reflect.Slice && r.Kind() != reflect.Array) { + return out + } + for i := 0; i < r.Len(); i++ { + if s, ok := r.Index(i).Interface().(string); ok { + out = append(out, s) + } + } + return out +} +func outputStage(s string) string { + switch s { + case "quality", "quality_review", "split_quality", "cut_review", "semantic_review": + return "quality" + case "acceptance", "prepackage_acceptance", "final_acceptance": + return "acceptance" + } + return "" +} + +// BuildOutput interprets successful immutable facts. It never infers completion +// or authorship from a task's current assignee or update timestamp. +func BuildOutput(d OutputSnapshot, q OutputQuery) OutputReport { + if q.Location == nil { + q.Location = time.UTC + } + if q.Now.IsZero() { + q.Now = time.Now() + } + projects, users, suppliers := map[string]string{}, map[string]string{}, map[string]string{} + for _, x := range d.Projects { + projects[x.ID] = x.Name + } + for _, x := range d.Users { + users[x.ID] = x.Name + } + for _, x := range d.Suppliers { + suppliers[x.ID] = x.Name + } + runs := map[string]domain.RunRecord{} + for _, r := range d.Runs { + runs[r.ID] = r + } + captures := map[string]domain.CaptureRecord{} + runCaptures := map[string]domain.CaptureRecord{} + for _, c := range d.Captures { + captures[c.ID] = c + if c.RunID != "" { + runCaptures[c.RunID] = c + } + } + for _, r := range d.Runs { + if c, ok := captures[r.TaskID]; ok { + runCaptures[r.ID] = c + } + } + items := map[string]domain.HumanItemRecord{} + for _, i := range d.Items { + items[i.ID] = i + } + revisions := map[string]domain.AnnotationRevisionRecord{} + for _, r := range d.Revisions { + revisions[revisionKey(r.TaskID, r.Revision)] = r + } + operations := map[string]domain.ReviewOperation{} + reviewOps := map[string]domain.ReviewOperation{} + worklogOps := map[string]domain.ReviewOperation{} + for _, o := range d.Operations { + operations[o.ID] = o + operations[o.OperationID] = o + for _, e := range o.Effects { + if e.Review != nil { + reviewOps[e.Review.ID] = o + } + if e.Worklog != nil { + worklogOps[e.Worklog.ID] = o + } + } + } + units := map[string]*outputUnit{} + for _, t := range d.Tasks { + if runs[t.RunID].Mode == domain.RunModeDebug { + continue + } + c, ok := runCaptures[t.RunID] + if !ok { + c, ok = captures[t.DataTaskID] + } + key, video := "task:"+t.ProjectID+":"+t.ID, t.ID + if ok && c.Prefix != "" { + key = outputPartKey(c) + video = c.CaptureID + if video == "" { + video = path.Base(path.Dir(strings.TrimRight(c.Prefix, "/"))) + "/" + path.Base(strings.TrimRight(c.Prefix, "/")) + } + } + attempt := t.RunID + if attempt == "" { + attempt = t.ID + } + u := &outputUnit{task: t, part: key, video: video, attempt: attempt, evidence: []OutputEvidence{}} + if t.SourceIsFullVideo { + u.seconds = outputSeconds(t.SourceDurationSeconds) + } + units[t.ID] = u + } + // Deduplicate persisted retries by task/revision, choosing the real earliest success. + submissions := map[string]domain.AnnotationWorklogEventRecord{} + for _, e := range d.Events { + if !submission(e) || e.Mode == domain.RunModeDebug || e.Temporary || units[e.TaskID] == nil { + continue + } + if o, ok := worklogOps[e.ID]; ok && o.Status != "completed" { + continue + } + if outputTime(e.CreatedAt).IsZero() || outputTime(e.CreatedAt).After(q.Now) { + continue + } + key := submissionKey(e) + if e.Revision > 0 { + key = revisionKey(e.TaskID, e.Revision) + } + old, ok := submissions[key] + if !ok || outputEarlier(e.CreatedAt, old.CreatedAt) || (e.CreatedAt == old.CreatedAt && e.ID < old.ID) { + submissions[key] = e + } + } + ownership := map[string]outputOwner{} + for _, e := range submissions { + r, has := revisions[revisionKey(e.TaskID, e.Revision)] + author := e.UserID + if has && r.AuthorUserID != "" { + author = r.AuthorUserID + } + owner := outputOwner{author, e.SupplierID} + ownership[revisionKey(e.TaskID, e.Revision)] = owner + var rev *domain.AnnotationRevisionRecord + if has { + rev = &r + } + u := units[e.TaskID] + u.events = append(u.events, outputEvent{OutputEvidence: OutputEvidence{e.ID, e.TaskID, "submitted", "approved", e.CreatedAt, e.Revision}, owner: owner, first: SubmissionKind(e, rev) == "first"}) + } + // Completed quality plans carry a frozen supplier even when the task changed owner. + for _, o := range d.Operations { + if o.Status != "completed" { + continue + } + supplier := "" + for _, e := range o.Effects { + if e.BeforeTask != nil { + supplier = e.BeforeTask.SupplierID + } + } + for _, e := range o.Effects { + if e.Review != nil { + key := revisionKey(e.Review.TaskID, e.Review.Revision) + owner := ownership[key] + if r := revisions[key]; r.AuthorUserID != "" { + owner.user = r.AuthorUserID + } + if supplier != "" { + owner.supplier = supplier + } + ownership[key] = owner + } + } + } + for _, r := range d.Reviews { + u := units[r.TaskID] + stage := outputStage(string(r.Stage)) + if u == nil || stage == "" { + continue + } + at := r.CreatedAt + if o, ok := reviewOps[r.ID]; ok { + if o.Status != "completed" { + continue + } + at = o.CompletedAt + } + owner := ownership[revisionKey(r.TaskID, r.Revision)] + if owner.user == "" { + owner.user = revisions[revisionKey(r.TaskID, r.Revision)].AuthorUserID + } + u.events = append(u.events, outputEvent{OutputEvidence: OutputEvidence{r.ID, r.TaskID, stage, string(r.Decision), at, r.Revision}, owner: owner}) + } + // Aggregate acceptance addresses source items, not the accepting reviewer. + addAcceptance := func(id, at, decision string, sourceIDs []string) { + for _, itemID := range sourceIDs { + task := items[itemID].LegacyTaskID + if u := units[task]; u != nil { + u.events = append(u.events, outputEvent{OutputEvidence: OutputEvidence{id, task, "acceptance", decision, at, 0}}) + } + } + } + covered := map[string]bool{} + for _, o := range d.Operations { + if outputStage(o.StageID) != "acceptance" { + continue + } + covered[o.ID] = true + covered[o.OperationID] = true + if o.Status == "completed" { + addAcceptance(o.ID, o.CompletedAt, o.Decision, o.SourceItemIDs) + } + } + for _, e := range d.HumanEvents { + if outputStage(e.StageID) != "acceptance" { + continue + } + opID, _ := e.Payload["operationId"].(string) + if covered[opID] { + continue + } + if o, ok := operations[opID]; ok && o.Status != "completed" { + continue + } + addAcceptance(e.ID, e.CreatedAt, e.EventType, outputIDs(e.Payload["sourceItemIds"])) + } + for _, p := range d.Packages { + if p.Mode == domain.RunModeDebug || p.Temporary { + continue + } + if _, copied := p.QualitySummary["prePackageAcceptance"]; copied { + continue + } + for _, r := range p.AcceptanceReviews { + addAcceptance(r.ID, r.CreatedAt, string(r.Decision), p.ItemIDs) + } + } + groups := map[string]map[string][]*outputUnit{} + for _, u := range units { + sort.Slice(u.events, func(i, j int) bool { + a, b := u.events[i], u.events[j] + ta, tb := outputTime(a.At), outputTime(b.At) + if !ta.Equal(tb) { + return ta.Before(tb) + } + rank := func(e outputEvent) int { + if e.Stage == "submitted" { + return 0 + } + if e.Stage == "quality" { + return 1 + } + return 2 + } + if rank(a) != rank(b) { + return rank(a) < rank(b) + } + return a.ID < b.ID + }) + seen := map[string]bool{} + currentRevision := 0 + for _, e := range u.events { + at := outputTime(e.At) + if at.IsZero() || at.After(q.Now) || seen[e.ID+":"+e.Stage] { + continue + } + seen[e.ID+":"+e.Stage] = true + if e.Stage == "submitted" { + if e.first && u.first == nil { + u.first = &outputFact{e.At, e.owner, u.seconds} + } + if u.submitted == nil || e.Revision != currentRevision { + u.quality = nil + u.acceptance = nil + u.qualityStatus = "pending" + u.acceptanceStatus = "pending" + u.status = "pending_quality" + } + u.submitted = &outputFact{e.At, e.owner, u.seconds} + currentRevision = e.Revision + } else { + if e.Revision > 0 && currentRevision > 0 && e.Revision != currentRevision { + continue + } + if e.owner.user == "" && u.submitted != nil { + e.owner = u.submitted.owner + } + f := &outputFact{e.At, e.owner, u.seconds} + if e.Stage == "quality" { + if e.Decision == "approved" { + if u.quality == nil { + u.quality = f + } + u.qualityStatus = "approved" + u.status = "pending_acceptance" + } else if e.Decision == "rejected" { + u.quality = nil + u.acceptance = nil + u.qualityStatus = "rejected" + u.acceptanceStatus = "pending" + u.status = "rework" + } + } else if e.Stage == "acceptance" { + if e.Decision == "approved" { + if u.acceptance == nil { + u.acceptance = f + } + u.acceptanceStatus = "approved" + u.status = "accepted" + } else if e.Decision == "rejected" { + u.acceptance = nil + u.quality = nil + u.qualityStatus = "pending" + u.acceptanceStatus = "rejected" + u.status = "rework" + } + } + } + u.evidence = append(u.evidence, e.OutputEvidence) + } + if groups[u.part] == nil { + groups[u.part] = map[string][]*outputUnit{} + } + groups[u.part][u.attempt] = append(groups[u.part][u.attempt], u) + } + parts := []outputPart{} + combine := func(us []*outputUnit, get func(*outputUnit) *outputFact) *outputFact { + var f *outputFact + for _, u := range us { + v := get(u) + if v == nil { + return nil + } + if f == nil { + copy := *v + f = © + } else { + if f.owner.user != v.owner.user { + f.owner.user = "" + } + if f.owner.supplier != v.owner.supplier { + f.owner.supplier = "" + } + if outputEarlier(f.at, v.at) { + f.at = v.at + } + } + } + return f + } + for id, attempts := range groups { + var selected []*outputUnit + selectedAt := "" + var first *outputFact + for attempt, us := range attempts { + sort.Slice(us, func(i, j int) bool { return us[i].task.ID < us[j].task.ID }) + at := runs[attempt].CreatedAt + if at == "" { + at = us[0].task.CreatedAt + } + if outputTime(at).After(q.Now) { + continue + } + if selected == nil || outputEarlier(selectedAt, at) || (selectedAt == at && attempt < selected[0].attempt) { + selected = us + selectedAt = at + } + if f := combine(us, func(u *outputUnit) *outputFact { return u.first }); f != nil && (first == nil || outputEarlier(f.at, first.at)) { + first = f + } + } + if len(selected) == 0 { + continue + } + u := selected[0] + var seconds *float64 + for _, x := range selected { + if x.task.SourceIsFullVideo && outputSeconds(x.task.SourceDurationSeconds) != nil { + if seconds != nil && math.Abs(*seconds-x.task.SourceDurationSeconds) > .001 { + seconds = nil + break + } + seconds = outputSeconds(x.task.SourceDurationSeconds) + } + } + if seconds == nil && len(selected) == 1 { + seconds = u.seconds + } + quality := combine(selected, func(u *outputUnit) *outputFact { return u.quality }) + acceptance := combine(selected, func(u *outputUnit) *outputFact { return u.acceptance }) + owner := outputOwner{} + if f := combine(selected, func(u *outputUnit) *outputFact { return u.submitted }); f != nil { + owner = f.owner + } + row := OutputTask{ID: id, VideoID: u.video, ProjectID: u.task.ProjectID, ProjectName: projects[u.task.ProjectID], Seconds: seconds, UserID: owner.user, UserName: users[owner.user], SupplierID: owner.supplier, SupplierName: suppliers[owner.supplier], Evidence: []OutputEvidence{}, QualityStatus: "pending", AcceptanceStatus: "pending", Status: "pending_quality"} + if first != nil { + first.seconds = seconds + row.SubmittedAt = first.at + } + if quality != nil { + quality.seconds = seconds + row.QualityAt = quality.at + row.QualityStatus = "approved" + row.Status = "pending_acceptance" + } + if acceptance != nil { + acceptance.seconds = seconds + row.AcceptedAt = acceptance.at + row.AcceptanceStatus = "approved" + row.Status = "accepted" + } + var rework *outputFact + for _, x := range selected { + row.Evidence = append(row.Evidence, x.evidence...) + if x.status == "rework" { + row.Status = "rework" + if x.submitted != nil { + rework = &outputFact{q.Now.Format(time.RFC3339Nano), x.submitted.owner, seconds} + } + } + if x.qualityStatus == "rejected" { + row.QualityStatus = "rejected" + } + if x.acceptanceStatus == "rejected" { + row.AcceptanceStatus = "rejected" + } + } + parts = append(parts, outputPart{row, first, quality, acceptance, rework}) + } + return outputSummarize(parts, q, projects, users, suppliers) +} + +type outputAccumulator struct { + seen map[string]bool + seconds float64 + unknown bool +} + +func (a *outputAccumulator) add(id string, seconds *float64) { + if a.seen == nil { + a.seen = map[string]bool{} + } + if a.seen[id] { + return + } + a.seen[id] = true + if seconds == nil { + a.unknown = true + } else { + a.seconds += *seconds + } +} +func (a outputAccumulator) metric() OutputMetric { + m := OutputMetric{Count: len(a.seen)} + if !a.unknown { + n := a.seconds + m.Seconds = &n + } + return m +} +func outputSummarize(parts []outputPart, q OutputQuery, projects, users, suppliers map[string]string) OutputReport { + out := OutputReport{Tasks: []OutputTask{}, Suppliers: []OutputSupplier{}, Days: []OutputDay{}, Projects: []OutputOption{}, SupplierOptions: []OutputOption{}, Users: []OutputOption{}} + policy := func(f *outputFact) bool { + return f != nil && (!q.MatchUser || f.owner.user == q.ScopeUserID) && (!q.MatchSupplier || f.owner.supplier == q.ScopeSupplierID) + } + matches := func(f *outputFact) bool { + return policy(f) && (q.UserID == "" || f.owner.user == q.UserID) && (q.SupplierID == "" || f.owner.supplier == q.SupplierID) + } + inRange := func(f *outputFact) bool { + return f != nil && !outputTime(f.at).Before(q.From) && outputTime(f.at).Before(q.To) + } + totals := make([]outputAccumulator, 4) + vendor := map[string][]outputAccumulator{} + members := map[string]map[string]bool{} + days := map[string]*OutputDay{} + dailyMetrics := map[string][]outputAccumulator{} + cumulative := map[string][]outputAccumulator{} + projectSet, supplierSet, userSet := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, p := range parts { + facts := []*outputFact{p.submitted, p.quality, p.acceptance, p.rework} + for _, f := range facts { + if policy(f) { + projectSet[p.row.ProjectID] = true + if q.ProjectID == "" || q.ProjectID == p.row.ProjectID { + supplierSet[f.owner.supplier] = true + if q.SupplierID == "" || q.SupplierID == f.owner.supplier { + userSet[f.owner.user] = true + } + } + } + } + if q.ProjectID != "" && q.ProjectID != p.row.ProjectID { + continue + } + selected := false + var owner *outputFact + for i, f := range facts { + if !matches(f) { + continue + } + + memberKey := f.owner.supplier + "\x00" + f.owner.user + if i == 0 || i == 2 { + if outputTime(f.at).Before(q.To) { + if cumulative[memberKey] == nil { + cumulative[memberKey] = make([]outputAccumulator, 2) + } + slot := 0 + if i == 2 { + slot = 1 + } + cumulative[memberKey][slot].add(p.row.ID, f.seconds) + } + } + if i != 3 && !inRange(f) { + continue + } + selected = true + owner = f + totals[i].add(p.row.ID, f.seconds) + if vendor[f.owner.supplier] == nil { + vendor[f.owner.supplier] = make([]outputAccumulator, 4) + members[f.owner.supplier] = map[string]bool{} + } + vendor[f.owner.supplier][i].add(p.row.ID, f.seconds) + if f.owner.user != "" { + members[f.owner.supplier][f.owner.user] = true + } + if i == 0 || i == 2 { + day := outputTime(f.at).In(q.Location).Format("2006-01-02") + key := day + "\x00" + memberKey + if days[key] == nil { + days[key] = &OutputDay{Date: day, UserID: f.owner.user, UserName: users[f.owner.user], SupplierID: f.owner.supplier, SupplierName: suppliers[f.owner.supplier]} + dailyMetrics[key] = make([]outputAccumulator, 2) + } + slot := 0 + if i == 2 { + slot = 1 + } + dailyMetrics[key][slot].add(p.row.ID, f.seconds) + } + } + if selected { + row := p.row + row.Attribution = map[string]OutputAttribution{} + for i, stage := range []string{"submitted", "quality", "acceptance"} { + if f := facts[i]; matches(f) { + row.Attribution[stage] = OutputAttribution{UserID: f.owner.user, UserName: users[f.owner.user], SupplierID: f.owner.supplier, SupplierName: suppliers[f.owner.supplier]} + } + } + if owner != nil { + row.UserID = owner.owner.user + row.UserName = users[row.UserID] + row.SupplierID = owner.owner.supplier + row.SupplierName = suppliers[row.SupplierID] + } + if !matches(p.submitted) { + row.SubmittedAt = "" + } + if !matches(p.quality) { + row.QualityAt = "" + } + if !matches(p.acceptance) { + row.AcceptedAt = "" + } + // Only expose evidence belonging to the selected author's version. Aggregate + // decisions have no customer text or credentials in this read model. + if q.MatchUser || q.MatchSupplier { + row.Evidence = nil + } + out.Tasks = append(out.Tasks, row) + } + } + out.Totals = OutputTotals{totals[0].metric(), totals[1].metric(), totals[2].metric(), totals[3].metric()} + for id, ms := range vendor { + out.Suppliers = append(out.Suppliers, OutputSupplier{OutputTotals: OutputTotals{ms[0].metric(), ms[1].metric(), ms[2].metric(), ms[3].metric()}, ID: id, Name: suppliers[id], MemberCount: len(members[id])}) + } + for key, day := range days { + ms := dailyMetrics[key] + day.Submitted = ms[0].metric() + day.Acceptance = ms[1].metric() + c := cumulative[day.SupplierID+"\x00"+day.UserID] + day.CumulativeSubmitted = c[0].metric() + day.CumulativeAcceptance = c[1].metric() + out.Days = append(out.Days, *day) + } + options := func(ids map[string]bool, names map[string]string) []OutputOption { + xs := []OutputOption{} + for id := range ids { + if id != "" { + xs = append(xs, OutputOption{id, names[id]}) + } + } + sort.Slice(xs, func(i, j int) bool { + if xs[i].Name != xs[j].Name { + return xs[i].Name < xs[j].Name + } + return xs[i].ID < xs[j].ID + }) + return xs + } + out.Projects = options(projectSet, projects) + out.SupplierOptions = options(supplierSet, suppliers) + out.Users = options(userSet, users) + sort.Slice(out.Tasks, func(i, j int) bool { + a, b := out.Tasks[i], out.Tasks[j] + if a.SubmittedAt != b.SubmittedAt { + return outputEarlier(b.SubmittedAt, a.SubmittedAt) + } + return a.ID < b.ID + }) + sort.Slice(out.Suppliers, func(i, j int) bool { return out.Suppliers[i].ID < out.Suppliers[j].ID }) + sort.Slice(out.Days, func(i, j int) bool { + a, b := out.Days[i], out.Days[j] + if a.Date != b.Date { + return a.Date > b.Date + } + return a.SupplierID+a.UserID < b.SupplierID+b.UserID + }) + return out +} diff --git a/internal/workforcestats/output_test.go b/internal/workforcestats/output_test.go new file mode 100644 index 00000000..0e942738 --- /dev/null +++ b/internal/workforcestats/output_test.go @@ -0,0 +1,224 @@ +package workforcestats + +import ( + "droboticflow/internal/domain" + "reflect" + "testing" + "time" +) + +func outputFixture() OutputSnapshot { + return OutputSnapshot{ + Projects: []domain.ProjectRecord{{ID: "p", Name: "项目"}}, Users: []domain.UserRecord{{ID: "a", Name: "作者", Role: domain.UserRoleQualityReviewer}, {ID: "b", Name: "另一作者"}}, Suppliers: []domain.SupplierRecord{{ID: "v", Name: "供应商"}, {ID: "other", Name: "另一供应商"}}, + Captures: []domain.CaptureRecord{{ID: "c", ProjectID: "p", ConnectionID: "bos", Bucket: "b", Prefix: "source/run/part0001", CaptureID: "run/part0001", RunID: "r"}}, + Runs: []domain.RunRecord{{ID: "r", ProjectID: "p", TaskID: "c", CreatedAt: "2026-09-25T00:00:00Z", Mode: domain.RunModeProduction}}, + Tasks: []domain.AnnotationTaskRecord{{ID: "t", ProjectID: "p", RunID: "r", SourceIsFullVideo: true, SourceDurationSeconds: 900, CurrentRevision: 4, AssignedUserID: "changed", SupplierID: "other", CreatedAt: "2026-09-26T00:00:00Z"}}, + Items: []domain.HumanItemRecord{{ID: "i", LegacyTaskID: "t", ProjectID: "p"}}, + Revisions: []domain.AnnotationRevisionRecord{{TaskID: "t", Revision: 4, AuthorUserID: "a", CountsAsOutput: true}}, + Events: []domain.AnnotationWorklogEventRecord{{ID: "s", TaskID: "t", UserID: "a", SupplierID: "v", EventType: "combined_submitted", Revision: 4, CreatedAt: "2026-09-28T00:00:00Z"}}, + Reviews: []domain.AnnotationReviewRecord{{ID: "q", TaskID: "t", Revision: 4, Stage: domain.AnnotationReviewStageQuality, Decision: domain.AnnotationReviewApproved, CreatedAt: "2026-10-01T00:00:00Z"}}, + Operations: []domain.ReviewOperation{{ID: "ac", StageID: "prepackage_acceptance", Decision: "approved", Status: "completed", SourceItemIDs: []string{"i"}, CompletedAt: "2026-10-03T00:00:00Z"}}, + } +} +func outputQuery() OutputQuery { + from, _ := time.Parse(time.RFC3339, "2026-10-01T00:00:00Z") + to := from.AddDate(0, 0, 20) + return OutputQuery{From: from, To: to, Now: to, Location: time.FixedZone("Asia/Shanghai", 8*3600)} +} +func assertOutputSeconds(t *testing.T, m OutputMetric, count int, seconds float64) { + t.Helper() + if m.Count != count || m.Seconds == nil || *m.Seconds != seconds { + t.Fatalf("metric %+v, want %d / %v", m, count, seconds) + } +} +func TestAnnotationOutputDatesAttributionAndCumulative(t *testing.T) { + d := outputFixture() + before := outputFixture() + r := BuildOutput(d, outputQuery()) + assertOutputSeconds(t, r.Totals.Submitted, 0, 0) + assertOutputSeconds(t, r.Totals.Quality, 1, 900) + assertOutputSeconds(t, r.Totals.Acceptance, 1, 900) + if len(r.Tasks) != 1 || r.Tasks[0].UserID != "a" || r.Tasks[0].SupplierID != "v" || r.Tasks[0].Status != "accepted" { + t.Fatalf("historical owner replaced: %+v", r.Tasks) + } + if len(r.Days) != 1 || r.Days[0].Date != "2026-10-03" { + t.Fatalf("day %+v", r.Days) + } + assertOutputSeconds(t, r.Days[0].CumulativeSubmitted, 1, 900) + if !reflect.DeepEqual(d, before) { + t.Fatal("report mutated its source") + } + q := outputQuery() + q.MatchUser = true + q.ScopeUserID = "b" + if len(BuildOutput(d, q).Tasks) != 0 { + t.Fatal("cross-author report") + } + q.ScopeUserID = "a" + if BuildOutput(d, q).Totals.Acceptance.Count != 1 { + t.Fatal("account role change erased production") + } + q.MatchSupplier = true + q.ScopeSupplierID = "other" + if len(BuildOutput(d, q).Tasks) != 0 { + t.Fatal("current supplier exposed historical output") + } +} +func TestAnnotationOutputRetriesReworkAndInvalidation(t *testing.T) { + d := outputFixture() + q := outputQuery() + q.From = q.From.AddDate(0, 0, -10) + retry := d.Events[0] + retry.ID = "retry" + retry.SubmissionID = "different" + retry.CreatedAt = "2026-10-02T00:00:00Z" + retry.SupplierID = "other" + d.Events = append(d.Events, retry) + r := BuildOutput(d, q) + assertOutputSeconds(t, r.Totals.Submitted, 1, 900) + if r.Tasks[0].SupplierID != "v" { + t.Fatal("retry changed original supplier") + } + d.Reviews = append(d.Reviews, domain.AnnotationReviewRecord{ID: "reject", TaskID: "t", Revision: 4, Stage: domain.AnnotationReviewStageQuality, Decision: domain.AnnotationReviewRejected, CreatedAt: "2026-10-05T00:00:00Z"}) + r = BuildOutput(d, q) + assertOutputSeconds(t, r.Totals.Rework, 1, 900) + assertOutputSeconds(t, r.Totals.Acceptance, 0, 0) + d.Revisions = append(d.Revisions, domain.AnnotationRevisionRecord{TaskID: "t", Revision: 5, AuthorUserID: "a", IsRework: true}) + d.Events = append(d.Events, domain.AnnotationWorklogEventRecord{ID: "resubmit", TaskID: "t", UserID: "a", SupplierID: "v", EventType: "rework_submitted", Revision: 5, CreatedAt: "2026-10-06T00:00:00Z"}) + r = BuildOutput(d, q) + assertOutputSeconds(t, r.Totals.Submitted, 1, 900) + assertOutputSeconds(t, r.Totals.Rework, 0, 0) + assertOutputSeconds(t, r.Totals.Quality, 0, 0) + d.Reviews = append(d.Reviews, domain.AnnotationReviewRecord{ID: "new-quality", TaskID: "t", Revision: 5, Stage: domain.AnnotationReviewStageQuality, Decision: domain.AnnotationReviewApproved, CreatedAt: "2026-10-07T00:00:00Z"}) + r = BuildOutput(d, q) + assertOutputSeconds(t, r.Totals.Quality, 1, 900) + if r.Tasks[0].QualityAt != "2026-10-07T00:00:00Z" { + t.Fatal("invalidated quality evidence reused") + } +} +func TestAnnotationOutputIncompletePlansMissingDurationAndDebug(t *testing.T) { + d := outputFixture() + d.Operations[0].Status = "applying" + d.Tasks[0].SourceDurationSeconds = 0 + d.Operations = append(d.Operations, domain.ReviewOperation{ID: "pending-q", Status: "applying", Effects: []domain.ReviewDecisionEffect{{Review: &domain.AnnotationReviewRecord{ID: "q"}}}}) + r := BuildOutput(d, outputQuery()) + if r.Totals.Quality.Count != 0 || r.Totals.Acceptance.Count != 0 { + t.Fatal("unfinished decision counted") + } + q := outputQuery() + q.From = q.From.AddDate(0, 0, -10) + r = BuildOutput(d, q) + if r.Totals.Submitted.Count != 1 || r.Totals.Submitted.Seconds != nil { + t.Fatal("unknown video seconds invented") + } + d.Runs[0].Mode = domain.RunModeDebug + if BuildOutput(d, q).Totals.Submitted.Count != 0 { + t.Fatal("debug output counted") + } +} +func TestAnnotationOutputPartIdentityAndMultipleTasks(t *testing.T) { + d := outputFixture() + second := d.Tasks[0] + second.ID = "t2" + d.Tasks = append(d.Tasks, second) + d.Revisions = append(d.Revisions, domain.AnnotationRevisionRecord{TaskID: "t2", Revision: 4, AuthorUserID: "a", CountsAsOutput: true}) + r := BuildOutput(d, outputQuery()) + if r.Totals.Quality.Count != 0 || r.Totals.Acceptance.Count != 0 { + t.Fatal("partial task set counted as a part") + } + e := d.Events[0] + e.ID = "s2" + e.TaskID = "t2" + d.Events = append(d.Events, e) + review := d.Reviews[0] + review.ID = "q2" + review.TaskID = "t2" + d.Reviews = append(d.Reviews, review) + d.Items = append(d.Items, domain.HumanItemRecord{ID: "i2", LegacyTaskID: "t2"}) + d.Operations[0].SourceItemIDs = append(d.Operations[0].SourceItemIDs, "i2") + r = BuildOutput(d, outputQuery()) + assertOutputSeconds(t, r.Totals.Acceptance, 1, 900) + c := d.Captures[0] + c.ID = "c2" + c.Prefix = "different/run/part0001" + c.RunID = "r2" + d.Captures = append(d.Captures, c) + run := d.Runs[0] + run.ID = "r2" + run.TaskID = "c2" + d.Runs = append(d.Runs, run) + third := second + third.ID = "t3" + third.RunID = "r2" + d.Tasks = append(d.Tasks, third) + e.ID = "s3" + e.TaskID = "t3" + e.SubmissionKind = "first" + e.CreatedAt = "2026-10-02T00:00:00Z" + d.Events = append(d.Events, e) + r = BuildOutput(d, outputQuery()) + assertOutputSeconds(t, r.Totals.Submitted, 1, 900) + if len(r.Tasks) != 2 { + t.Fatal("same display name from different directories collapsed") + } +} +func TestAnnotationOutputLatestAttemptDoesNotReuseApproval(t *testing.T) { + d := outputFixture() + run := d.Runs[0] + run.ID = "new" + run.CreatedAt = "2026-10-10T00:00:00Z" + d.Runs = append(d.Runs, run) + task := d.Tasks[0] + task.ID = "new-task" + task.RunID = "new" + d.Tasks = append(d.Tasks, task) + q := outputQuery() + q.From = q.From.AddDate(0, 0, -10) + r := BuildOutput(d, q) + assertOutputSeconds(t, r.Totals.Submitted, 1, 900) + assertOutputSeconds(t, r.Totals.Acceptance, 0, 0) +} + +func TestAnnotationOutputChangedAuthorKeepsStageAttribution(t *testing.T) { + d := outputFixture() + d.Operations = nil + d.Reviews = append(d.Reviews, domain.AnnotationReviewRecord{ID: "reject", TaskID: "t", Revision: 4, Stage: domain.AnnotationReviewStageQuality, Decision: domain.AnnotationReviewRejected, CreatedAt: "2026-10-02T00:00:00Z"}) + d.Revisions = append(d.Revisions, domain.AnnotationRevisionRecord{TaskID: "t", Revision: 5, AuthorUserID: "b", IsRework: true}) + d.Events = append(d.Events, domain.AnnotationWorklogEventRecord{ID: "s-new", TaskID: "t", Revision: 5, UserID: "b", SupplierID: "other", EventType: "rework_submitted", CreatedAt: "2026-10-03T00:00:00Z"}) + d.Reviews = append(d.Reviews, domain.AnnotationReviewRecord{ID: "q-new", TaskID: "t", Revision: 5, Stage: domain.AnnotationReviewStageQuality, Decision: domain.AnnotationReviewApproved, CreatedAt: "2026-10-04T00:00:00Z"}) + d.Operations = []domain.ReviewOperation{{ID: "ac-new", StageID: "prepackage_acceptance", Decision: "approved", Status: "completed", SourceItemIDs: []string{"i"}, CompletedAt: "2026-10-05T00:00:00Z"}} + q := outputQuery() + q.From = q.From.AddDate(0, 0, -10) + r := BuildOutput(d, q) + if r.Tasks[0].Attribution["submitted"].UserID != "a" || r.Tasks[0].Attribution["acceptance"].UserID != "b" { + t.Fatal("stage attribution conflated") + } + q.MatchUser = true + q.ScopeUserID = "a" + r = BuildOutput(d, q) + assertOutputSeconds(t, r.Totals.Acceptance, 0, 0) + if _, leaked := r.Tasks[0].Attribution["acceptance"]; leaked { + t.Fatal("other author's attribution exposed") + } + q.ScopeUserID = "b" + r = BuildOutput(d, q) + assertOutputSeconds(t, r.Totals.Submitted, 0, 0) + assertOutputSeconds(t, r.Totals.Acceptance, 1, 900) +} + +func TestAnnotationOutputNeverUsesSliceWorklogAsWholeVideoDuration(t *testing.T) { + for _, full := range []bool{true, false} { + d := outputFixture() + d.Tasks[0].SourceDurationSeconds = 0 + d.Tasks[0].SourceIsFullVideo = full + d.Tasks[0].ClipEnd = 120 + keptSeconds := 90.0 + d.Events[0].VideoSeconds = &keptSeconds + q := outputQuery() + q.From = q.From.AddDate(0, 0, -10) + r := BuildOutput(d, q) + if r.Totals.Submitted.Count != 1 || r.Totals.Submitted.Seconds != nil || r.Tasks[0].Seconds != nil { + t.Fatalf("slice/worklog duration used as whole video: %+v", r) + } + } +} diff --git a/internal/workforcestats/output_types.go b/internal/workforcestats/output_types.go new file mode 100644 index 00000000..d882730d --- /dev/null +++ b/internal/workforcestats/output_types.go @@ -0,0 +1,105 @@ +package workforcestats + +import ( + "droboticflow/internal/domain" + "time" +) + +// OutputSnapshot contains reporting facts only. Loading it must never recover, +// claim or modify a task. Mutable drafts and media payloads are not needed. +type OutputSnapshot struct { + Captures []domain.CaptureRecord + Runs []domain.RunRecord + Tasks []domain.AnnotationTaskRecord + Items []domain.HumanItemRecord + Events []domain.AnnotationWorklogEventRecord + Reviews []domain.AnnotationReviewRecord + Operations []domain.ReviewOperation + HumanEvents []domain.HumanEventRecord + Revisions []domain.AnnotationRevisionRecord + Packages []domain.DeliveryPackageRecord + Users []domain.UserRecord + Projects []domain.ProjectRecord + Suppliers []domain.SupplierRecord +} + +type OutputQuery struct { + ScopeUserID, ScopeSupplierID string + From, To, Now time.Time + Location *time.Location + ProjectID, SupplierID, UserID string + MatchSupplier, MatchUser bool +} +type OutputOption struct { + ID string `json:"id"` + Name string `json:"name"` +} +type OutputMetric struct { + Count int `json:"count"` + Seconds *float64 `json:"seconds"` +} +type OutputTotals struct { + Submitted OutputMetric `json:"submitted"` + Quality OutputMetric `json:"quality"` + Acceptance OutputMetric `json:"acceptance"` + Rework OutputMetric `json:"rework"` +} +type OutputEvidence struct { + ID string `json:"id"` + TaskID string `json:"taskId"` + Stage string `json:"stage"` + Decision string `json:"decision"` + At string `json:"at"` + Revision int `json:"revision"` +} +type OutputAttribution struct { + UserID string `json:"userId"` + UserName string `json:"userName"` + SupplierID string `json:"supplierId"` + SupplierName string `json:"supplierName"` +} +type OutputTask struct { + Attribution map[string]OutputAttribution `json:"attribution"` + ID string `json:"id"` + VideoID string `json:"videoId"` + ProjectID string `json:"projectId"` + ProjectName string `json:"projectName"` + UserID string `json:"userId"` + UserName string `json:"userName"` + SupplierID string `json:"supplierId"` + SupplierName string `json:"supplierName"` + Seconds *float64 `json:"seconds"` + SubmittedAt string `json:"submittedAt"` + QualityAt string `json:"qualityAt"` + AcceptedAt string `json:"acceptedAt"` + QualityStatus string `json:"qualityStatus"` + AcceptanceStatus string `json:"acceptanceStatus"` + Status string `json:"status"` + Evidence []OutputEvidence `json:"evidence"` +} +type OutputSupplier struct { + OutputTotals + ID string `json:"id"` + Name string `json:"name"` + MemberCount int `json:"memberCount"` +} +type OutputDay struct { + Date string `json:"date"` + UserID string `json:"userId"` + UserName string `json:"userName"` + SupplierID string `json:"supplierId"` + SupplierName string `json:"supplierName"` + Submitted OutputMetric `json:"submitted"` + Acceptance OutputMetric `json:"acceptance"` + CumulativeSubmitted OutputMetric `json:"cumulativeSubmitted"` + CumulativeAcceptance OutputMetric `json:"cumulativeAcceptance"` +} +type OutputReport struct { + Totals OutputTotals `json:"totals"` + Tasks []OutputTask `json:"tasks"` + Suppliers []OutputSupplier `json:"suppliers"` + Days []OutputDay `json:"days"` + Projects []OutputOption `json:"projects"` + SupplierOptions []OutputOption `json:"supplierOptions"` + Users []OutputOption `json:"users"` +} diff --git a/package.json b/package.json index 08a781b9..ce046e8d 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,8 @@ "dev:api:replica": "bash scripts/local/start-replica-api.sh", "test:browser:editing": "node tests/browser/run-editing-regression.mjs", "test:browser:review-permissions": "node tests/browser/review-authoring-permissions.mjs", - "test:production:boot": "node tests/browser/run-production-boot.mjs" + "test:production:boot": "node tests/browser/run-production-boot.mjs", + "test:browser:annotation-output": "node tests/browser/annotation-output-replica.mjs" }, "dependencies": { "@tailwindcss/vite": "^4.1.14", diff --git a/src/components/AnnotationOutputDashboard.tsx b/src/components/AnnotationOutputDashboard.tsx new file mode 100644 index 00000000..7cc69fee --- /dev/null +++ b/src/components/AnnotationOutputDashboard.tsx @@ -0,0 +1,590 @@ +import React, { useEffect, useMemo, useRef, useState } from "react"; +import { Download, RefreshCw, X } from "lucide-react"; +import { + annotationOutputApi, + type AnnotationOutput, + type OutputDay, + type OutputFilters, + type OutputMetric, + type OutputPage, + type OutputRequest, + type OutputTask, +} from "../services/api/annotation-output"; +import { formatCompactDateTime } from "../lib/datetime"; +import { getActionErrorMessage } from "../lib/action-errors.js"; +import "../styles/annotation-output.css"; + +const dateInZone = (date: Date, timeZone: string) => { + const parts = new Intl.DateTimeFormat("en-CA", { + timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + }).formatToParts(date); + const read = (type: string) => parts.find((p) => p.type === type)?.value; + return `${read("year")}-${read("month")}-${read("day")}`; +}; +const defaults = (timeZone: string): OutputFilters => ({ + from: dateInZone(new Date(Date.now() - 29 * 86400000), timeZone), + to: dateInZone(new Date(), timeZone), + projectId: "", + supplierId: "", + userId: "", + timeZone, +}); +const minutes = (seconds: number | null | undefined) => + seconds == null ? "—" : `${(seconds / 60).toLocaleString("zh-CN", { maximumFractionDigits: 1 })} 分钟`; +const statusLabels: Record = { + approved: "通过", + rejected: "未通过", + pending: "待审核", + pending_quality: "待质检", + pending_acceptance: "待最终验收", + accepted: "已验收", + rework: "返工中", +}; +function Status({ value }: { value: string }) { + return value ? ( + {statusLabels[value] || "—"} + ) : ( + <>— + ); +} +function Pagination({ + label, + page, + onChange, + disabled, +}: { + label: string; + page: OutputPage; + onChange: (page: number, size: number) => void; + disabled: boolean; +}) { + const pages = Math.max(1, Math.ceil(page.total / page.pageSize)); + const numbers = + pages <= 7 + ? Array.from({ length: pages }, (_, i) => i + 1) + : [...new Set([1, page.page - 1, page.page, page.page + 1, pages])] + .filter((n) => n > 0 && n <= pages) + .sort((a, b) => a - b); + return ( + + ); +} +const emptyPage = { items: [], total: 0, page: 1, pageSize: 5 }; +export function AnnotationOutputDashboard({ + audience, + timeZone, +}: { + audience: "self" | "vendor" | "platform"; + timeZone: string; +}) { + const manager = audience !== "self"; + const [draft, setDraft] = useState(() => defaults(timeZone)); + const [query, setQuery] = useState(() => defaults(timeZone)); + const [result, setResult] = useState(null); + const [loading, setLoading] = useState(true), + [error, setError] = useState(""); + const [exporting, setExporting] = useState(false), + [refresh, setRefresh] = useState(0); + const [expandedSupplier, setExpandedSupplier] = useState(null); + const [task, setTask] = useState(null); + const sequence = useRef(0), + closeRef = useRef(null), + restoreFocus = useRef(null); + const key = JSON.stringify(query); + useEffect(() => { + setDraft(defaults(timeZone)); + setQuery(defaults(timeZone)); + setExpandedSupplier(null); + setTask(null); + }, [timeZone, audience]); + useEffect(() => { + const id = ++sequence.current, + controller = new AbortController(); + setLoading(true); + setError(""); + setResult(null); + annotationOutputApi + .get(query, controller.signal) + .then((data) => { + if (id === sequence.current) setResult(data); + }) + .catch((reason) => { + if (id === sequence.current && !controller.signal.aborted) + setError(getActionErrorMessage(reason, "看板加载失败,请重试")); + }) + .finally(() => { + if (id === sequence.current) setLoading(false); + }); + return () => { + sequence.current++; + controller.abort(); + }; + }, [key, refresh]); + useEffect(() => { + if (!task) return; + closeRef.current?.focus(); + const previous = document.body.style.overflow; + document.body.style.overflow = "hidden"; + const onKey = (event: KeyboardEvent) => { + if (event.key === "Escape") setTask(null); + if (event.key === "Tab") { + event.preventDefault(); + closeRef.current?.focus(); + } + }; + document.addEventListener("keydown", onKey); + return () => { + document.body.style.overflow = previous; + document.removeEventListener("keydown", onKey); + restoreFocus.current?.focus(); + }; + }, [task]); + const changePage = (name: "tasks" | "suppliers" | "days", page: number, size: number) => + setQuery((current) => ({ ...current, [`${name}Page`]: page, [`${name}PageSize`]: size })); + const apply = (next: OutputFilters) => { + setQuery({ ...next }); + setExpandedSupplier(null); + setTask(null); + }; + const exportRows = async () => { + if (exporting || !result) return; + setExporting(true); + setError(""); + try { + const blob = await annotationOutputApi.export({ + from: query.from, + to: query.to, + projectId: query.projectId, + supplierId: query.supplierId, + userId: query.userId, + timeZone: query.timeZone, + }); + const url = URL.createObjectURL(blob), + link = document.createElement("a"); + link.href = url; + link.download = `产出明细_${query.from}_${query.to}.csv`; + link.click(); + setTimeout(() => URL.revokeObjectURL(url), 1000); + } catch (reason) { + setError(getActionErrorMessage(reason, "导出失败,请重试")); + } finally { + setExporting(false); + } + }; + const daily = useMemo(() => result?.dailyOutput || emptyPage, [result]); + const openTask = (row: OutputTask, event: React.MouseEvent) => { + restoreFocus.current = event.currentTarget; + setTask(row); + }; + const focusPerson = (row: OutputDay) => { + const next = { ...query, supplierId: row.supplierId, userId: row.userId, tasksPage: 1, daysPage: 1 }; + setDraft((current) => ({ ...current, supplierId: row.supplierId, userId: row.userId })); + setQuery(next); + }; + return ( +
+

{audience === "self" ? "我的工作记录" : audience === "vendor" ? "供应商产出" : "生产总览"}

+
{ + event.preventDefault(); + if (!draft.from || !draft.to || draft.from > draft.to) { + setError("请选择有效日期范围"); + return; + } + apply(draft); + }} + > +
+ +
+ setDraft({ ...draft, from: e.target.value })} + /> + — + setDraft({ ...draft, to: e.target.value })} + /> +
+
+ {manager ? ( + <> +
+ + +
+ {audience === "platform" ? ( +
+ + +
+ ) : null} +
+ + +
+ + ) : null} + + + + {manager && (query.projectId || query.supplierId || query.userId) ? ( + + ) : null} +
+ {error ? ( +
+ {error} + +
+ ) : null} +
+ {( + [ + ["提交视频", "submitted"], + ["质检通过", "quality"], + ["最终验收通过", "acceptance"], + ["返工中", "rework"], + ] as const + ).map(([name, id]) => { + const metric = result?.totals[id]; + return ( +
+ {name} + {metric ? metric.count : "—"} + {minutes(metric?.seconds)} +
+ ); + })} + {!manager ? ( +
+ 可结算参考 + {minutes(result?.totals.acceptance.seconds)} +
+ ) : null} +
+ {loading ? ( +

+ 正在加载… +

+ ) : null} + {manager ? ( +
+

供应商总体产出

+
+ + + + {["供应商", "标注人数", "提交时长", "质检通过时长", "最终验收通过时长", "返工中时长", ""].map( + (h, i) => ( + + ), + )} + + + + {result?.supplierOutput.items.map((row) => ( + + + + {(["submitted", "quality", "acceptance", "rework"] as const).map((k) => ( + + ))} + + + ))} + +
{h}
{row.name || "—"}{row.memberCount}{minutes(row[k].seconds)} + +
+
+ {!loading && result?.supplierOutput.total === 0 ?

暂无数据

: null} + changePage("suppliers", p, s)} + /> +
+ ) : null} + {manager && expandedSupplier ? ( +
+
+

标注员每日及累计产出{expandedSupplier.name ? ` · ${expandedSupplier.name}` : ""}

+ +
+
+ + + + {["日期", "标注员", "当日提交", "当日验收通过", "累计提交", "累计验收通过", ""].map((h, i) => ( + + ))} + + + + {daily.items.map((row) => ( + + + + + + + + + + ))} + +
{h}
{row.date}{row.userName || "—"}{minutes(row.submitted.seconds)}{minutes(row.acceptance.seconds)}{minutes(row.cumulativeSubmitted.seconds)}{minutes(row.cumulativeAcceptance.seconds)} + +
+
+ {!loading && daily.total === 0 ?

暂无数据

: null} + changePage("days", p, s)} + /> +
+ ) : null} +
+
+

{manager ? "任务产出明细" : "我的任务明细"}

+ + {query.from} 至 {query.to} + +
+
+ + + + {["视频编号", "整段视频时长", "首次提交时间", "质检状态", "最终验收状态", "当前状态", ""].map( + (h, i) => ( + + ), + )} + + + + {result?.tasks.items.map((row) => ( + + + + + + + + + + ))} + +
{h}
{row.videoId}{minutes(row.seconds)}{formatCompactDateTime(row.submittedAt, timeZone, "—")} + + + + + + + +
+
+ {!loading && result?.tasks.total === 0 ?

暂无数据

: null} + changePage("tasks", p, s)} + /> +
+ {task ? ( +
{ + if (e.target === e.currentTarget) setTask(null); + }} + > +
+
+

任务详情

+ +
+

{task.videoId}

+
+
项目
+
{task.projectName || "—"}
+
供应商
+
{task.supplierName || "—"}
+
标注员
+
{task.userName || "—"}
+
视频时长
+
{minutes(task.seconds)}
+
当前状态
+
+ +
+
+

流转记录

+ {( + [ + ["首次成功送检", task.submittedAt, task.attribution?.submitted?.userName], + ["质检通过", task.qualityAt, task.attribution?.quality?.userName], + ["最终验收通过", task.acceptedAt, task.attribution?.acceptance?.userName], + ] as const + ).map(([name, at, author]) => ( +
+ {name} + +
+ ))} +
+
+ ) : null} +
+ ); +} +type OutputOptionForDrawer = { id: string; name: string }; diff --git a/src/pages/Dashboard.tsx b/src/pages/Dashboard.tsx index e5cb0a09..ee8a018c 100644 --- a/src/pages/Dashboard.tsx +++ b/src/pages/Dashboard.tsx @@ -1,3 +1,4 @@ +import { AnnotationOutputDashboard } from "../components/AnnotationOutputDashboard"; import { userErrorMessage } from "../lib/user-error-message.js"; import { WorkforceStatisticsPanel, type WorkforceFilters } from "../components/WorkforceStatisticsPanel"; import { @@ -74,6 +75,7 @@ const gateTone = (status?: string) => export default function Dashboard() { const navigate = useNavigate(); + const [legacyExpanded, setLegacyExpanded] = useState(false); const [summary, setSummary] = useState(null); const [workbench, setWorkbench] = useState(emptyWorkbench); const [loading, setLoading] = useState(true); @@ -159,10 +161,17 @@ export default function Dashboard() { }, []); useEffect(() => { + if ( + (currentUser?.role === "annotator" || + currentUser?.role === "supplier_manager" || + isOrganizationAdminRole(currentUser?.role)) && + !legacyExpanded + ) + return; loadSummary(); loadWorkbench(); if (isOrganizationAdminRole(currentUser?.role)) void loadServices(); - }, [currentUser?.role, loadSummary, loadWorkbench, loadServices]); + }, [currentUser?.role, loadSummary, loadWorkbench, loadServices, legacyExpanded]); const staleAuditRuns = workbench.waitingForAudit.filter( (run) => Date.now() - new Date(run.createdAt).getTime() >= 24 * 60 * 60 * 1000, @@ -187,6 +196,11 @@ export default function Dashboard() { }; const poc = summary?.poc; + if (currentUser?.role === "annotator" || currentUser?.role === "supplier_manager") { + return ( + + ); + } if (currentUser && !isOrganizationAdminRole(currentUser.role)) { return (
@@ -252,365 +266,386 @@ export default function Dashboard() { return (
-
-

Drobodata 仪表盘

-
- + +
setLegacyExpanded(event.currentTarget.open)}> + 生产进度与运行概览 + {legacyExpanded ? ( +
+
+

Drobodata 仪表盘

+
+ - {errorMessage && ( -
-
- - {userErrorMessage(errorMessage)} -
- -
- )} - - {loading && !summary && ( -
- - 正在加载真实执行概览... -
- )} - - {workbenchError && ( -
- {userErrorMessage(workbenchError)} -
- )} - -
-
-

- - 服务状态 -

-
- {[ - ["API", serviceStatus.api], - ["Worker", serviceStatus.worker], - ["Runner", serviceStatus.runner], - ].map(([label, ready]) => ( - - {label}: {ready ? "ready" : "offline"} - - ))} -
-
- {serviceStatus.commit || "version unknown"} -
-
-
-

活跃项目

-
{summary?.projects.active ?? "--"}
-
- {summary?.projects.archived ?? 0} 已归档项目 -
-
-
-

- 已注册算子 -

-
{summary?.operators.total ?? "--"}
-
- - {summary?.operators.builtin ?? 0} 当前内置 - - - {summary?.operators.custom ?? 0} 自定义 - - - {summary?.operators.historical ?? 0}{" "} - 历史记录 - -
-
-
-

- 今日 Job 执行数 -

-
{summary?.jobs.today ?? "--"}
-
- {summary?.jobs.successRate ?? 0}% 成功率 -
-
-
- -
- {[ - ["累计采集时长", `${((summary?.data.captureDurationSeconds || 0) / 3600).toFixed(1)} h`], - ["已入库视频", summary?.data.videoCount ?? 0], - ["成品数据包", summary?.data.packageCount ?? 0], - ["质量通过率", `${(summary?.tasks.qualityPassRate || 0).toFixed(1)}%`], - ].map(([label, value]) => ( -
-
{label}
-
{value}
-
- ))} -
- -
-
最近 7 日任务趋势
-
- {(summary?.trends || []).map((point) => { - const height = Math.max(8, Math.min(96, point.runCount * 12)); - return ( -
-
{point.runCount}
-
-
{point.date.slice(5)}
-
- ); - })} -
-
- 待标注 {summary?.tasks.pendingAnnotation || 0} · 标注中 {summary?.tasks.inAnnotation || 0} · 待质检{" "} - {summary?.tasks.pendingQuality || 0} · 待验收 {summary?.tasks.pendingAcceptance || 0} · 拒绝{" "} - {summary?.tasks.rejected || 0} · 升级 {summary?.tasks.escalated || 0} -
-
- - - -
-
-
-

任务工作台

-

- 优先处理正在运行、失败、过期待审核与最近产物。 -

-
- {isOrganizationAdminRole(currentUser?.role) && staleAuditRuns.length > 0 && ( - - )} -
-
- {[ - { - key: "running", - title: "运行中", - icon: Play, - items: workbench.running, - tone: "text-blue-600 dark:text-blue-300", - }, - { - key: "failed", - title: "失败待处理", - icon: XCircle, - items: workbench.failed, - tone: "text-red-600 dark:text-red-300", - }, - { - key: "audit", - title: "过期待审核", - icon: Clock3, - items: staleAuditRuns, - tone: "text-amber-600 dark:text-amber-300", - }, - ].map((section) => ( -
-
-
- - {section.title} + {errorMessage && ( +
+
+ + {userErrorMessage(errorMessage)}
- - {section.items.length} - +
-
- {section.items.map((run) => ( -
-
- + + 正在加载真实执行概览... +
+ )} + + {workbenchError && ( +
+ {userErrorMessage(workbenchError)} +
+ )} + +
+
+

+ + 服务状态 +

+
+ {[ + ["API", serviceStatus.api], + ["Worker", serviceStatus.worker], + ["Runner", serviceStatus.runner], + ].map(([label, ready]) => ( + + {label}: {ready ? "ready" : "offline"} + + ))} +
+
+ {serviceStatus.commit || "version unknown"} +
+
+
+

活跃项目

+
{summary?.projects.active ?? "--"}
+
+ {summary?.projects.archived ?? 0} 已归档项目 +
+
+
+

+ 已注册算子 +

+
+ {summary?.operators.total ?? "--"} +
+
+ + {summary?.operators.builtin ?? 0}{" "} + 当前内置 + + + {summary?.operators.custom ?? 0}{" "} + 自定义 + + + {summary?.operators.historical ?? 0}{" "} + 历史记录 + +
+
+
+

+ 今日 Job 执行数 +

+
{summary?.jobs.today ?? "--"}
+
+ {summary?.jobs.successRate ?? 0}%{" "} + 成功率 +
+
+
+ +
+ {[ + ["累计采集时长", `${((summary?.data.captureDurationSeconds || 0) / 3600).toFixed(1)} h`], + ["已入库视频", summary?.data.videoCount ?? 0], + ["成品数据包", summary?.data.packageCount ?? 0], + ["质量通过率", `${(summary?.tasks.qualityPassRate || 0).toFixed(1)}%`], + ].map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+ +
+
最近 7 日任务趋势
+
+ {(summary?.trends || []).map((point) => { + const height = Math.max(8, Math.min(96, point.runCount * 12)); + return ( +
+
{point.runCount}
+
- {run.mode === "production" ? ( - - - 正式 - - ) : ( - - - 调试 - +
+ {point.date.slice(5)} +
+
+ ); + })} +
+
+ 待标注 {summary?.tasks.pendingAnnotation || 0} · 标注中 {summary?.tasks.inAnnotation || 0} · 待质检{" "} + {summary?.tasks.pendingQuality || 0} · 待验收 {summary?.tasks.pendingAcceptance || 0} · 拒绝{" "} + {summary?.tasks.rejected || 0} · 升级 {summary?.tasks.escalated || 0} +
+
+ + + +
+
+
+

任务工作台

+

+ 优先处理正在运行、失败、过期待审核与最近产物。 +

+
+ {isOrganizationAdminRole(currentUser?.role) && staleAuditRuns.length > 0 && ( + + )} +
+
+ {[ + { + key: "running", + title: "运行中", + icon: Play, + items: workbench.running, + tone: "text-blue-600 dark:text-blue-300", + }, + { + key: "failed", + title: "失败待处理", + icon: XCircle, + items: workbench.failed, + tone: "text-red-600 dark:text-red-300", + }, + { + key: "audit", + title: "过期待审核", + icon: Clock3, + items: staleAuditRuns, + tone: "text-amber-600 dark:text-amber-300", + }, + ].map((section) => ( +
+
+
+ + {section.title} +
+ + {section.items.length} + +
+
+ {section.items.map((run) => ( +
+
+ + {run.mode === "production" ? ( + + + 正式 + + ) : ( + + + 调试 + + )} +
+ {run.progress && ( +
+ {run.progress.stage || "running"} + {typeof run.progress.step === "number" + ? ` · step ${run.progress.step}${run.progress.maxStep ? `/${run.progress.maxStep}` : ""}` + : ""} +
+ )} + {run.status === "failed" && (run.failure?.message || run.error) && ( +
+ {userErrorMessage(run.failure?.message || run.error)} +
+ )} +
+ + {formatCompactDateTime(run.updatedAt || run.createdAt, timeZone)} + + +
+ {section.key === "failed" && ( + + )} +
+ ))} + {section.items.length === 0 && ( +
暂无待处理项
)}
- {run.progress && ( -
- {run.progress.stage || "running"} - {typeof run.progress.step === "number" - ? ` · step ${run.progress.step}${run.progress.maxStep ? `/${run.progress.maxStep}` : ""}` - : ""} -
- )} - {run.status === "failed" && (run.failure?.message || run.error) && ( -
- {userErrorMessage(run.failure?.message || run.error)} -
- )} -
- - {formatCompactDateTime(run.updatedAt || run.createdAt, timeZone)} - - -
- {section.key === "failed" && ( - - )}
))} - {section.items.length === 0 && ( -
暂无待处理项
- )} -
-
- ))} -
-
-
- - 最近产物 -
- - {workbench.recentArtifacts.length} - -
-
- {workbench.recentArtifacts.map(({ run, artifact }) => ( - - ))} - {workbench.recentArtifacts.length === 0 && ( -
暂无近期产物
- )} -
-
-
-
- -
-
-

POC readiness

-

- 服务、GPU、业务验收与算子覆盖分开判断,不再用一个绿色状态混在一起。 -

-
-
- {readinessGates.map(({ title, gate }) => ( -
-
{title}
-
- {gate.status === "passed" || gate.status === "ready" ? ( - - ) : ( - - )} - {gate.label} +
+ {workbench.recentArtifacts.map(({ run, artifact }) => ( + + ))} + {workbench.recentArtifacts.length === 0 && ( +
暂无近期产物
+ )} +
+
-
{gate.message}
-
- ))} -
- {poc && ( -
-
- freshness: {poc.freshness || "unknown"} -
-
- source: {poc.sourceCommit || "unknown"} -
-
- catalog:{" "} - - {poc.catalogCount ?? poc.operatorRepositoryCount} - -
-
- completed:{" "} - - {poc.completedAt ? formatCompactDateTime(poc.completedAt, timeZone) : "unknown"} - -
+ + +
+
+

POC readiness

+

+ 服务、GPU、业务验收与算子覆盖分开判断,不再用一个绿色状态混在一起。 +

+
+
+ {readinessGates.map(({ title, gate }) => ( +
+
{title}
+
+ {gate.status === "passed" || gate.status === "ready" ? ( + + ) : ( + + )} + {gate.label} +
+
{gate.message}
+
+ ))} +
+ {poc && ( +
+
+ freshness:{" "} + {poc.freshness || "unknown"} +
+
+ source:{" "} + {poc.sourceCommit || "unknown"} +
+
+ catalog:{" "} + + {poc.catalogCount ?? poc.operatorRepositoryCount} + +
+
+ completed:{" "} + + {poc.completedAt ? formatCompactDateTime(poc.completedAt, timeZone) : "unknown"} + +
+
+ )} +
- )} - + ) : null} +
); } diff --git a/src/services/api/annotation-output.ts b/src/services/api/annotation-output.ts new file mode 100644 index 00000000..5ec40922 --- /dev/null +++ b/src/services/api/annotation-output.ts @@ -0,0 +1,95 @@ +import { apiFetch, apiRequest } from "./client"; + +export type OutputMetric = { count: number; seconds: number | null }; +export type OutputTotals = Record<"submitted" | "quality" | "acceptance" | "rework", OutputMetric>; +export type OutputOption = { id: string; name: string }; +export type OutputTask = { + attribution: Partial< + Record< + "submitted" | "quality" | "acceptance", + { userId: string; userName: string; supplierId: string; supplierName: string } + > + >; + id: string; + videoId: string; + projectId: string; + projectName: string; + userId: string; + userName: string; + supplierId: string; + supplierName: string; + seconds: number | null; + submittedAt: string; + qualityAt: string; + acceptedAt: string; + qualityStatus: string; + acceptanceStatus: string; + status: string; + evidence: { id: string; taskId: string; stage: string; decision: string; at: string; revision: number }[] | null; +}; +export type OutputSupplier = OutputTotals & { id: string; name: string; memberCount: number }; +export type OutputDay = { + date: string; + userId: string; + userName: string; + supplierId: string; + supplierName: string; + submitted: OutputMetric; + acceptance: OutputMetric; + cumulativeSubmitted: OutputMetric; + cumulativeAcceptance: OutputMetric; +}; +export type OutputPage = { items: T[]; total: number; page: number; pageSize: number }; +export type AnnotationOutput = { + asOf: string; + audienceScope: "platform" | "vendor" | "self"; + from: string; + to: string; + timeZone: string; + totals: OutputTotals; + projects: OutputOption[]; + suppliers: OutputOption[]; + users: OutputOption[]; + tasks: OutputPage; + supplierOutput: OutputPage; + dailyOutput: OutputPage; +}; +export type OutputFilters = { + from: string; + to: string; + projectId: string; + supplierId: string; + userId: string; + timeZone: string; +}; +export type OutputRequest = OutputFilters & { + tasksPage?: number; + tasksPageSize?: number; + suppliersPage?: number; + suppliersPageSize?: number; + daysPage?: number; + daysPageSize?: number; + dailySupplierId?: string; + dailyUserId?: string; +}; +const search = (query: OutputRequest) => { + const params = new URLSearchParams(); + Object.entries(query).forEach(([key, value]) => { + if (value !== "" && value !== undefined) params.set(key, String(value)); + }); + return params; +}; +export const annotationOutputApi = { + get: (query: OutputRequest, signal?: AbortSignal) => + apiRequest(`/api/dashboard/annotation-output?${search(query)}`, { signal }), + export: async (query: OutputFilters) => { + const params = search(query); + params.set("format", "csv"); + const response = await apiFetch(`/api/dashboard/annotation-output?${params}`); + if (!response.ok) { + const body = await response.json().catch(() => null); + throw new Error(body?.error || "导出失败,请重试"); + } + return response.blob(); + }, +}; diff --git a/src/styles/annotation-output.css b/src/styles/annotation-output.css new file mode 100644 index 00000000..15aa7116 --- /dev/null +++ b/src/styles/annotation-output.css @@ -0,0 +1,312 @@ +.annotation-output { + color: var(--app-text); + display: flex; + flex-direction: column; + gap: 20px; +} +.annotation-output h1 { + font-size: 28px; + font-weight: 700; + line-height: 1.3; + margin: 0 0 4px; +} +.annotation-output h2 { + font-size: 19px; + font-weight: 650; + margin: 0; +} +.annotation-output h3 { + font-size: 16px; + font-weight: 600; + margin: 22px 0; +} +.annotation-output button, +.annotation-output input, +.annotation-output select { + font: inherit; + border: 1px solid var(--app-border); + background: var(--app-panel); + color: var(--app-text); + border-radius: 8px; + padding: 9px 13px; + font-size: 13px; +} +.annotation-output button { + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 7px; +} +.annotation-output button:hover:enabled { + border-color: #fb923c; +} +.annotation-output button:disabled { + opacity: 0.4; + cursor: default; +} +.annotation-output :focus-visible { + outline: 2px solid #fb923c; + outline-offset: 2px; +} +.annotation-output .annotation-output-primary { + background: #ff780a; + color: white; + border-color: #ff780a; +} +.annotation-output-filters, +.annotation-output-panel, +.annotation-output-card { + background: var(--app-panel); + border: 1px solid var(--app-border); + border-radius: 12px; +} +.annotation-output-filters { + display: flex; + flex-wrap: wrap; + gap: 10px; + align-items: end; + padding: 20px; +} +.annotation-output-field { + display: flex; + flex-direction: column; + gap: 8px; + min-width: 120px; +} +.annotation-output-field label { + font-size: 12px; + opacity: 0.65; +} +.annotation-output-date { + display: flex; + align-items: center; + gap: 8px; +} +.annotation-output-date input { + width: 152px; +} +.annotation-output-cards { + display: grid; + grid-template-columns: repeat(5, minmax(0, 1fr)); + gap: 12px; +} +.annotation-output-cards-manager { + grid-template-columns: repeat(4, minmax(0, 1fr)); +} +.annotation-output-card { + display: flex; + flex-direction: column; + padding: 20px 18px; + min-height: 127px; +} +.annotation-output-card > span { + font-size: 14px; + opacity: 0.65; +} +.annotation-output-card > strong { + font-size: 27px; + letter-spacing: -0.5px; + margin-top: 16px; +} +.annotation-output-card > small { + font-size: 12px; + opacity: 0.55; + margin-top: 6px; +} +.annotation-output-panel { + padding: 20px; + min-width: 0; +} +.annotation-output-panel > h2 { + margin-bottom: 18px; +} +.annotation-output-panel header { + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; + margin-bottom: 18px; +} +.annotation-output-panel header > span { + font-size: 12px; + opacity: 0.5; +} +.annotation-output-scroll { + overflow-x: auto; +} +.annotation-output table { + border-collapse: collapse; + width: 100%; + text-align: left; + white-space: nowrap; + font-size: 13px; +} +.annotation-output th { + background: var(--app-inset); + padding: 13px 12px; + font-weight: 500; + opacity: 0.7; +} +.annotation-output td { + padding: 15px 12px; + border-bottom: 1px solid var(--app-border); +} +.annotation-output tr:last-child td { + border: 0; +} +.annotation-output .annotation-output-link { + background: none; + border: 0; + padding: 0; + color: #ed791b; + border-radius: 3px; +} +.annotation-output-pagination { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + gap: 12px; + align-items: center; + margin-top: 18px; + padding-top: 15px; + border-top: 1px solid var(--app-border); + font-size: 12px; +} +.annotation-output-pagination > span { + opacity: 0.55; +} +.annotation-output-pagination > div { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; +} +.annotation-output-pagination button, +.annotation-output-pagination select { + font-size: 12px; + padding: 6px 9px; + min-width: 30px; +} +.annotation-output-pagination button[aria-current="page"] { + background: #ff780a; + border-color: #ff780a; + color: white; +} +.annotation-output-empty { + text-align: center; + padding: 30px; + opacity: 0.5; + font-size: 13px; +} +.annotation-output-badge { + display: inline-block; + border-radius: 18px; + padding: 4px 9px; + font-size: 12px; + background: #e8f1ff; + color: #4678be; +} +.annotation-output-badge-approved, +.annotation-output-badge-accepted { + background: #e5f8e8; + color: #358347; +} +.annotation-output-badge-rework { + background: #fff0e3; + color: #c7762b; +} +.annotation-output-badge-rejected { + background: #f4eded; + color: #a66060; +} +.annotation-output-loading { + font-size: 13px; + opacity: 0.6; +} +.annotation-output-error { + display: flex; + align-items: center; + gap: 15px; + color: #c45342; + font-size: 13px; +} +.annotation-output-backdrop { + position: fixed; + inset: 0; + background: #17243b55; + z-index: 80; + display: flex; + justify-content: flex-end; +} +.annotation-output-drawer { + width: min(530px, 95vw); + height: 100%; + overflow: auto; + padding: 28px; + background: var(--app-panel); + box-shadow: -12px 0 35px #0002; +} +.annotation-output-drawer header { + display: flex; + align-items: center; + justify-content: space-between; +} +.annotation-output-drawer dl { + display: grid; + grid-template-columns: 100px 1fr; + gap: 16px; + font-size: 13px; +} +.annotation-output-drawer dt { + opacity: 0.55; +} +.annotation-output-drawer dd { + margin: 0; + overflow-wrap: anywhere; +} +.annotation-output-event { + border-left: 2px solid var(--app-border); + padding: 0 0 20px 16px; + display: flex; + flex-direction: column; + gap: 7px; + font-size: 13px; +} +.annotation-output-event time { + opacity: 0.55; +} +@media (max-width: 1150px) { + .annotation-output-cards { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } + .annotation-output-cards-manager { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} +@media (max-width: 640px) { + .annotation-output h1 { + font-size: 24px; + } + .annotation-output-cards { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .annotation-output-filters, + .annotation-output-panel { + padding: 14px; + } + .annotation-output-date input { + width: 132px; + padding: 8px; + } + .annotation-output-card { + padding: 16px 13px; + } + .annotation-output-card > strong { + font-size: 23px; + } + .annotation-output-panel header { + align-items: start; + flex-wrap: wrap; + } +} diff --git a/tests/browser/annotation-output-replica.mjs b/tests/browser/annotation-output-replica.mjs new file mode 100644 index 00000000..bdbb1d88 --- /dev/null +++ b/tests/browser/annotation-output-replica.mjs @@ -0,0 +1,113 @@ +// Real HTTP router + isolated local Mongo. Start the documented replica server +// and Vite first; this test never mocks a successful statistics response. +import { chromium } from "playwright"; +import fs from "node:fs"; +import path from "node:path"; +import assert from "node:assert/strict"; +const out = path.resolve("local-data/verification/customer-dashboard-20261020"); +const accounts = JSON.parse(fs.readFileSync(path.join(out, "accounts.json"), "utf8")); +assert.match(accounts.database, /^annotation_output_browser_\d+$/); +assert.equal(accounts.origin, "http://127.0.0.1:5196"); +const browser = await chromium.launch({ + executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + headless: true, + args: ["--no-proxy-server"], +}); +const report = { database: accounts.database, roles: [], failures: [] }; +try { + for (const id of ["admin", "u1", "manager"]) { + const a = accounts.accounts.find((x) => x.id === id), + ctx = await browser.newContext({ + viewport: { width: 1500, height: 1100 }, + acceptDownloads: true, + serviceWorkers: "block", + }); + const row = { id, pageErrors: [], unexpectedWrites: [], checks: [] }; + report.roles.push(row); + try { + console.log("Role", id, "started"); + const login = await ctx.request.post(accounts.origin + "/api/auth/login", { + data: { phone: a.phone, password: a.password }, + }); + assert.equal(login.status(), 200, "local login"); + await login.dispose(); + const page = await ctx.newPage(); + page.on("pageerror", (e) => row.pageErrors.push(e.message)); + page.on("request", (r) => { + if (!["GET", "HEAD", "OPTIONS"].includes(r.method()) && !new URL(r.url()).pathname.startsWith("/api/auth/")) + row.unexpectedWrites.push(new URL(r.url()).pathname); + }); + await page.goto(accounts.origin + "/?view=dashboard", { waitUntil: "domcontentloaded" }); + console.log("Role", id, "page loaded"); + const dashboard = page.getByRole("region", { name: id === "u1" ? "我的工作记录" : "生产产出看板", exact: true }); + await dashboard.waitFor(); + const tasks = dashboard.getByRole("region", { name: "任务明细", exact: true }); + await tasks.locator("tbody tr").first().waitFor({ timeout: 30000 }); + assert.equal(await dashboard.getByText(/个 part|待核对/).count(), 0); + row.checks.push("no internal terminology"); + const firstVideo = await tasks.locator("tbody tr").first().locator("td").first().innerText(); + const cardNumbers = await dashboard.locator(".annotation-output-card strong").allTextContents(); + const pager = tasks.getByRole("navigation", { name: "任务明细分页" }); + await pager.getByRole("button", { name: "下一页", exact: true }).click(); + await tasks.getByText(/第 6–/).waitFor(); + assert.notEqual(await tasks.locator("tbody tr").first().locator("td").first().innerText(), firstVideo); + assert.deepEqual(await dashboard.locator(".annotation-output-card strong").allTextContents(), cardNumbers); + row.checks.push("pagination preserves totals"); + const expectedRows = Number((await pager.locator("span").first().innerText()).match(/共 (\d+)/)[1]); + const downloaded = page.waitForEvent("download"); + await dashboard.getByRole("button", { name: "导出明细", exact: true }).click(); + const download = await downloaded; + const file = await download.path(), + csv = fs.readFileSync(file, "utf8"); + assert.equal(csv.trim().split(/\r?\n/).length, expectedRows + 1); + row.checks.push("CSV exports whole filtered scope from page 2"); + await pager.getByRole("combobox").selectOption("10"); + await page.waitForFunction( + () => + document.querySelector('[aria-label="任务明细每页条数"]')?.value === "10" && + document.querySelector('[aria-label="任务明细分页"]')?.textContent.includes("第 1–"), + ); + row.checks.push("page size resets page"); + await tasks.getByRole("button", { name: "查看详情" }).first().click(); + const dialog = page.getByRole("dialog", { name: "任务详情" }); + await dialog.waitFor(); + await page.keyboard.press("Escape"); + await dialog.waitFor({ state: "hidden" }); + row.checks.push("read-only detail and Escape close"); + if (id !== "u1") { + const vendors = dashboard.getByRole("region", { name: "供应商总体产出", exact: true }); + await vendors.getByRole("button", { name: "查看人员" }).first().click(); + const daily = dashboard.getByRole("region", { name: "标注员每日及累计产出", exact: true }); + await daily.locator("tbody tr").first().waitFor(); + await daily.getByRole("navigation", { name: "人员每日产出分页" }).waitFor(); + row.checks.push("supplier/member/daily drilldown"); + await dashboard.getByLabel("项目", { exact: true }).selectOption("p1"); + await dashboard.getByRole("button", { name: "查询", exact: true }).click(); + await tasks.locator("tbody tr").first().waitFor(); + assert.match(await tasks.getByRole("navigation").innerText(), /第 1–/); + row.checks.push("project filtering resets pagination"); + if (id === "admin") { + assert.equal(await dashboard.getByLabel("项目", { exact: true }).locator("option").count(), 3); + await dashboard.getByLabel("项目", { exact: true }).selectOption("p2"); + await dashboard.getByRole("button", { name: "查询", exact: true }).click(); + await tasks.locator("tbody tr").first().waitFor(); + row.checks.push("switch between projects without reset"); + } + } + await page.screenshot({ path: path.join(out, `${id}-implementation.png`), fullPage: true }); + assert.deepEqual(row.pageErrors, []); + assert.deepEqual(row.unexpectedWrites, []); + await page.close(); + } catch (e) { + row.error = e.message; + report.failures.push({ id, error: e.message }); + } finally { + await ctx.close(); + fs.writeFileSync(path.join(out, "browser-report.json"), JSON.stringify(report, null, 2)); + } + } + assert.equal(report.failures.length, 0, JSON.stringify(report.failures)); + console.log(JSON.stringify(report)); +} finally { + await browser.close(); +} diff --git a/tests/browser/support/annotation-output-replica/main.go b/tests/browser/support/annotation-output-replica/main.go new file mode 100644 index 00000000..80a392a3 --- /dev/null +++ b/tests/browser/support/annotation-output-replica/main.go @@ -0,0 +1,109 @@ +// Local browser fixture server. Hard-coded loopback Mongo; a fresh synthetic +// database per run. It never connects to the snapshot or a company cluster. +package main + +import ( + "context" + "crypto/rand" + "droboticflow/internal/api" + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/store" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/crypto/bcrypt" + "log" + "net/http" + "os" + "path/filepath" + "time" +) + +func main() { + accountFile := flag.String("accounts", "local-data/verification/customer-dashboard-20261020/accounts.json", "untracked credentials output") + flag.Parse() + if filepath.Base(filepath.Dir(*accountFile)) != "customer-dashboard-20261020" { + log.Fatal("use the isolated verification directory") + } + ctx := context.Background() + client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018")) + must(err) + db := client.Database(fmt.Sprintf("annotation_output_browser_%d", time.Now().UnixNano())) + s := store.NewMongoStore(db) + org := "output-browser-org" + must(s.UpsertOrganization(ctx, domain.OrganizationRecord{ID: org, Name: "看板本地验收", Slug: org, Status: domain.OrganizationStatusActive})) + for i := 1; i <= 2; i++ { + must(s.UpsertSupplier(ctx, domain.SupplierRecord{ID: fmt.Sprintf("v%d", i), OrganizationID: org, Name: fmt.Sprintf("示例供应商%d", i), Status: "active"})) + must(s.UpsertProject(ctx, domain.ProjectRecord{ID: fmt.Sprintf("p%d", i), OrganizationID: org, Name: fmt.Sprintf("示例项目%d", i)})) + } + accounts := []map[string]string{} + for _, a := range []struct { + id, name, supplier string + role domain.UserRole + }{{"admin", "本地管理员", "", domain.UserRoleAdmin}, {"u1", "示例张三", "v1", domain.UserRoleAnnotator}, {"u2", "示例李四", "v1", domain.UserRoleAnnotator}, {"u3", "示例王五", "v2", domain.UserRoleAnnotator}, {"manager", "本地供应商管理员", "v1", domain.UserRoleSupplierManager}} { + b := make([]byte, 18) + _, err = rand.Read(b) + must(err) + password := hex.EncodeToString(b) + hash, e := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost) + must(e) + phone := "local-output-" + a.id + must(s.UpsertUser(ctx, domain.UserRecord{ID: a.id, OrganizationID: org, Name: a.name, Phone: phone, Role: a.role, SupplierID: a.supplier, Status: domain.UserStatusActive, PasswordHash: string(hash)})) + accounts = append(accounts, map[string]string{"id": a.id, "role": string(a.role), "phone": phone, "password": password}) + } + _, err = s.SaveProductionStatsScope(ctx, org, productionstats.Scope{ProjectIDs: []string{"p1", "p2"}, Revision: 1}, 0) + must(err) + for i := 0; i < 25; i++ { + author, supplier, project := "u1", "v1", "p1" + if i%3 == 1 { + author = "u2" + } + if i%3 == 2 { + author, supplier, project = "u3", "v2", "p2" + } + run, task, item, capture := fmt.Sprintf("run-%02d", i), fmt.Sprintf("task-%02d", i), fmt.Sprintf("item-%02d", i), fmt.Sprintf("capture-%02d", i) + at := time.Now().UTC().Add(-time.Duration(2+i%7) * 24 * time.Hour).Truncate(time.Hour) + submittedAt := at + if i == 24 { + submittedAt = at.Add(-40 * 24 * time.Hour) + } + must(s.UpsertRun(ctx, domain.RunRecord{ID: run, OrganizationID: org, ProjectID: project, TaskID: capture, Mode: domain.RunModeProduction, CreatedAt: submittedAt.Add(-time.Hour).Format(time.RFC3339)})) + must(s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: capture, OrganizationID: org, ProjectID: project, RunID: run, ConnectionID: "local", Bucket: "fixture", Prefix: fmt.Sprintf("DEMO_%02d/part0001", i), CaptureID: fmt.Sprintf("DEMO_%02d/part0001", i)})) + must(s.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: task, OrganizationID: org, ProjectID: project, RunID: run, SourceIsFullVideo: true, SourceDurationSeconds: 900 + float64(i*10), CurrentRevision: 1, CreatedAt: submittedAt.Format(time.RFC3339)})) + must(s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: item, OrganizationID: org, ProjectID: project, RunID: run, LegacyTaskID: task, Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("DEMO_%02d/part0001", i)}})) + must(s.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: task + "-revision", OrganizationID: org, TaskID: task, Revision: 1, AuthorUserID: author, CountsAsOutput: true})) + must(s.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: task + "-submitted", OrganizationID: org, ProjectID: project, TaskID: task, UserID: author, SupplierID: supplier, EventType: "combined_submitted", Revision: 1, CreatedAt: submittedAt.Format(time.RFC3339)})) + if i%4 != 3 { + decision := domain.AnnotationReviewApproved + if i%4 == 2 { + decision = domain.AnnotationReviewRejected + } + must(s.UpsertAnnotationReview(ctx, domain.AnnotationReviewRecord{ID: task + "-quality", OrganizationID: org, TaskID: task, Revision: 1, Stage: domain.AnnotationReviewStageQuality, Decision: decision, CreatedAt: at.Add(time.Hour).Format(time.RFC3339)})) + } + if i%4 == 0 { + _, e := s.CompareAndSwapReviewOperation(ctx, domain.ReviewOperation{ID: task + "-accepted", OrganizationID: org, ProjectID: project, StageID: "prepackage_acceptance", SourceItemIDs: []string{item}, Decision: "approved", Status: "completed", CreatedAt: at.Add(2 * time.Hour).Format(time.RFC3339), CompletedAt: at.Add(2 * time.Hour).Format(time.RFC3339)}, 0) + must(e) + } + } + must(os.MkdirAll(filepath.Dir(*accountFile), 0700)) + payload, e := json.Marshal(map[string]any{"origin": "http://127.0.0.1:5196", "database": db.Name(), "accounts": accounts}) + must(e) + must(os.WriteFile(*accountFile, payload, 0600)) + must(os.Setenv("MONGO_URI", "mongodb://127.0.0.1:27018")) + must(os.Setenv("MONGO_DB", db.Name())) + must(os.Setenv("PORT", "3026")) + must(os.Setenv("DROBOTICFLOW_REQUIRE_SESSION_AUTH", "true")) + must(os.Setenv("DROBOTICFLOW_ENABLE_LEGACY_ORGANIZATION_ADMIN", "true")) + must(os.Setenv("ANNOTATION_PROCESSOR_MODE", "external")) + log.Println("Local dashboard fixture ready on 127.0.0.1:3026; isolated database", db.Name()) + must(http.ListenAndServe("127.0.0.1:3026", api.NewHandler(s, nil))) +} +func must(err error) { + if err != nil { + log.Fatal(err) + } +} From 962ea6da3d6932254e1bae293188128db8cbc6ce Mon Sep 17 00:00:00 2001 From: d-robotics Date: Tue, 20 Oct 2026 22:47:45 +0800 Subject: [PATCH 02/54] =?UTF-8?q?=E2=9A=A1=20Share=20bounded=20dashboard?= =?UTF-8?q?=20snapshots=20across=20API=20instances?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/customer-dashboard-20261020/spec.md | 10 +- .scratch/dashboard-snapshots-20261020/spec.md | 28 ++ .../dashboard-snapshots-20261020/report.md | 60 ++++ internal/api/annotation_output.go | 35 ++- .../annotation_output_snapshot_scale_test.go | 120 ++++++++ .../api/annotation_output_snapshot_test.go | 162 ++++++++++ internal/api/annotation_output_test.go | 4 +- internal/api/database_browser.go | 4 + internal/api/server.go | 3 + internal/outputsnapshot/memory.go | 96 ++++++ internal/outputsnapshot/reader.go | 286 ++++++++++++++++++ internal/outputsnapshot/reader_test.go | 269 ++++++++++++++++ internal/outputsnapshot/types.go | 126 ++++++++ internal/store/annotation_output.go | 58 +++- internal/store/annotation_output_snapshots.go | 117 +++++++ .../store/annotation_output_snapshots_test.go | 167 ++++++++++ internal/store/memory.go | 8 + internal/store/mongo.go | 7 + internal/store/store.go | 2 + internal/workforcestats/output.go | 13 +- internal/workforcestats/output_dataset.go | 70 +++++ .../workforcestats/output_dataset_test.go | 45 +++ src/components/AnnotationOutputDashboard.tsx | 50 ++- src/services/api/annotation-output.ts | 39 ++- tests/browser/annotation-output-replica.mjs | 82 ++++- 25 files changed, 1826 insertions(+), 35 deletions(-) create mode 100644 .scratch/dashboard-snapshots-20261020/spec.md create mode 100644 docs/verification/dashboard-snapshots-20261020/report.md create mode 100644 internal/api/annotation_output_snapshot_scale_test.go create mode 100644 internal/api/annotation_output_snapshot_test.go create mode 100644 internal/outputsnapshot/memory.go create mode 100644 internal/outputsnapshot/reader.go create mode 100644 internal/outputsnapshot/reader_test.go create mode 100644 internal/outputsnapshot/types.go create mode 100644 internal/store/annotation_output_snapshots.go create mode 100644 internal/store/annotation_output_snapshots_test.go create mode 100644 internal/workforcestats/output_dataset.go create mode 100644 internal/workforcestats/output_dataset_test.go diff --git a/.scratch/customer-dashboard-20261020/spec.md b/.scratch/customer-dashboard-20261020/spec.md index f54fe679..3ca6364b 100644 --- a/.scratch/customer-dashboard-20261020/spec.md +++ b/.scratch/customer-dashboard-20261020/spec.md @@ -43,7 +43,15 @@ Status: resolved - Mongo 仅投影读取已有事实;统计源使用正式生产范围配置,不添加客户数据、不回填历史、不改变交付 JSON。时长只使用明确的整段视频源时长,不使用保留切片时长或按 15 分钟估算。 - 个人与管理员/供应商管理员接入真实 React 页面;旧的源采集生产进度/运行概览保留在管理员折叠区;质检与验收角色原有工作看板保留。 - 已验证:真实 router 与隔离合成 Mongo 的三角色浏览器流程;权限、返工/重试去重、跨期、冻结归属、缺失时长、207 条分页与完整导出;前端/Go 既有失败基线无新增;生产前端启动与 Go 构建通过。 -- 当前本地预览使用独立合成数据。生产历史完整性和数据规模下的性能尚未线上核实;历史补录与公司集群发布均未执行。 +- 上述初版已按用户授权发布为 `65c19de0`,线上三角色只读读取通过;看板请求实测约 13.7–14.7 秒。生产管理员/供应商管理员专用账号验收和历史完整性覆盖仍未完成,未执行历史补录。 + +## 性能优化:共享按需快照(本地完成,未发布) + +用户选择方案 2 后,改为所有 API 副本复用按组织/正式项目范围隔离的 part 统计事实,日期、人员和分页请求不再重复读取完整历史。访问触发刷新,默认间隔 120 秒;保留完整可用版本,失败退避,超过 15 分钟不再继续提供陈旧结果。卡片、明细和 CSV 固定同一快照版本,失效后整页恢复;保持确认版式。 + +每次请求仍重新校验当前身份和正式项目范围。业务记录、历史补录、交付 JSON 不变。该 GET 在缺少快照或刷新时会写入两个内部派生统计集合,不能再描述为服务端零写入;普通数据库浏览/写入入口禁止访问这些集合。原始审核事件保留在业务存储,快照不复制事件列表,报表详情提供阶段时间与归属。 + +本地验证与资源限制见 [`共享快照验证报告`](../../docs/verification/dashboard-snapshots-20261020/report.md),实现契约见 [`共享按需看板快照`](../dashboard-snapshots-20261020/spec.md)。本次优化没有执行公司镜像上传或部署。 验证记录:[`docs/verification/customer-dashboard-20261020/report.md`](../../docs/verification/customer-dashboard-20261020/report.md)。 diff --git a/.scratch/dashboard-snapshots-20261020/spec.md b/.scratch/dashboard-snapshots-20261020/spec.md new file mode 100644 index 00000000..f0d8ae46 --- /dev/null +++ b/.scratch/dashboard-snapshots-20261020/spec.md @@ -0,0 +1,28 @@ +# 共享按需看板快照 + +Status: resolved + +2026-10-20 用户选择方案 2,授权本地实现。线上基线三次读取为 13.7–14.7 秒,分页在读历史/重算之后执行。当前不执行生产派生记录写入或新版本发布。 + +## 契约 + +- 保留 part、整段时长、正式阶段时间、冻结作者/供应商、跨期累计、返工快照、空值和服务端权限规则。 +- 共享已解释的 part 统计事实,按组织/正式项目范围/算法版本区分。所有 API 副本通过 Mongo 复用同一代快照,不按人员、日期或页码重复读历史。 +- 请求时校验当前身份与范围;首次缺快照返回构建中,后台生成。有完整快照时立即读取,过期按需触发一个后台构建;无人访问不刷新。刷新默认间隔 120 秒,失败重试退避 120 秒,15 分钟后不把陈旧结果继续当作可用结果。 +- 全局一个有效重建租约,续租与发布按 owner/epoch 条件更新。失效工作者不能覆盖新结果。崩溃/重启后可回收过期租约。快照完整生成、校验成功后才原子替换可见指针。 +- 独立内部统计集合,不写业务记录、客户描述、媒体或交付 JSON;不把原始审核历史复制到统计快照。保留原有事实读取与纯统计算法作重建来源。 +- 固定 16 个存储槽,每槽压缩载荷最多 4 MiB;总压缩载荷最多 64 MiB。每个范围保留当前和前一代,最多 8 个范围;有限复用槽位,不能无限积累历史代次。解压限制 32 MiB、重建源读取限制 64 MiB/250000 文档、同时至多两条来源查询、重建 90 秒超时。字节预算不冒充进程 RSS 硬上限。 +- 卡片/明细/导出按 snapshotId 固定同一代;过期或被回收的 token 明确返回冲突,整页切到新版本,不能混用两代或静默重算导出。 +- 页面保留确认版式,不加口径长提示;冷启动只显示正常加载状态并有界轮询,取消切换停止轮询。普通刷新不绕过全局限频。 + +## 验证 + +真实 router 多 handler/分页共用一次历史重建的回归;Memory/Mongo 两个 Adapter;跨 API 并发、过期刷新、失败保留、租约回收、失效 owner 发布拒绝、槽位/字节上限、固定版本导出、权限变更与范围变化;隔离合成 Mongo 三角色浏览器与有规模的计时/扫描次数;按既有前端/Go 基线门禁核对新增失败。 + +## 实现结果 + +已接入真实 API、Mongo/Memory Adapter 和正式看板。统计结果按版本分页、下钻及导出,旧客户端继续获得完整 200 响应;新客户端冷启动采用可取消的构建中轮询。内部集合 `annotationOutputSnapshotState`、`annotationOutputSnapshotSlots` 不通过普通数据库浏览/写入接口开放。 + +本地 5000 part、25000 条历史事件、4 个 API 实例及 32 次热请求:历史重建 1 次,首次 224 ms,热请求 P50 15 ms / P95 17 ms。该数据来自隔离合成 Mongo,不是生产性能承诺。 + +验证和兼容边界见 [`report.md`](../../docs/verification/dashboard-snapshots-20261020/report.md)。本次只完成本地实现,尚未发布;上线会新增内部派生集合写入,应在具体发布计划中明确列出。 diff --git a/docs/verification/dashboard-snapshots-20261020/report.md b/docs/verification/dashboard-snapshots-20261020/report.md new file mode 100644 index 00000000..95b59933 --- /dev/null +++ b/docs/verification/dashboard-snapshots-20261020/report.md @@ -0,0 +1,60 @@ +# 共享按需看板快照:本地验证 + +日期:2026-10-20。分支 `codex/customer-dashboard-20261020`,基于已发布的 `65c19de0bf402cd8202714ebbfd8a05551cd5a25`。用户选择方案 2,本次完成本地实现,没有上传镜像或写入公司环境。 + +## 结果 + +原路由每次加载并解释业务历史,之后才分页;已发布版三次读取实测 14275、14703、13703 ms。本版共享经过解释的 part 统计事实,API 副本、日期、人员和页码复用同一代数据;请求仍按当前身份和正式项目范围过滤。旧缓存因其他 POST 失效时不再连带重建本看板。 + +后台重建由访问触发,默认间隔 120 秒,无访问不刷新。构建成功并校验后才替换可见指针;失败保留此前完整版本并退避 120 秒。超过 15 分钟不再提供旧结果,返回正常加载/暂不可用状态,不用空统计冒充成功。卡片、三个分页表、人员下钻和全量 CSV 使用同一 snapshotId;被回收、过旧或范围变化的 token 返回 409,页面重置分页后整体加载新版本。 + +产品页面沿用已确认版式,不新增口径长提示、资源参数或调试信息。初次构建显示正常加载状态,可取消轮询;已部署旧客户端没有请求 shared 模式时,由服务器等待同一共享构建并返回原有 200 结构,避免 202 破坏旧页面。 + +## 存储与兼容边界 + +新增内部集合 `annotationOutputSnapshotState` 和 `annotationOutputSnapshotSlots`,分别保存小型全局协调记录及压缩后的不可变统计事实;只依赖现有 Mongo `_id` 索引,不增加启动迁移。普通数据库概览、读取和写入入口拒绝这两个集合,避免全组织快照绕过报表权限。 + +快照只含 part 标识、项目、明确整段时长、状态、阶段时间及冻结作者/供应商,并含显示名称字典;不复制描述正文、草稿、媒体、凭证或原始事件列表。原始审核和工作日志保留不动。响应字段兼容既有客户端并新增 snapshot 元数据;原始 evidence 列表不复制到缓存报表,详情继续显示阶段时间与归属。保留原有事实读取和纯统计计算作为重建路径。 + +本接口对业务数据只读,但 GET 可能产生内部派生统计写入。后续公司发布计划必须明确列出这两个集合及其写入范围,不能沿用“零写入验收”的描述。生产统计范围变化使旧 token 失效,角色与供应商变更按每次请求重新授权。 + +业务文档、任务状态、工作流、交付 JSON 均无需迁移。回退到 65c19de0 不依赖这些快照,但旧版数据库浏览器不认识这两个保留名称;若以后回退,需在回退计划中单独处理遗留内部集合的可见性。本次没有执行任何生产清理。 + +## 资源和故障边界 + +| 项目 | 实现限制 | +| --- | --- | +| 历史重建 | 每个数据库一个有效全局租约;owner + epoch 条件更新,60 秒租期、10 秒心跳、90 秒构建超时。 | +| 历史读取 | 最多两条并行来源查询,每条 Mongo 查询 maxTime 15 秒;跨集合累计投影 BSON 最多 64 MiB、250000 文档,超预算终止,不发布部分结果。 | +| 共享存储 | 最多 8 个范围,16 个固定槽;单槽压缩载荷最多 4 MiB,总压缩载荷最多 64 MiB。通常保留当前/前一代;满槽时可提前回收前一代,旧 token 明确失效。 | +| API 本机 | 压缩缓存最多 8 MiB / 16 项;每个 Reader 最多两次完整查询并发,解压载荷最多 32 MiB。 | +| 故障恢复 | 重建不依赖发起请求存活;失败退避,重启后读取共享结果,过期租约可接管。旧 owner 不能发布覆盖新代或释放新 owner 的租约。压缩长度、摘要和解压上限校验失败时不提供部分结果。 | + +这些是载荷和并发预算,不是进程 RSS、CPU 或 Mongo 实际扫描文档数的硬配额;业务查询索引和线上数据分布仍影响冷启动。租约使用应用时间,与现有租约模式一致,节点时钟需正常同步;未模拟任意长暂停和跨节点时钟偏差。无后台定时全表扫描,未引入 Redis、新服务或新依赖。 + +## 验证证据 + +原始日志与截图位于当前 worktree 未跟踪目录 `local-data/verification/dashboard-snapshots-20261020/`。 + +| 验证 | 结果 | +| --- | --- | +| 先失败再修复 | `red-repeated-history.txt` 记录两个 handler/分页历史加载 2 次的失败;修复后共享 1 次,非相关 POST 不再触发重建。 | +| Memory / Mongo 与并发 | 真实 Mongo 多 Store/Reader 及重建后新 Reader 共用同一代;四实例并发冷启动仅 1 次来源加载。覆盖失败保留、退避、旧 owner fencing、遗弃租约接管、取消请求/关闭、损坏代次、容量/槽位限制和字节预算。 | +| 统计口径 | 序列化前后与原算法等价,覆盖日期、人员、供应商筛选;既有冻结归属、跨期、返工/重试去重、缺少时长、207 条分页及全量导出回归通过。 | +| 授权与兼容 | 当前角色或项目范围变更及时拒绝;相同数据范围跨身份共享后仍隔离结果;内部集合不能通过通用接口绕过权限。旧客户端等待 200,新客户端冷启动 202;固定版本导出头一致,失效版本 409。 | +| Race 检查 | outputsnapshot/store/workforcestats/api 定向 `go test -race` 通过,包含本机合成 Mongo。 | +| 三角色浏览器 | 管理员、标注员、供应商管理员真实 HTTP + 本机合成 Mongo。通过分页与下钻、固定版本全量 CSV、真实 409 后整页恢复、详情及项目切换;无页面异常、无非认证业务写请求。 | +| 全量门禁 | 前端 13 项失败全部属于 14 项既有基线;Go 70 项失败全部属于 70 项既有基线,无新增失败,未改基线。TypeScript 与格式检查通过。 | +| 构建 | Web 与 Go API 构建通过,生产静态包在真实浏览器显示登录表且无启动异常;保留既有 bundle 大小提醒。 | + +故障测试通过受控失败、模拟租约遗弃/时间推进以及重建 Store/Reader 验证恢复,没有宣称做过操作系统强杀、生产 Mongo 故障切换或公司多节点压测。最初浏览器测试的占位行等待/页大小同步问题记录在 `browser-initial-fixture-failures.json`,修正测试同步后通过。 + +## 本地规模计时 + +`RUN_OUTPUT_SNAPSHOT_BENCHMARK=1` 的隔离合成 Mongo 测试:5000 个 part、25000 条历史事件、4 个 API 实例、32 次热请求。首次请求 224 ms,热请求 P50 15 ms / P95 17 ms,来源重建 1 次,1 个已写槽,压缩载荷 44634 字节。原始输出 `scale-tests.txt`。 + +该测试验证共享和负载方向,不能与生产的 13.7–14.7 秒直接计算加速倍数,也不代表生产时延或 RSS 承诺。生产四角色完整验收、冷启动时间与数据库负载仍待具体发布后核验。 + +## 本地预览 + +入口 `http://127.0.0.1:5196/?view=dashboard`,Vite 代理到隔离 API `127.0.0.1:3026`;测试仅使用本机 Mongo `27018` 下新建的 `annotation_output_browser_*` 数据库。随机合成账号保存于 `local-data/verification/customer-dashboard-20261020/accounts.json`,不提交账号口令。复现命令沿用 [初版验证记录](../customer-dashboard-20261020/report.md#本地复现与预览),额外设置 `ANNOTATION_OUTPUT_VERIFY_DIR` 可指定本次浏览器证据目录。 diff --git a/internal/api/annotation_output.go b/internal/api/annotation_output.go index caa40dcf..f919cbf6 100644 --- a/internal/api/annotation_output.go +++ b/internal/api/annotation_output.go @@ -3,8 +3,10 @@ package api import ( "context" "droboticflow/internal/domain" + "droboticflow/internal/outputsnapshot" "droboticflow/internal/workforcestats" "encoding/csv" + "errors" "fmt" "net/http" "strconv" @@ -164,17 +166,40 @@ func (s *server) handleAnnotationOutput(w http.ResponseWriter, r *http.Request) } // Do not narrow the load to a requested person: a review may address sources // attributed to several authors. The read model applies the principal fence. - snapshot, err := s.store.LoadAnnotationOutput(ctx, org, ids) - if err != nil { - renderError(w, 500, err) + s.outputOnce.Do(func() { s.outputReader = outputsnapshot.NewReader(s.store, s.store.LoadAnnotationOutput) }) + report, info, err := s.outputReader.Query(ctx, org, ids, r.URL.Query().Get("snapshotId"), q) + // Older deployed clients cannot consume 202. They wait for the same shared + // background generation; they never start a private synchronous rebuild. + for err == nil && report == nil && r.URL.Query().Get("snapshotMode") != "shared" { + timer := time.NewTimer(100 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + err = ctx.Err() + case <-timer.C: + report, info, err = s.outputReader.Query(ctx, org, ids, r.URL.Query().Get("snapshotId"), q) + } + } + if errors.Is(err, outputsnapshot.ErrExpired) { + renderJSON(w, 409, map[string]string{"error": "数据已更新,请刷新后重试", "code": "snapshot_expired"}) + return + } + if err != nil { + renderJSON(w, 503, map[string]string{"error": "看板暂时未能加载,请稍后重试", "code": "snapshot_unavailable"}) + return + } + w.Header().Set("Cache-Control", "no-store") + if report == nil { + w.Header().Set("Retry-After", "2") + renderJSON(w, 202, map[string]any{"status": "building", "retryAfterMs": 2000}) return } - report := workforcestats.BuildOutput(snapshot, q) if format == "csv" { if len(report.Tasks) > 100000 { renderError(w, 400, fmt.Errorf("export exceeds 100000 rows; narrow the filters")) return } + w.Header().Set("X-Output-Snapshot", info.ID) writeOutputCSV(w, report.Tasks, loc) return } @@ -197,7 +222,7 @@ func (s *server) handleAnnotationOutput(w http.ResponseWriter, r *http.Request) days = filtered } w.Header().Set("Cache-Control", "no-store") - renderJSON(w, 200, map[string]any{"asOf": now.Format(time.RFC3339Nano), "audienceScope": audience, "from": from.In(loc).Format("2006-01-02"), "to": to.In(loc).AddDate(0, 0, -1).Format("2006-01-02"), "timeZone": loc.String(), "totals": report.Totals, "projects": report.Projects, "suppliers": report.SupplierOptions, "users": report.Users, "tasks": paginateOutput(report.Tasks, tasksPage, tasksSize), "supplierOutput": paginateOutput(report.Suppliers, suppliersPage, suppliersSize), "dailyOutput": paginateOutput(days, daysPage, daysSize)}) + renderJSON(w, 200, map[string]any{"asOf": info.AsOf.Format(time.RFC3339Nano), "snapshot": info, "audienceScope": audience, "from": from.In(loc).Format("2006-01-02"), "to": to.In(loc).AddDate(0, 0, -1).Format("2006-01-02"), "timeZone": loc.String(), "totals": report.Totals, "projects": report.Projects, "suppliers": report.SupplierOptions, "users": report.Users, "tasks": paginateOutput(report.Tasks, tasksPage, tasksSize), "supplierOutput": paginateOutput(report.Suppliers, suppliersPage, suppliersSize), "dailyOutput": paginateOutput(days, daysPage, daysSize)}) } func outputCSVCell(s string) string { v := strings.TrimLeftFunc(s, unicode.IsSpace) diff --git a/internal/api/annotation_output_snapshot_scale_test.go b/internal/api/annotation_output_snapshot_scale_test.go new file mode 100644 index 00000000..f0eee800 --- /dev/null +++ b/internal/api/annotation_output_snapshot_scale_test.go @@ -0,0 +1,120 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/store" + "encoding/json" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "sort" + "testing" + "time" +) + +func TestAnnotationOutputSnapshotScale(t *testing.T) { + if os.Getenv("RUN_OUTPUT_SNAPSHOT_BENCHMARK") != "1" || os.Getenv("TEST_WORKFORCE_MONGO_URI") != "mongodb://127.0.0.1:27018" { + t.Skip("explicit isolated local timing run") + } + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) + defer cancel() + c, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018")) + if e != nil { + t.Fatal(e) + } + db := c.Database(fmt.Sprintf("annotation_output_scale_%d", time.Now().UnixNano())) + defer c.Disconnect(context.Background()) + defer db.Drop(context.Background()) + f := newCombinedDraftFixture(t) + s := store.NewMongoStore(db) + for _, u := range []domain.UserRecord{f.admin, f.annotator} { + if e = s.UpsertUser(ctx, u); e != nil { + t.Fatal(e) + } + } + if e = s.UpsertProject(ctx, domain.ProjectRecord{ID: "scale", OrganizationID: domain.DefaultOrganizationID, Name: "Synthetic scale project"}); e != nil { + t.Fatal(e) + } + if _, e = s.SaveProductionStatsScope(ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{"scale"}, Revision: 1}, 0); e != nil { + t.Fatal(e) + } + n := 5000 + at := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano) + collections := map[string][]any{"annotationTasks": {}, "annotationWorklogEvents": {}} + for i := 0; i < n; i++ { + id := fmt.Sprintf("scale-%d", i) + collections["annotationTasks"] = append(collections["annotationTasks"], bson.M{"id": id, "organizationId": domain.DefaultOrganizationID, "projectId": "scale", "sourceIsFullVideo": true, "sourceDurationSeconds": 900, "createdAt": at}) + for j := 0; j < 5; j++ { + collections["annotationWorklogEvents"] = append(collections["annotationWorklogEvents"], bson.M{"id": fmt.Sprintf("%s-%d", id, j), "organizationId": domain.DefaultOrganizationID, "projectId": "scale", "taskId": id, "userId": f.annotator.ID, "supplierId": f.annotator.SupplierID, "revision": 1, "submissionKind": "first", "eventType": "combined_submitted", "createdAt": at}) + } + } + for name, docs := range collections { + for len(docs) > 0 { + size := 1000 + if size > len(docs) { + size = len(docs) + } + if _, e = db.Collection(name).InsertMany(ctx, docs[:size]); e != nil { + t.Fatal(e) + } + docs = docs[size:] + } + } + wrapped := &countingOutputStore{Store: s} + f.handler = NewHandler(wrapped, nil) + start := time.Now() + cold := outputReady(t, f, "/api/dashboard/annotation-output", f.admin) + coldMS := time.Since(start).Milliseconds() + var body struct { + Snapshot struct { + ID string `json:"id"` + } `json:"snapshot"` + Tasks struct { + Total int `json:"total"` + } `json:"tasks"` + } + if e = json.Unmarshal(cold.Body.Bytes(), &body); e != nil || body.Tasks.Total != n { + t.Fatalf("scale setup %d %v", body.Tasks.Total, e) + } + samples := []int64{} + // Four independent handlers share only Mongo, including after a restart. + for instance := 0; instance < 4; instance++ { + proxy := &countingOutputStore{Store: store.NewMongoStore(db)} + if instance == 0 { + proxy = wrapped + } + f.handler = NewHandler(proxy, nil) + for page := 1; page <= 8; page++ { + start := time.Now() + r := f.request("GET", fmt.Sprintf("/api/dashboard/annotation-output?snapshotId=%s&tasksPage=%d", body.Snapshot.ID, page), f.admin, nil) + samples = append(samples, time.Since(start).Milliseconds()) + if r.Code != 200 { + t.Fatalf("warm %d", r.Code) + } + } + if instance > 0 && proxy.loads.Load() != 0 { + t.Fatal("another API reloaded source history") + } + } + sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] }) + p95 := samples[(len(samples)*95-1)/100] + if p95 > 1000 { + t.Fatalf("warm report p95 %dms exceeds local target", p95) + } + if wrapped.loads.Load() != 1 { + t.Fatalf("%d raw source reads", wrapped.loads.Load()) + } + sizes := db.Collection("annotationOutputSnapshotSlots") + count, _ := sizes.CountDocuments(ctx, bson.M{}) + var slot struct { + Data []byte `bson:"data"` + } + _ = sizes.FindOne(ctx, bson.M{}).Decode(&slot) + report := map[string]any{"environment": "local isolated Mongo; synthetic metadata; no production performance claim", "parts": n, "historyEvents": n * 5, "apiInstances": 4, "warmRequests": len(samples), "coldMs": coldMS, "warmP50Ms": samples[len(samples)/2], "warmP95Ms": p95, "sourceBuilds": wrapped.loads.Load(), "storedSlots": count, "compressedBytes": len(slot.Data)} + encoded, _ := json.Marshal(report) + t.Log(string(encoded)) +} diff --git a/internal/api/annotation_output_snapshot_test.go b/internal/api/annotation_output_snapshot_test.go new file mode 100644 index 00000000..e6540689 --- /dev/null +++ b/internal/api/annotation_output_snapshot_test.go @@ -0,0 +1,162 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/outputsnapshot" + "droboticflow/internal/productionstats" + "droboticflow/internal/store" + "droboticflow/internal/workforcestats" + "encoding/json" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +type countingOutputStore struct { + store.Store + loads atomic.Int32 + gate chan struct{} +} + +func (s *countingOutputStore) LoadAnnotationOutput(ctx context.Context, org string, projects []string) (workforcestats.OutputSnapshot, error) { + s.loads.Add(1) + if s.gate != nil { + select { + case <-s.gate: + case <-ctx.Done(): + return workforcestats.OutputSnapshot{}, ctx.Err() + } + } + return s.Store.LoadAnnotationOutput(ctx, org, projects) +} +func outputReady(t *testing.T, f combinedDraftFixture, target string, user domain.UserRecord) *httptest.ResponseRecorder { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for { + r := f.request("GET", target, user, nil) + if r.Code != 202 { + if r.Code != 200 { + t.Fatalf("read %d: %s", r.Code, r.Body.String()) + } + return r + } + if time.Now().After(deadline) { + t.Fatal("snapshot never became ready") + } + time.Sleep(time.Millisecond) + } +} +func TestAnnotationOutputSnapshotSharedAcrossHandlersAndPages(t *testing.T) { + f := newCombinedDraftFixture(t) + if _, err := f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{f.task.ProjectID}, Revision: 1}, 0); err != nil { + t.Fatal(err) + } + wrapped := &countingOutputStore{Store: f.store} + f.handler = NewHandler(wrapped, nil) + outputReady(t, f, "/api/dashboard/annotation-output?tasksPage=1", f.admin) + // A second API handler must use the already published shared snapshot. + f.handler = NewHandler(wrapped, nil) + outputReady(t, f, "/api/dashboard/annotation-output?tasksPage=2", f.annotator) + f.request("POST", "/api/nonexistent-output-test", f.admin, nil) + outputReady(t, f, "/api/dashboard/annotation-output?tasksPage=3", f.annotator) + if n := wrapped.loads.Load(); n != 1 { + t.Fatalf("two handlers/pages rebuilt history %d times; want one shared build", n) + } +} + +func TestAnnotationOutputSnapshotPinAndCurrentPermissions(t *testing.T) { + f := newCombinedDraftFixture(t) + _, e := f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{f.task.ProjectID}, Revision: 1}, 0) + if e != nil { + t.Fatal(e) + } + r := outputReady(t, f, "/api/dashboard/annotation-output", f.admin) + var data struct { + Snapshot struct { + ID string `json:"id"` + } `json:"snapshot"` + } + if e = json.Unmarshal(r.Body.Bytes(), &data); e != nil { + t.Fatal(e) + } + pin := data.Snapshot.ID + if pin == "" { + t.Fatal("missing snapshot version") + } + export := f.request("GET", "/api/dashboard/annotation-output?format=csv&snapshotId="+pin, f.admin, nil) + if export.Code != 200 || export.Header().Get("X-Output-Snapshot") != pin { + t.Fatal("CSV did not use selected generation") + } + if r := f.request("GET", "/api/dashboard/annotation-output?snapshotId=expired", f.admin, nil); r.Code != 409 { + t.Fatal("unknown generation silently replaced") + } + changed := f.admin + changed.Role = domain.UserRoleQualityReviewer + if r := f.request("GET", "/api/dashboard/annotation-output?snapshotId="+pin, changed, nil); r.Code != 403 { + t.Fatal("cached generation bypassed current role") + } + if _, e = f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{}, Revision: 2}, 1); e != nil { + t.Fatal(e) + } + if r := f.request("GET", "/api/dashboard/annotation-output?snapshotId="+pin, f.admin, nil); r.Code != 409 { + t.Fatal("scope removal kept old cached coverage") + } +} + +func TestAnnotationOutputColdSharedResponseAndLegacyCompatibility(t *testing.T) { + f := newCombinedDraftFixture(t) + _, e := f.store.SaveProductionStatsScope(f.ctx, domain.DefaultOrganizationID, productionstats.Scope{ProjectIDs: []string{f.task.ProjectID}, Revision: 1}, 0) + if e != nil { + t.Fatal(e) + } + gate := make(chan struct{}) + var release sync.Once + defer release.Do(func() { close(gate) }) + wrapped := &countingOutputStore{Store: f.store, gate: gate} + f.handler = NewHandler(wrapped, nil) + started := time.Now() + cold := f.request("GET", "/api/dashboard/annotation-output?snapshotMode=shared", f.admin, nil) + if cold.Code != 202 || time.Since(started) > 250*time.Millisecond { + t.Fatalf("cold request blocked or returned fake numbers: %d", cold.Code) + } + legacy := make(chan *httptest.ResponseRecorder, 1) + go func() { legacy <- f.request("GET", "/api/dashboard/annotation-output", f.admin, nil) }() + select { + case r := <-legacy: + t.Fatalf("legacy client received premature %d", r.Code) + case <-time.After(20 * time.Millisecond): + } + release.Do(func() { close(gate) }) + select { + case r := <-legacy: + if r.Code != 200 { + t.Fatalf("legacy client failed %d", r.Code) + } + case <-time.After(2 * time.Second): + t.Fatal("legacy client not released") + } + if wrapped.loads.Load() != 1 { + t.Fatal("legacy and shared requests duplicated source read") + } +} + +func TestAnnotationOutputPrivateCollectionsRejectBrowserAndWrites(t *testing.T) { + f := newCombinedDraftFixture(t) + for _, name := range []string{outputsnapshot.StateCollection, outputsnapshot.SlotsCollection} { + for _, operation := range []struct { + method, target string + body any + }{ + {"GET", "/api/database/collections/" + name + "/documents?target=platform", nil}, + {"POST", "/api/database/collections?target=business", map[string]any{"name": name}}, + } { + r := f.request(operation.method, operation.target, f.admin, operation.body) + if r.Code < 400 { + t.Fatalf("internal collection exposed through %s %s: %d", operation.method, operation.target, r.Code) + } + } + } +} diff --git a/internal/api/annotation_output_test.go b/internal/api/annotation_output_test.go index 88c6652d..cd2dfc61 100644 --- a/internal/api/annotation_output_test.go +++ b/internal/api/annotation_output_test.go @@ -34,7 +34,7 @@ func TestAnnotationOutputAPIAndCSVScope(t *testing.T) { } before, _ := f.store.GetHumanItem(f.ctx, f.item.ID) for _, user := range []domain.UserRecord{f.admin, f.annotator} { - r := f.request("GET", "/api/dashboard/annotation-output?tasksPage=42&tasksPageSize=5", user, nil) + r := outputReady(t, f, "/api/dashboard/annotation-output?tasksPage=42&tasksPageSize=5", user) if r.Code != 200 { t.Fatalf("read %d %s", r.Code, r.Body.String()) } @@ -46,7 +46,7 @@ func TestAnnotationOutputAPIAndCSVScope(t *testing.T) { if d.Tasks.Total != 207 || len(d.Tasks.Items) != 2 || d.Totals.Submitted.Count != 207 { t.Fatalf("pagination changed totals %s", r.Body.String()) } - exported := f.request("GET", "/api/dashboard/annotation-output?format=csv&tasksPage=42&tasksPageSize=5", user, nil) + exported := outputReady(t, f, "/api/dashboard/annotation-output?format=csv&tasksPage=42&tasksPageSize=5", user) if exported.Code != 200 { t.Fatalf("export %s", exported.Body.String()) } diff --git a/internal/api/database_browser.go b/internal/api/database_browser.go index 23e244b4..45fd42f7 100644 --- a/internal/api/database_browser.go +++ b/internal/api/database_browser.go @@ -10,6 +10,7 @@ import ( "time" "droboticflow/internal/domain" + "droboticflow/internal/outputsnapshot" "droboticflow/internal/store" "github.com/go-chi/chi/v5" @@ -147,6 +148,9 @@ func normalizeDatabaseBrowserTarget(raw string) databaseBrowserTarget { func validateDatabaseCollectionName(field string, value string) *validationIssue { name := strings.TrimSpace(value) + if outputsnapshot.IsPrivateCollection(name) { + return &validationIssue{Field: field, Message: "internal statistics collection is not accessible"} + } label := "collection name" if field == "newName" { label = "new collection name" diff --git a/internal/api/server.go b/internal/api/server.go index 8ccc1d3c..32397f8c 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -32,6 +32,7 @@ import ( "droboticflow/internal/domain" "droboticflow/internal/k8s" "droboticflow/internal/lightweight" + "droboticflow/internal/outputsnapshot" "droboticflow/internal/store" "github.com/baidubce/bce-sdk-go/services/bos" @@ -47,6 +48,8 @@ type validationIssue struct { } type server struct { + outputOnce sync.Once + outputReader *outputsnapshot.Reader humanEvidenceIndexes humanEvidenceCache dashboardResponses dashboardResponseCache store store.Store diff --git a/internal/outputsnapshot/memory.go b/internal/outputsnapshot/memory.go new file mode 100644 index 00000000..bb69c317 --- /dev/null +++ b/internal/outputsnapshot/memory.go @@ -0,0 +1,96 @@ +package outputsnapshot + +import ( + "context" + "fmt" + "sync" + "time" +) + +// MemoryBackend is the in-memory Adapter used by API fixtures. Its coordinator +// and bounded slots have the same ownership/publication rules as Mongo. +type MemoryBackend struct { + mu sync.Mutex + state State + slots [SlotCount]memoryBlob +} +type memoryBlob struct { + epoch int64 + id string + data []byte +} + +func (b *MemoryBackend) OutputSnapshotState(context.Context) (State, error) { + b.mu.Lock() + defer b.mu.Unlock() + s := b.state + s.Heads = map[string]Head{} + for k, h := range b.state.Heads { + s.Heads[k] = h + } + return s, nil +} +func (b *MemoryBackend) AcquireOutputSnapshot(_ context.Context, key string, now time.Time) (Lease, bool, error) { + b.mu.Lock() + defer b.mu.Unlock() + h := b.state.Heads[key] + if b.state.Owner != "" && now.Before(b.state.Until) || now.Before(h.NextAttempt) || h.Current != nil && now.Sub(h.Current.AsOf) < RefreshInterval { + return Lease{}, false, nil + } + b.state.Epoch++ + b.state.Owner = fmt.Sprintf("memory-%d", b.state.Epoch) + b.state.Until = now.Add(LeaseDuration) + l, err := PrepareLease(&b.state, key, now) + if err != nil { + b.state.Owner = "" + return Lease{}, false, err + } + return l, true, nil +} +func (b *MemoryBackend) owns(l Lease, now time.Time) bool { + return b.state.Owner == l.Owner && b.state.Epoch == l.Epoch && now.Before(b.state.Until) +} +func (b *MemoryBackend) RenewOutputSnapshot(_ context.Context, l Lease, now time.Time) (bool, error) { + b.mu.Lock() + defer b.mu.Unlock() + if !b.owns(l, now) { + return false, nil + } + b.state.Until = now.Add(LeaseDuration) + return true, nil +} +func (b *MemoryBackend) PublishOutputSnapshot(_ context.Context, l Lease, g Generation, data []byte, now time.Time) (bool, error) { + b.mu.Lock() + defer b.mu.Unlock() + if !b.owns(l, now) { + return false, nil + } + if l.Slot < 0 || l.Slot >= SlotCount || g.Slot != l.Slot || g.Epoch != l.Epoch || g.Bytes != len(data) || len(data) > MaxEncodedBytes { + return false, ErrCapacity + } + b.slots[l.Slot] = memoryBlob{l.Epoch, g.ID, append([]byte(nil), data...)} + b.state.Heads[l.Key] = Head{Current: &g, Previous: l.Previous, NextAttempt: g.AsOf.Add(RefreshInterval)} + b.state.Owner = "" + return true, nil +} +func (b *MemoryBackend) FailOutputSnapshot(_ context.Context, l Lease, now time.Time) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.state.Owner == l.Owner && b.state.Epoch == l.Epoch { + h := b.state.Heads[l.Key] + h.Failed = true + h.NextAttempt = now.Add(RetryInterval) + b.state.Heads[l.Key] = h + b.state.Owner = "" + } + return nil +} +func (b *MemoryBackend) OutputSnapshotBlob(_ context.Context, g Generation) ([]byte, error) { + b.mu.Lock() + defer b.mu.Unlock() + v := b.slots[g.Slot] + if v.epoch != g.Epoch || v.id != g.ID { + return nil, ErrExpired + } + return append([]byte(nil), v.data...), nil +} diff --git a/internal/outputsnapshot/reader.go b/internal/outputsnapshot/reader.go new file mode 100644 index 00000000..5b83c387 --- /dev/null +++ b/internal/outputsnapshot/reader.go @@ -0,0 +1,286 @@ +package outputsnapshot + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "droboticflow/internal/workforcestats" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sort" + "sync" + "time" +) + +type Reader struct { + querySlots chan struct{} + backend Backend + source Source + now func() time.Time + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + mu sync.Mutex + closed bool + attempts map[string]time.Time + cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache. + cacheBytes int + reads chan struct{} +} + +func NewReader(b Backend, source Source) *Reader { + ctx, cancel := context.WithCancel(context.Background()) + return &Reader{querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)} +} +func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() } +func scopeKey(org string, projects []string) string { + ids := append([]string(nil), projects...) + sort.Strings(ids) + data, _ := json.Marshal([]any{Schema, org, ids}) + h := sha256.Sum256(data) + return hex.EncodeToString(h[:]) +} + +// Read returns nil while the initial generation is being built. Existing complete +// generations remain available during refresh. Pins never silently change data. +func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) { + key, now := scopeKey(org, projects), r.now() + state, err := r.backend.OutputSnapshotState(ctx) + if err != nil { + return nil, Info{}, err + } + head := state.Heads[key] + if _, exists := state.Heads[key]; !exists { + live := 0 + for _, h := range state.Heads { + if h.Current != nil && now.Sub(h.Current.AsOf) <= MaxAge || h.Current == nil && now.Before(h.NextAttempt) { + live++ + } + } + if live >= MaxScopes { + return nil, Info{}, ErrCapacity + } + } + var gen *Generation + if pin != "" { + for _, g := range []*Generation{head.Current, head.Previous} { + if g != nil && g.ID == pin { + gen = g + } + } + if gen == nil { + return nil, Info{}, ErrExpired + } + } else { + gen = head.Current + } + if state.Owner == "" || !now.Before(state.Until) { + r.request(key, org, projects, head, now) + } + info := Info{Refreshing: state.Owner != "" && now.Before(state.Until)} + if gen == nil { + if head.Failed && now.Before(head.NextAttempt) { + return nil, info, ErrUnavailable + } + return nil, info, nil + } + if now.Sub(gen.AsOf) > MaxAge { + if pin != "" { + return nil, info, ErrExpired + } + if head.Failed && now.Before(head.NextAttempt) { + return nil, info, ErrUnavailable + } + return nil, info, nil + } + select { + case r.reads <- struct{}{}: + case <-ctx.Done(): + return nil, info, ctx.Err() + } + defer func() { <-r.reads }() + data, err := r.load(ctx, *gen) + if err != nil && pin == "" && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge { + gen = head.Previous + data, err = r.load(ctx, *gen) + } + if err != nil { + return nil, info, ErrUnavailable + } + info.ID = gen.ID + info.AsOf = gen.AsOf + info.Stale = now.Sub(gen.AsOf) >= RefreshInterval + return data, info, nil +} +func (r *Reader) request(key, org string, projects []string, head Head, now time.Time) { + if head.Current != nil && now.Sub(head.Current.AsOf) < RefreshInterval { + return + } + if now.Before(head.NextAttempt) { + return + } + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return + } + if t := r.attempts[key]; now.Before(t) { + r.mu.Unlock() + return + } + if len(r.attempts) >= MaxScopes*2 { + r.attempts = map[string]time.Time{} + } + r.attempts[key] = now.Add(2 * time.Second) + r.wg.Add(1) + r.mu.Unlock() + ids := append([]string(nil), projects...) + go func() { defer r.wg.Done(); r.build(key, org, ids) }() +} +func (r *Reader) build(key, org string, projects []string) { + ctx, cancel := context.WithTimeout(r.ctx, BuildTimeout) + defer cancel() + lease, ok, err := r.backend.AcquireOutputSnapshot(ctx, key, r.now()) + if err != nil || !ok { + return + } + published := false + defer func() { + if !published { + cleanup, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + _ = r.backend.FailOutputSnapshot(cleanup, lease, r.now()) + } + }() + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ctx.Done(): + return + case <-ticker.C: + renewal, finish := context.WithTimeout(ctx, 3*time.Second) + ok, err := r.backend.RenewOutputSnapshot(renewal, lease, r.now()) + finish() + if err != nil || !ok { + cancel() + return + } + } + } + }() + defer func() { close(stop); <-done }() + asOf := r.now().UTC().Truncate(time.Millisecond) + raw, err := r.source(ctx, org, projects) + if err != nil || ctx.Err() != nil { + return + } + data := workforcestats.BuildOutputDataset(raw, asOf) + if ctx.Err() != nil { + return + } + payload, err := encode(data) + if err != nil { + return + } + hash := sha256.Sum256(payload) + gen := Generation{ID: fmt.Sprintf("of1-%s-%d", lease.Owner, lease.Epoch), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)} + published, err = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now()) + _ = err +} +func encode(d workforcestats.OutputDataset) ([]byte, error) { + raw, err := json.Marshal(d) + if err != nil { + return nil, err + } + if len(raw) > MaxDecodedBytes { + return nil, ErrCapacity + } + var b bytes.Buffer + w, _ := gzip.NewWriterLevel(&b, gzip.BestSpeed) + _, err = w.Write(raw) + if err != nil { + return nil, err + } + if err = w.Close(); err != nil { + return nil, err + } + if b.Len() > MaxEncodedBytes { + return nil, ErrCapacity + } + return b.Bytes(), nil +} +func (r *Reader) load(ctx context.Context, g Generation) (*workforcestats.OutputDataset, error) { + r.mu.Lock() + payload := r.cache[g.ID] + r.mu.Unlock() + if payload == nil { + var err error + payload, err = r.backend.OutputSnapshotBlob(ctx, g) + if err != nil { + return nil, err + } + if len(payload) > MaxEncodedBytes || len(payload) != g.Bytes { + return nil, ErrCapacity + } + hash := sha256.Sum256(payload) + if hex.EncodeToString(hash[:]) != g.SHA256 { + return nil, ErrUnavailable + } + r.mu.Lock() + if _, exists := r.cache[g.ID]; !exists { + if r.cacheBytes+len(payload) > 8<<20 || len(r.cache) >= MaxScopes*2 { + r.cache = map[string][]byte{} + r.cacheBytes = 0 + } + r.cache[g.ID] = payload + r.cacheBytes += len(payload) + } + r.mu.Unlock() + } + z, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + return nil, err + } + defer z.Close() + raw, err := io.ReadAll(io.LimitReader(z, MaxDecodedBytes+1)) + if err != nil { + return nil, err + } + if len(raw) > MaxDecodedBytes { + return nil, ErrCapacity + } + var d workforcestats.OutputDataset + if err = json.Unmarshal(raw, &d); err != nil { + return nil, err + } + if !d.AsOf.Equal(g.AsOf) { + return nil, ErrUnavailable + } + return &d, nil +} + +// Query bounds the entire decode + report calculation, not only Mongo reads. +func (r *Reader) Query(ctx context.Context, org string, projects []string, pin string, q workforcestats.OutputQuery) (*workforcestats.OutputReport, Info, error) { + select { + case r.querySlots <- struct{}{}: + case <-ctx.Done(): + return nil, Info{}, ctx.Err() + } + defer func() { <-r.querySlots }() + d, info, err := r.Read(ctx, org, projects, pin) + if err != nil || d == nil { + return nil, info, err + } + result := workforcestats.QueryOutputDataset(*d, q) + return &result, info, nil +} diff --git a/internal/outputsnapshot/reader_test.go b/internal/outputsnapshot/reader_test.go new file mode 100644 index 00000000..f4528eec --- /dev/null +++ b/internal/outputsnapshot/reader_test.go @@ -0,0 +1,269 @@ +package outputsnapshot + +import ( + "context" + "crypto/sha256" + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "encoding/hex" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func testData(now time.Time) workforcestats.OutputSnapshot { + d := workforcestats.OutputSnapshot{Projects: []domain.ProjectRecord{{ID: "p", Name: "project"}}, Users: []domain.UserRecord{{ID: "u", Name: "user"}}, Suppliers: []domain.SupplierRecord{{ID: "v", Name: "vendor"}}} + for i := 0; i < 12; i++ { + id := fmt.Sprintf("t%d", i) + d.Tasks = append(d.Tasks, domain.AnnotationTaskRecord{ID: id, ProjectID: "p", SourceIsFullVideo: true, SourceDurationSeconds: 900}) + d.Events = append(d.Events, domain.AnnotationWorklogEventRecord{ID: id + "-event", TaskID: id, UserID: "u", SupplierID: "v", Revision: 1, SubmissionKind: "first", EventType: "combined_submitted", CreatedAt: now.Add(-time.Hour).Format(time.RFC3339Nano)}) + } + return d +} +func eventually(t *testing.T, f func() bool) { + t.Helper() + until := time.Now().Add(3 * time.Second) + for !f() { + if time.Now().After(until) { + t.Fatal("condition did not converge") + } + time.Sleep(time.Millisecond) + } +} +func ready(t *testing.T, r *Reader, pin string) (*workforcestats.OutputDataset, Info) { + t.Helper() + var data *workforcestats.OutputDataset + var info Info + eventually(t, func() bool { + var e error + data, info, e = r.Read(t.Context(), "org", []string{"p"}, pin) + if e != nil { + t.Fatal(e) + } + return data != nil + }) + return data, info +} +func TestSharedReadersCoalesceColdBuildAndReuseForPages(t *testing.T) { + var calls atomic.Int32 + gate := make(chan struct{}) + b := &MemoryBackend{} + now := time.Now().UTC().Truncate(time.Millisecond) + source := func(ctx context.Context, _ string, _ []string) (workforcestats.OutputSnapshot, error) { + calls.Add(1) + select { + case <-gate: + return testData(now), nil + case <-ctx.Done(): + return workforcestats.OutputSnapshot{}, ctx.Err() + } + } + readers := []*Reader{} + for i := 0; i < 4; i++ { + r := NewReader(b, source) + readers = append(readers, r) + t.Cleanup(r.Close) + } + var wg sync.WaitGroup + for _, r := range readers { + wg.Add(1) + go func(r *Reader) { + defer wg.Done() + d, _, e := r.Read(t.Context(), "org", []string{"p"}, "") + if e != nil || d != nil { + t.Errorf("cold read: %v %v", d, e) + } + }(r) + } + wg.Wait() + eventually(t, func() bool { return calls.Load() == 1 }) + close(gate) + _, first := ready(t, readers[0], "") + for _, r := range readers { + for i := 0; i < 8; i++ { + q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), MatchUser: true, ScopeUserID: "u"} + report, info, e := r.Query(t.Context(), "org", []string{"p"}, first.ID, q) + if e != nil || info.ID != first.ID || report.Totals.Submitted.Count != 12 { + t.Fatalf("shared read %+v %v", info, e) + } + } + } + if calls.Load() != 1 { + t.Fatalf("%d source builds", calls.Load()) + } +} +func TestRefreshFailureKeepsCompleteGenerationAndPinsExpire(t *testing.T) { + b := &MemoryBackend{} + base := time.Now().UTC().Truncate(time.Millisecond) + var clock atomic.Int64 + clock.Store(base.UnixMilli()) + var calls atomic.Int32 + var failing atomic.Bool + r := NewReader(b, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) { + calls.Add(1) + if failing.Load() { + return workforcestats.OutputSnapshot{}, errors.New("simulated read failure") + } + return testData(base), nil + }) + r.now = func() time.Time { return time.UnixMilli(clock.Load()) } + defer r.Close() + _, one := ready(t, r, "") + failing.Store(true) + clock.Add((RefreshInterval + time.Second).Milliseconds()) + data, info, e := r.Read(t.Context(), "org", []string{"p"}, one.ID) + if e != nil || data == nil || info.ID != one.ID { + t.Fatal("stale refresh blocked or lost complete data") + } + eventually(t, func() bool { + s, _ := b.OutputSnapshotState(t.Context()) + return s.Heads[scopeKey("org", []string{"p"})].Failed + }) + for i := 0; i < 10; i++ { + _, _, _ = r.Read(t.Context(), "org", []string{"p"}, "") + } + if calls.Load() != 2 { + t.Fatalf("failure retry storm %d", calls.Load()) + } + failing.Store(false) + clock.Add((RetryInterval + time.Second).Milliseconds()) + _, _, _ = r.Read(t.Context(), "org", []string{"p"}, "") + var two Info + eventually(t, func() bool { _, two = ready(t, r, ""); return two.ID != one.ID }) + _, pin, e := r.Read(t.Context(), "org", []string{"p"}, one.ID) + if e != nil || pin.ID != one.ID { + t.Fatal("previous generation pin lost") + } + clock.Add((RefreshInterval + time.Second).Milliseconds()) + _, _, _ = r.Read(t.Context(), "org", []string{"p"}, "") + eventually(t, func() bool { _, g := ready(t, r, ""); return g.ID != two.ID }) + if _, _, e = r.Read(t.Context(), "org", []string{"p"}, one.ID); !errors.Is(e, ErrExpired) { + t.Fatal("recycled pin silently changed version") + } + if _, _, e = r.Read(t.Context(), "org", []string{"different"}, two.ID); !errors.Is(e, ErrExpired) { + t.Fatal("scope change reused old snapshot") + } +} +func TestLeaseRecoveryRejectsOldOwnerAndStorageIsBounded(t *testing.T) { + b := &MemoryBackend{} + now := time.Now().UTC().Truncate(time.Millisecond) + first, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now) + if e != nil || !ok { + t.Fatal(e) + } + // Failed process never releases. A new owner can take over after the lease. + now = now.Add(LeaseDuration + time.Second) + second, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now) + if e != nil || !ok { + t.Fatal(e) + } + g := Generation{ID: "second", Epoch: second.Epoch, Slot: second.Slot, AsOf: now, Bytes: 1} + if ok, e = b.PublishOutputSnapshot(t.Context(), first, Generation{Slot: first.Slot, Epoch: first.Epoch}, []byte{1}, now); e != nil || ok { + t.Fatal("old owner published") + } + _ = b.FailOutputSnapshot(t.Context(), first, now) + if ok, e = b.PublishOutputSnapshot(t.Context(), second, g, []byte{1}, now); e != nil || !ok { + t.Fatalf("old failure disturbed successor: %v", e) + } + for i := 0; i < 50; i++ { + now = now.Add(RefreshInterval + time.Second) + l, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now) + if e != nil || !ok { + t.Fatal(e) + } + g := Generation{ID: fmt.Sprint(i), Epoch: l.Epoch, Slot: l.Slot, AsOf: now, Bytes: 1} + if ok, e = b.PublishOutputSnapshot(t.Context(), l, g, []byte{1}, now); e != nil || !ok { + t.Fatal(e) + } + } + b.mu.Lock() + defer b.mu.Unlock() + populated := 0 + for _, slot := range b.slots { + if slot.data != nil { + populated++ + } + } + if populated > 3 { + t.Fatalf("unbounded generations: %d", populated) + } +} +func TestCorruptCurrentFallsBackButPinnedDataNeverChanges(t *testing.T) { + b := &MemoryBackend{} + now := time.Now().UTC().Truncate(time.Millisecond) + key := scopeKey("org", []string{"p"}) + var old Generation + for i := 0; i < 2; i++ { + l, _, e := b.AcquireOutputSnapshot(t.Context(), key, now) + if e != nil { + t.Fatal(e) + } + payload, _ := encode(workforcestats.BuildOutputDataset(testData(now), now)) + h := sha256.Sum256(payload) + g := Generation{ID: fmt.Sprint(i), Epoch: l.Epoch, Slot: l.Slot, AsOf: now, Bytes: len(payload), SHA256: hex.EncodeToString(h[:])} + _, _ = b.PublishOutputSnapshot(t.Context(), l, g, payload, now) + if i == 0 { + old = g + } + now = now.Add(RefreshInterval + time.Second) + } + state, _ := b.OutputSnapshotState(t.Context()) + current := state.Heads[key].Current + b.mu.Lock() + b.slots[current.Slot].data[0] ^= 1 + b.mu.Unlock() + r := NewReader(b, nil) + r.now = func() time.Time { return current.AsOf.Add(time.Second) } + defer r.Close() + d, info, e := r.Read(t.Context(), "org", []string{"p"}, "") + if e != nil || d == nil || info.ID != old.ID { + t.Fatal("did not retain previous valid snapshot") + } + if _, _, e = r.Read(t.Context(), "org", []string{"p"}, current.ID); !errors.Is(e, ErrUnavailable) { + t.Fatal("corrupt pin silently substituted") + } +} + +func TestBuildCancellationAndPayloadLimits(t *testing.T) { + b := &MemoryBackend{} + started := make(chan struct{}) + r := NewReader(b, func(ctx context.Context, _ string, _ []string) (workforcestats.OutputSnapshot, error) { + close(started) + <-ctx.Done() + return workforcestats.OutputSnapshot{}, ctx.Err() + }) + _, _, _ = r.Read(t.Context(), "org", []string{"p"}, "") + <-started + r.Close() + state, _ := b.OutputSnapshotState(t.Context()) + head := state.Heads[scopeKey("org", []string{"p"})] + if state.Owner != "" || head.Current != nil || !head.Failed { + t.Fatal("cancel left a visible incomplete generation or live owner") + } + // Limit the decoded data too, not only a highly compressible stored blob. + huge := workforcestats.OutputDataset{Projects: map[string]string{"p": strings.Repeat("x", MaxDecodedBytes+1)}} + if _, e := encode(huge); !errors.Is(e, ErrCapacity) { + t.Fatal("decoded payload limit not enforced") + } +} +func TestSnapshotScopeCountCannotGrowWithoutBound(t *testing.T) { + b := &MemoryBackend{} + now := time.Now().UTC().Truncate(time.Millisecond) + for i := 0; i < MaxScopes; i++ { + l, ok, e := b.AcquireOutputSnapshot(t.Context(), fmt.Sprint(i), now) + if e != nil || !ok { + t.Fatal(e) + } + g := Generation{ID: fmt.Sprint(i), Slot: l.Slot, Epoch: l.Epoch, AsOf: now, Bytes: 1} + if ok, e = b.PublishOutputSnapshot(t.Context(), l, g, []byte{1}, now); e != nil || !ok { + t.Fatal(e) + } + } + if _, _, e := b.AcquireOutputSnapshot(t.Context(), "extra", now); !errors.Is(e, ErrCapacity) { + t.Fatal("scope limit not enforced") + } +} diff --git a/internal/outputsnapshot/types.go b/internal/outputsnapshot/types.go new file mode 100644 index 00000000..6851574e --- /dev/null +++ b/internal/outputsnapshot/types.go @@ -0,0 +1,126 @@ +// Package outputsnapshot shares bounded, immutable reporting generations across +// API instances. The Backend owns atomic lease fencing and generation publication. +package outputsnapshot + +import ( + "context" + "droboticflow/internal/workforcestats" + "errors" + "time" +) + +const ( + Schema = "annotation-output-facts/v1" + RefreshInterval = 120 * time.Second + RetryInterval = 120 * time.Second + MaxAge = 15 * time.Minute + LeaseDuration = 60 * time.Second + BuildTimeout = 90 * time.Second + MaxScopes = 8 + SlotCount = 16 + MaxEncodedBytes = 4 << 20 + MaxDecodedBytes = 32 << 20 +) + +var ( + ErrExpired = errors.New("snapshot expired") + ErrUnavailable = errors.New("snapshot unavailable") + ErrCapacity = errors.New("snapshot capacity exceeded") +) + +type Generation struct { + ID string `bson:"id" json:"id"` + Slot int `bson:"slot" json:"-"` + Epoch int64 `bson:"epoch" json:"-"` + AsOf time.Time `bson:"asOf" json:"asOf"` + SHA256 string `bson:"sha256" json:"-"` + Bytes int `bson:"bytes" json:"-"` +} +type Head struct { + Current *Generation `bson:"current,omitempty"` + Previous *Generation `bson:"previous,omitempty"` + NextAttempt time.Time `bson:"nextAttempt"` + Failed bool `bson:"failed"` +} +type State struct { + Heads map[string]Head `bson:"heads"` + Owner string `bson:"owner"` + Epoch int64 `bson:"epoch"` + Until time.Time `bson:"until"` +} +type Lease struct { + Key, Owner string + Epoch int64 + Slot int + Previous *Generation +} +type Backend interface { + OutputSnapshotState(context.Context) (State, error) + AcquireOutputSnapshot(context.Context, string, time.Time) (Lease, bool, error) + RenewOutputSnapshot(context.Context, Lease, time.Time) (bool, error) + PublishOutputSnapshot(context.Context, Lease, Generation, []byte, time.Time) (bool, error) + FailOutputSnapshot(context.Context, Lease, time.Time) error + OutputSnapshotBlob(context.Context, Generation) ([]byte, error) +} +type Info struct { + ID string `json:"id"` + AsOf time.Time `json:"asOf"` + Refreshing bool `json:"refreshing"` + Stale bool `json:"stale"` +} +type Source func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) + +// PrepareLease mutates only the newly acquired coordinator state, never a +// visible generation. A recycled previous generation yields ErrExpired to pins. +func PrepareLease(s *State, key string, now time.Time) (Lease, error) { + if s.Heads == nil { + s.Heads = map[string]Head{} + } + for k, h := range s.Heads { + if k != key && (h.Current != nil && now.Sub(h.Current.AsOf) > MaxAge || h.Current == nil && !now.Before(h.NextAttempt)) { + delete(s.Heads, k) + } + } + if _, ok := s.Heads[key]; !ok && len(s.Heads) >= MaxScopes { + return Lease{}, ErrCapacity + } + used := map[int]bool{} + for _, h := range s.Heads { + if h.Current != nil { + used[h.Current.Slot] = true + } + if h.Previous != nil { + used[h.Previous.Slot] = true + } + } + slot := -1 + for i := 0; i < SlotCount; i++ { + if !used[i] { + slot = i + break + } + } + if slot < 0 { + for k, h := range s.Heads { + if h.Previous != nil { + slot = h.Previous.Slot + h.Previous = nil + s.Heads[k] = h + break + } + } + } + if slot < 0 { + return Lease{}, ErrCapacity + } + h := s.Heads[key] + old := h.Current + h.Failed = false + s.Heads[key] = h + return Lease{Key: key, Owner: s.Owner, Epoch: s.Epoch, Slot: slot, Previous: old}, nil +} + +const StateCollection = "annotationOutputSnapshotState" +const SlotsCollection = "annotationOutputSnapshotSlots" + +func IsPrivateCollection(name string) bool { return name == StateCollection || name == SlotsCollection } diff --git a/internal/store/annotation_output.go b/internal/store/annotation_output.go index ef56af71..382e6946 100644 --- a/internal/store/annotation_output.go +++ b/internal/store/annotation_output.go @@ -4,10 +4,13 @@ import ( "context" "droboticflow/internal/domain" "droboticflow/internal/workforcestats" + "fmt" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "golang.org/x/sync/errgroup" + "reflect" + "sync" "time" ) @@ -105,7 +108,12 @@ func (s *MemoryStore) LoadAnnotationOutput(_ context.Context, org string, projec // Project/organization-scoped projections keep history sufficient for cumulative // output while excluding descriptions, secrets, drafts and media. No UI list cap. +type outputSourceBudget struct{ bytes, documents int } + func (s *MongoStore) LoadAnnotationOutput(ctx context.Context, org string, projects []string) (workforcestats.OutputSnapshot, error) { + return s.loadAnnotationOutput(ctx, org, projects, outputSourceBudget{64 << 20, 250000}) +} +func (s *MongoStore) loadAnnotationOutput(ctx context.Context, org string, projects []string, budget outputSourceBudget) (workforcestats.OutputSnapshot, error) { d := workforcestats.OutputSnapshot{} if len(projects) == 0 { return d, nil @@ -118,27 +126,59 @@ func (s *MongoStore) LoadAnnotationOutput(ctx context.Context, org string, proje } return m } - read := func(c *mongo.Collection, filter, projection bson.M, out any) error { - cursor, err := c.Find(ctx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second)) + readCtx, cancel := context.WithCancel(ctx) + defer cancel() + sem := make(chan struct{}, 2) + var budgetMu sync.Mutex + totalBytes, totalDocs := 0, 0 + read := func(c *mongo.Collection, filter, projection bson.M, out any) (err error) { + select { + case sem <- struct{}{}: + case <-readCtx.Done(): + return readCtx.Err() + } + defer func() { + <-sem + if err != nil { + cancel() + } + }() + cursor, err := c.Find(readCtx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second)) if err != nil { return err } - defer cursor.Close(ctx) - return cursor.All(ctx, out) + defer cursor.Close(readCtx) + target := reflect.ValueOf(out).Elem() + for cursor.Next(readCtx) { + budgetMu.Lock() + totalBytes += len(cursor.Current) + totalDocs++ + exceeded := totalBytes > budget.bytes || totalDocs > budget.documents + budgetMu.Unlock() + if exceeded { + return fmt.Errorf("annotation output source budget exceeded") + } + v := reflect.New(target.Type().Elem()) + if err = cursor.Decode(v.Interface()); err != nil { + return err + } + target.Set(reflect.Append(target, v.Elem())) + } + return cursor.Err() } var g errgroup.Group g.Go(func() error { - return read(s.annotationTasks, f, fields("id", "organizationId", "projectId", "dataTaskId", "runId", "sourceIsFullVideo", "sourceDurationSeconds", "clipStart", "clipEnd", "supplierId", "status", "currentRevision", "createdAt", "updatedAt"), &d.Tasks) + return read(s.annotationTasks, f, fields("id", "organizationId", "projectId", "dataTaskId", "runId", "sourceIsFullVideo", "sourceDurationSeconds", "createdAt"), &d.Tasks) }) g.Go(func() error { - return read(s.humanItems, f, fields("id", "projectId", "runId", "legacyTaskId", "subject.itemKey", "currentStageId", "status", "currentRevision", "createdAt", "updatedAt"), &d.Items) + return read(s.humanItems, f, fields("id", "projectId", "legacyTaskId"), &d.Items) }) g.Go(func() error { return read(s.captureRecords, f, fields("id", "projectId", "connectionId", "bucket", "prefix", "captureId", "runId"), &d.Captures) }) g.Go(func() error { - return read(s.annotationWorklogEvents, f, fields("id", "taskId", "userId", "supplierId", "eventType", "submissionId", "submissionKind", "videoSeconds", "outputDelta", "revision", "createdAt", "mode", "temporary"), &d.Events) + return read(s.annotationWorklogEvents, f, fields("id", "taskId", "userId", "supplierId", "eventType", "submissionId", "submissionKind", "outputDelta", "revision", "createdAt", "mode", "temporary"), &d.Events) }) g.Go(func() error { return read(s.db.Collection("reviewOperations"), f, fields("_id", "operationId", "projectId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id"), &d.Operations) @@ -167,7 +207,7 @@ func (s *MongoStore) LoadAnnotationOutput(ctx context.Context, org string, proje captureIDs = append(captureIDs, c.ID) } g.Go(func() error { - return read(s.runs, organizationScopedFilter(org, bson.M{"$or": bson.A{bson.M{"projectId": bson.M{"$in": projects}}, bson.M{"id": bson.M{"$in": runIDs}}, bson.M{"taskId": bson.M{"$in": captureIDs}}}}), fields("id", "taskId", "parentRunId", "mode", "createdAt"), &d.Runs) + return read(s.runs, organizationScopedFilter(org, bson.M{"$or": bson.A{bson.M{"projectId": bson.M{"$in": projects}}, bson.M{"id": bson.M{"$in": runIDs}}, bson.M{"taskId": bson.M{"$in": captureIDs}}}}), fields("id", "taskId", "mode", "createdAt"), &d.Runs) }) ids := []string{} for _, t := range d.Tasks { @@ -178,7 +218,7 @@ func (s *MongoStore) LoadAnnotationOutput(ctx context.Context, org string, proje return read(s.annotationReviews, related, fields("id", "taskId", "revision", "stage", "decision", "createdAt"), &d.Reviews) }) g.Go(func() error { - return read(s.annotationRevisions, related, fields("id", "taskId", "revision", "authorUserId", "isRework", "countsAsOutput", "createdAt"), &d.Revisions) + return read(s.annotationRevisions, related, fields("taskId", "revision", "authorUserId", "isRework", "countsAsOutput"), &d.Revisions) }) itemIDs := []string{} for _, i := range d.Items { diff --git a/internal/store/annotation_output_snapshots.go b/internal/store/annotation_output_snapshots.go new file mode 100644 index 00000000..911629d1 --- /dev/null +++ b/internal/store/annotation_output_snapshots.go @@ -0,0 +1,117 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/outputsnapshot" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +func (s *MemoryStore) OutputSnapshotState(c context.Context) (outputsnapshot.State, error) { + return s.outputSnapshots.OutputSnapshotState(c) +} +func (s *MemoryStore) AcquireOutputSnapshot(c context.Context, k string, t time.Time) (outputsnapshot.Lease, bool, error) { + return s.outputSnapshots.AcquireOutputSnapshot(c, k, t) +} +func (s *MemoryStore) RenewOutputSnapshot(c context.Context, l outputsnapshot.Lease, t time.Time) (bool, error) { + return s.outputSnapshots.RenewOutputSnapshot(c, l, t) +} +func (s *MemoryStore) PublishOutputSnapshot(c context.Context, l outputsnapshot.Lease, g outputsnapshot.Generation, b []byte, t time.Time) (bool, error) { + return s.outputSnapshots.PublishOutputSnapshot(c, l, g, b, t) +} +func (s *MemoryStore) FailOutputSnapshot(c context.Context, l outputsnapshot.Lease, t time.Time) error { + return s.outputSnapshots.FailOutputSnapshot(c, l, t) +} +func (s *MemoryStore) OutputSnapshotBlob(c context.Context, g outputsnapshot.Generation) ([]byte, error) { + return s.outputSnapshots.OutputSnapshotBlob(c, g) +} + +// These collections contain derived statistics only. No startup migration or +// customer document is changed. Fixed slot _ids bound retained payload storage; +// only the small coordinator is renewed. The normal Mongo _id indexes suffice. +func (s *MongoStore) outputCoordinator() *mongo.Collection { + return s.db.Collection(outputsnapshot.StateCollection) +} +func (s *MongoStore) outputBlobs() *mongo.Collection { + return s.db.Collection(outputsnapshot.SlotsCollection) +} +func (s *MongoStore) OutputSnapshotState(ctx context.Context) (outputsnapshot.State, error) { + var st outputsnapshot.State + err := s.outputCoordinator().FindOne(ctx, bson.M{"_id": "global"}).Decode(&st) + if err == mongo.ErrNoDocuments { + err = nil + } + return st, err +} +func outputLeaseFilter(l outputsnapshot.Lease, now time.Time) bson.M { + return bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch, "until": bson.M{"$gt": now}} +} +func (s *MongoStore) AcquireOutputSnapshot(ctx context.Context, key string, now time.Time) (outputsnapshot.Lease, bool, error) { + owner := domain.NewID("output") + prefix := "heads." + key + filter := bson.M{"_id": "global", "$and": bson.A{ + bson.M{"$or": bson.A{bson.M{"owner": ""}, bson.M{"owner": bson.M{"$exists": false}}, bson.M{"until": bson.M{"$lte": now}}}}, + bson.M{"$or": bson.A{bson.M{prefix + ".nextAttempt": bson.M{"$exists": false}}, bson.M{prefix + ".nextAttempt": bson.M{"$lte": now}}}}, + bson.M{"$or": bson.A{bson.M{prefix + ".current.asOf": bson.M{"$exists": false}}, bson.M{prefix + ".current.asOf": bson.M{"$lte": now.Add(-outputsnapshot.RefreshInterval)}}}}, + }} + update := bson.M{"$set": bson.M{"owner": owner, "until": now.Add(outputsnapshot.LeaseDuration)}, "$inc": bson.M{"epoch": int64(1)}, "$setOnInsert": bson.M{"heads": bson.M{}}} + var st outputsnapshot.State + err := s.outputCoordinator().FindOneAndUpdate(ctx, filter, update, options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After)).Decode(&st) + if mongo.IsDuplicateKeyError(err) { + return outputsnapshot.Lease{}, false, nil + } + if err != nil { + return outputsnapshot.Lease{}, false, err + } + l, err := outputsnapshot.PrepareLease(&st, key, now) + if err != nil { + _, _ = s.outputCoordinator().UpdateOne(ctx, bson.M{"_id": "global", "owner": owner, "epoch": st.Epoch}, bson.M{"$set": bson.M{"owner": "", "until": now}}) + return outputsnapshot.Lease{}, false, err + } + res, err := s.outputCoordinator().UpdateOne(ctx, outputLeaseFilter(l, now), bson.M{"$set": bson.M{"heads": st.Heads}}) + return l, err == nil && res.MatchedCount == 1, err +} +func (s *MongoStore) RenewOutputSnapshot(ctx context.Context, l outputsnapshot.Lease, now time.Time) (bool, error) { + res, err := s.outputCoordinator().UpdateOne(ctx, outputLeaseFilter(l, now), bson.M{"$set": bson.M{"until": now.Add(outputsnapshot.LeaseDuration)}}) + return err == nil && res.MatchedCount == 1, err +} +func (s *MongoStore) PublishOutputSnapshot(ctx context.Context, l outputsnapshot.Lease, g outputsnapshot.Generation, data []byte, now time.Time) (bool, error) { + if g.Slot != l.Slot || g.Epoch != l.Epoch || len(data) > outputsnapshot.MaxEncodedBytes || g.Bytes != len(data) || l.Slot < 0 || l.Slot >= outputsnapshot.SlotCount { + return false, outputsnapshot.ErrCapacity + } + ok, err := s.RenewOutputSnapshot(ctx, l, now) + if err != nil || !ok { + return false, err + } + write, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + // An expired writer cannot overwrite a slot already reused by a newer epoch. + f := bson.M{"_id": l.Slot, "$or": bson.A{bson.M{"epoch": bson.M{"$lte": l.Epoch}}, bson.M{"epoch": bson.M{"$exists": false}}}} + _, err = s.outputBlobs().ReplaceOne(write, f, bson.M{"_id": l.Slot, "epoch": l.Epoch, "generationId": g.ID, "data": data}, options.Replace().SetUpsert(true)) + if mongo.IsDuplicateKeyError(err) { + return false, nil + } + if err != nil { + return false, err + } + head := outputsnapshot.Head{Current: &g, Previous: l.Previous, NextAttempt: g.AsOf.Add(outputsnapshot.RefreshInterval)} + res, err := s.outputCoordinator().UpdateOne(write, outputLeaseFilter(l, now), bson.M{"$set": bson.M{"heads." + l.Key: head, "owner": "", "until": now}}) + return err == nil && res.MatchedCount == 1, err +} +func (s *MongoStore) FailOutputSnapshot(ctx context.Context, l outputsnapshot.Lease, now time.Time) error { + _, err := s.outputCoordinator().UpdateOne(ctx, bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch}, bson.M{"$set": bson.M{"heads." + l.Key + ".failed": true, "heads." + l.Key + ".nextAttempt": now.Add(outputsnapshot.RetryInterval), "owner": "", "until": now}}) + return err +} +func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Generation) ([]byte, error) { + var doc struct { + Data []byte `bson:"data"` + } + err := s.outputBlobs().FindOne(ctx, bson.M{"_id": g.Slot, "epoch": g.Epoch, "generationId": g.ID}, options.FindOne().SetProjection(bson.M{"data": 1})).Decode(&doc) + if err == mongo.ErrNoDocuments { + err = outputsnapshot.ErrExpired + } + return doc.Data, err +} diff --git a/internal/store/annotation_output_snapshots_test.go b/internal/store/annotation_output_snapshots_test.go new file mode 100644 index 00000000..2476e7b9 --- /dev/null +++ b/internal/store/annotation_output_snapshots_test.go @@ -0,0 +1,167 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/outputsnapshot" + "droboticflow/internal/workforcestats" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "strings" + "sync/atomic" + "testing" + "time" +) + +func localOutputMongo(t *testing.T) *mongo.Database { + t.Helper() + if os.Getenv("TEST_WORKFORCE_MONGO_URI") != "mongodb://127.0.0.1:27018" { + t.Skip("isolated local Mongo required") + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + c, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018")) + if e != nil { + t.Fatal(e) + } + db := c.Database(fmt.Sprintf("annotation_output_snapshots_test_%d", time.Now().UnixNano())) + t.Cleanup(func() { _ = db.Drop(context.Background()); _ = c.Disconnect(context.Background()) }) + return db +} +func TestAnnotationOutputMongoSharedAcrossInstancesAndRestart(t *testing.T) { + db := localOutputMongo(t) + s1, s2 := NewMongoStore(db), NewMongoStore(db) + seedAnnotationOutput(t, s1) + var loads atomic.Int32 + source := func(c context.Context, o string, p []string) (workforcestats.OutputSnapshot, error) { + loads.Add(1) + return s1.LoadAnnotationOutput(c, o, p) + } + a, b := outputsnapshot.NewReader(s1, source), outputsnapshot.NewReader(s2, source) + defer a.Close() + defer b.Close() + q := workforcestats.OutputQuery{From: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), To: time.Date(2026, 10, 21, 0, 0, 0, 0, time.UTC)} + var first outputsnapshot.Info + until := time.Now().Add(5 * time.Second) + for { + r, info, e := a.Query(t.Context(), "out", []string{"p"}, "", q) + if e != nil { + t.Fatal(e) + } + if r != nil { + first = info + if r.Totals.Acceptance.Count != 1 { + t.Fatalf("wrong persisted facts %+v", r) + } + break + } + _, _, e = b.Query(t.Context(), "out", []string{"p"}, "", q) + if e != nil { + t.Fatal(e) + } + if time.Now().After(until) { + t.Fatal("Mongo generation did not publish") + } + time.Sleep(5 * time.Millisecond) + } + // A fresh Reader + MongoStore models another pod or process restart. + c := outputsnapshot.NewReader(NewMongoStore(db), source) + defer c.Close() + for _, r := range []*outputsnapshot.Reader{b, c} { + result, info, e := r.Query(t.Context(), "out", []string{"p"}, first.ID, q) + if e != nil || result == nil || info.ID != first.ID || result.Totals.Acceptance.Count != 1 { + t.Fatalf("restart/pin mismatch %+v %v", info, e) + } + } + if n := loads.Load(); n != 1 { + t.Fatalf("cross instance history rebuilt %d times", n) + } +} +func TestAnnotationOutputLeaseFencingAndFixedSlots(t *testing.T) { + db := localOutputMongo(t) + for name, backend := range map[string]outputsnapshot.Backend{"memory": NewMemoryStore(), "mongo": NewMongoStore(db)} { + t.Run(name, func(t *testing.T) { + now := time.Now().UTC().Truncate(time.Millisecond) + old, ok, e := backend.AcquireOutputSnapshot(t.Context(), "scope", now) + if e != nil || !ok { + t.Fatal(e) + } + now = now.Add(outputsnapshot.LeaseDuration + time.Second) + next, ok, e := backend.AcquireOutputSnapshot(t.Context(), "scope", now) + if e != nil || !ok { + t.Fatalf("lease not recovered: %v", e) + } + g := outputsnapshot.Generation{ID: "next", Slot: next.Slot, Epoch: next.Epoch, AsOf: now, Bytes: 1} + if ok, e = backend.PublishOutputSnapshot(t.Context(), old, outputsnapshot.Generation{Slot: old.Slot, Epoch: old.Epoch, Bytes: 1}, []byte{1}, now); e != nil || ok { + t.Fatal("expired owner published") + } + _ = backend.FailOutputSnapshot(t.Context(), old, now) + if ok, e = backend.PublishOutputSnapshot(t.Context(), next, g, []byte{1}, now); e != nil || !ok { + t.Fatalf("successor disturbed %v", e) + } + for i := 0; i < 40; i++ { + now = now.Add(outputsnapshot.RefreshInterval + time.Second) + l, ok, e := backend.AcquireOutputSnapshot(t.Context(), "scope", now) + if e != nil || !ok { + t.Fatal(e) + } + g := outputsnapshot.Generation{ID: fmt.Sprint(i), Slot: l.Slot, Epoch: l.Epoch, AsOf: now, Bytes: 1} + if ok, e = backend.PublishOutputSnapshot(t.Context(), l, g, []byte{1}, now); e != nil || !ok { + t.Fatal(e) + } + } + }) + } + n, e := db.Collection("annotationOutputSnapshotSlots").CountDocuments(t.Context(), map[string]any{}) + if e != nil || n > 3 { + t.Fatalf("unbounded generation slots %d %v", n, e) + } +} + +func TestAnnotationOutputSourceBudgetAndPayloadProjection(t *testing.T) { + db := localOutputMongo(t) + s := NewMongoStore(db) + seedAnnotationOutput(t, s) + // Large draft text must not be loaded into the statistics source projection. + _, e := db.Collection("humanItems").UpdateOne(t.Context(), bson.M{"id": "i"}, bson.M{"$set": bson.M{"subject.payload.annotationDraft": strings.Repeat("private draft ", 10000)}}) + if e != nil { + t.Fatal(e) + } + d, e := s.loadAnnotationOutput(t.Context(), "out", []string{"p"}, outputSourceBudget{8192, 100}) + if e != nil || len(d.Tasks) != 1 || d.Items[0].Subject.Payload != nil { + t.Fatalf("payload projection/budget %v", e) + } + if _, e = s.loadAnnotationOutput(t.Context(), "out", []string{"p"}, outputSourceBudget{1, 100}); e == nil { + t.Fatal("byte budget did not stop the cursor") + } + if _, e = s.loadAnnotationOutput(t.Context(), "out", []string{"p"}, outputSourceBudget{8192, 1}); e == nil { + t.Fatal("document budget did not stop the cursor") + } +} + +func TestAnnotationOutputPrivateCollectionsCannotBypassReportScopes(t *testing.T) { + db := localOutputMongo(t) + s := NewMongoStore(db) + for _, name := range []string{outputsnapshot.StateCollection, outputsnapshot.SlotsCollection} { + if _, e := db.Collection(name).InsertOne(t.Context(), bson.M{"private": "cross organization derived stats"}); e != nil { + t.Fatal(e) + } + } + overview, e := s.GetDatabaseOverview(t.Context()) + if e != nil { + t.Fatal(e) + } + for _, c := range overview.Collections { + if outputsnapshot.IsPrivateCollection(c.Name) { + t.Fatal("internal snapshot leaked in database overview") + } + } + for _, name := range []string{outputsnapshot.StateCollection, outputsnapshot.SlotsCollection} { + if _, e := s.QueryDatabaseCollection(t.Context(), domain.DatabaseQueryInput{Collection: name}); e == nil { + t.Fatal("database browser bypassed dashboard scope") + } + } +} diff --git a/internal/store/memory.go b/internal/store/memory.go index f4c5b764..62dcbad5 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -9,10 +9,12 @@ import ( "time" "droboticflow/internal/domain" + "droboticflow/internal/outputsnapshot" "droboticflow/internal/productionstats" ) type MemoryStore struct { + outputSnapshots outputsnapshot.MemoryBackend acceptanceAuthorizations map[string]domain.SupplierAcceptanceAuthorization productionStatsScopes map[string]productionstats.Scope reviewContinuations map[string]domain.ReviewContinuation @@ -3258,6 +3260,9 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)), } for _, name := range names { + if outputsnapshot.IsPrivateCollection(name) { + continue + } documents := collectionsMap[name] summary := domain.DatabaseCollectionSummary{ Name: name, @@ -3272,6 +3277,9 @@ func (store *MemoryStore) GetDatabaseOverview(context.Context) (domain.DatabaseO } func (store *MemoryStore) QueryDatabaseCollection(_ context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) { + if outputsnapshot.IsPrivateCollection(input.Collection) { + return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", input.Collection) + } store.mu.Lock() defer store.mu.Unlock() diff --git a/internal/store/mongo.go b/internal/store/mongo.go index eb7b5118..9524a4e7 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -11,6 +11,7 @@ import ( "time" "droboticflow/internal/domain" + "droboticflow/internal/outputsnapshot" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -2793,6 +2794,9 @@ func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.Databa Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)), } for _, name := range names { + if outputsnapshot.IsPrivateCollection(name) { + continue + } summary, err := mongoBrowserCollectionSummary(ctx, store.db.Collection(name), name) if err != nil { return domain.DatabaseOverview{}, err @@ -2807,6 +2811,9 @@ func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input doma if collectionName == "" { return domain.DatabaseQueryResult{}, fmt.Errorf("collection is required") } + if outputsnapshot.IsPrivateCollection(collectionName) { + return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", collectionName) + } limit := input.Limit if limit <= 0 { limit = 50 diff --git a/internal/store/store.go b/internal/store/store.go index a94665cb..c15e8173 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2,6 +2,7 @@ package store import ( "context" + "droboticflow/internal/outputsnapshot" "errors" "time" @@ -33,6 +34,7 @@ type WorkerRunCounts struct { } type Store interface { + outputsnapshot.Backend InsertClientMoviStat(context.Context, domain.ClientMoviStatRecord) error ListClientMoviStats(context.Context, string, int) ([]domain.ClientMoviStatRecord, error) LoadWorkforceStats(context.Context, string, workforcestats.Query) (workforcestats.Snapshot, error) diff --git a/internal/workforcestats/output.go b/internal/workforcestats/output.go index e66f7434..ebf7f699 100644 --- a/internal/workforcestats/output.go +++ b/internal/workforcestats/output.go @@ -76,6 +76,17 @@ func outputStage(s string) string { // BuildOutput interprets successful immutable facts. It never infers completion // or authorship from a task's current assignee or update timestamp. func BuildOutput(d OutputSnapshot, q OutputQuery) OutputReport { + if q.Location == nil { + q.Location = time.UTC + } + if q.Now.IsZero() { + q.Now = time.Now() + } + parts, projects, users, suppliers := buildOutputParts(d, q) + return outputSummarize(parts, q, projects, users, suppliers) +} + +func buildOutputParts(d OutputSnapshot, q OutputQuery) ([]outputPart, map[string]string, map[string]string, map[string]string) { if q.Location == nil { q.Location = time.UTC } @@ -476,7 +487,7 @@ func BuildOutput(d OutputSnapshot, q OutputQuery) OutputReport { } parts = append(parts, outputPart{row, first, quality, acceptance, rework}) } - return outputSummarize(parts, q, projects, users, suppliers) + return parts, projects, users, suppliers } type outputAccumulator struct { diff --git a/internal/workforcestats/output_dataset.go b/internal/workforcestats/output_dataset.go new file mode 100644 index 00000000..3cff4d9f --- /dev/null +++ b/internal/workforcestats/output_dataset.go @@ -0,0 +1,70 @@ +package workforcestats + +import "time" + +// OutputDataset contains interpreted part facts, never drafts, media or the +// source event history. Authorization and date filters are applied on every read. +type OutputDataset struct { + AsOf time.Time `json:"asOf"` + Parts []OutputPartFact `json:"parts"` + Projects map[string]string `json:"projects"` + Users map[string]string `json:"users"` + Suppliers map[string]string `json:"suppliers"` +} +type OutputStageFact struct { + At string `json:"at"` + User string `json:"user"` + Supplier string `json:"supplier"` +} +type OutputPartFact struct { + ID string `json:"id"` + Video string `json:"video"` + Project string `json:"project"` + Seconds *float64 `json:"seconds"` + Status string `json:"status"` + QualityStatus string `json:"qualityStatus"` + AcceptanceStatus string `json:"acceptanceStatus"` + Stages [4]*OutputStageFact `json:"stages"` +} + +func BuildOutputDataset(d OutputSnapshot, asOf time.Time) OutputDataset { + parts, projects, users, suppliers := buildOutputParts(d, OutputQuery{Now: asOf}) + out := OutputDataset{AsOf: asOf, Projects: projects, Users: users, Suppliers: suppliers, Parts: make([]OutputPartFact, 0, len(parts))} + for _, p := range parts { + v := OutputPartFact{ID: p.row.ID, Video: p.row.VideoID, Project: p.row.ProjectID, Seconds: p.row.Seconds, Status: p.row.Status, QualityStatus: p.row.QualityStatus, AcceptanceStatus: p.row.AcceptanceStatus} + for i, f := range []*outputFact{p.submitted, p.quality, p.acceptance, p.rework} { + if f != nil { + v.Stages[i] = &OutputStageFact{At: f.at, User: f.owner.user, Supplier: f.owner.supplier} + } + } + out.Parts = append(out.Parts, v) + } + return out +} +func QueryOutputDataset(d OutputDataset, q OutputQuery) OutputReport { + if q.Location == nil { + q.Location = time.UTC + } + q.Now = d.AsOf + parts := make([]outputPart, 0, len(d.Parts)) + for _, v := range d.Parts { + row := OutputTask{ID: v.ID, VideoID: v.Video, ProjectID: v.Project, ProjectName: d.Projects[v.Project], Seconds: v.Seconds, Status: v.Status, QualityStatus: v.QualityStatus, AcceptanceStatus: v.AcceptanceStatus, Evidence: nil} + fs := [4]*outputFact{} + for i, f := range v.Stages { + if f != nil { + fs[i] = &outputFact{f.At, outputOwner{f.User, f.Supplier}, v.Seconds} + } + } + if fs[0] != nil { + row.SubmittedAt = fs[0].at + } + if fs[1] != nil { + row.QualityAt = fs[1].at + } + if fs[2] != nil { + row.AcceptedAt = fs[2].at + } + parts = append(parts, outputPart{row, fs[0], fs[1], fs[2], fs[3]}) + } + return outputSummarize(parts, q, d.Projects, d.Users, d.Suppliers) +} diff --git a/internal/workforcestats/output_dataset_test.go b/internal/workforcestats/output_dataset_test.go new file mode 100644 index 00000000..7f12ef72 --- /dev/null +++ b/internal/workforcestats/output_dataset_test.go @@ -0,0 +1,45 @@ +package workforcestats + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestAnnotationOutputDatasetPreservesRulesAcrossFilters(t *testing.T) { + d := outputFixture() + base := outputQuery() + base.From = base.From.AddDate(0, 0, -10) + // Dataset must survive persistence before being queried with another scope/date. + data := BuildOutputDataset(d, base.Now) + raw, e := json.Marshal(data) + if e != nil { + t.Fatal(e) + } + var persisted OutputDataset + if e = json.Unmarshal(raw, &persisted); e != nil { + t.Fatal(e) + } + for _, scope := range []string{"", "a", "b"} { + for _, vendor := range []string{"", "v", "other"} { + for _, from := range []int{0, 10} { + q := base + q.From = q.From.AddDate(0, 0, from) + q.ScopeUserID = scope + q.MatchUser = scope != "" + q.SupplierID = vendor + old := BuildOutput(d, q) + for i := range old.Tasks { + old.Tasks[i].Evidence = nil + } + got := QueryOutputDataset(persisted, q) + if !reflect.DeepEqual(old, got) { + t.Fatalf("query changed (%s,%s,%d)\nold: %+v\nnew: %+v", scope, vendor, from, old, got) + } + } + } + } + if len(data.Parts) != 1 || len(data.Parts[0].Stages) != 4 { + t.Fatal("part facts missing") + } +} diff --git a/src/components/AnnotationOutputDashboard.tsx b/src/components/AnnotationOutputDashboard.tsx index 7cc69fee..10dc0665 100644 --- a/src/components/AnnotationOutputDashboard.tsx +++ b/src/components/AnnotationOutputDashboard.tsx @@ -10,6 +10,7 @@ import { type OutputRequest, type OutputTask, } from "../services/api/annotation-output"; +import { APIError } from "../services/api/client"; import { formatCompactDateTime } from "../lib/datetime"; import { getActionErrorMessage } from "../lib/action-errors.js"; import "../styles/annotation-output.css"; @@ -160,8 +161,11 @@ export function AnnotationOutputDashboard({ if (id === sequence.current) setResult(data); }) .catch((reason) => { - if (id === sequence.current && !controller.signal.aborted) - setError(getActionErrorMessage(reason, "看板加载失败,请重试")); + if (id === sequence.current && !controller.signal.aborted) { + if (reason instanceof APIError && reason.code === "snapshot_expired") { + setQuery((current) => ({ ...current, snapshotId: undefined, tasksPage: 1, suppliersPage: 1, daysPage: 1 })); + } else setError(getActionErrorMessage(reason, "看板加载失败,请重试")); + } }) .finally(() => { if (id === sequence.current) setLoading(false); @@ -191,7 +195,12 @@ export function AnnotationOutputDashboard({ }; }, [task]); const changePage = (name: "tasks" | "suppliers" | "days", page: number, size: number) => - setQuery((current) => ({ ...current, [`${name}Page`]: page, [`${name}PageSize`]: size })); + setQuery((current) => ({ + ...current, + snapshotId: result?.snapshot?.id, + [`${name}Page`]: page, + [`${name}PageSize`]: size, + })); const apply = (next: OutputFilters) => { setQuery({ ...next }); setExpandedSupplier(null); @@ -209,6 +218,7 @@ export function AnnotationOutputDashboard({ supplierId: query.supplierId, userId: query.userId, timeZone: query.timeZone, + snapshotId: result.snapshot?.id, }); const url = URL.createObjectURL(blob), link = document.createElement("a"); @@ -228,7 +238,14 @@ export function AnnotationOutputDashboard({ setTask(row); }; const focusPerson = (row: OutputDay) => { - const next = { ...query, supplierId: row.supplierId, userId: row.userId, tasksPage: 1, daysPage: 1 }; + const next = { + ...query, + snapshotId: result?.snapshot?.id, + supplierId: row.supplierId, + userId: row.userId, + tasksPage: 1, + daysPage: 1, + }; setDraft((current) => ({ ...current, supplierId: row.supplierId, userId: row.userId })); setQuery(next); }; @@ -324,7 +341,15 @@ export function AnnotationOutputDashboard({ {exporting ? "导出中…" : "导出明细"} - {manager && (query.projectId || query.supplierId || query.userId) ? ( @@ -343,7 +368,13 @@ export function AnnotationOutputDashboard({ {error ? (
{error} -
@@ -410,7 +441,12 @@ export function AnnotationOutputDashboard({ disabled={!row.id} onClick={() => { setExpandedSupplier({ id: row.id, name: row.name }); - setQuery((current) => ({ ...current, dailySupplierId: row.id, daysPage: 1 })); + setQuery((current) => ({ + ...current, + snapshotId: result?.snapshot?.id, + dailySupplierId: row.id, + daysPage: 1, + })); }} > 查看人员 diff --git a/src/services/api/annotation-output.ts b/src/services/api/annotation-output.ts index 5ec40922..daa35f9c 100644 --- a/src/services/api/annotation-output.ts +++ b/src/services/api/annotation-output.ts @@ -1,4 +1,4 @@ -import { apiFetch, apiRequest } from "./client"; +import { APIError, apiFetch, apiRequest } from "./client"; export type OutputMetric = { count: number; seconds: number | null }; export type OutputTotals = Record<"submitted" | "quality" | "acceptance" | "rework", OutputMetric>; @@ -41,6 +41,7 @@ export type OutputDay = { }; export type OutputPage = { items: T[]; total: number; page: number; pageSize: number }; export type AnnotationOutput = { + snapshot?: { id: string; asOf: string; refreshing: boolean; stale: boolean }; asOf: string; audienceScope: "platform" | "vendor" | "self"; from: string; @@ -63,6 +64,7 @@ export type OutputFilters = { timeZone: string; }; export type OutputRequest = OutputFilters & { + snapshotId?: string; tasksPage?: number; tasksPageSize?: number; suppliersPage?: number; @@ -80,15 +82,42 @@ const search = (query: OutputRequest) => { return params; }; export const annotationOutputApi = { - get: (query: OutputRequest, signal?: AbortSignal) => - apiRequest(`/api/dashboard/annotation-output?${search(query)}`, { signal }), - export: async (query: OutputFilters) => { + get: async (query: OutputRequest, signal?: AbortSignal): Promise => { + const params = search(query); + params.set("snapshotMode", "shared"); + const deadline = Date.now() + 120_000; + while (true) { + const data = await apiRequest( + `/api/dashboard/annotation-output?${params}`, + { signal }, + ); + if (!("status" in data)) return data; + if (Date.now() >= deadline) throw new Error("看板正在准备,请稍后重试"); + await new Promise((resolve, reject) => { + if (signal?.aborted) return reject(new DOMException("Aborted", "AbortError")); + const abort = () => { + window.clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }; + const timer = window.setTimeout( + () => { + signal?.removeEventListener("abort", abort); + resolve(); + }, + Math.max(500, Math.min(data.retryAfterMs, 5000)), + ); + signal?.addEventListener("abort", abort, { once: true }); + }); + } + }, + export: async (query: OutputFilters & { snapshotId?: string }) => { const params = search(query); params.set("format", "csv"); const response = await apiFetch(`/api/dashboard/annotation-output?${params}`); + if (response.status === 202) throw new Error("看板正在准备,请稍后重试"); if (!response.ok) { const body = await response.json().catch(() => null); - throw new Error(body?.error || "导出失败,请重试"); + throw new APIError(body?.error || "导出失败,请重试", response.status, body?.code || ""); } return response.blob(); }, diff --git a/tests/browser/annotation-output-replica.mjs b/tests/browser/annotation-output-replica.mjs index bdbb1d88..08dd5415 100644 --- a/tests/browser/annotation-output-replica.mjs +++ b/tests/browser/annotation-output-replica.mjs @@ -4,8 +4,13 @@ import { chromium } from "playwright"; import fs from "node:fs"; import path from "node:path"; import assert from "node:assert/strict"; -const out = path.resolve("local-data/verification/customer-dashboard-20261020"); -const accounts = JSON.parse(fs.readFileSync(path.join(out, "accounts.json"), "utf8")); +const out = path.resolve( + process.env.ANNOTATION_OUTPUT_VERIFY_DIR || "local-data/verification/customer-dashboard-20261020", +); +fs.mkdirSync(out, { recursive: true }); +const accounts = JSON.parse( + fs.readFileSync(path.resolve("local-data/verification/customer-dashboard-20261020/accounts.json"), "utf8"), +); assert.match(accounts.database, /^annotation_output_browser_\d+$/); assert.equal(accounts.origin, "http://127.0.0.1:5196"); const browser = await chromium.launch({ @@ -37,26 +42,42 @@ try { if (!["GET", "HEAD", "OPTIONS"].includes(r.method()) && !new URL(r.url()).pathname.startsWith("/api/auth/")) row.unexpectedWrites.push(new URL(r.url()).pathname); }); + const initialData = page.waitForResponse( + (r) => + r.url().includes("/api/dashboard/annotation-output?") && + r.status() === 200 && + !r.url().includes("format=csv"), + ); await page.goto(accounts.origin + "/?view=dashboard", { waitUntil: "domcontentloaded" }); console.log("Role", id, "page loaded"); const dashboard = page.getByRole("region", { name: id === "u1" ? "我的工作记录" : "生产产出看板", exact: true }); await dashboard.waitFor(); const tasks = dashboard.getByRole("region", { name: "任务明细", exact: true }); - await tasks.locator("tbody tr").first().waitFor({ timeout: 30000 }); + await tasks.getByRole("button", { name: "查看详情" }).first().waitFor({ timeout: 30000 }); assert.equal(await dashboard.getByText(/个 part|待核对/).count(), 0); row.checks.push("no internal terminology"); + const snapshotId = (await (await initialData).json()).snapshot?.id; + assert.ok(snapshotId, "server returned a shared generation"); const firstVideo = await tasks.locator("tbody tr").first().locator("td").first().innerText(); const cardNumbers = await dashboard.locator(".annotation-output-card strong").allTextContents(); const pager = tasks.getByRole("navigation", { name: "任务明细分页" }); + const nextData = page.waitForResponse((r) => r.url().includes("tasksPage=2") && r.status() === 200); await pager.getByRole("button", { name: "下一页", exact: true }).click(); + const nextResponse = await nextData; + assert.equal(new URL(nextResponse.url()).searchParams.get("snapshotId"), snapshotId); + assert.equal((await nextResponse.json()).snapshot.id, snapshotId); + row.checks.push("pagination pins shared generation"); await tasks.getByText(/第 6–/).waitFor(); assert.notEqual(await tasks.locator("tbody tr").first().locator("td").first().innerText(), firstVideo); assert.deepEqual(await dashboard.locator(".annotation-output-card strong").allTextContents(), cardNumbers); row.checks.push("pagination preserves totals"); const expectedRows = Number((await pager.locator("span").first().innerText()).match(/共 (\d+)/)[1]); + const exported = page.waitForResponse((r) => r.url().includes("format=csv") && r.status() === 200); const downloaded = page.waitForEvent("download"); await dashboard.getByRole("button", { name: "导出明细", exact: true }).click(); const download = await downloaded; + assert.equal((await exported).headers()["x-output-snapshot"], snapshotId); + row.checks.push("CSV pins same generation"); const file = await download.path(), csv = fs.readFileSync(file, "utf8"); assert.equal(csv.trim().split(/\r?\n/).length, expectedRows + 1); @@ -74,6 +95,49 @@ try { await page.keyboard.press("Escape"); await dialog.waitFor({ state: "hidden" }); row.checks.push("read-only detail and Escape close"); + if (id === "u1") { + await pager.getByRole("combobox").selectOption("5"); + await page.waitForFunction( + () => + document.querySelector('[aria-label="任务明细每页条数"]')?.value === "5" && + !!document.querySelector('[aria-label="任务明细分页"] [aria-current="page"]'), + ); + + let expiredOnce = false; + await page.route("**/api/dashboard/annotation-output?*", async (route) => { + const url = new URL(route.request().url()); + if (!expiredOnce && url.searchParams.get("tasksPage") === "2" && url.searchParams.has("snapshotId")) { + expiredOnce = true; + url.searchParams.set("snapshotId", "expired-test-generation"); + return route.continue({ url: url.toString() }); + } + return route.continue(); + }); + const expiredResponse = page.waitForResponse( + (r) => r.url().includes("/api/dashboard/annotation-output?") && r.status() === 409, + ); + const recoveredResponse = page.waitForResponse((r) => { + const u = new URL(r.url()); + return ( + u.pathname === "/api/dashboard/annotation-output" && + r.status() === 200 && + u.searchParams.get("tasksPage") === "1" && + !u.searchParams.has("snapshotId") + ); + }); + await pager.getByRole("button", { name: "下一页", exact: true }).click(); + await expiredResponse; + await recoveredResponse; + await page.waitForFunction( + () => + document.querySelector('[aria-label="任务明细分页"] [aria-current="page"]')?.textContent === "1" && + !document.querySelector(".annotation-output-loading"), + ); + assert.equal(expiredOnce, true); + assert.equal(await dashboard.locator(".annotation-output-error").count(), 0); + await page.unroute("**/api/dashboard/annotation-output?*"); + row.checks.push("expired pin resets whole dashboard without mixed generations"); + } if (id !== "u1") { const vendors = dashboard.getByRole("region", { name: "供应商总体产出", exact: true }); await vendors.getByRole("button", { name: "查看人员" }).first().click(); @@ -82,15 +146,23 @@ try { await daily.getByRole("navigation", { name: "人员每日产出分页" }).waitFor(); row.checks.push("supplier/member/daily drilldown"); await dashboard.getByLabel("项目", { exact: true }).selectOption("p1"); + const projectResult = page.waitForResponse( + (r) => new URL(r.url()).searchParams.get("projectId") === "p1" && r.status() === 200, + ); await dashboard.getByRole("button", { name: "查询", exact: true }).click(); - await tasks.locator("tbody tr").first().waitFor(); + await projectResult; + await tasks.getByRole("button", { name: "查看详情" }).first().waitFor(); assert.match(await tasks.getByRole("navigation").innerText(), /第 1–/); row.checks.push("project filtering resets pagination"); if (id === "admin") { assert.equal(await dashboard.getByLabel("项目", { exact: true }).locator("option").count(), 3); await dashboard.getByLabel("项目", { exact: true }).selectOption("p2"); + const switched = page.waitForResponse( + (r) => new URL(r.url()).searchParams.get("projectId") === "p2" && r.status() === 200, + ); await dashboard.getByRole("button", { name: "查询", exact: true }).click(); - await tasks.locator("tbody tr").first().waitFor(); + await switched; + await tasks.getByRole("button", { name: "查看详情" }).first().waitFor(); row.checks.push("switch between projects without reset"); } } From bc00467d24e018ec145529710b88378b7ebcd0c1 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 21 Oct 2026 00:07:55 +0800 Subject: [PATCH 03/54] =?UTF-8?q?=F0=9F=93=9D=20Record=20dashboard=20snaps?= =?UTF-8?q?hot=20deployment=20status=20and=20acceptance=20gaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/customer-dashboard-20261020/spec.md | 5 +++-- .scratch/dashboard-snapshots-20261020/spec.md | 4 ++-- .../customer-dashboard-20261020/report.md | 2 +- .../dashboard-snapshots-20261020/report.md | 12 +++++++++++- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.scratch/customer-dashboard-20261020/spec.md b/.scratch/customer-dashboard-20261020/spec.md index 3ca6364b..c48e752c 100644 --- a/.scratch/customer-dashboard-20261020/spec.md +++ b/.scratch/customer-dashboard-20261020/spec.md @@ -44,14 +44,15 @@ Status: resolved - 个人与管理员/供应商管理员接入真实 React 页面;旧的源采集生产进度/运行概览保留在管理员折叠区;质检与验收角色原有工作看板保留。 - 已验证:真实 router 与隔离合成 Mongo 的三角色浏览器流程;权限、返工/重试去重、跨期、冻结归属、缺失时长、207 条分页与完整导出;前端/Go 既有失败基线无新增;生产前端启动与 Go 构建通过。 - 上述初版已按用户授权发布为 `65c19de0`,线上三角色只读读取通过;看板请求实测约 13.7–14.7 秒。生产管理员/供应商管理员专用账号验收和历史完整性覆盖仍未完成,未执行历史补录。 +- 该初版随后被同分支 `962ea6da`(共享按需快照)取代;线上当前构建为 `962ea6da`,状态仍为 `deployed_pending_acceptance`,六类业务验收证据为 `unverified`。 -## 性能优化:共享按需快照(本地完成,未发布) +## 性能优化:共享按需快照(已发布,业务验收待补) 用户选择方案 2 后,改为所有 API 副本复用按组织/正式项目范围隔离的 part 统计事实,日期、人员和分页请求不再重复读取完整历史。访问触发刷新,默认间隔 120 秒;保留完整可用版本,失败退避,超过 15 分钟不再继续提供陈旧结果。卡片、明细和 CSV 固定同一快照版本,失效后整页恢复;保持确认版式。 每次请求仍重新校验当前身份和正式项目范围。业务记录、历史补录、交付 JSON 不变。该 GET 在缺少快照或刷新时会写入两个内部派生统计集合,不能再描述为服务端零写入;普通数据库浏览/写入入口禁止访问这些集合。原始审核事件保留在业务存储,快照不复制事件列表,报表详情提供阶段时间与归属。 -本地验证与资源限制见 [`共享快照验证报告`](../../docs/verification/dashboard-snapshots-20261020/report.md),实现契约见 [`共享按需看板快照`](../dashboard-snapshots-20261020/spec.md)。本次优化没有执行公司镜像上传或部署。 +本地验证与资源限制见 [`共享快照验证报告`](../../docs/verification/dashboard-snapshots-20261020/report.md),实现契约见 [`共享按需看板快照`](../dashboard-snapshots-20261020/spec.md)。本次优化已按用户授权于 2026-10-20 22:56 发布为 `962ea6da`;线上发布与有限验收见同一报告「线上发布状态」一节。发布记录为 Git 忽略路径 `local-data/releases/dashboard-snapshots-20261020-962ea6da/`,六类业务验收仍为 `unverified`。 验证记录:[`docs/verification/customer-dashboard-20261020/report.md`](../../docs/verification/customer-dashboard-20261020/report.md)。 diff --git a/.scratch/dashboard-snapshots-20261020/spec.md b/.scratch/dashboard-snapshots-20261020/spec.md index f0d8ae46..e3b7221e 100644 --- a/.scratch/dashboard-snapshots-20261020/spec.md +++ b/.scratch/dashboard-snapshots-20261020/spec.md @@ -2,7 +2,7 @@ Status: resolved -2026-10-20 用户选择方案 2,授权本地实现。线上基线三次读取为 13.7–14.7 秒,分页在读历史/重算之后执行。当前不执行生产派生记录写入或新版本发布。 +2026-10-20 用户选择方案 2。线上基线三次读取为 13.7–14.7 秒,分页在读历史/重算之后执行。同日 22:56 用户另行授权镜像上传、具体部署及内部派生统计写入,候选 `962ea6da` 已发布为线上构建;6 类业务验收证据仍为 `unverified`,发布记录状态为 `deployed_pending_acceptance`,不宣称完整业务验收通过。 ## 契约 @@ -25,4 +25,4 @@ Status: resolved 本地 5000 part、25000 条历史事件、4 个 API 实例及 32 次热请求:历史重建 1 次,首次 224 ms,热请求 P50 15 ms / P95 17 ms。该数据来自隔离合成 Mongo,不是生产性能承诺。 -验证和兼容边界见 [`report.md`](../../docs/verification/dashboard-snapshots-20261020/report.md)。本次只完成本地实现,尚未发布;上线会新增内部派生集合写入,应在具体发布计划中明确列出。 +验证和兼容边界见 [`report.md`](../../docs/verification/dashboard-snapshots-20261020/report.md),线上发布与有限验收状态见该报告「线上发布状态」一节。本次已发布为 `962ea6da`;新增内部派生集合 `annotationOutputSnapshotState` / `annotationOutputSnapshotSlots` 的写入已列入发布计划并获得用户明确授权。发布记录与原始证据位于工作树内 Git 忽略的 `local-data/releases/dashboard-snapshots-20261020-962ea6da/` 和 `local-data/verification/dashboard-snapshots-release-20261020/`。 diff --git a/docs/verification/customer-dashboard-20261020/report.md b/docs/verification/customer-dashboard-20261020/report.md index dcb42f99..01726cb5 100644 --- a/docs/verification/customer-dashboard-20261020/report.md +++ b/docs/verification/customer-dashboard-20261020/report.md @@ -1,6 +1,6 @@ # 客户可对账看板:本地实现验证 -日期:2026-10-20。实现分支:`codex/customer-dashboard-20261020`,基于 `d6fe5e9be3f9e0002694cffdf776ae7c886235e5`。本次为本地修改,不含公司集群发布或客户历史数据补录。 +日期:2026-10-20。实现分支:`codex/customer-dashboard-20261020`,基于 `d6fe5e9be3f9e0002694cffdf776ae7c886235e5`。本报告完成时只做本地修改,不含公司集群发布或客户历史数据补录;此后初版按用户授权发布为 `65c19de0`,并被 `962ea6da` 取代,线上发布状态见 [`共享快照验证报告`](../dashboard-snapshots-20261020/report.md)。 ## 最终行为 diff --git a/docs/verification/dashboard-snapshots-20261020/report.md b/docs/verification/dashboard-snapshots-20261020/report.md index 95b59933..b89019fb 100644 --- a/docs/verification/dashboard-snapshots-20261020/report.md +++ b/docs/verification/dashboard-snapshots-20261020/report.md @@ -1,6 +1,16 @@ # 共享按需看板快照:本地验证 -日期:2026-10-20。分支 `codex/customer-dashboard-20261020`,基于已发布的 `65c19de0bf402cd8202714ebbfd8a05551cd5a25`。用户选择方案 2,本次完成本地实现,没有上传镜像或写入公司环境。 +日期:2026-10-20。分支 `codex/customer-dashboard-20261020`,基于已发布的 `65c19de0bf402cd8202714ebbfd8a05551cd5a25`。用户选择方案 2;本报告完成时只做了本地实现,没有上传镜像或写入公司环境。同日 22:56 按用户单独授权的镜像上传、具体部署和内部统计写入完成发布,线上状态见下节。 + +## 线上发布状态(2026-10-20 22:56) + +权威发布记录位于 Git 忽略的本地路径 `local-data/releases/dashboard-snapshots-20261020-962ea6da/release.json`,原始证据在 `local-data/verification/dashboard-snapshots-release-20261020/`。 + +- 候选与线上构建:`962ea6da3d6932254e1bae293188128db8cbc6ce`;镜像 `ccr-29eug8s3-pub.cnc.bj.baidubce.com/fanyu/droboticflow-api@sha256:5915ff069632516942f80244c03fd9f6d087eee4d60987287def7a5ee21ac113`;计划 `0c2a7c9657fd67b0d4600075b392b2da28b8f9a285514759a9495a37ac398824`;目标 Fanyu / `droboticflow-fanyu`。 +- 部署与复核:处理器 2/2、API 4/4 就绪后才通过基础设施复核;Worker、validator、coordinator 及受保护配置、PVC、Service、资源设定与发布前一致。 +- 线上看板实测:首次共享快照 18,472 ms(经历 9 次 202 轮询);随后 6 次 92–97 ms;到期刷新触发请求 125 ms 返回旧完整结果,刷新期间查询 87–102 ms。均为不同时点的观察值,不当作容量承诺。 +- 业务验收:`role-login`、`supplier-permissions`、`annotation-delivery`、`review-acceptance`、`slice-recovery`、`video-playback` 六项均如实记录为 `unverified`(缺生产管理员/供应商管理员账号,未取得客户任务写测试、生产故障注入和播放验收授权);发布记录状态为 `deployed_pending_acceptance`,不宣称完整业务验收通过。 +- 同日早先的三条发布记录(`63560220`、`86a1d7a3`、`65c19de0`)线上镜像已被本版本取代;记录仍保持各自状态,不追认其业务验收。 ## 结果 From 9546810895bf77af9b4361c1d97ce752dad9dac2 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 21 Oct 2026 09:24:44 +0800 Subject: [PATCH 04/54] =?UTF-8?q?=E2=9C=A8=20Preserve=20accepted=20classif?= =?UTF-8?q?ications=20and=20separate=20private=20delivery=20audits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../delivery-classification-20261021/spec.md | 22 +++ CONTEXT.md | 4 + backend/operators/builtin/package/operator.py | 83 ++++++++++-- docs/adr/0005-part-index-delivery-shape.md | 8 ++ docs/delivery-classification-v4.zh-CN.md | 19 +++ .../annotation_invalid_slice_consolidation.go | 3 +- internal/api/annotation_labels.go | 32 ++++- internal/api/annotation_metadata_revision.go | 5 +- internal/api/annotation_operations.go | 13 +- internal/api/annotation_source_revision.go | 8 +- internal/api/builtin_operators_catalog.json | 16 +++ internal/api/delivery_classification_audit.go | 125 ++++++++++++++++++ internal/api/delivery_index_batches.go | 37 +++++- internal/api/delivery_index_batches_test.go | 47 ++++++- internal/api/delivery_packages.go | 2 +- internal/api/dictionary_selections.go | 97 ++++++++++++++ internal/api/dictionary_selections_test.go | 121 +++++++++++++++++ internal/api/discard_reason_dictionary.go | 2 +- .../api/discard_reason_dictionary_test.go | 4 + internal/api/human_slice_materialization.go | 8 +- internal/api/rework_mongo_http_test.go | 13 ++ internal/api/slice_dictionary_names.go | 82 ++++++------ internal/api/slice_disposition.go | 5 + internal/api/slice_edit_application.go | 3 + internal/api/slice_edit_restoration.go | 2 + internal/deliveryindex/batch.go | 109 +++++++++++++-- internal/deliveryindex/classification_test.go | 56 ++++++++ internal/domain/annotation_labels.go | 22 +++ internal/domain/annotation_operations.go | 51 +++---- internal/domain/delivery_package.go | 6 + internal/worker/annotation_gate.go | 35 ++++- internal/worker/delivery_audit.go | 121 +++++++++++++++++ internal/worker/delivery_audit_test.go | 73 ++++++++++ internal/worker/human_release.go | 2 +- .../builtin/package/src/operator.py | 83 ++++++++++-- src/components/AnnotationLabelsPanel.tsx | 44 +++++- src/components/CaptureAnnotationWorkbench.tsx | 46 ++++++- src/components/SliceEditPanel.tsx | 6 +- src/components/SliceNameDictionaryPicker.tsx | 10 +- src/hooks/useProjectSliceDictionary.ts | 17 ++- src/lib/annotation-operations.js | 1 + src/lib/dictionary-selection.ts | 15 +++ src/lib/slice-edit-panel-ops.js | 4 +- src/pages/AnnotationOperations.tsx | 26 +++- src/types.ts | 16 +++ tests/backend/export_delivery_v4_fixture.py | 39 ++++++ tests/backend/test_delivery_v4_package.py | 101 ++++++++++++++ .../dictionary-classification-tags.mjs | 61 +++++++++ .../dictionary-classification-tags.html | 11 ++ .../dictionary-classification-tags.tsx | 34 +++++ tests/browser/project-dictionary-reuse.mjs | 7 +- .../frontend/project-ui-regressions.test.mjs | 6 +- tests/frontend/slice-dictionary.test.mjs | 6 +- 53 files changed, 1616 insertions(+), 153 deletions(-) create mode 100644 .scratch/delivery-classification-20261021/spec.md create mode 100644 docs/delivery-classification-v4.zh-CN.md create mode 100644 internal/api/delivery_classification_audit.go create mode 100644 internal/api/dictionary_selections.go create mode 100644 internal/api/dictionary_selections_test.go create mode 100644 internal/deliveryindex/classification_test.go create mode 100644 internal/worker/delivery_audit.go create mode 100644 internal/worker/delivery_audit_test.go create mode 100644 src/lib/dictionary-selection.ts create mode 100644 tests/backend/export_delivery_v4_fixture.py create mode 100644 tests/backend/test_delivery_v4_package.py create mode 100644 tests/browser/dictionary-classification-tags.mjs create mode 100644 tests/browser/fixtures/dictionary-classification-tags.html create mode 100644 tests/browser/fixtures/dictionary-classification-tags.tsx diff --git a/.scratch/delivery-classification-20261021/spec.md b/.scratch/delivery-classification-20261021/spec.md new file mode 100644 index 00000000..758f1c50 --- /dev/null +++ b/.scratch/delivery-classification-20261021/spec.md @@ -0,0 +1,22 @@ +# 最终分类交付与内部追溯分离 + +Status: claimed + +2026-10-21 用户要求将两轮总 JSON 核查的问题一起本地修复,并明确选择独立 `tags` 列表、管理员维护。依据为马迪 10-09 至 10-14 的格式对话与本会话补充。原始核查证据在主仓 `local-data/diagnostics/delivery-json-audit-20261021/`。 + +## 验收条件 + +- 字典词条独立 tags,不从 category/aliases 猜补。选择字典后 ID、类型、版本、名称、编码、tags 经服务器校验并冻结,切片/动作修订、物化、审查及最终 JSON 保持一致;字典后改不改写已确认结果。 +- 未写动作描述仍保留切片基础分类和全部区间。显式无效与舍弃独立,不从空文本推断;舍弃原因是分类事实,保留。纯留空、混合、全无效及全舍弃均可表达。 +- 新分类交付使用显式 v4 协议及默认关闭的 DELIVERY_JSON_V4(要求 PART_INDEX_DELIVERY)。客户主 JSON 保持 capture/part、slices/taskSplit/segments 的约定形态,只含定位、最终分类、字典快照、边界及必要源引用;动作 contentKind 使用马迪约定的 annotated/invalid_marked/gap_placeholder。 +- v4 全舍弃 part 仍输出分类 JSON。旧目录/v1/v2/v3 的 no_deliverables 行为、历史文件和在途任务兼容保留,不静默重写旧批次。 +- 交付只取最终验收通过的固定物化修订与摘要;返工正式修正须重新通过验收。批注、人员、轮次变化不能改分类,审核者不能通过标注接口修改业务结果。 +- 全部已提交追溯留内部 JSON,按 capture/part/结果版本关联:审核人、批注、轮次、退回/复审及正式处理记录。客户主 JSON 不带审核历史、lineage、抽检过程或内部路径引用,客户下载不包含内部追溯文件。 +- 不可变批次的选择/身份/SHA/范围/条件发布/重试规则保留;版本必须显式识别,拒绝混搭和不一致证据。历史缺少字典标签不得按最新字典猜补,保留已有名称事实和未知字段边界。 +- 舍弃 reasonNote/at 与 note/discardedAt 兼容映射;原始 meta/标定路径基准显式且一致。规范、真实入口样例与校验器枚举一致。 + +## 验证 + +定向 Go/Python/前端回归,隔离本机 Mongo 与真实 router 的保存→返工→复审→验收→原生打包→批次及下载,检查最终分类/标签和完整内部追溯;覆盖失败重试、重启/旧版本读取、字典更新、未授权篡改、全舍弃、缺失历史证据。前端与 Go 全量门禁按既有基线对照;真实浏览器检查字典 tags 管理和选择,Web/Go 构建及生产包启动。 + +不上传镜像、不部署、不写公司数据、不补写历史交付、不 push/PR。本地完成后以新分支提交。 diff --git a/CONTEXT.md b/CONTEXT.md index 9d87430d..bae63421 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -2,6 +2,10 @@ 以下词汇描述人工标注与审核中的业务对象。 +**分类标签**:管理员为字典词条明确维护的 tags 集合,用于按已确认类别归组;它不同于词条分类 category、别名 aliases 或动作描述文本。确认后的词条身份、版本和标签共同构成当次分类快照,字典后续更新不改写该快照。 + +**最终分类结果**:最终验收通过的固定版本中,视频各区间的基础分类、动作细分、显式无效及舍弃判断。未写动作描述不撤销基础分类;审核批注和返工过程不属于分类结果。 + ## 标注对象 **Capture(源采集)**:一次源采集所关联的双目视频、时间戳与传感器数据,是本流程组织人工处理和审核的对象。 diff --git a/backend/operators/builtin/package/operator.py b/backend/operators/builtin/package/operator.py index cb8ae7e9..4ee57b03 100644 --- a/backend/operators/builtin/package/operator.py +++ b/backend/operators/builtin/package/operator.py @@ -752,9 +752,11 @@ def _part_index_discarded_slice( "reason": str(evidence.get("reason") or ""), "reasonLabelId": str(evidence.get("reasonLabelId") or ""), "reasonName": str(evidence.get("reasonName") or ""), - "note": str(evidence.get("note") or ""), + "note": str(evidence.get("note") or evidence.get("reasonNote") or ""), + "reasonVersion": evidence.get("reasonVersion"), + "dictionary": evidence.get("dictionary"), "discardedByUserId": str(evidence.get("discardedByUserId") or ""), - "discardedAt": str(evidence.get("discardedAt") or ""), + "discardedAt": str(evidence.get("discardedAt") or evidence.get("at") or ""), }, }, "segments": [], @@ -843,13 +845,14 @@ def _description_status(actions: list[dict[str, Any]]) -> str: return "empty_confirmed" -def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): +def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None): """Use the established immutable directory publisher for the two JSON files. Legacy single-file publishers deliberately accept only v1/v2. A directory artifact preserves their ownership/conditional-publication protocol without teaching an old worker to misinterpret a new single-file schema. """ + v4 = internal_audit is not None capture = dict(payload["capture"]) prefix = pathlib.PurePosixPath(str(capture.get("prefix") or "")) part_id = prefix.name @@ -860,7 +863,7 @@ def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3: raise RuntimeError("unsafe v3 capture identity prefix") capture.update(captureId=capture_id, prefix=str(prefix.parent)) - payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id) + payload.update(schemaVersion="droboticflow.delivery.part-index/v4" if v4 else "droboticflow.delivery.part-index/v3", capture=capture, partId=part_id) payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"} payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"} history = { @@ -869,7 +872,8 @@ def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): "qualityReviews": payload["acceptance"].pop("qualityReviews", []), } payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"} - payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None) + if not v4: + payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None) for item in payload["calibration"]: if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]: raise RuntimeError("unsafe calibration filename") @@ -889,6 +893,52 @@ def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): classification["contentCounts"] = counts if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]: classification["descriptionStatus"] = "invalid_and_blank" + if v4: + if not isinstance(internal_audit.get("trace"), list): + raise RuntimeError("DELIVERY_JSON_V4 requires complete internal trace input") + if _output(node_data, helpers, "deliveryAudit") is None: + raise RuntimeError("DELIVERY_JSON_V4 requires an internal deliveryAudit output") + audit = { + "schemaVersion": "droboticflow.delivery.review-history/v2", + "captureId": capture_id, "partId": part_id, "revision": payload["revision"], + "acceptance": payload.pop("acceptance"), "lineage": payload.pop("lineage"), + "qualityReviews": internal_audit.get("qualityReviews", []), + "trace": internal_audit.get("trace", []), + "acceptedMaterializations": internal_audit.get("acceptedMaterializations", []), + } + audit["acceptance"]["finalAcceptance"] = internal_audit.get("finalAcceptance", {}) + payload.pop("reviewHistory", None) + payload["capture"] = {"captureId": capture_id, "prefix": str(prefix.parent)} + for item in payload["slices"] + payload["discardedSlices"]: + classification = item["classification"] + # Blank descriptions retain the independently selected base class. + if classification.get("dictionary") is None: + classification.pop("dictionary", None) + evidence = classification.get("discardEvidence", {}) + evidence.pop("discardedAt", None) + if evidence.get("dictionary") is None: + evidence.pop("dictionary", None) + counts = classification.get("contentCounts") + if counts is not None: + counts["annotated"] = counts.pop("described") + for action in item["segments"]: + if action["contentKind"] == "described": action["contentKind"] = "annotated" + for key in ("source", "confidence", "phrases", "labelId"): + action.pop(key, None) + if action.get("dictionary") is None: action.pop("dictionary", None) + body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8") + if len(json.dumps(audit, ensure_ascii=False).encode("utf8")) > 8 * 1024 * 1024: + raise RuntimeError("internal delivery audit exceeds 8 MiB; refusing incomplete export") + audit["partSha256"] = hashlib.sha256(body).hexdigest() + staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v4-", dir=output)) + (staging / "part.json").write_bytes(body) + # A sibling artifact stays in the internal run, never in the customer tree. + audit_artifact = helpers["write_json_artifact"](run_id, node_id, output, "review-history.json", audit, output=_output(node_data, helpers, "deliveryAudit")) + directory = helpers["build_virtual_artifact"]( + run_id, node_id, "delivery.directory.index", str(staging), "file", 0, + data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"]), "classificationOnly": True}, + output=_output(node_data, helpers, "deliveryDirectory")) + return [directory, audit_artifact] staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output)) for name, document in (("part.json", payload), ("review-history.json", history)): (staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8") @@ -944,6 +994,7 @@ def _part_index_slice( "clipStartMs": _optional_float(action.get("clipStartMs")), "clipEndMs": _optional_float(action.get("clipEndMs")), "labelId": str(action.get("descriptionTemplateId") or ""), + "dictionary": action.get("dictionary"), "phrases": [str(item) for item in action.get("phrases", [])] if isinstance(action.get("phrases"), list) else [], "source": str(action.get("source") or ""), "confidence": _optional_float(action.get("confidence")), @@ -976,6 +1027,7 @@ def _part_index_slice( "deliveryDisposition": declared, "taskLabel": str(synchronization.get("taskLabel") or ""), "taskLabelId": str(synchronization.get("taskLabelId") or ""), + "dictionary": synchronization.get("dictionary"), "descriptionStatus": _description_status(actions), }, "segments": actions, @@ -999,9 +1051,11 @@ def _part_index_slice( "reason": str(evidence.get("reason") or ""), "reasonLabelId": str(evidence.get("reasonLabelId") or ""), "reasonName": str(evidence.get("reasonName") or ""), - "note": str(evidence.get("note") or ""), + "note": str(evidence.get("note") or evidence.get("reasonNote") or ""), + "reasonVersion": evidence.get("reasonVersion"), + "dictionary": evidence.get("dictionary"), "discardedByUserId": str(evidence.get("discardedByUserId") or ""), - "discardedAt": str(evidence.get("discardedAt") or ""), + "discardedAt": str(evidence.get("discardedAt") or evidence.get("at") or ""), } # Pose frames deliberately stay out of the index: the customer contract for # this mode is the split information, and embedded per-frame pose would @@ -1651,6 +1705,12 @@ def _run_accepted_slice_directory( raise RuntimeError("acceptedSliceDirectory does not prove gap-free full-video coverage") summary_artifact = next((item for item in inputs if _port(item) == "acceptanceSummary"), None) summary_payload = summary_artifact.get("data") if isinstance((summary_artifact or {}).get("data"), dict) else {} + if summary_artifact is not None and not isinstance(summary_payload.get("finalAcceptance"), dict): + summary_path = _materialize_input_file(summary_artifact, output / "_acceptance", helpers) + if summary_path.stat().st_size > 8 * 1024 * 1024: + raise RuntimeError("acceptance summary exceeds 8 MiB") + summary_payload = json.loads(summary_path.read_text(encoding="utf8")) + if not isinstance(summary_payload, dict): raise RuntimeError("invalid acceptance summary reference") final_acceptance = summary_payload.get("finalAcceptance") if isinstance(summary_payload.get("finalAcceptance"), dict) else {} capture_record = _part_index_capture_record(summary_payload, revision_manifest) all_slices = revision_manifest.get("items", []) @@ -1679,7 +1739,8 @@ def _run_accepted_slice_directory( raise RuntimeError("slice dispositions are not pinned to the accepted manifest revision") if int(coverage.get("sliceCount", -1)) != len(all_slices): raise RuntimeError("review coverage slice count mismatch") - if disposition_contract and not kept_slices: + params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + if disposition_contract and not kept_slices and not _bool(params, "DELIVERY_JSON_V4", False): if any(p.get("key") == "PROCESSING_PROFILE" and p.get("value") == "raw_human" for p in node_data.get("hyperparams", [])) and (summary_payload.get("processingProfile") != "raw_human" or final_acceptance.get("processingProfile") != "raw_human"): raise RuntimeError("raw-human package requires matching accepted task profile") # Internal evidence only: deliberately no deliveryDirectory output and @@ -1715,8 +1776,10 @@ def _run_accepted_slice_directory( for item in payload["slices"] + payload["discardedSlices"]: item["sourceFrames"] = frames[item["sliceId"]] body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8") + if _bool(params, "DELIVERY_JSON_V4", False): + return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload) if _bool(params, "DELIVERY_JSON_V3", False): - return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload) + return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload) return _emit_part_index(run_id, node_id, output, node_data, helpers, body) if source_index: source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output) @@ -1953,6 +2016,8 @@ def _run_multiport( def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + if _bool(params, "DELIVERY_JSON_V4", False) and (not _bool(params, "PART_INDEX_DELIVERY", False) or _bool(params, "DELIVERY_JSON_V3", False)): + raise RuntimeError("DELIVERY_JSON_V4 requires PART_INDEX_DELIVERY and cannot combine with DELIVERY_JSON_V3") if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False): raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY") # A part-index delivery intentionally has no directory port; it must still diff --git a/docs/adr/0005-part-index-delivery-shape.md b/docs/adr/0005-part-index-delivery-shape.md index f2f816df..a1f88f5d 100644 --- a/docs/adr/0005-part-index-delivery-shape.md +++ b/docs/adr/0005-part-index-delivery-shape.md @@ -43,3 +43,11 @@ status: accepted 路径基准显式写入:总索引相对本批次项目根,采集清单相对采集交付目录,原始 meta 和标定文件相对原始采集目录;已固定 workspace 来源另标基准。采集清单记录原始 `meta_.json` 的 SHA/大小,expected parts 来自原始目录完整分页列举,不用已验收 part 数推定交齐。最终验收结论及最大 acceptedAt 的聚合仅覆盖本批次选择的修订。 总索引生成由管理员显式预览、按预览 snapshotId 条件发布,不随普通 GET/下载自动写对象存储。旧已完成 part 目录不能原地补发 v3,历史迁移仍需另行规划新交付目标。已有调用者和在途任务默认沿用旧协议;回退 API/开关不删除新批次,也不能把新格式任务改成旧格式后重用已完成目录。 + +## 2026-10-21 修订:分类结果与内部追溯独立 + +用户明确交付价值为最终人工确认的分类及标签,而非过程记录。新 v4 主 JSON 只包含最终验收通过的分类、区间及必要源定位;审核批注、人员、轮次、返工和抽检过程不进入主 JSON,也不默认进入客户批次。完整正式追溯保存为内部 JSON,并与固定结果修订关联。返工经过重新验收的分类修正应进入新交付版本,单纯修改批注不能改变分类。 + +独立字典 tags 由管理员维护,与 ID/版本/名称一起在确认时冻结;category 与 aliases 不当作 tags。未写动作描述不等于没有动作,也不撤销切片已有分类。历史没有标签快照时不反查最新字典拼造历史。全舍弃也属于人工分类结果,因此 v4 仍交付该 part 的分类 JSON;这显式替代 v4 路径的旧 no_deliverables 规则,旧媒体目录和 v1–v3 路径继续保持原行为。 + +采用独立 v4 协议与默认关闭的开关,避免同一 v3 schema 悄悄改变字段、枚举或文件集合。保持已约定的 capture/part、slices/taskSplit/segments 层级及不可变批次身份,动作有描述使用约定的 annotated。保留旧版本读取与发布恢复,不原地修补历史完成目录。单独写内部追溯会增加一份内部产物,但避免客户为读取分类被迫接触人员、批注和平台流程字段。 diff --git a/docs/delivery-classification-v4.zh-CN.md b/docs/delivery-classification-v4.zh-CN.md new file mode 100644 index 00000000..d0af3195 --- /dev/null +++ b/docs/delivery-classification-v4.zh-CN.md @@ -0,0 +1,19 @@ +# 最终分类总 JSON v4 + +本地实现对应 [ADR-0005 的 2026-10-21 修订](adr/0005-part-index-delivery-shape.md)。历史 v1/v2/v3 和已交付文件保持原状。开关默认关闭;本文不授权启用工作流或发布。 + +`DELIVERY_JSON_V4=true` 要求 `PART_INDEX_DELIVERY=true`,与 `DELIVERY_JSON_V3` 互斥;源索引修订另需 `SOURCE_INDEX_DELIVERY=true`。新工作流须有内部 `deliveryAudit` JSON 输出,以及正常的 `deliveryDirectory` 输出。配套 Worker 负责固定审核追溯,大于 inline 限额的验收摘要走现有内部 artifact 存储,打包节点读取固定引用;必须先验证内部存储可用。旧 Worker/旧算子不能启用新协议。 + +客户格式保持 `capture`、`partId`、`slices[].taskSplit`、`slices[].segments[]` 和 `discardedSlices`。按马迪约定的采集目录/part 组织文件:每个 part 一个总分类 JSON;批次索引 `delivery_index.json` 和采集索引 `annotation_manifest.json` 只负责目录定位、文件校验及汇总,不额外携带审核流水。正式批次目录不可变。 + +- `sliceName` 是已确认的基础分类名称;`classification.dictionary` 是可获得的字典快照。动作描述为空也保留这个分类,不代表无动作。 +- 字典快照固定 `id/kind/version/code/name/tags`。`tags` 是独立分类标签,与 `category`、别名和描述原义不混用。新选择由服务端从项目字典校验,客户不能伪造标签;历史缺失快照不按当前词条补填。 +- 动作 `contentKind` 为 `annotated`(有描述)、`invalid_marked`(明确无效)、`gap_placeholder`(保留的留空区间)。`descriptionStatus` 描述文字覆盖情况,不能用于推断整个切片是否有动作。 +- `discardedSlices[].classification.discardEvidence` 保留业务舍弃原因、原因词条/版本/标签和补充说明。全舍弃 part 也输出总分类 JSON。审核退回原因不放在这里。 +- `sourceMetadata.path` 与标定路径相对原采集目录。源索引 `sources` 的 workspace 路径基准显式标记。内部审核文件不在客户目录内,也不从主 JSON 引用。 + +内部 `droboticflow.delivery.review-history/v2` JSON 按 capture/part/revision 关联,以 `partSha256` 固定原生主文件字节;包含最终验收、抽检、审核人/批注/轮次/确认项、已有正式退回基线、处理记录和阶段事件。该 artifact 留在内部 Run 存储,不进入最终交付列表、不可变客户批次或客户下载。旧记录没有的追溯不制造补写。追溯超出限制时拒绝交付,不静默截断。 + +最终分类取通过验收的固定物化修订。返工正式修改分类后必须重新送检、验收;查看或处理意见本身不修改分类。相同名称而字典版本/标签不同的切片不能自动合并成一个分类。 + +本地可用 `python -m tests.backend.export_delivery_v4_fixture <全新输出目录>` 生成明确标记的合成样例。目录中 `partNNNN/part.json` 是客户主文件;`internal/partNNNN/review-history.json` 是分离的内部示例,不属于客户包。不要把整个验证输出目录直接交付给客户。 diff --git a/internal/api/annotation_invalid_slice_consolidation.go b/internal/api/annotation_invalid_slice_consolidation.go index ae8ec142..5a361130 100644 --- a/internal/api/annotation_invalid_slice_consolidation.go +++ b/internal/api/annotation_invalid_slice_consolidation.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "reflect" "sort" "strings" @@ -19,7 +20,7 @@ func consolidateInvalidActionSlices(input []domain.SliceSegment, actions []domai eligible := map[string]int{} for i := 1; i < len(ordered)-1; i++ { l, c, r := ordered[i-1], ordered[i], ordered[i+1] - if l.IsDiscarded() || c.IsDiscarded() || r.IsDiscarded() || strings.TrimSpace(c.Name) == "" || l.Name != c.Name || r.Name != c.Name || l.End != c.Start || c.End != r.Start { + if l.IsDiscarded() || c.IsDiscarded() || r.IsDiscarded() || strings.TrimSpace(c.Name) == "" || l.Name != c.Name || r.Name != c.Name || !reflect.DeepEqual(l.Dictionary, c.Dictionary) || !reflect.DeepEqual(r.Dictionary, c.Dictionary) || l.End != c.Start || c.End != r.Start { continue } cursor, has, valid := c.Start, false, true diff --git a/internal/api/annotation_labels.go b/internal/api/annotation_labels.go index 53f7a686..d0772a52 100644 --- a/internal/api/annotation_labels.go +++ b/internal/api/annotation_labels.go @@ -14,6 +14,7 @@ import ( ) type annotationLabelInput struct { + Tags []string `json:"tags"` RequiresNote *bool `json:"requiresNote"` Kind string `json:"kind"` Code string `json:"code"` @@ -46,7 +47,7 @@ func annotationLabelRevision(item domain.AnnotationLabelRecord) domain.Annotatio return domain.AnnotationLabelRevision{ RequiresNote: item.RequiresNote, Version: item.Version, Code: item.Code, Name: item.Name, Category: item.Category, Description: item.Description, Example: item.Example, Color: item.Color, - Aliases: append([]string(nil), item.Aliases...), Status: item.Status, + Tags: append([]string(nil), item.Tags...), Aliases: append([]string(nil), item.Aliases...), Status: item.Status, UpdatedByUserID: item.UpdatedByUserID, UpdatedAt: item.UpdatedAt, } } @@ -79,6 +80,19 @@ func normalizeAnnotationLabelInput(input annotationLabelInput) annotationLabelIn seen[value] = struct{}{} aliases = append(aliases, value) } + + if input.Tags != nil { + tags := []string{} + seenTags := map[string]bool{} + for _, tag := range input.Tags { + tag = strings.TrimSpace(tag) + if tag != "" && !seenTags[tag] { + tags = append(tags, tag) + seenTags[tag] = true + } + } + input.Tags = tags + } input.Aliases = aliases return input } @@ -97,6 +111,15 @@ func validateAnnotationLabelInput(input annotationLabelInput) []validationIssue if input.Status != domain.AnnotationLabelActive && input.Status != domain.AnnotationLabelInactive { issues = append(issues, validationIssue{Field: "status", Message: "标签状态必须是 active 或 inactive"}) } + if len(input.Tags) > 32 { + issues = append(issues, validationIssue{Field: "tags", Message: "每个词条最多 32 个标签"}) + } + for _, tag := range input.Tags { + if len([]rune(tag)) > 80 { + issues = append(issues, validationIssue{Field: "tags", Message: "标签最多 80 字"}) + break + } + } return issues } @@ -165,7 +188,7 @@ func (server *server) handleAnnotationLabelCreate(writer http.ResponseWriter, re ID: domain.NewID("annotation-label"), OrganizationID: organizationID, Code: payload.Code, Name: payload.Name, Category: payload.Category, Description: payload.Description, Example: payload.Example, Color: payload.Color, - Aliases: payload.Aliases, Status: payload.Status, Version: 1, + Tags: payload.Tags, Aliases: payload.Aliases, Status: payload.Status, Version: 1, CreatedByUserID: actorID, UpdatedByUserID: actorID, CreatedAt: now, UpdatedAt: now, } item.History = []domain.AnnotationLabelRevision{annotationLabelRevision(item)} @@ -225,6 +248,9 @@ func (server *server) handleAnnotationLabelPatch(writer http.ResponseWriter, req item.Code, item.Name, item.Category = payload.Code, payload.Name, payload.Category item.Description, item.Example, item.Color = payload.Description, payload.Example, payload.Color item.Aliases, item.Status = payload.Aliases, payload.Status + if payload.Tags != nil { + item.Tags = payload.Tags + } item.Version++ item.UpdatedByUserID, item.UpdatedAt = principal.User.ID, nowISO() item.History = append(item.History, annotationLabelRevision(*item)) @@ -399,7 +425,7 @@ func (server *server) projectAnnotationLabelDefinitions(ctx context.Context, org if !exists || item.Status != domain.AnnotationLabelActive || dictionaryKind(item.Kind) != "action" { continue } - labels = append(labels, domain.SemanticLabelDefinition{ID: item.Code, Name: item.Name, Description: item.Description, Aliases: item.Aliases, Color: item.Color}) + labels = append(labels, domain.SemanticLabelDefinition{Dictionary: item.Selection(), ID: item.Code, Name: item.Name, Description: item.Description, Aliases: item.Aliases, Color: item.Color}) } return labels, nil } diff --git a/internal/api/annotation_metadata_revision.go b/internal/api/annotation_metadata_revision.go index 727be5c9..82e8e7d1 100644 --- a/internal/api/annotation_metadata_revision.go +++ b/internal/api/annotation_metadata_revision.go @@ -134,6 +134,7 @@ func (s *server) tryAnnotationMetadataRevision(ctx context.Context, task domain. } entry["sourceSliceId"], entry["sliceName"], entry["deliveryDisposition"] = slice.ID, slice.Name, sliceDisposition(*slice) synchronized["sliceName"], synchronized["deliveryDisposition"] = slice.Name, sliceDisposition(*slice) + entry["dictionary"], synchronized["dictionary"] = slice.Dictionary, slice.Dictionary delete(entry, "discardEvidence") delete(synchronized, "discardEvidence") if slice.IsDiscarded() { @@ -146,9 +147,9 @@ func (s *server) tryAnnotationMetadataRevision(ctx context.Context, task domain. if slice.IsDiscarded() || segment.End <= slice.Start || segment.Start >= slice.End { continue } - actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": (segment.Start - slice.Start) * 1000, "clipEndMs": (segment.End - slice.Start) * 1000, "startTimestampUs": segment.StartTimestampUS, "endTimestampUs": segment.EndTimestampUS}) + actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "dictionary": segment.Dictionary, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": (segment.Start - slice.Start) * 1000, "clipEndMs": (segment.End - slice.Start) * 1000, "startTimestampUs": segment.StartTimestampUS, "endTimestampUs": segment.EndTimestampUS}) } - annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": entry["sliceId"], "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions} + annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": entry["sliceId"], "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": map[string]any{"startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions} if e = os.MkdirAll(filepath.Join(stage, oldDir), 0755); e != nil { return nil, e } diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index b4146954..3f13aca0 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -1825,6 +1825,7 @@ func (server *server) handleCombinedAnnotationDraftPut(writer http.ResponseWrite return } template = annotationTemplateWithTaskSnapshot(template, task) + issues = append(issues, server.canonicalActionDictionaries(request.Context(), humanItem, payload.Segments)...) issues = append(issues, validateDraftFanyuV5CombinedSemanticSegments(payload.Segments, payload.SliceSegments, template)...) if len(issues) > 0 { renderValidationError(writer, issues...) @@ -1971,6 +1972,7 @@ func (server *server) handleAnnotationConfirmationCreate(writer http.ResponseWri return } template = annotationTemplateWithTaskSnapshot(template, task) + issues = append(issues, server.canonicalActionDictionaries(request.Context(), item, payload.Segments)...) issues = append(issues, validateDraftFanyuV5CombinedSemanticSegments(payload.Segments, slices, template)...) if len(issues) > 0 { renderValidationError(writer, issues...) @@ -2463,6 +2465,10 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri return } template = annotationTemplateWithTaskSnapshot(template, task) + if issues := server.canonicalActionDictionaries(request.Context(), humanItem, payload.Segments); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if issues := validateFanyuV5CombinedSemanticSegmentsWithRequireDesc(payload.Segments, payload.SliceSegments, template, !continuousActions); len(issues) > 0 { renderValidationError(writer, issues...) return @@ -3071,13 +3077,18 @@ func (server *server) handleAnnotationRevisionCreate(writer http.ResponseWriter, renderJSON(writer, http.StatusForbidden, map[string]string{"error": "annotator assignment is required"}) return } - if _, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "semantic_annotation"); !claimed { + humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "semantic_annotation") + if !claimed { return } var payload annotationRevisionInput if !decodeJSON(writer, request, &payload) { return } + if issues := server.canonicalActionDictionaries(request.Context(), humanItem, payload.Segments); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } if issues := validateAnnotationSegments(payload.Segments, *task); len(issues) > 0 { renderValidationError(writer, issues...) return diff --git a/internal/api/annotation_source_revision.go b/internal/api/annotation_source_revision.go index 5f54f631..31ad9752 100644 --- a/internal/api/annotation_source_revision.go +++ b/internal/api/annotation_source_revision.go @@ -85,10 +85,10 @@ func (s *server) materializeSourceIndexRevision(ctx context.Context, t domain.An if segment.Start < slice.Start || segment.End > slice.End { return nil, fmt.Errorf("动作不能跨越切片边界") } - actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": (segment.Start - slice.Start) * 1000, "clipEndMs": (segment.End - slice.Start) * 1000, "startTimestampUs": segment.StartTimestampUS, "endTimestampUs": segment.EndTimestampUS}) + actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "dictionary": segment.Dictionary, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": (segment.Start - slice.Start) * 1000, "clipEndMs": (segment.End - slice.Start) * 1000, "startTimestampUs": segment.StartTimestampUS, "endTimestampUs": segment.EndTimestampUS}) } window := map[string]any{"startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "semantics": "half-open"} - annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": id, "sliceName": slice.Name, "window": window, "actions": actions, "descriptions": actions} + annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": id, "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": window, "actions": actions, "descriptions": actions} seconds, rightSeconds, stamps := []float64{}, []float64{}, []int64{} for j := a; j <= b; j++ { seconds = append(seconds, grid.Seconds[j]-slice.Start) @@ -96,8 +96,8 @@ func (s *server) materializeSourceIndexRevision(ctx context.Context, t domain.An stamps = append(stamps, grid.Timestamps[j]) } frames := map[string]any{"left": []int{leftStart, leftEnd}, "right": []int{rightStart, rightEnd}} - sync := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "sourceIndexSchema": sourceIndexSchema, "boundaryContract": "hardware-frame-v1", "collectionId": cert.CollectionID, "sliceIndex": i + 1, "sliceCount": len(slices), "sliceName": slice.Name, "window": window, "boundarySeconds": seconds, "rightBoundarySeconds": rightSeconds, "boundaryTimestampUs": stamps, "sourceFrames": frames, "sourceMediaOffsets": map[string]any{"left": slice.Start, "right": cert.Timeline.RightSeconds[a]}, "deliveryDisposition": sliceDisposition(*slice), "left": map[string]any{"pts": map[string]any{"rowCount": leftEnd - leftStart}}, "right": map[string]any{"pts": map[string]any{"rowCount": rightEnd - rightStart}}, "imu": map[string]any{"sampleCount": cert.IMURowsBeforeBoundary[b] - cert.IMURowsBeforeBoundary[a]}, "commonRange": map[string]any{"startTimestampUs": grid.Timestamps[0], "endTimestampUs": grid.Timestamps[len(grid.Timestamps)-1], "startSecond": grid.Seconds[0], "endSecond": grid.Seconds[len(grid.Seconds)-1]}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true}} - entry := map[string]any{"sliceId": id, "sourceSliceId": slice.ID, "sliceName": slice.Name, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "directory": filepath.ToSlash(filepath.Join("slices", id)), "sourceFrames": frames, "deliveryDisposition": sliceDisposition(*slice)} + sync := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "sourceIndexSchema": sourceIndexSchema, "boundaryContract": "hardware-frame-v1", "collectionId": cert.CollectionID, "sliceIndex": i + 1, "sliceCount": len(slices), "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": window, "boundarySeconds": seconds, "rightBoundarySeconds": rightSeconds, "boundaryTimestampUs": stamps, "sourceFrames": frames, "sourceMediaOffsets": map[string]any{"left": slice.Start, "right": cert.Timeline.RightSeconds[a]}, "deliveryDisposition": sliceDisposition(*slice), "left": map[string]any{"pts": map[string]any{"rowCount": leftEnd - leftStart}}, "right": map[string]any{"pts": map[string]any{"rowCount": rightEnd - rightStart}}, "imu": map[string]any{"sampleCount": cert.IMURowsBeforeBoundary[b] - cert.IMURowsBeforeBoundary[a]}, "commonRange": map[string]any{"startTimestampUs": grid.Timestamps[0], "endTimestampUs": grid.Timestamps[len(grid.Timestamps)-1], "startSecond": grid.Seconds[0], "endSecond": grid.Seconds[len(grid.Seconds)-1]}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true}} + entry := map[string]any{"sliceId": id, "sourceSliceId": slice.ID, "sliceName": slice.Name, "dictionary": slice.Dictionary, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": slice.StartTimestampUS, "endTimestampUs": slice.EndTimestampUS, "directory": filepath.ToSlash(filepath.Join("slices", id)), "sourceFrames": frames, "deliveryDisposition": sliceDisposition(*slice)} if slice.IsDiscarded() { entry["discardEvidence"] = discardReasonEvidence(*slice) sync["discardEvidence"] = discardReasonEvidence(*slice) diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index 4c72fb03..0db3b859 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -7860,6 +7860,12 @@ "type": "boolean", "value": "false", "description": "原生 v3:part.json 与独立审核历史,通过目录发布;须同时启用 PART_INDEX_DELIVERY" + }, + { + "key": "DELIVERY_JSON_V4", + "type": "boolean", + "value": "false", + "description": "最终分类总 JSON;审核返工追溯单独保存在内部,须启用 PART_INDEX_DELIVERY" } ], "id": "operator-builtin-package", @@ -8170,6 +8176,16 @@ "role": "summary", "transmission": "oss", "type": "json" + }, + { + "id": "deliveryAudit", + "name": "deliveryAudit", + "type": "json", + "container": "single", + "role": "internal-audit", + "payloadMode": "inline", + "transmission": "bus", + "description": "内部审核与返工追溯,不属于客户交付文件" } ], "publicationStatus": "built", diff --git a/internal/api/delivery_classification_audit.go b/internal/api/delivery_classification_audit.go new file mode 100644 index 00000000..27cce771 --- /dev/null +++ b/internal/api/delivery_classification_audit.go @@ -0,0 +1,125 @@ +package api + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "droboticflow/internal/deliveryindex" + "droboticflow/internal/domain" +) + +// Read only the audit produced by the same completed, tenant-scoped package. +// Customer JSON deliberately contains no private run pointers. +func (s *server) classificationPackageAudit(r *http.Request, packages []domain.DeliveryPackageRecord, target deliveryDownloadTarget, partBytes []byte) ([]byte, string, error) { + var part struct { + Capture struct { + ID string `json:"captureId"` + } `json:"capture"` + PartID string `json:"partId"` + Revision int `json:"revision"` + } + if json.Unmarshal(partBytes, &part) != nil { + return nil, "", fmt.Errorf("invalid classification part") + } + var matched []byte + sourceRun := "" + for _, pkg := range packages { + run, err := s.store.GetRun(r.Context(), pkg.ChildRunID) + if err != nil { + return nil, "", err + } + if run == nil || run.OrganizationID != s.requestOrganizationID(r) || run.ProjectID != pkg.ProjectID || run.Status != domain.RunStatusCompleted { + continue + } + ownsTarget := false + for _, a := range run.Artifacts { + if !deliveryPackageArtifactAllowed(&pkg, a.ID) || a.RunID != run.ID || a.Payload == nil || a.Payload.Ref == nil { + continue + } + ref := a.Payload.Ref + if ref.ConnectionID == target.connectionID && ref.Bucket == target.bucket && strings.Trim(ref.Prefix, "/") == target.prefix && ref.ManifestKey == target.prefix+"/manifest.json" { + ownsTarget = true + } + } + if !ownsTarget { + continue + } + for _, a := range run.Artifacts { + if a.RunID != run.ID || !domain.IsInternalDeliveryAudit(a) { + continue + } + data, readErr := s.readClassificationAudit(r, run, a) + if readErr != nil { + return nil, "", readErr + } + raw, err := json.Marshal(data) + if err != nil || len(raw) > 8<<20 { + return nil, "", fmt.Errorf("内部追溯不可读或超过 8 MiB") + } + var audit struct { + Schema string `json:"schemaVersion"` + CaptureID string `json:"captureId"` + PartID string `json:"partId"` + Revision int `json:"revision"` + PartSHA string `json:"partSha256"` + Lineage struct { + RunID string `json:"sourceRunId"` + } `json:"lineage"` + } + if json.Unmarshal(raw, &audit) != nil || audit.Schema != "droboticflow.delivery.review-history/v2" || audit.CaptureID != part.Capture.ID || audit.PartID != part.PartID || audit.Revision != part.Revision || audit.PartSHA != deliveryindex.Hash(partBytes) || audit.Lineage.RunID != pkg.ParentRunID { + return nil, "", fmt.Errorf("内部追溯与固定验收结果不一致") + } + if matched != nil { + return nil, "", fmt.Errorf("交付分类对应多个内部追溯") + } + matched, sourceRun = raw, audit.Lineage.RunID + } + } + if matched == nil || sourceRun == "" { + return nil, "", fmt.Errorf("缺少同一正式交付任务的内部验收证据") + } + return matched, sourceRun, nil +} + +func (s *server) readClassificationAudit(r *http.Request, run *domain.RunRecord, a domain.ExecutionArtifact) (map[string]any, error) { + if a.Payload != nil && a.Payload.Ref != nil { + ref := a.Payload.Ref + if ref.Kind != domain.ArtifactRefKindObject || ref.SizeBytes <= 0 || ref.SizeBytes > 8<<20 || len(ref.SHA256) != 64 { + return nil, fmt.Errorf("内部追溯引用缺少完整校验") + } + connection, err := s.resolveArtifactStorageConnection(r.Context(), run, a, ref.Bucket) + if err != nil || connection == nil || connection.ID != ref.ConnectionID { + return nil, fmt.Errorf("内部追溯连接不可访问") + } + browser, cleanup, err := storageConnectionBrowserResolver(r.Context(), s.storageConnectionWithResolvedEndpoint(connection)) + if err != nil { + return nil, err + } + if cleanup != nil { + defer cleanup() + } + raw, _, err := browser.ReadObjectPreview(r.Context(), ref.Bucket, ref.Key, 8<<20+1) + if err != nil || int64(len(raw)) != ref.SizeBytes || deliveryindex.Hash(raw) != ref.SHA256 { + return nil, fmt.Errorf("内部追溯文件校验失败") + } + var data map[string]any + if json.Unmarshal(raw, &data) != nil { + return nil, fmt.Errorf("内部追溯 JSON 无效") + } + return data, nil + } + if a.Payload != nil && a.Payload.Mode == domain.ArtifactTransferModeInline { + raw, err := json.Marshal(a.Payload.Inline) + if err != nil || len(raw) > 8<<20 { + return nil, fmt.Errorf("内部追溯过大") + } + var data map[string]any + if json.Unmarshal(raw, &data) != nil { + return nil, fmt.Errorf("内部追溯 JSON 无效") + } + return data, nil + } + return artifactStructuredData(&a), nil +} diff --git a/internal/api/delivery_index_batches.go b/internal/api/delivery_index_batches.go index c9ee435f..4d0ca75a 100644 --- a/internal/api/delivery_index_batches.go +++ b/internal/api/delivery_index_batches.go @@ -99,7 +99,7 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla if e != nil { return nil, e } - if len(files) != 3 { + if len(files) != 2 && len(files) != 3 { return nil, fmt.Errorf("原生 v3 part 包必须只含 part.json 与独立审核历史") } var partBytes, historyBytes []byte @@ -124,10 +124,11 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla historyBytes = raw } } - if len(partBytes) == 0 || len(historyBytes) == 0 { + if len(partBytes) == 0 { return nil, fmt.Errorf("选择的包尚未由原生 v3 算子交付;旧格式必须另行重交,不会覆盖旧文件") } var part struct { + Schema string `json:"schemaVersion"` Capture struct { ID string `json:"captureId"` Prefix string `json:"prefix"` @@ -140,6 +141,22 @@ func (s *server) planIndexBatch(r *http.Request, q indexBatchRequest) (*batchPla if json.Unmarshal(partBytes, &part) != nil || target.prefix != root+"/"+part.Capture.ID+"/"+part.PartID { return nil, fmt.Errorf("part 身份与冻结交付目标不一致") } + switch part.Schema { + case deliveryindex.ClassificationPartSchema: + if len(files) != 2 || len(historyBytes) != 0 { + return nil, fmt.Errorf("分类交付包中包含额外文件") + } + historyBytes, part.Lineage.RunID, e = s.classificationPackageAudit(r, selected, target, partBytes) + if e != nil { + return nil, e + } + case deliveryindex.PartSchema: + if len(files) != 3 || len(historyBytes) == 0 { + return nil, fmt.Errorf("旧 v3 交付缺少审核历史") + } + default: + return nil, fmt.Errorf("不支持的交付格式;历史交付不会被覆盖") + } record, e := s.store.GetCaptureRecordByRunID(r.Context(), part.Lineage.RunID) if e != nil || record == nil || record.OrganizationID != project.OrganizationID || record.ProjectID != project.ID || record.Prefix != part.Capture.Prefix+"/"+part.PartID || record.Delivery == nil || record.Delivery.Prefix != target.prefix || record.Delivery.ConnectionID != t.connectionID || record.Delivery.Bucket != t.bucket { return nil, fmt.Errorf("part 来源与采集记录不一致") @@ -425,14 +442,14 @@ func readIndexBatchDownloadManifest(ctx context.Context, b objectStorageBrowser, } `json:"project"` Entries []deliveryindex.Entry `json:"entries"` } - if json.Unmarshal(raw, &index) != nil || index.Schema != 3 || index.Kind != "part-index" || index.ID != id || index.Project.Root != t.prefix || index.Count != len(index.Entries) || index.Count < 1 || index.Count > 400 { + if json.Unmarshal(raw, &index) != nil || (index.Schema != 3 && index.Schema != 4) || index.Kind != "part-index" || index.ID != id || index.Project.Root != t.prefix || index.Count != len(index.Entries) || index.Count < 1 || index.Count > 400 { return nil, fmt.Errorf("批次总索引身份或成员数量无效") } files := []deliveryDownloadFile{} var size int64 for _, entry := range index.Entries { sha, shaErr := hex.DecodeString(entry.SHA256) - if !validDeliveryDownloadPath(entry.Path) || entry.Path == "delivery_index.json" || strings.HasPrefix(entry.Path, ".") || entry.Key != t.prefix+"/"+entry.Path || entry.SizeBytes < 0 || entry.SizeBytes > 8<<20 || shaErr != nil || len(sha) != 32 { + if (index.Schema == 4 && !validClassificationBatchPath(entry.Path)) || !validDeliveryDownloadPath(entry.Path) || entry.Path == "delivery_index.json" || strings.HasPrefix(entry.Path, ".") || entry.Key != t.prefix+"/"+entry.Path || entry.SizeBytes < 0 || entry.SizeBytes > 8<<20 || shaErr != nil || len(sha) != 32 { return nil, fmt.Errorf("批次成员校验信息无效") } size += entry.SizeBytes @@ -444,3 +461,15 @@ func readIndexBatchDownloadManifest(ctx context.Context, b objectStorageBrowser, files = append(files, deliveryDownloadFile{RelativePath: t.relative + "/delivery_index.json", SizeBytes: int64(len(raw)), SHA256: deliveryindex.Hash(raw), key: t.prefix + "/delivery_index.json", bucket: t.bucket, connectionID: t.connectionID}) return files, nil } + +func validClassificationBatchPath(value string) bool { + parts := strings.Split(value, "/") + if len(parts) != 2 { + return false + } + if parts[1] == "annotation_manifest.json" { + return true + } + name := strings.TrimSuffix(parts[1], ".json") + return strings.HasSuffix(parts[1], ".json") && len(name) >= 8 && strings.HasPrefix(name, "part") && strings.Trim(name[4:], "0123456789") == "" +} diff --git a/internal/api/delivery_index_batches_test.go b/internal/api/delivery_index_batches_test.go index 191355c2..4b645fd9 100644 --- a/internal/api/delivery_index_batches_test.go +++ b/internal/api/delivery_index_batches_test.go @@ -108,6 +108,10 @@ func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authent must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: "project-A", OrganizationID: org, Name: "项目A"})) must(ds.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "index-storage", OrganizationID: org, Provider: domain.StorageProviderBOS, Bucket: "test-bucket", Endpoint: "https://bj.bcebos.com"})) dir := os.Getenv("DELIVERY_V3_FIXTURE_DIR") + v4 := os.Getenv("DELIVERY_CLASSIFICATION_TEST") == "1" + if v4 { + dir = os.Getenv("DELIVERY_V4_FIXTURE_DIR") + } for _, part := range []string{"part0001", "part0002"} { prefix := root + "/capture-A/" + part sourceRun := "source-run-" + part @@ -119,19 +123,43 @@ func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authent var e error body, e = os.ReadFile(filepath.Join(dir, part, "part.json")) must(e) - history, e = os.ReadFile(filepath.Join(dir, part, "review-history.json")) + historyPath := filepath.Join(dir, part, "review-history.json") + if v4 { + historyPath = filepath.Join(dir, "internal", part, "review-history.json") + } + history, e = os.ReadFile(historyPath) must(e) } + if v4 && dir == "" { + var doc map[string]any + must(json.Unmarshal(body, &doc)) + acceptance := doc["acceptance"] + delete(doc, "acceptance") + delete(doc, "lineage") + doc["schemaVersion"] = deliveryindex.ClassificationPartSchema + body, _ = json.Marshal(doc) + history, _ = json.Marshal(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2", "captureId": "capture-A", "partId": part, "revision": 1, "partSha256": deliveryindex.Hash(body), "acceptance": acceptance, "lineage": map[string]any{"sourceRunId": sourceRun}, "qualityReviews": []any{map[string]any{"comment": "PRIVATE_REWORK"}}, "trace": []any{}}) + } b.objects[prefix+"/.delivery-owner.json"] = []byte(`{"owner":"synthetic-source"}`) b.objects[prefix+"/part.json"] = body b.objects[prefix+"/review-history.json"] = history entries := []deliveryindex.Entry{{Path: "part.json", Key: prefix + "/part.json", SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)}, {Path: "review-history.json", Key: prefix + "/review-history.json", SizeBytes: int64(len(history)), SHA256: deliveryindex.Hash(history)}} b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 2, "totalBytes": len(body) + len(history), "entries": entries}) + if v4 { + delete(b.objects, prefix+"/review-history.json") + entries = entries[:1] + b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 1, "totalBytes": len(body), "entries": entries}) + } b.objects["incoming/capture-A/"+part+"/left.mp4"] = []byte("synthetic") run := domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: "project-A", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: artifactID, RunID: runID, Name: "delivery.directory.index", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindDirectory, Provider: domain.StorageProviderBOS, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix, ManifestKey: prefix + "/manifest.json"}}}}} + if v4 { + var audit map[string]any + must(json.Unmarshal(history, &audit)) + run.Artifacts = append(run.Artifacts, domain.ExecutionArtifact{ID: "audit-" + part, RunID: runID, PortID: "deliveryAudit", Type: domain.PortTypeJSON, Data: audit}) + } run.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "package", Artifacts: run.Artifacts}} must(ds.UpsertRun(ctx, run)) - must(ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "package-" + part, OrganizationID: org, ProjectID: "project-A", ChildRunID: runID, Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{artifactID}})) + must(ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "package-" + part, OrganizationID: org, ProjectID: "project-A", ParentRunID: sourceRun, ChildRunID: runID, Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{artifactID}})) must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, OrganizationID: org, ProjectID: "project-A", RunID: sourceRun, CaptureID: "capture-A/" + part, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A/" + part, SourcePrefix: "incoming/capture-A", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix}})) } b.objects["incoming/capture-A/meta_capture-A.json"] = []byte(`{"synthetic":true}`) @@ -207,6 +235,16 @@ func exerciseIndexBatch(t *testing.T, ds store.Store) { if download.Code != 200 || !strings.Contains(download.Body.String(), "delivery_index.json") { t.Fatal(download.Code, download.Body.String()) } + if os.Getenv("DELIVERY_CLASSIFICATION_TEST") == "1" { + for _, file := range files { + if strings.Contains(file.RelativePath, "history") || bytes.Contains(b.objects[file.key], []byte("PRIVATE_REWORK")) { + t.Fatal("private audit delivered") + } + } + if strings.Contains(download.Body.String(), "history") { + t.Fatal("download exposes private audit") + } + } // A later partial snapshot does not alter the old full snapshot. first := indexCall(h, p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil) var partial struct { @@ -299,3 +337,8 @@ func TestIndexBatchDelegationKeepsSourceAndDeliveryScopesSeparate(t *testing.T) t.Fatal("sibling batch prefix escaped delivery grant", w.Code, w.Body.String()) } } + +func TestClassificationBatchHTTPPublicationRecoveryAndDownload(t *testing.T) { + t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1") + exerciseIndexBatch(t, store.NewMemoryStore()) +} diff --git a/internal/api/delivery_packages.go b/internal/api/delivery_packages.go index 15b361b8..fbe84733 100644 --- a/internal/api/delivery_packages.go +++ b/internal/api/delivery_packages.go @@ -114,7 +114,7 @@ func (server *server) handleDeliveryPackageContents(writer http.ResponseWriter, artifacts := make([]domain.ExecutionArtifact, 0, len(allowed)) var manifest map[string]any for _, artifact := range run.Artifacts { - if _, exists := allowed[artifact.ID]; !exists || artifact.RunID != run.ID { + if _, exists := allowed[artifact.ID]; !exists || artifact.RunID != run.ID || domain.IsInternalDeliveryAudit(artifact) { continue } copy := artifact diff --git a/internal/api/dictionary_selections.go b/internal/api/dictionary_selections.go new file mode 100644 index 00000000..a554ad97 --- /dev/null +++ b/internal/api/dictionary_selections.go @@ -0,0 +1,97 @@ +package api + +import ( + "context" + "fmt" + "strings" + + "droboticflow/internal/domain" +) + +// Always resolve new selections on the server. Existing saved selections are +// copied from the workspace/revision, so dictionary maintenance cannot rewrite +// a submitted classification and a client cannot forge tags or another tenant. +func (s *server) projectDictionarySelections(ctx context.Context, item *domain.HumanItemRecord, kind string) (map[string]domain.AnnotationLabelRecord, error) { + selection, err := s.store.GetProjectAnnotationLabels(ctx, item.ProjectID) + if err != nil { + return nil, err + } + result := map[string]domain.AnnotationLabelRecord{} + if selection == nil { + return result, nil + } + if selection.OrganizationID != item.OrganizationID { + return nil, fmt.Errorf("项目字典不属于当前组织") + } + ids := selection.LabelIDs + if kind == "slice" { + ids = selection.SliceLabelIDs + } + for _, id := range ids { + label, err := s.store.GetAnnotationLabel(ctx, id) + if err != nil { + return nil, err + } + if label != nil && label.OrganizationID == item.OrganizationID && dictionaryKind(label.Kind) == kind && label.Status == domain.AnnotationLabelActive { + result[id] = *label + } + } + return result, nil +} + +func sameDictionaryVersion(a, b *domain.DictionarySelection) bool { + return a != nil && b != nil && a.ID == b.ID && a.Version == b.Version +} + +func (s *server) canonicalActionDictionaries(ctx context.Context, item *domain.HumanItemRecord, segments []domain.AnnotationSegment) []validationIssue { + previous := decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(item)["segments"]) + if len(previous) == 0 && item.LegacyTaskID != "" { + revisions, err := s.store.ListAnnotationRevisions(ctx, item.OrganizationID, item.LegacyTaskID) + if err != nil { + return []validationIssue{{Field: "segments", Message: "无法读取已保存的动作分类"}} + } + for _, rev := range revisions { + if rev.Revision == item.CurrentRevision { + previous = rev.Segments + } + } + } + old := map[string]domain.AnnotationSegment{} + for _, seg := range previous { + old[seg.ID] = seg + } + var allowed map[string]domain.AnnotationLabelRecord + for i := range segments { + seg := &segments[i] + p := old[seg.ID] + if seg.ClearDictionary { + seg.Dictionary = nil + seg.ClearDictionary = false + continue + } + if seg.IsInvalid || strings.TrimSpace(seg.Description) == "" { + seg.Dictionary = nil + continue + } + if p.Dictionary != nil && (sameDictionaryVersion(p.Dictionary, seg.Dictionary) || (seg.Dictionary == nil && seg.Description == p.Description && !p.IsInvalid)) { + seg.Dictionary = p.Dictionary + continue + } + if seg.Dictionary == nil { + continue + } // Legacy/free text is never guessed into a category. + if allowed == nil { + var err error + allowed, err = s.projectDictionarySelections(ctx, item, "action") + if err != nil { + return []validationIssue{{Field: "segments", Message: "无法读取动作字典,请重试"}} + } + } + label, ok := allowed[seg.Dictionary.ID] + if !ok || label.Version != seg.Dictionary.Version { + return []validationIssue{{Field: fmt.Sprintf("segments.%d.dictionary", i), Code: "dictionary_selection_stale", Message: "动作词条已变化,请从项目字典重新选择"}} + } + seg.Dictionary = label.Selection() + } + return nil +} diff --git a/internal/api/dictionary_selections_test.go b/internal/api/dictionary_selections_test.go new file mode 100644 index 00000000..e642f34d --- /dev/null +++ b/internal/api/dictionary_selections_test.go @@ -0,0 +1,121 @@ +package api + +import ( + "encoding/json" + "net/http" + "reflect" + "testing" + + "droboticflow/internal/domain" +) + +func TestDictionarySelectionsPersistThroughHTTPAndDictionaryMaintenance(t *testing.T) { + f := newCombinedDraftFixture(t) + labels := []domain.AnnotationLabelRecord{ + {ID: "base", OrganizationID: f.item.OrganizationID, Kind: "slice", Code: "base", Name: "物品操作", Tags: []string{"商品"}, Version: 2, Status: domain.AnnotationLabelActive}, + {ID: "action", OrganizationID: f.item.OrganizationID, Kind: "action", Code: "pick", Name: "拿取", Tags: []string{"抓取"}, Version: 3, Status: domain.AnnotationLabelActive}, + } + for _, label := range labels { + if err := f.store.UpsertAnnotationLabel(f.ctx, label); err != nil { + t.Fatal(err) + } + } + selection := domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID, LabelIDs: []string{"action"}, SliceLabelIDs: []string{"base"}} + if err := f.store.UpsertProjectAnnotationLabels(f.ctx, selection); err != nil { + t.Fatal(err) + } + if err := f.store.UpsertProjectSliceLabels(f.ctx, selection); err != nil { + t.Fatal(err) + } + forgedBase := *labels[0].Selection() + forgedBase.Tags = []string{"FORGED"} + forgedAction := *labels[1].Selection() + forgedAction.Tags = []string{"FORGED"} + payload := combinedAnnotationDraftInput{Version: f.item.Version, SliceSegments: []domain.SliceSegment{{ID: "s", Name: "物品操作", Start: 0, End: 10, Dictionary: &forgedBase}}, Segments: []domain.AnnotationSegment{{ID: "a", Start: 1, End: 2, Description: "拿起杯子", Dictionary: &forgedAction}}} + endpoint := "/api/plugins/annotation-qa/tasks/" + f.task.ID + "/drafts" + save := func(actor domain.UserRecord) *domain.HumanItemRecord { + t.Helper() + r := f.request(http.MethodPut, endpoint, actor, payload) + if r.Code != 200 { + t.Fatalf("save: %d %s", r.Code, r.Body.String()) + } + item, err := f.store.GetHumanItem(f.ctx, f.item.ID) + if err != nil { + t.Fatal(err) + } + return item + } + saved := save(f.annotator) + draft := annotationWorkspacePayload(saved) + slices := decodeMaterializationValue[domain.SliceSegment](draft["sliceSegments"]) + actions := decodeMaterializationValue[domain.AnnotationSegment](draft["segments"]) + if !reflect.DeepEqual(slices[0].Dictionary, labels[0].Selection()) || !reflect.DeepEqual(actions[0].Dictionary, labels[1].Selection()) { + t.Fatal("server trusted forged tags") + } + // Empty action text never clears the independent base classification. + payload.Version = saved.Version + payload.SliceSegments = slices + payload.Segments = nil + saved = save(f.annotator) + slices = decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(saved)["sliceSegments"]) + if slices[0].Dictionary == nil || slices[0].Dictionary.Tags[0] != "商品" { + t.Fatal("blank description lost base class") + } + for i := range labels { + labels[i].Version++ + labels[i].Name += "新版" + labels[i].Tags = []string{"新标签"} + labels[i].Status = domain.AnnotationLabelInactive + f.store.UpsertAnnotationLabel(f.ctx, labels[i]) + } + f.store.UpsertProjectSliceLabels(f.ctx, domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID}) + payload.Version = saved.Version + payload.SliceSegments = slices + saved = save(f.annotator) + // A fresh handler recovers the frozen snapshot, independent of active dictionaries. + f.handler = NewHandler(f.store, nil) + payload.Version = saved.Version + saved = save(f.annotator) + slices = decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(saved)["sliceSegments"]) + if slices[0].Dictionary.Version != 2 || slices[0].Dictionary.Name != "物品操作" { + t.Fatal("historical classification rewritten") + } + payload.Version = saved.Version + payload.Segments = []domain.AnnotationSegment{{ID: "new-action", Start: 1, End: 2, Description: "拿取", Dictionary: &forgedAction}} + if r := f.request(http.MethodPut, endpoint, f.annotator, payload); r.Code != 400 { + t.Fatalf("stale/new action accepted %d %s", r.Code, r.Body.String()) + } + if r := f.request(http.MethodPut, endpoint, f.reviewer, payload); r.Code != 403 { + t.Fatal("reviewer can change classification") + } +} + +func TestDictionaryTagsHaveIndependentRevisionAndAdminAuthorization(t *testing.T) { + f := newCombinedDraftFixture(t) + body := map[string]any{"kind": "slice", "name": "分拣", "code": "sort", "category": "business-category", "aliases": []string{"别名"}, "tags": []string{" 商品 ", "商品", "分拣"}} + r := f.request("POST", "/api/annotation-labels", f.annotator, body) + if r.Code != 403 { + t.Fatal("annotator maintained dictionary") + } + r = f.request("POST", "/api/annotation-labels", f.admin, body) + if r.Code != 201 { + t.Fatalf("%d %s", r.Code, r.Body.String()) + } + var label domain.AnnotationLabelRecord + if err := json.Unmarshal(r.Body.Bytes(), &label); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(label.Tags, []string{"商品", "分拣"}) || label.Category != "business-category" || len(label.Aliases) != 1 { + t.Fatal("tags confused with other fields") + } + body["tags"] = []string{} + r = f.request("PATCH", "/api/annotation-labels/"+label.ID, f.admin, body) + if r.Code != 200 { + t.Fatalf("%d %s", r.Code, r.Body.String()) + } + var updated domain.AnnotationLabelRecord + json.Unmarshal(r.Body.Bytes(), &updated) + if updated.Version != 2 || len(updated.Tags) != 0 || !reflect.DeepEqual(updated.History[0].Tags, label.Tags) { + t.Fatal("tag history lost") + } +} diff --git a/internal/api/discard_reason_dictionary.go b/internal/api/discard_reason_dictionary.go index b5629fda..b2ffe81a 100644 --- a/internal/api/discard_reason_dictionary.go +++ b/internal/api/discard_reason_dictionary.go @@ -73,5 +73,5 @@ func describeSliceNameIssues(issues []validationIssue, slices []domain.SliceSegm } func discardReasonEvidence(slice domain.SliceSegment) map[string]any { - return map[string]any{"reason": slice.DiscardReason, "reasonLabelId": slice.DiscardReasonLabelID, "reasonVersion": slice.DiscardReasonVersion, "reasonName": slice.DiscardReasonName, "reasonNote": slice.DiscardReasonNote, "at": slice.DiscardedAt, "previousAnnotations": slice.DiscardedAnnotations} + return map[string]any{"dictionary": slice.DiscardReasonDictionary, "reason": slice.DiscardReason, "reasonLabelId": slice.DiscardReasonLabelID, "reasonVersion": slice.DiscardReasonVersion, "reasonName": slice.DiscardReasonName, "reasonNote": slice.DiscardReasonNote, "at": slice.DiscardedAt, "previousAnnotations": slice.DiscardedAnnotations} } diff --git a/internal/api/discard_reason_dictionary_test.go b/internal/api/discard_reason_dictionary_test.go index f36eed4c..b226bafb 100644 --- a/internal/api/discard_reason_dictionary_test.go +++ b/internal/api/discard_reason_dictionary_test.go @@ -30,6 +30,7 @@ func TestDiscardReasonDictionaryValidationAndSnapshots(t *testing.T) { case "other_without_note": label.RequiresNote = true case "success": + label.Tags = []string{"不可用"} body.Reason = "forged" body.ReasonNote = "遮挡持续两秒" } @@ -54,6 +55,9 @@ func TestDiscardReasonDictionaryValidationAndSnapshots(t *testing.T) { t.Fatal(err) } slice := got.Workspace.SliceSegments[1] + if slice.DiscardReasonDictionary == nil || len(slice.DiscardReasonDictionary.Tags) != 1 || slice.DiscardReasonDictionary.Tags[0] != "不可用" { + t.Fatal("discard tags missing") + } if slice.DiscardReasonName != "相机被遮挡" || slice.DiscardReasonVersion != 1 || slice.DiscardReasonLabelID != label.ID || slice.DiscardReasonNote != body.ReasonNote || strings.Contains(slice.DiscardReason, "forged") { t.Fatalf("snapshot %+v", slice) } diff --git a/internal/api/human_slice_materialization.go b/internal/api/human_slice_materialization.go index db805543..0d1015a7 100644 --- a/internal/api/human_slice_materialization.go +++ b/internal/api/human_slice_materialization.go @@ -948,9 +948,9 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta } relativeStart := segmentStart - slice.Start relativeEnd := segmentEnd - slice.Start - actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": relativeStart * 1000, "clipEndMs": relativeEnd * 1000, "startTimestampUs": mediaSecondToTimestamp(segmentStart), "endTimestampUs": mediaSecondToTimestamp(segmentEnd)}) + actions = append(actions, map[string]any{"id": segment.ID, "status": effectiveActionStatus(segment), "isInvalid": segment.IsInvalid, "description": segment.Description, "descriptionTemplateId": segment.LabelID, "dictionary": segment.Dictionary, "phrases": segment.Phrases, "source": segment.Source, "confidence": segment.Confidence, "clipStartMs": relativeStart * 1000, "clipEndMs": relativeEnd * 1000, "startTimestampUs": mediaSecondToTimestamp(segmentStart), "endTimestampUs": mediaSecondToTimestamp(segmentEnd)}) } - annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": sliceID, "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions} + annotation := map[string]any{"schemaVersion": "droboticflow.timestamped-description/v3", "boundaryContract": "hardware-frame-v1", "sliceId": sliceID, "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "descriptions": actions, "actions": actions} annotationSHA, annotationBytes, err := writeJSONFile(filepath.Join(directory, "annotation.json"), annotation) if err != nil { return nil, err @@ -968,7 +968,7 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta boundaryTimes = append(boundaryTimes, grid.Seconds[k]-slice.Start) rightBoundaryTimes = append(rightBoundaryTimes, float64(grid.RightFrames[k]-rightStart)*rightMediaDuration/float64(len(rightPTS))) } - synchronization := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "collectionId": collectionID, "sliceIndex": index + 1, "sliceCount": len(normalizedSlices), "sliceName": slice.Name, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "commonRange": map[string]any{"startTimestampUs": commonStart, "endTimestampUs": commonEnd, "startSecond": commonMediaStart, "endSecond": commonMediaEnd}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true, "previousEndTimestampUs": func() any { + synchronization := map[string]any{"schemaVersion": "droboticflow.synchronized-slice/v1", "collectionId": collectionID, "sliceIndex": index + 1, "sliceCount": len(normalizedSlices), "sliceName": slice.Name, "dictionary": slice.Dictionary, "window": map[string]any{"startTimestampUs": startUS, "endTimestampUs": endUS, "semantics": "half-open"}, "commonRange": map[string]any{"startTimestampUs": commonStart, "endTimestampUs": commonEnd, "startSecond": commonMediaStart, "endSecond": commonMediaEnd}, "coverage": map[string]any{"mode": "full_common_range", "contiguous": true, "previousEndTimestampUs": func() any { if index == 0 { return nil } @@ -991,7 +991,7 @@ func (server *server) materializeCombinedSliceSubmission(ctx context.Context, ta if err != nil { return nil, err } - manifestItems = append(manifestItems, map[string]any{"sliceId": sliceID, "sourceSliceId": slice.ID, "sliceName": slice.Name, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": startUS, "endTimestampUs": endUS, "directory": filepath.ToSlash(filepath.Join("slices", sliceID)), "files": map[string]any{"leftVideo": leftVideo, "rightVideo": rightVideo, "leftPts": leftManifest, "rightPts": rightManifest, "imu": imuManifest, "annotation": map[string]any{"name": "annotation.json", "sha256": annotationSHA, "bytes": annotationBytes}, "synchronization": map[string]any{"name": "synchronized-slice.json", "sha256": syncSHA, "bytes": syncBytes}}}) + manifestItems = append(manifestItems, map[string]any{"sliceId": sliceID, "sourceSliceId": slice.ID, "sliceName": slice.Name, "dictionary": slice.Dictionary, "startSecond": slice.Start, "endSecond": slice.End, "startTimestampUs": startUS, "endTimestampUs": endUS, "directory": filepath.ToSlash(filepath.Join("slices", sliceID)), "files": map[string]any{"leftVideo": leftVideo, "rightVideo": rightVideo, "leftPts": leftManifest, "rightPts": rightManifest, "imu": imuManifest, "annotation": map[string]any{"name": "annotation.json", "sha256": annotationSHA, "bytes": annotationBytes}, "synchronization": map[string]any{"name": "synchronized-slice.json", "sha256": syncSHA, "bytes": syncBytes}}}) manifestItems[len(manifestItems)-1]["deliveryDisposition"] = sliceDisposition(slice) manifestItems[len(manifestItems)-1]["mediaKey"] = mediaKey if slice.IsDiscarded() { diff --git a/internal/api/rework_mongo_http_test.go b/internal/api/rework_mongo_http_test.go index 17bc13d5..fe09bbe3 100644 --- a/internal/api/rework_mongo_http_test.go +++ b/internal/api/rework_mongo_http_test.go @@ -93,12 +93,20 @@ func TestReworkMongoHTTPRepeatedReturnAcrossRestart(t *testing.T) { seed("annotationTasks", f.task) slices, e := f.store.ListSliceRevisions(ctx, f.item.OrganizationID, f.task.ID) must(e) + baseClass := &domain.DictionarySelection{ID: "historical-base", Kind: "slice", Version: 2, Code: "sort", Name: "物品操作", Tags: []string{"商品"}} + actionClass := &domain.DictionarySelection{ID: "historical-action", Kind: "action", Version: 3, Code: "pick", Name: "拿取", Tags: []string{"抓取"}} for _, r := range slices { + for i := range r.Segments { + r.Segments[i].Dictionary = baseClass + } seed("sliceRevisions", r) } actions, e := f.store.ListAnnotationRevisions(ctx, f.item.OrganizationID, f.task.ID) must(e) for _, r := range actions { + for i := range r.Segments { + r.Segments[i].Dictionary = actionClass + } seed("annotationRevisions", r) } f.handler = NewHandler(ds, nil) @@ -128,6 +136,11 @@ func TestReworkMongoHTTPRepeatedReturnAcrossRestart(t *testing.T) { if w.Code != 200 || len(out.Items) != 1 || out.Items[0].AssignedUserID != f.annotator.ID { t.Fatalf("persisted return missing from author pool: %d %s", w.Code, w.Body.String()) } + baseline, e := ds.GetReworkBaseline(ctx, store.ReworkBaselineID(item.ID, out.Items[0].ReviewRound)) + must(e) + if baseline == nil || len(baseline.Slices) == 0 || baseline.Slices[0].Dictionary == nil || baseline.Slices[0].Dictionary.Version != 2 || baseline.Slices[0].Dictionary.Tags[0] != "商品" { + t.Fatal("base classification lost across Mongo review/restart") + } t.Logf("%s: round %d persisted and visible after restart", operation, out.Items[0].ReviewRound) } item, e := ds.GetHumanItem(ctx, f.item.ID) diff --git a/internal/api/slice_dictionary_names.go b/internal/api/slice_dictionary_names.go index 5f8f3592..41a5d7d7 100644 --- a/internal/api/slice_dictionary_names.go +++ b/internal/api/slice_dictionary_names.go @@ -7,8 +7,8 @@ import ( "strings" ) -// Unnamed drafts are allowed, but new confirmations need project dictionary -// names. Unchanged historical names remain valid during annotation/review. +// Binding happens before persistence, and never at delivery time. Name-only +// historical records remain name-only until a new selection is made. func (s *server) validateProjectSliceNames(ctx context.Context, item *domain.HumanItemRecord, slices []domain.SliceSegment, mode string) []validationIssue { if item == nil { return nil @@ -22,57 +22,53 @@ func (s *server) validateProjectSliceNames(ctx context.Context, item *domain.Hum } previous := decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(item)["sliceSegments"]) if len(previous) == 0 { - revisions, e := s.store.ListSliceRevisions(ctx, item.OrganizationID, item.LegacyTaskID) - if e != nil { + revisions, err := s.store.ListSliceRevisions(ctx, item.OrganizationID, item.LegacyTaskID) + if err != nil { return []validationIssue{{Field: "sliceSegments", Message: "无法读取历史切片名称"}} } - if len(revisions) > 0 { - previous = revisions[len(revisions)-1].Segments - } - } - old := map[string]string{} - for _, p := range previous { - old[p.ID] = p.Name - } - needs := []int{} - for i, c := range slices { - if c.IsDiscarded() { - continue - } - if mode != "confirm" { - if name, ok := old[c.ID]; ok && name == c.Name { - continue + for _, rev := range revisions { + if rev.Revision == item.CurrentRevision || len(previous) == 0 { + previous = rev.Segments } } - if mode == "draft" && strings.TrimSpace(c.Name) == "" { + } + old := map[string]domain.SliceSegment{} + for _, p := range previous { + old[p.ID] = p + } + var allowed map[string]domain.AnnotationLabelRecord + for i := range slices { + c := &slices[i] + p, exists := old[c.ID] + if c.IsDiscarded() { + c.Dictionary = nil continue } - needs = append(needs, i) - } - if len(needs) == 0 { - return nil - } - selection, err := s.store.GetProjectAnnotationLabels(ctx, item.ProjectID) - if err != nil { - return []validationIssue{{Field: "sliceSegments", Message: "无法读取项目切片字典,请重试"}} - } - allowed := map[string]bool{} - if selection != nil { - for _, id := range selection.SliceLabelIDs { - label, e := s.store.GetAnnotationLabel(ctx, id) - if e != nil { + if exists && p.Name == c.Name && (c.Dictionary == nil || sameDictionaryVersion(p.Dictionary, c.Dictionary)) && (mode != "confirm" || p.Dictionary != nil) { + c.Dictionary = p.Dictionary + continue + } + if (mode == "restore" && c.Dictionary == nil) || (mode == "draft" && strings.TrimSpace(c.Name) == "") { + c.Dictionary = nil + continue + } + if allowed == nil { + allowed, err = s.projectDictionarySelections(ctx, item, "slice") + if err != nil { return []validationIssue{{Field: "sliceSegments", Message: "无法读取项目切片字典,请重试"}} } - if label != nil && label.OrganizationID == item.OrganizationID && dictionaryKind(label.Kind) == "slice" && label.Status == domain.AnnotationLabelActive { - allowed[label.Name] = true + } + var chosen *domain.DictionarySelection + for _, label := range allowed { + if label.Name == c.Name && (c.Dictionary == nil || (label.ID == c.Dictionary.ID && label.Version == c.Dictionary.Version)) { + chosen = label.Selection() + break } } - } - issues := []validationIssue{} - for _, i := range needs { - if !allowed[slices[i].Name] { - issues = append(issues, validationIssue{Field: fmt.Sprintf("sliceSegments.%d.name", i), Code: "slice_dictionary_name_required", Message: "请从本项目切片字典选择名称;不能自由填写或修改。未配置字典请联系管理员"}) + if chosen == nil { + return []validationIssue{{Field: fmt.Sprintf("sliceSegments.%d.name", i), Code: "slice_dictionary_name_required", Message: "请从本项目切片字典选择名称;词条已更新时请重新选择"}} } + c.Dictionary = chosen } - return issues + return nil } diff --git a/internal/api/slice_disposition.go b/internal/api/slice_disposition.go index 2a23d01f..c3fef44c 100644 --- a/internal/api/slice_disposition.go +++ b/internal/api/slice_disposition.go @@ -142,6 +142,7 @@ func canonicalDispositions(previous, slices []domain.SliceSegment) ([]domain.Sli next[i].DiscardReasonLabelID = p.DiscardReasonLabelID next[i].DiscardReasonVersion = p.DiscardReasonVersion next[i].DiscardReasonName = p.DiscardReasonName + next[i].DiscardReasonDictionary = p.DiscardReasonDictionary next[i].DiscardReasonNote = p.DiscardReasonNote next[i].DiscardedByUserID = p.DiscardedByUserID next[i].DiscardedAt = p.DiscardedAt @@ -480,6 +481,7 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, } issues = append(issues, validateSliceSegments(slices, *task)...) issues = append(issues, validateCompleteSliceCoverage(slices, *task)...) + issues = append(issues, s.canonicalActionDictionaries(r.Context(), item, body.Segments)...) issues = append(issues, validateDraftCombinedAnnotationEvents(body.Segments, *task)...) idx := -1 for i, v := range slices { @@ -540,10 +542,12 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, next.DiscardReasonLabelID = "" next.DiscardReasonVersion = 0 next.DiscardReasonName = "" + next.DiscardReasonDictionary = nil if reasonLabel != nil { next.DiscardReasonLabelID = reasonLabel.ID next.DiscardReasonVersion = reasonLabel.Version next.DiscardReasonName = reasonLabel.Name + next.DiscardReasonDictionary = reasonLabel.Selection() } next.DiscardReasonNote = body.ReasonNote next.DiscardedByUserID = actorID @@ -627,6 +631,7 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, next.DiscardReasonLabelID = "" next.DiscardReasonVersion = 0 next.DiscardReasonName = "" + next.DiscardReasonDictionary = nil next.DiscardReasonNote = "" next.DiscardedByUserID = "" next.DiscardedAt = "" diff --git a/internal/api/slice_edit_application.go b/internal/api/slice_edit_application.go index 16c8c990..fbb74401 100644 --- a/internal/api/slice_edit_application.go +++ b/internal/api/slice_edit_application.go @@ -239,10 +239,12 @@ func (s *server) handleSliceEditApplication(w http.ResponseWriter, r *http.Reque for _, candidate := range proposed { if candidate.ID == slices[i].ID && !slices[i].IsDiscarded() { slices[i].Name = candidate.Name + slices[i].Dictionary = candidate.Dictionary } } } issues = s.validateProjectSliceNames(r.Context(), &item, slices, "draft") + issues = append(issues, s.canonicalActionDictionaries(r.Context(), stored, segments)...) issues = append(issues, s.validateSliceEditFrameBounds(r, task, &item, slices, segments)...) issues = append(issues, validateDraftCombinedAnnotationEvents(segments, *task)...) issues = append(issues, validateDraftFanyuV5CombinedSemanticSegments(segments, slices, annotationTemplateWithTaskSnapshot(template, task))...) @@ -302,6 +304,7 @@ func sliceEditPolicyBaseline(candidate, old domain.SliceSegment) domain.SliceSeg candidate.DiscardReasonLabelID = old.DiscardReasonLabelID candidate.DiscardReasonVersion = old.DiscardReasonVersion candidate.DiscardReasonName = old.DiscardReasonName + candidate.DiscardReasonDictionary = old.DiscardReasonDictionary candidate.DiscardReasonNote = old.DiscardReasonNote candidate.DiscardedAt = old.DiscardedAt candidate.DiscardedByUserID = old.DiscardedByUserID diff --git a/internal/api/slice_edit_restoration.go b/internal/api/slice_edit_restoration.go index b74b2b44..5e1f33c4 100644 --- a/internal/api/slice_edit_restoration.go +++ b/internal/api/slice_edit_restoration.go @@ -122,6 +122,8 @@ func (s *server) handleSliceEditRestoration(w http.ResponseWriter, r *http.Reque issues = append(issues, validateCompleteSliceCoverage(slices, *task)...) issues = append(issues, validateDraftCombinedAnnotationEvents(body.Segments, *task)...) issues = append(issues, validateDraftStatuses(body.normalizedStatuses(), slices)...) + issues = append(issues, s.validateProjectSliceNames(r.Context(), item, slices, "restore")...) + issues = append(issues, s.canonicalActionDictionaries(r.Context(), item, body.Segments)...) template, err := s.store.GetAnnotationTemplate(r.Context(), task.TemplateID) if err != nil { renderError(w, 500, err) diff --git a/internal/deliveryindex/batch.go b/internal/deliveryindex/batch.go index 84463db5..6ce33db1 100644 --- a/internal/deliveryindex/batch.go +++ b/internal/deliveryindex/batch.go @@ -16,6 +16,7 @@ import ( ) const PartSchema = "droboticflow.delivery.part-index/v3" +const ClassificationPartSchema = "droboticflow.delivery.part-index/v4" var partName = regexp.MustCompile(`^part[0-9]{4,}$`) @@ -48,7 +49,7 @@ type Summary struct { DiscardedCount int `json:"discardedSliceCount"` KeptDuration float64 `json:"keptDurationSeconds"` DiscardedDuration float64 `json:"discardedDurationSeconds"` - Acceptance map[string]string `json:"acceptance"` + Acceptance map[string]string `json:"acceptance,omitempty"` } type part struct { Schema string `json:"schemaVersion"` @@ -117,6 +118,9 @@ func addSummary(a *Summary, b Summary) { a.DiscardedCount += b.DiscardedCount a.KeptDuration += b.KeptDuration a.DiscardedDuration += b.DiscardedDuration + if b.Acceptance == nil { + return + } if a.Acceptance == nil { a.Acceptance = map[string]string{"decision": "approved"} } @@ -128,7 +132,7 @@ func addSummary(a *Summary, b Summary) { } func checkPart(p part) (Summary, error) { s := Summary{PartCount: 1, Acceptance: map[string]string{"decision": "approved", "acceptedAt": p.Acceptance.Final.AcceptedAt}} - if p.Schema != PartSchema || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 { + if (p.Schema != PartSchema && p.Schema != ClassificationPartSchema) || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 { return s, fmt.Errorf("part must pin an accepted native v3 revision without embedded review history") } if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil { @@ -168,6 +172,9 @@ func checkPart(p part) (Summary, error) { invalid = true } else if strings.TrimSpace(a.Description) != "" { kind = "described" + if p.Schema == ClassificationPartSchema { + kind = "annotated" + } described = true } else { blank = true @@ -191,7 +198,7 @@ func checkPart(p part) (Summary, error) { } } } - if s.KeptCount == 0 { + if s.KeptCount == 0 && (p.Schema != ClassificationPartSchema || s.DiscardedCount == 0) { return s, fmt.Errorf("no deliverable slices") } if math.IsInf(s.KeptDuration, 0) || math.IsInf(s.DiscardedDuration, 0) { @@ -206,6 +213,9 @@ func checkPart(p part) (Summary, error) { return s, fmt.Errorf("slice coverage gap or overlap") } } + if p.Schema == ClassificationPartSchema { + s.Acceptance = nil + } return s, nil } @@ -223,6 +233,22 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh return out, fmt.Errorf("source JSON exceeds 64 MiB") } } + schema := "" + for _, src := range sources { + var p part + if err := json.Unmarshal(src.Part, &p); err != nil { + return out, err + } + if schema != "" && schema != p.Schema { + return out, fmt.Errorf("cannot mix part schema versions in one batch") + } + schema = p.Schema + if p.Schema == ClassificationPartSchema { + if _, err := classificationAudit(src); err != nil { + return out, err + } + } + } // Normalize JSON before hashing to ignore transport whitespace/map ordering. for i := range sources { var d, h any @@ -235,7 +261,17 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh sort.Strings(sources[i].ExpectedParts) } sort.Slice(sources, func(i, j int) bool { return string(sources[i].Part) < string(sources[j].Part) }) - identity := encode([]any{"droboticflow.delivery.batch/v1", projectID, projectName, projectRoot, sources}) + identitySources := append([]Source(nil), sources...) + if schema == ClassificationPartSchema { + for i := range identitySources { + identitySources[i].History = nil + } + } + identity := encode([]any{"droboticflow.delivery.batch/v1", projectID, projectName, projectRoot, identitySources}) + version := 3 + if schema == ClassificationPartSchema { + version = 4 + } out.ID = Hash(identity) out.Root = projectRoot + "/index-batches/" + out.ID type group struct { @@ -259,6 +295,15 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh if e := json.Unmarshal(src.Part, &p); e != nil { return out, e } + if p.Schema == ClassificationPartSchema { + // The original bytes were checked before canonicalization above. + var audit classificationHistory + if json.Unmarshal(src.History, &audit) != nil { + return out, fmt.Errorf("invalid internal audit") + } + p.Acceptance.Final.Confirmed = audit.Acceptance.Final.Confirmed + p.Acceptance.Final.AcceptedAt = audit.Acceptance.Final.AcceptedAt + } s, e := checkPart(p) if e != nil { return out, e @@ -290,7 +335,7 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh Revision int `json:"revision"` Reviews []json.RawMessage `json:"qualityReviews"` } - if json.Unmarshal(src.History, &history) != nil || history.Schema != "droboticflow.delivery.review-history/v1" || history.CaptureID != p.Capture.ID || history.PartID != p.PartID || history.Revision != p.Revision || history.Reviews == nil { + if p.Schema != ClassificationPartSchema && (json.Unmarshal(src.History, &history) != nil || history.Schema != "droboticflow.delivery.review-history/v1" || history.CaptureID != p.Capture.ID || history.PartID != p.PartID || history.Revision != p.Revision || history.Reviews == nil) { return out, fmt.Errorf("review history revision mismatch") } g := groups[p.Capture.ID] @@ -302,10 +347,17 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh } var doc map[string]any json.Unmarshal(src.Part, &doc) - doc["reviewHistory"] = map[string]string{"path": p.PartID + ".review-history.json", "pathBase": "capture_delivery"} doc["delivery"] = map[string]string{"kind": "part_index", "fileName": p.PartID + ".json", "artifactName": "delivery.part-index"} - g.entries = append(g.entries, add(name+".json", encode(doc)), add(name+".review-history.json", src.History)) - g.parts = append(g.parts, map[string]any{"part": p.PartID, "revision": p.Revision, "path": p.PartID + ".json", "reviewHistoryPath": p.PartID + ".review-history.json"}) + g.entries = append(g.entries, add(name+".json", encode(doc))) + reference := map[string]any{"part": p.PartID, "revision": p.Revision, "path": p.PartID + ".json"} + if p.Schema != ClassificationPartSchema { + doc["reviewHistory"] = map[string]string{"path": p.PartID + ".review-history.json", "pathBase": "capture_delivery"} + totalBytes -= len(out.Files[name+".json"]) + g.entries[len(g.entries)-1] = add(name+".json", encode(doc)) + g.entries = append(g.entries, add(name+".review-history.json", src.History)) + reference["reviewHistoryPath"] = p.PartID + ".review-history.json" + } + g.parts = append(g.parts, reference) addSummary(&g.summary, s) if math.IsInf(g.summary.KeptDuration, 0) || math.IsInf(g.summary.DiscardedDuration, 0) { return out, fmt.Errorf("capture duration overflow") @@ -343,7 +395,7 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh size += local[i].SizeBytes } metadata := map[string]any{"path": "meta_" + n + ".json", "pathBase": "original_capture", "sha256": g.metadata.MetadataSHA256, "sizeBytes": g.metadata.MetadataBytes} - manifest := map[string]any{"schemaVersion": 3, "kind": "part-index", "capture": g.capture, "pathBase": "capture_delivery", "sourceMetadata": metadata, "summary": g.summary, "parts": g.parts, "entries": local, "fileCount": len(local), "totalBytes": size} + manifest := map[string]any{"schemaVersion": version, "kind": "part-index", "capture": g.capture, "pathBase": "capture_delivery", "sourceMetadata": metadata, "summary": g.summary, "parts": g.parts, "entries": local, "fileCount": len(local), "totalBytes": size} allEntries = append(allEntries, g.entries...) allEntries = append(allEntries, add(n+"/annotation_manifest.json", encode(manifest))) captures = append(captures, map[string]any{"captureId": n, "prefix": g.capture["prefix"], "manifestPath": n + "/annotation_manifest.json", "summary": g.summary}) @@ -360,7 +412,44 @@ func Build(projectID, projectName, projectRoot string, sources []Source) (Snapsh for _, v := range allEntries { size += v.SizeBytes } - out.Index = encode(map[string]any{"schemaVersion": 3, "kind": "part-index", "snapshotId": out.ID, "pathBase": "batch_project", "project": map[string]string{"projectId": projectID, "deliveryName": projectName, "rootKey": out.Root}, "summary": total, "captures": captures, "entries": allEntries, "fileCount": len(allEntries), "totalBytes": size}) + out.Index = encode(map[string]any{"schemaVersion": version, "kind": "part-index", "snapshotId": out.ID, "pathBase": "batch_project", "project": map[string]string{"projectId": projectID, "deliveryName": projectName, "rootKey": out.Root}, "summary": total, "captures": captures, "entries": allEntries, "fileCount": len(allEntries), "totalBytes": size}) out.Files["delivery_index.json"] = out.Index return out, nil } + +// classificationHistory is internal acceptance proof, excluded from v4 batch +// identity and customer files. The digest binds it to the original native bytes. +type classificationHistory struct { + Schema string `json:"schemaVersion"` + CaptureID string `json:"captureId"` + PartID string `json:"partId"` + Revision int `json:"revision"` + PartSHA string `json:"partSha256"` + Acceptance struct { + Final struct { + Confirmed bool `json:"confirmed"` + AcceptedAt string `json:"acceptedAt"` + } `json:"finalAcceptance"` + } `json:"acceptance"` +} + +func classificationAudit(src Source) (classificationHistory, error) { + var audit classificationHistory + var p part + var doc map[string]json.RawMessage + if json.Unmarshal(src.Part, &p) != nil || json.Unmarshal(src.Part, &doc) != nil || json.Unmarshal(src.History, &audit) != nil { + return audit, fmt.Errorf("invalid classification/audit JSON") + } + for _, field := range []string{"acceptance", "lineage", "reviewHistory"} { + if _, ok := doc[field]; ok { + return audit, fmt.Errorf("customer classification contains internal trace") + } + } + if audit.Schema != "droboticflow.delivery.review-history/v2" || audit.CaptureID != p.Capture.ID || audit.PartID != p.PartID || audit.Revision != p.Revision || audit.PartSHA != Hash(src.Part) || !audit.Acceptance.Final.Confirmed { + return audit, fmt.Errorf("internal audit does not pin the accepted classification") + } + if _, err := time.Parse(time.RFC3339Nano, audit.Acceptance.Final.AcceptedAt); err != nil { + return audit, fmt.Errorf("missing internal acceptance timestamp") + } + return audit, nil +} diff --git a/internal/deliveryindex/classification_test.go b/internal/deliveryindex/classification_test.go new file mode 100644 index 00000000..365a021f --- /dev/null +++ b/internal/deliveryindex/classification_test.go @@ -0,0 +1,56 @@ +package deliveryindex + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func classificationFixture(allDiscard bool) Source { + src := fixture("part0001", 7) + var p map[string]any + json.Unmarshal(src.Part, &p) + acceptance := p["acceptance"] + delete(p, "acceptance") + p["schemaVersion"] = ClassificationPartSchema + if allDiscard { + p["slices"] = []any{} + p["discardedSlices"] = []any{map[string]any{"sliceId": "all", "taskSplit": map[string]any{"startSecond": 0, "endSecond": 3, "semantics": "half-open"}, "classification": map[string]any{"deliveryDisposition": "discard", "discardEvidence": map[string]any{"reason": "遮挡"}}, "segments": []any{}}} + p["deliveryClassification"] = map[string]any{"keptSliceCount": 0, "discardedSliceCount": 1, "keptDurationSeconds": 0, "discardedDurationSeconds": 3} + } + src.Part = encode(p) + src.History = encode(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2", "captureId": "capture-A", "partId": "part0001", "revision": 7, "partSha256": Hash(src.Part), "acceptance": acceptance, "qualityReviews": []any{map[string]string{"comment": "PRIVATE_REWORK"}}, "trace": []any{}}) + return src +} +func TestClassificationBatchNeverIncludesPrivateTrace(t *testing.T) { + for _, all := range []bool{false, true} { + src := classificationFixture(all) + out := build(t, src) + if len(out.Files) != 3 { + t.Fatalf("customer files %v", out.Files) + } + for name, body := range out.Files { + if strings.Contains(name, "history") || bytes.Contains(body, []byte("PRIVATE_REWORK")) || bytes.Contains(body, []byte(`"acceptance"`)) { + t.Fatalf("private trace in %s", name) + } + } + var h map[string]any + json.Unmarshal(src.History, &h) + h["qualityReviews"] = []any{map[string]string{"comment": "changed"}} + src.History = encode(h) + again := build(t, src) + if out.ID != again.ID || !bytes.Equal(out.Index, again.Index) { + t.Fatal("audit-only edit changed customer batch") + } + src.History = bytes.ReplaceAll(src.History, []byte(Hash(src.Part)), []byte(strings.Repeat("0", 64))) + if _, err := Build("project-A", "项目A", "delivery/项目A__project-A", []Source{src}); err == nil { + t.Fatal("wrong acceptance digest accepted") + } + } +} +func TestClassificationBatchRejectsMixedVersions(t *testing.T) { + if _, err := Build("project-A", "项目A", "delivery/项目A__project-A", []Source{classificationFixture(false), fixture("part0002", 1)}); err == nil { + t.Fatal("mixed versions accepted") + } +} diff --git a/internal/domain/annotation_labels.go b/internal/domain/annotation_labels.go index 67eb64b0..ac7641f1 100644 --- a/internal/domain/annotation_labels.go +++ b/internal/domain/annotation_labels.go @@ -29,6 +29,7 @@ type AnnotationLabelRecord struct { Description string `json:"description,omitempty" bson:"description,omitempty"` Example string `json:"example,omitempty" bson:"example,omitempty"` Color string `json:"color,omitempty" bson:"color,omitempty"` + Tags []string `json:"tags,omitempty" bson:"tags,omitempty"` Aliases []string `json:"aliases,omitempty" bson:"aliases,omitempty"` Status AnnotationLabelStatus `json:"status" bson:"status"` Version int `json:"version" bson:"version"` @@ -48,6 +49,7 @@ type AnnotationLabelRevision struct { Description string `json:"description,omitempty" bson:"description,omitempty"` Example string `json:"example,omitempty" bson:"example,omitempty"` Color string `json:"color,omitempty" bson:"color,omitempty"` + Tags []string `json:"tags,omitempty" bson:"tags,omitempty"` Aliases []string `json:"aliases,omitempty" bson:"aliases,omitempty"` Status AnnotationLabelStatus `json:"status" bson:"status"` UpdatedByUserID string `json:"updatedByUserId" bson:"updatedByUserId"` @@ -64,3 +66,23 @@ type ProjectAnnotationLabelsRecord struct { UpdatedByUserID string `json:"updatedByUserId" bson:"updatedByUserId"` UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } + +// DictionarySelection is an immutable business classification, never a lookup +// against the current dictionary during review or delivery. +type DictionarySelection struct { + ID string `json:"id" bson:"id"` + Kind string `json:"kind" bson:"kind"` + Version int `json:"version" bson:"version"` + Code string `json:"code" bson:"code"` + Name string `json:"name" bson:"name"` + Tags []string `json:"tags" bson:"tags"` +} + +func (label AnnotationLabelRecord) Selection() *DictionarySelection { + kind := label.Kind + if kind == "" { + kind = "action" + } + return &DictionarySelection{ID: label.ID, Kind: kind, Version: label.Version, + Code: label.Code, Name: label.Name, Tags: append([]string{}, label.Tags...)} +} diff --git a/internal/domain/annotation_operations.go b/internal/domain/annotation_operations.go index df4ee5d8..b106ae23 100644 --- a/internal/domain/annotation_operations.go +++ b/internal/domain/annotation_operations.go @@ -66,11 +66,12 @@ type ProjectAnnotationMembershipRecord struct { } type SemanticLabelDefinition struct { - ID string `json:"id" bson:"id"` - Name string `json:"name" bson:"name"` - Description string `json:"description,omitempty" bson:"description,omitempty"` - Aliases []string `json:"aliases,omitempty" bson:"aliases,omitempty"` - Color string `json:"color,omitempty" bson:"color,omitempty"` + Dictionary *DictionarySelection `json:"dictionary,omitempty" bson:"dictionary,omitempty"` + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + Aliases []string `json:"aliases,omitempty" bson:"aliases,omitempty"` + Color string `json:"color,omitempty" bson:"color,omitempty"` } type AnnotationPrelabelSettings struct { @@ -99,20 +100,22 @@ type AnnotationTemplateRecord struct { } type AnnotationSegment struct { - Status string `json:"status,omitempty" bson:"status,omitempty"` - ID string `json:"id" bson:"id"` - LabelID string `json:"labelId" bson:"labelId"` - Label string `json:"label" bson:"label"` - Description string `json:"description,omitempty" bson:"description,omitempty"` - Handedness string `json:"handedness,omitempty" bson:"handedness,omitempty"` - Start float64 `json:"start" bson:"start"` - End float64 `json:"end" bson:"end"` - StartTimestampUS int64 `json:"startTimestampUs,omitempty" bson:"startTimestampUs,omitempty"` - EndTimestampUS int64 `json:"endTimestampUs,omitempty" bson:"endTimestampUs,omitempty"` - Confidence float64 `json:"confidence,omitempty" bson:"confidence,omitempty"` - Phrases []string `json:"phrases,omitempty" bson:"phrases,omitempty"` - Source string `json:"source,omitempty" bson:"source,omitempty"` - IsInvalid bool `json:"isInvalid,omitempty" bson:"isInvalid,omitempty"` + ClearDictionary bool `json:"clearDictionary,omitempty" bson:"-"` + Dictionary *DictionarySelection `json:"dictionary,omitempty" bson:"dictionary,omitempty"` + Status string `json:"status,omitempty" bson:"status,omitempty"` + ID string `json:"id" bson:"id"` + LabelID string `json:"labelId" bson:"labelId"` + Label string `json:"label" bson:"label"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + Handedness string `json:"handedness,omitempty" bson:"handedness,omitempty"` + Start float64 `json:"start" bson:"start"` + End float64 `json:"end" bson:"end"` + StartTimestampUS int64 `json:"startTimestampUs,omitempty" bson:"startTimestampUs,omitempty"` + EndTimestampUS int64 `json:"endTimestampUs,omitempty" bson:"endTimestampUs,omitempty"` + Confidence float64 `json:"confidence,omitempty" bson:"confidence,omitempty"` + Phrases []string `json:"phrases,omitempty" bson:"phrases,omitempty"` + Source string `json:"source,omitempty" bson:"source,omitempty"` + IsInvalid bool `json:"isInvalid,omitempty" bson:"isInvalid,omitempty"` } type SliceAnnotationStatus string @@ -124,10 +127,12 @@ const ( ) type SliceSegment struct { - DiscardReasonLabelID string `json:"discardReasonLabelId,omitempty" bson:"discardReasonLabelId,omitempty"` - DiscardReasonVersion int `json:"discardReasonVersion,omitempty" bson:"discardReasonVersion,omitempty"` - DiscardReasonName string `json:"discardReasonName,omitempty" bson:"discardReasonName,omitempty"` - DiscardReasonNote string `json:"discardReasonNote,omitempty" bson:"discardReasonNote,omitempty"` + Dictionary *DictionarySelection `json:"dictionary,omitempty" bson:"dictionary,omitempty"` + DiscardReasonDictionary *DictionarySelection `json:"discardReasonDictionary,omitempty" bson:"discardReasonDictionary,omitempty"` + DiscardReasonLabelID string `json:"discardReasonLabelId,omitempty" bson:"discardReasonLabelId,omitempty"` + DiscardReasonVersion int `json:"discardReasonVersion,omitempty" bson:"discardReasonVersion,omitempty"` + DiscardReasonName string `json:"discardReasonName,omitempty" bson:"discardReasonName,omitempty"` + DiscardReasonNote string `json:"discardReasonNote,omitempty" bson:"discardReasonNote,omitempty"` DeliveryDisposition string `json:"deliveryDisposition,omitempty" bson:"deliveryDisposition,omitempty"` DiscardReason string `json:"discardReason,omitempty" bson:"discardReason,omitempty"` diff --git a/internal/domain/delivery_package.go b/internal/domain/delivery_package.go index 73e3269d..366529b3 100644 --- a/internal/domain/delivery_package.go +++ b/internal/domain/delivery_package.go @@ -65,3 +65,9 @@ type DeliveryPackageQuery struct { Status DeliveryPackageStatus Mode RunMode } + +// IsInternalDeliveryAudit also recognizes the schema so historical workflow +// output bindings cannot accidentally expose this internal artifact. +func IsInternalDeliveryAudit(artifact ExecutionArtifact) bool { + return artifact.PortID == "deliveryAudit" || artifact.PortName == "deliveryAudit" || artifact.Data["schemaVersion"] == "droboticflow.delivery.review-history/v2" +} diff --git a/internal/worker/annotation_gate.go b/internal/worker/annotation_gate.go index 752e28d7..0481f22c 100644 --- a/internal/worker/annotation_gate.go +++ b/internal/worker/annotation_gate.go @@ -1292,6 +1292,7 @@ func (runtime *Runtime) releasePartitionArtifacts( } for _, review := range reviews { qualityReviews = append(qualityReviews, map[string]any{ + "id": review.ID, "reviewRound": review.ReviewRound, "stage": review.Stage, "confirmations": review.Confirmations, "nextRevision": review.NextRevision, "rejectTargets": review.RejectTargets, "taskId": review.TaskID, "revision": review.Revision, "decision": review.Decision, "reasonCodes": review.ReasonCodes, "problemSliceIds": review.ProblemSliceIDs, "comment": review.Comment, "reviewerUserId": review.ReviewerUserID, "createdAt": review.CreatedAt, @@ -1308,12 +1309,44 @@ func (runtime *Runtime) releasePartitionArtifacts( } } summary := map[string]any{"schemaVersion": "droboticflow.acceptance-summary/v2", "decision": "approved", "partitionId": partition.ID, "itemCount": len(selected), "capture": captureEvidence, "qualityReviews": qualityReviews, "finalAcceptance": acceptanceEvidence} + classificationMode, err := runtime.classificationDeliveryEnabled(ctx, run.WorkflowID) + if err != nil { + return nil, err + } + if classificationMode { + auditItems := []domain.HumanItemRecord{} + for _, itemID := range partition.ItemIDs { + auditItems = append(auditItems, selected[itemID]) + } + if acceptanceID, ok := acceptanceEvidence["humanItemId"].(string); ok && acceptanceID != "" { + acceptedItem, err := runtime.store.GetHumanItem(ctx, acceptanceID) + if err != nil { + return nil, err + } + if acceptedItem == nil { + return nil, fmt.Errorf("acceptance audit item missing") + } + auditItems = append(auditItems, *acceptedItem) + } + trace, err := runtime.deliveryAuditTrace(ctx, batch.OrganizationID, auditItems) + if err != nil { + return nil, err + } + summary["trace"] = trace + } summary["acceptedMaterializations"] = acceptedMaterializations if batch.ProcessingProfile == domain.ProcessingProfileRawHuman { summary["processingProfile"] = domain.ProcessingProfileRawHuman } encoded, _ := json.Marshal(summary) - artifacts = append(artifacts, domain.ExecutionArtifact{ID: fmt.Sprintf("%s-%s-acceptance-summary", run.ID, node.ID), RunID: run.ID, NodeID: node.ID, PortID: summaryPortID, PortName: summaryPortName, Name: "acceptance-summary.json", Type: domain.PortTypeJSON, Size: annotationArtifactSize(int64(len(encoded))), SizeBytes: int64(len(encoded)), Data: summary, Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: summary}}) + summaryArtifact := domain.ExecutionArtifact{ID: fmt.Sprintf("%s-%s-acceptance-summary", run.ID, node.ID), RunID: run.ID, NodeID: node.ID, PortID: summaryPortID, PortName: summaryPortName, Name: "acceptance-summary.json", Type: domain.PortTypeJSON, Size: annotationArtifactSize(int64(len(encoded))), SizeBytes: int64(len(encoded)), Data: summary, Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: summary}} + if classificationMode { + summaryArtifact, err = runtime.persistClassificationSummary(ctx, run, node, partition.ID, summaryArtifact, encoded) + if err != nil { + return nil, err + } + } + artifacts = append(artifacts, summaryArtifact) return artifacts, nil } clips := annotationClipItems(execution.InputArtifacts) diff --git a/internal/worker/delivery_audit.go b/internal/worker/delivery_audit.go new file mode 100644 index 00000000..ae4b9ff5 --- /dev/null +++ b/internal/worker/delivery_audit.go @@ -0,0 +1,121 @@ +package worker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +// Internal evidence is pinned with the accepted package, independently from +// customer classification JSON. Never resolve it from a later live workspace. +func (runtime *Runtime) deliveryAuditTrace(ctx context.Context, org string, items []domain.HumanItemRecord) ([]map[string]any, error) { + trace := []map[string]any{} + seen := map[string]bool{} + for _, item := range items { + if seen[item.ID] { + continue + } + seen[item.ID] = true + if item.OrganizationID != org || item.ReviewRound > 10000 { + return nil, fmt.Errorf("invalid audit item scope/round") + } + attempts, err := runtime.store.ListHumanStageAttempts(ctx, org, item.ID) + if err != nil { + return nil, err + } + events, err := runtime.store.ListHumanEvents(ctx, org, item.ID) + if err != nil { + return nil, err + } + rejects, err := runtime.store.ListHumanRejects(ctx, org, domain.HumanRejectQuery{ItemID: item.ID}) + if err != nil { + return nil, err + } + baselines := []domain.ReworkBaselineRecord{} + handlingRecords := []domain.ReworkHandlingRecord{} + for round := 1; round <= item.ReviewRound; round++ { + baseline, err := runtime.store.GetReworkBaseline(ctx, store.ReworkBaselineID(item.ID, round)) + if err != nil { + return nil, err + } + handling, err := runtime.store.GetReworkHandling(ctx, store.ReworkHandlingID(item.ID, round)) + if err != nil { + return nil, err + } + if handling != nil { + handlingRecords = append(handlingRecords, *handling) + } + if baseline != nil { + baselines = append(baselines, *baseline) + } + } + trace = append(trace, map[string]any{"humanItemId": item.ID, "taskId": item.LegacyTaskID, "revision": item.CurrentRevision, "reviewRound": item.ReviewRound, "stageAttempts": attempts, "events": events, "rejects": rejects, "reworkBaselines": baselines, "reworkHandling": handlingRecords}) + } + body, err := json.Marshal(trace) + if err != nil { + return nil, err + } + if len(body) > 6<<20 { + return nil, fmt.Errorf("internal delivery trace exceeds 6 MiB; refusing incomplete export") + } + return trace, nil +} + +func (runtime *Runtime) classificationDeliveryEnabled(ctx context.Context, workflowID string) (bool, error) { + workflow, err := runtime.store.GetWorkflow(ctx, workflowID) + if err != nil { + return false, err + } + if workflow == nil { + return false, nil + } + for _, node := range workflow.Nodes { + if node.Data.OperatorID != "operator-builtin-package" { + continue + } + for _, param := range node.Data.Hyperparams { + if param.Key == "DELIVERY_JSON_V4" && param.Value == "true" { + return true, nil + } + } + } + return false, nil +} + +// Reuse normal internal-artifact persistence rather than exceeding the 64 KiB +// inline transport limit with a multi-round audit. Customer publication is a +// different path and never sees this source artifact. +func (runtime *Runtime) persistClassificationSummary(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, partitionID string, artifact domain.ExecutionArtifact, body []byte) (domain.ExecutionArtifact, error) { + if len(body) <= domain.MaxInlinePayloadBytes && !structuredJSONRequiresReference(artifact, artifact.Data) { + return artifact, nil + } + if runtime.workspaceRoot == "" { + return artifact, fmt.Errorf("classification audit requires a persistent workspace") + } + sum := sha256.Sum256(body) + directory := filepath.Join(runtime.workspaceRoot, "system", "runs", run.ID, "release-audits", artifactSlug(partitionID), hex.EncodeToString(sum[:])) + if err := os.MkdirAll(directory, 0700); err != nil { + return artifact, err + } + artifact.Path = filepath.Join(directory, "acceptance-summary.json") + if err := os.WriteFile(artifact.Path, body, 0600); err != nil { + return artifact, err + } + artifact.Payload = nil + workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) + if err != nil { + return artifact, err + } + result, err := runtime.normalizeOutputArtifacts(ctx, workflow, run, node.ID, []domain.ExecutionArtifact{artifact}) + if err != nil { + return artifact, err + } + return result[0], nil +} diff --git a/internal/worker/delivery_audit_test.go b/internal/worker/delivery_audit_test.go new file mode 100644 index 00000000..e77576c8 --- /dev/null +++ b/internal/worker/delivery_audit_test.go @@ -0,0 +1,73 @@ +package worker + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func TestClassificationAuditUsesInternalStorageBeyondInlineLimit(t *testing.T) { + ctx := context.Background() + ds := store.NewMemoryStore() + publisher := &memoryArtifactPublisher{} + runtime := &Runtime{store: ds, workspaceRoot: t.TempDir(), artifactPublisher: publisher} + c := domain.StorageConnectionRecord{ID: "audit-storage", OrganizationID: "audit-org", Provider: domain.StorageProviderBOS, Bucket: "internal-audit", Endpoint: "https://bos.example.test"} + if err := ds.UpsertStorageConnection(ctx, c); err != nil { + t.Fatal(err) + } + workflow := domain.WorkflowRecord{ID: "wf", OrganizationID: c.OrganizationID, ArtifactStorage: &domain.ArtifactStoragePolicy{ConnectionID: c.ID, Bucket: c.Bucket, PrefixTemplate: "artifacts/{runId}/{nodeId}"}} + if err := ds.UpsertWorkflow(ctx, workflow); err != nil { + t.Fatal(err) + } + run := &domain.RunRecord{ID: "run-audit", OrganizationID: c.OrganizationID, WorkflowID: workflow.ID} + data := map[string]any{"schemaVersion": "droboticflow.acceptance-summary/v2", "trace": []any{map[string]any{"comment": strings.Repeat("详细返工记录", 20000)}}} + body, _ := json.Marshal(data) + artifact := domain.ExecutionArtifact{ID: "summary", RunID: run.ID, NodeID: "human", Name: "acceptance-summary.json", PortID: "acceptanceSummary", Type: domain.PortTypeJSON, Data: data, SizeBytes: int64(len(body))} + saved, err := runtime.persistClassificationSummary(ctx, run, domain.WorkflowNode{ID: "human"}, "partition", artifact, body) + if err != nil { + t.Fatal(err) + } + if saved.Payload == nil || saved.Payload.Mode != domain.ArtifactTransferModeRef || saved.Payload.Ref == nil { + t.Fatal("oversized trace remained inline") + } + ref := saved.Payload.Ref + if !strings.HasPrefix(ref.Key, "artifacts/run-audit/human/") || strings.Contains(ref.Key, "deliveries/") { + t.Fatal("audit in customer location") + } + raw := publisher.objects[ref.Bucket+"/"+ref.Key] + if string(raw) != string(body) || ref.SizeBytes != int64(len(body)) || len(ref.SHA256) != 64 { + t.Fatal("internal trace truncated or unpinned") + } +} + +func TestClassificationAuditTraceRetainsFormalReworkAndScopedEvidence(t *testing.T) { + ctx := context.Background() + ds := store.NewMemoryStore() + runtime := &Runtime{store: ds} + item := domain.HumanItemRecord{ID: "item", OrganizationID: "audit-org", LegacyTaskID: "task", CurrentRevision: 3, ReviewRound: 2} + for round := 1; round <= 2; round++ { + if err := ds.UpsertReworkBaseline(ctx, domain.ReworkBaselineRecord{ID: store.ReworkBaselineID(item.ID, round), OrganizationID: item.OrganizationID, ItemID: item.ID, Round: round, Problems: []domain.ReviewProblem{{ID: "problem", Text: "正式退回意见"}}}); err != nil { + t.Fatal(err) + } + } + ds.AppendHumanEvent(ctx, domain.HumanEventRecord{ID: "event", OrganizationID: item.OrganizationID, ItemID: item.ID, EventType: "review_rejected"}) + trace, err := runtime.deliveryAuditTrace(ctx, item.OrganizationID, []domain.HumanItemRecord{item}) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(trace) + if !strings.Contains(string(raw), "正式退回意见") || !strings.Contains(string(raw), "review_rejected") { + t.Fatal("formal trace lost") + } + if _, err := runtime.deliveryAuditTrace(ctx, "foreign", []domain.HumanItemRecord{item}); err == nil { + t.Fatal("foreign trace accepted") + } + audit := domain.ExecutionArtifact{Data: compactArtifactData(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2"})} + if !domain.IsInternalDeliveryAudit(audit) { + t.Fatal("compacted audit no longer excluded from final export") + } +} diff --git a/internal/worker/human_release.go b/internal/worker/human_release.go index c510becc..5cdb04c9 100644 --- a/internal/worker/human_release.go +++ b/internal/worker/human_release.go @@ -256,7 +256,7 @@ func (runtime *Runtime) syncDeliveryPackage(ctx context.Context, batch domain.Hu handposeIDs := make([]string, 0) for _, artifact := range child.Artifacts { artifactIDs = append(artifactIDs, artifact.ID) - if artifact.RunID == child.ID { + if artifact.RunID == child.ID && !domain.IsInternalDeliveryAudit(artifact) { finalArtifactIDs = append(finalArtifactIDs, artifact.ID) } identity := strings.ToLower(artifact.Name + " " + artifact.Path + " " + artifact.NodeID) diff --git a/operator-repos/builtin/package/src/operator.py b/operator-repos/builtin/package/src/operator.py index cb8ae7e9..4ee57b03 100644 --- a/operator-repos/builtin/package/src/operator.py +++ b/operator-repos/builtin/package/src/operator.py @@ -752,9 +752,11 @@ def _part_index_discarded_slice( "reason": str(evidence.get("reason") or ""), "reasonLabelId": str(evidence.get("reasonLabelId") or ""), "reasonName": str(evidence.get("reasonName") or ""), - "note": str(evidence.get("note") or ""), + "note": str(evidence.get("note") or evidence.get("reasonNote") or ""), + "reasonVersion": evidence.get("reasonVersion"), + "dictionary": evidence.get("dictionary"), "discardedByUserId": str(evidence.get("discardedByUserId") or ""), - "discardedAt": str(evidence.get("discardedAt") or ""), + "discardedAt": str(evidence.get("discardedAt") or evidence.get("at") or ""), }, }, "segments": [], @@ -843,13 +845,14 @@ def _description_status(actions: list[dict[str, Any]]) -> str: return "empty_confirmed" -def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): +def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None): """Use the established immutable directory publisher for the two JSON files. Legacy single-file publishers deliberately accept only v1/v2. A directory artifact preserves their ownership/conditional-publication protocol without teaching an old worker to misinterpret a new single-file schema. """ + v4 = internal_audit is not None capture = dict(payload["capture"]) prefix = pathlib.PurePosixPath(str(capture.get("prefix") or "")) part_id = prefix.name @@ -860,7 +863,7 @@ def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): if prefix.is_absolute() or ".." in prefix.parts or len(prefix.parts) < 3: raise RuntimeError("unsafe v3 capture identity prefix") capture.update(captureId=capture_id, prefix=str(prefix.parent)) - payload.update(schemaVersion="droboticflow.delivery.part-index/v3", capture=capture, partId=part_id) + payload.update(schemaVersion="droboticflow.delivery.part-index/v4" if v4 else "droboticflow.delivery.part-index/v3", capture=capture, partId=part_id) payload["pathBases"] = {"sourceMetadata": "original_capture", "calibration": "original_capture", "sources": "workspace"} payload["sourceMetadata"] = {"path": f"meta_{capture_id}.json"} history = { @@ -869,7 +872,8 @@ def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): "qualityReviews": payload["acceptance"].pop("qualityReviews", []), } payload["reviewHistory"] = {"path": "review-history.json", "pathBase": "part_package"} - payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None) + if not v4: + payload["acceptance"]["finalAcceptance"].pop("confirmedByUserId", None) for item in payload["calibration"]: if pathlib.PurePosixPath(item["name"]).name != item["name"] or "\\" in item["name"]: raise RuntimeError("unsafe calibration filename") @@ -889,6 +893,52 @@ def _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload): classification["contentCounts"] = counts if not counts["described"] and counts["invalid_marked"] and counts["gap_placeholder"]: classification["descriptionStatus"] = "invalid_and_blank" + if v4: + if not isinstance(internal_audit.get("trace"), list): + raise RuntimeError("DELIVERY_JSON_V4 requires complete internal trace input") + if _output(node_data, helpers, "deliveryAudit") is None: + raise RuntimeError("DELIVERY_JSON_V4 requires an internal deliveryAudit output") + audit = { + "schemaVersion": "droboticflow.delivery.review-history/v2", + "captureId": capture_id, "partId": part_id, "revision": payload["revision"], + "acceptance": payload.pop("acceptance"), "lineage": payload.pop("lineage"), + "qualityReviews": internal_audit.get("qualityReviews", []), + "trace": internal_audit.get("trace", []), + "acceptedMaterializations": internal_audit.get("acceptedMaterializations", []), + } + audit["acceptance"]["finalAcceptance"] = internal_audit.get("finalAcceptance", {}) + payload.pop("reviewHistory", None) + payload["capture"] = {"captureId": capture_id, "prefix": str(prefix.parent)} + for item in payload["slices"] + payload["discardedSlices"]: + classification = item["classification"] + # Blank descriptions retain the independently selected base class. + if classification.get("dictionary") is None: + classification.pop("dictionary", None) + evidence = classification.get("discardEvidence", {}) + evidence.pop("discardedAt", None) + if evidence.get("dictionary") is None: + evidence.pop("dictionary", None) + counts = classification.get("contentCounts") + if counts is not None: + counts["annotated"] = counts.pop("described") + for action in item["segments"]: + if action["contentKind"] == "described": action["contentKind"] = "annotated" + for key in ("source", "confidence", "phrases", "labelId"): + action.pop(key, None) + if action.get("dictionary") is None: action.pop("dictionary", None) + body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8") + if len(json.dumps(audit, ensure_ascii=False).encode("utf8")) > 8 * 1024 * 1024: + raise RuntimeError("internal delivery audit exceeds 8 MiB; refusing incomplete export") + audit["partSha256"] = hashlib.sha256(body).hexdigest() + staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v4-", dir=output)) + (staging / "part.json").write_bytes(body) + # A sibling artifact stays in the internal run, never in the customer tree. + audit_artifact = helpers["write_json_artifact"](run_id, node_id, output, "review-history.json", audit, output=_output(node_data, helpers, "deliveryAudit")) + directory = helpers["build_virtual_artifact"]( + run_id, node_id, "delivery.directory.index", str(staging), "file", 0, + data={"schemaVersion": "droboticflow.directory-index/v1", "presentation": "collection", "datasetType": "directory", "dataSource": "FILESYSTEM", "rootPath": str(staging), "sliceCount": len(payload["slices"]), "classificationOnly": True}, + output=_output(node_data, helpers, "deliveryDirectory")) + return [directory, audit_artifact] staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v3-", dir=output)) for name, document in (("part.json", payload), ("review-history.json", history)): (staging / name).write_text(json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf8") @@ -944,6 +994,7 @@ def _part_index_slice( "clipStartMs": _optional_float(action.get("clipStartMs")), "clipEndMs": _optional_float(action.get("clipEndMs")), "labelId": str(action.get("descriptionTemplateId") or ""), + "dictionary": action.get("dictionary"), "phrases": [str(item) for item in action.get("phrases", [])] if isinstance(action.get("phrases"), list) else [], "source": str(action.get("source") or ""), "confidence": _optional_float(action.get("confidence")), @@ -976,6 +1027,7 @@ def _part_index_slice( "deliveryDisposition": declared, "taskLabel": str(synchronization.get("taskLabel") or ""), "taskLabelId": str(synchronization.get("taskLabelId") or ""), + "dictionary": synchronization.get("dictionary"), "descriptionStatus": _description_status(actions), }, "segments": actions, @@ -999,9 +1051,11 @@ def _part_index_slice( "reason": str(evidence.get("reason") or ""), "reasonLabelId": str(evidence.get("reasonLabelId") or ""), "reasonName": str(evidence.get("reasonName") or ""), - "note": str(evidence.get("note") or ""), + "note": str(evidence.get("note") or evidence.get("reasonNote") or ""), + "reasonVersion": evidence.get("reasonVersion"), + "dictionary": evidence.get("dictionary"), "discardedByUserId": str(evidence.get("discardedByUserId") or ""), - "discardedAt": str(evidence.get("discardedAt") or ""), + "discardedAt": str(evidence.get("discardedAt") or evidence.get("at") or ""), } # Pose frames deliberately stay out of the index: the customer contract for # this mode is the split information, and embedded per-frame pose would @@ -1651,6 +1705,12 @@ def _run_accepted_slice_directory( raise RuntimeError("acceptedSliceDirectory does not prove gap-free full-video coverage") summary_artifact = next((item for item in inputs if _port(item) == "acceptanceSummary"), None) summary_payload = summary_artifact.get("data") if isinstance((summary_artifact or {}).get("data"), dict) else {} + if summary_artifact is not None and not isinstance(summary_payload.get("finalAcceptance"), dict): + summary_path = _materialize_input_file(summary_artifact, output / "_acceptance", helpers) + if summary_path.stat().st_size > 8 * 1024 * 1024: + raise RuntimeError("acceptance summary exceeds 8 MiB") + summary_payload = json.loads(summary_path.read_text(encoding="utf8")) + if not isinstance(summary_payload, dict): raise RuntimeError("invalid acceptance summary reference") final_acceptance = summary_payload.get("finalAcceptance") if isinstance(summary_payload.get("finalAcceptance"), dict) else {} capture_record = _part_index_capture_record(summary_payload, revision_manifest) all_slices = revision_manifest.get("items", []) @@ -1679,7 +1739,8 @@ def _run_accepted_slice_directory( raise RuntimeError("slice dispositions are not pinned to the accepted manifest revision") if int(coverage.get("sliceCount", -1)) != len(all_slices): raise RuntimeError("review coverage slice count mismatch") - if disposition_contract and not kept_slices: + params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + if disposition_contract and not kept_slices and not _bool(params, "DELIVERY_JSON_V4", False): if any(p.get("key") == "PROCESSING_PROFILE" and p.get("value") == "raw_human" for p in node_data.get("hyperparams", [])) and (summary_payload.get("processingProfile") != "raw_human" or final_acceptance.get("processingProfile") != "raw_human"): raise RuntimeError("raw-human package requires matching accepted task profile") # Internal evidence only: deliberately no deliveryDirectory output and @@ -1715,8 +1776,10 @@ def _run_accepted_slice_directory( for item in payload["slices"] + payload["discardedSlices"]: item["sourceFrames"] = frames[item["sliceId"]] body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf8") + if _bool(params, "DELIVERY_JSON_V4", False): + return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, summary_payload) if _bool(params, "DELIVERY_JSON_V3", False): - return _emit_part_index_v3(run_id, node_id, output, node_data, helpers, payload) + return _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload) return _emit_part_index(run_id, node_id, output, node_data, helpers, body) if source_index: source_root = _source_index_physical_export(source_root, revision_manifest, kept_slices, output) @@ -1953,6 +2016,8 @@ def _run_multiport( def run(run_id: str, node_id: str, output_dir: Any, node_data: dict[str, Any], inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + if _bool(params, "DELIVERY_JSON_V4", False) and (not _bool(params, "PART_INDEX_DELIVERY", False) or _bool(params, "DELIVERY_JSON_V3", False)): + raise RuntimeError("DELIVERY_JSON_V4 requires PART_INDEX_DELIVERY and cannot combine with DELIVERY_JSON_V3") if _bool(params, "DELIVERY_JSON_V3", False) and not _bool(params, "PART_INDEX_DELIVERY", False): raise RuntimeError("DELIVERY_JSON_V3 requires PART_INDEX_DELIVERY") # A part-index delivery intentionally has no directory port; it must still diff --git a/src/components/AnnotationLabelsPanel.tsx b/src/components/AnnotationLabelsPanel.tsx index 5c475f49..f0f1470b 100644 --- a/src/components/AnnotationLabelsPanel.tsx +++ b/src/components/AnnotationLabelsPanel.tsx @@ -39,6 +39,7 @@ export function AnnotationLabelsPanel({ const [error, setError] = useState(""); const [editingLabelId, setEditingLabelId] = useState(""); const [inputText, setInputText] = useState(""); + const [tagText, setTagText] = useState(""); const [checkedIds, setCheckedIds] = useState([]); const [importOpen, setImportOpen] = useState(false); const [importText, setImportText] = useState(""); @@ -83,6 +84,7 @@ export function AnnotationLabelsPanel({ setEnabled([]); setEditingLabelId(""); setInputText(""); + setTagText(""); setCheckedIds([]); setImportOpen(false); setImportText(""); @@ -99,6 +101,7 @@ export function AnnotationLabelsPanel({ const resetEditing = () => { setEditingLabelId(""); setInputText(""); + setTagText(""); }; const startEdit = (label: AnnotationLabelRecord) => { @@ -106,6 +109,7 @@ export function AnnotationLabelsPanel({ setError(""); setEditingLabelId(label.id); setInputText(label.name || label.description || ""); + setTagText((label.tags || []).join(",")); }; const saveLabel = async (e?: React.FormEvent) => { @@ -133,7 +137,16 @@ export function AnnotationLabelsPanel({ try { const existing = labels.find((item) => item.id === editingLabelId); const code = `desc_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 6)}`; + const tags = [ + ...new Set( + tagText + .split(/[,,\n]/) + .map((tag) => tag.trim()) + .filter(Boolean), + ), + ]; const payload = { + tags, kind, code, name: text, @@ -143,7 +156,7 @@ export function AnnotationLabelsPanel({ status: existing?.status || ("active" as const), }; - const saved = await reuseOrCreateDictionaryLabel({ + let saved = await reuseOrCreateDictionaryLabel({ kind, name: text, preferredId: editingLabelId, @@ -154,6 +167,19 @@ export function AnnotationLabelsPanel({ }); if (scope.current !== generation) return; + // Explicit tag maintenance versions the shared dictionary word; saved + // selections retain their original tags. + if ( + (editingLabelId === saved.id || tags.length > 0) && + JSON.stringify(saved.tags || []) !== JSON.stringify(tags) + ) { + saved = await apiRequest( + `/api/annotation-labels/${saved.id}`, + { method: "PATCH" }, + { ...saved, tags }, + ); + if (scope.current !== generation) return; + } // Relink only this project; never rename a term shared by other projects. const selection = await apiRequest( selectionPath, @@ -321,6 +347,14 @@ export function AnnotationLabelsPanel({ } className="flex-1 min-w-[280px] rounded-lg border border-[color:var(--app-border)] bg-[color:var(--app-panel)] px-4 py-2.5 text-sm text-[color:var(--app-text)] shadow-sm focus:border-blue-500 focus:outline-none" /> + setTagText(e.target.value)} + disabled={loading || saving} + className="min-w-48 flex-1 rounded-lg border border-[color:var(--app-border)] bg-[color:var(--app-panel)] px-3 py-2 text-sm" + />
+ {!isInvalid && editingAnnotation.dictionary && ( +
+ 分类:{editingAnnotation.dictionary.name} + {editingAnnotation.dictionary.tags.map((tag) => ( + + {tag} + + ))} + +
+ )} {isInvalid ? (
@@ -6030,7 +6058,11 @@ export function CaptureAnnotationWorkbench({ ref={descriptionInputRef} aria-label="动作描述" value={editingAnnotation.description || ""} - onChange={(event) => updateEditingAnnotation({ description: event.target.value })} + onChange={(event) => + updateEditingAnnotation({ + description: event.target.value, + }) + } placeholder="直接填写详细动作描述..." rows={3} className="min-h-24 w-full resize-y rounded border border-slate-300 p-2.5 text-sm text-slate-800 outline-none focus:border-orange-400 dark:border-[#2a2b2f] dark:bg-[#1a1b1e] dark:text-gray-200" @@ -6566,9 +6598,9 @@ export function CaptureAnnotationWorkbench({ loading: sliceDictionaryLoading, error: userErrorMessage(sliceDictionaryError), refresh: onRefreshSliceDictionary, - onSelect: (name) => + onSelect: (name, dictionary) => applySliceEditPanelOp( - setPendingSliceName(sliceEditPanel.pendingSlices, sliceEditPanel.sliceId, name), + setPendingSliceName(sliceEditPanel.pendingSlices, sliceEditPanel.sliceId, name, dictionary), ), }} discard={{ diff --git a/src/components/SliceEditPanel.tsx b/src/components/SliceEditPanel.tsx index 51527d90..96575661 100644 --- a/src/components/SliceEditPanel.tsx +++ b/src/components/SliceEditPanel.tsx @@ -1,3 +1,5 @@ +import type { DictionaryItem } from "../lib/dictionary-selection"; +import type { DictionarySelection } from "../types"; import { useEffect, useLayoutEffect, useRef, useState } from "react"; import type { AnnotationLabelRecord, SliceSegment } from "../types"; import { formatAnnotationTime } from "../lib/annotation-time.js"; @@ -57,11 +59,11 @@ export function SliceEditPanel({ onCancel: () => void; error?: string; dictionary: { - items: { id: string; name: string }[]; + items: DictionaryItem[]; loading?: boolean; error?: string; refresh?: () => void; - onSelect: (name: string) => void; + onSelect: (name: string, dictionary?: DictionarySelection) => void; }; discard: { draft: SliceEditPanelDiscardDraft | null; diff --git a/src/components/SliceNameDictionaryPicker.tsx b/src/components/SliceNameDictionaryPicker.tsx index bc3c76e6..59213c86 100644 --- a/src/components/SliceNameDictionaryPicker.tsx +++ b/src/components/SliceNameDictionaryPicker.tsx @@ -1,3 +1,5 @@ +import { dictionarySelection, type DictionaryItem } from "../lib/dictionary-selection"; +import type { DictionarySelection } from "../types"; import { userErrorMessage } from "../lib/user-error-message.js"; import { useEffect, useLayoutEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; @@ -20,13 +22,13 @@ export function SliceNameDictionaryPicker({ inline = false, }: { key?: string; - items: { id: string; name: string }[]; + items: DictionaryItem[]; value: string; disabled?: boolean; loading?: boolean; error?: string; onRefresh?: () => void; - onSelect: (name: string) => void; + onSelect: (name: string, dictionary?: DictionarySelection) => void; triggerRef?: RefObject; onCloseFocus?: () => void; portal?: boolean; @@ -128,7 +130,7 @@ export function SliceNameDictionaryPicker({ } onSelect={(id) => { const item = items.find((item) => item.id === id); - if (item) onSelect(item.name); + if (item) onSelect(item.name, dictionarySelection(item)); close(); }} onClose={close} @@ -178,7 +180,7 @@ export function SliceNameDictionaryPicker({ type="button" key={item.id} onClick={() => { - onSelect(item.name); + onSelect(item.name, dictionarySelection(item)); close(); }} className={`flex w-full items-center justify-between gap-2 rounded px-2 py-2 text-left text-sm hover:bg-orange-50 dark:hover:bg-orange-950/40 ${value === item.name ? "bg-orange-50 text-orange-700 dark:bg-orange-950/40" : ""}`} diff --git a/src/hooks/useProjectSliceDictionary.ts b/src/hooks/useProjectSliceDictionary.ts index 926ffc8a..d828f31f 100644 --- a/src/hooks/useProjectSliceDictionary.ts +++ b/src/hooks/useProjectSliceDictionary.ts @@ -2,7 +2,7 @@ import { useEffect, useState } from "react"; import { apiRequest } from "../services/api/client"; import type { AnnotationLabelRecord } from "../types"; -export function useProjectSliceDictionary(projectId?: string, taskId?: string) { +export function useProjectSliceDictionary(projectId?: string, taskId?: string, kind: "slice" | "action" = "slice") { const [refreshKey, setRefreshKey] = useState(0); const [state, setState] = useState<{ owner: string; @@ -10,23 +10,26 @@ export function useProjectSliceDictionary(projectId?: string, taskId?: string) { loading: boolean; error: string; }>({ owner: "", items: [], loading: false, error: "" }); - const owner = `${projectId || ""}:${taskId || ""}`; + const owner = `${projectId || ""}:${taskId || ""}:${kind}`; useEffect(() => { const controller = new AbortController(); if (!projectId || !taskId) return () => controller.abort(); setState({ owner, items: [], loading: true, error: "" }); - void apiRequest<{ items: AnnotationLabelRecord[] }>(`/api/projects/${projectId}/slice-labels`, { - signal: controller.signal, - }) + void apiRequest<{ items: AnnotationLabelRecord[] }>( + `/api/projects/${projectId}/${kind === "slice" ? "slice-labels" : "annotation-labels"}`, + { + signal: controller.signal, + }, + ) .then((result) => { if (!controller.signal.aborted) setState({ owner, items: result.items || [], loading: false, error: "" }); }) .catch(() => { if (!controller.signal.aborted) - setState({ owner, items: [], loading: false, error: "切片字典加载失败,仍可手动输入名称。" }); + setState({ owner, items: [], loading: false, error: "字典加载失败,请刷新重试。" }); }); return () => controller.abort(); - }, [projectId, taskId, owner, refreshKey]); + }, [projectId, taskId, owner, refreshKey, kind]); return { ...(state.owner === owner ? state : { items: [], loading: Boolean(projectId && taskId), error: "" }), refresh: () => setRefreshKey((n) => n + 1), diff --git a/src/lib/annotation-operations.js b/src/lib/annotation-operations.js index b597ca25..272f8177 100644 --- a/src/lib/annotation-operations.js +++ b/src/lib/annotation-operations.js @@ -27,6 +27,7 @@ export function normalizeAnnotationSegments(segments, clipStart, clipEnd) { return (Array.isArray(segments) ? segments : []) .map((segment, index) => ({ id: String(segment?.id || `segment-${index + 1}`), + dictionary: segment?.dictionary, labelId: String(segment?.labelId || "").trim(), label: String(segment?.label || "").trim(), description: String(segment?.description || "").trim(), diff --git a/src/lib/dictionary-selection.ts b/src/lib/dictionary-selection.ts new file mode 100644 index 00000000..53d9dcea --- /dev/null +++ b/src/lib/dictionary-selection.ts @@ -0,0 +1,15 @@ +import type { AnnotationLabelRecord, DictionarySelection } from "../types"; + +export type DictionaryItem = Pick & Partial; + +export function dictionarySelection(item: DictionaryItem): DictionarySelection | undefined { + if (!item.code || !item.version) return undefined; + return { + id: item.id, + kind: item.kind || "action", + version: item.version, + code: item.code, + name: item.name, + tags: [...(item.tags || [])], + }; +} diff --git a/src/lib/slice-edit-panel-ops.js b/src/lib/slice-edit-panel-ops.js index 0db55153..19109d31 100644 --- a/src/lib/slice-edit-panel-ops.js +++ b/src/lib/slice-edit-panel-ops.js @@ -137,10 +137,10 @@ export function mergePendingSliceBoundary(slices, sliceId, side) { } /** ④ 切片字典描述(名称)编辑:只改目标片名称。 */ -export function setPendingSliceName(slices, sliceId, name) { +export function setPendingSliceName(slices, sliceId, name, dictionary = undefined) { const index = indexOfSliceId(slices, sliceId); if (index < 0) return { slices, issue: "未找到目标切片。" }; - return { slices: slices.map((slice, at) => (at === index ? { ...slice, name } : slice)) }; + return { slices: slices.map((slice, at) => (at === index ? { ...slice, name, dictionary } : slice)) }; } /** diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index a462f3b3..b6ee1eac 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -1,3 +1,5 @@ +import { dictionarySelection } from "../lib/dictionary-selection"; +import type { DictionarySelection } from "../types"; import { AnnotationWorkModeToggle } from "../components/AnnotationWorkModeToggle"; import { loadOwnedAnnotationProject, loadAnnotationSidebar } from "../lib/annotation-project-loading.js"; import { FailedSliceRecoveryNotice } from "../components/FailedSliceRecoveryNotice"; @@ -746,6 +748,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const [detail, setDetailState] = useState(null); const discardReasons = useProjectDiscardReasons(detail?.task.projectId, detail?.task.id); const sliceDictionary = useProjectSliceDictionary(detail?.task.projectId, detail?.task.id); + const actionDictionary = useProjectSliceDictionary(detail?.task.projectId, detail?.task.id, "action"); const detailRef = useRef(null); const annotationMutationScopeRef = useRef(createAuthoritativeMutationScope()); useEffect(() => { @@ -1839,7 +1842,17 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: null, [detail?.task.templateId, templates], ); - const selectedLabels = detail?.task.labelSnapshot || []; + const selectedLabels = + (detail?.task.templateVersion || 0) >= 6 + ? actionDictionary.items.map((item) => ({ + id: item.code, + name: item.name, + description: item.description, + aliases: item.aliases, + color: item.color, + dictionary: dictionarySelection(item), + })) + : detail?.task.labelSnapshot || []; const modelPrelabels = useMemo( () => detail?.revisions @@ -3599,12 +3612,17 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: applySliceEditQuickResolution(prompt, plan.resolution); }; - const updateSliceName = (id: string, name: string) => { + const updateSliceName = (id: string, name: string, dictionary?: DictionarySelection) => { if (!annotationAuthoringRef.current) return; if (!exactFrameEditing || saving || combinedSubmissionLocked || annotationMutationScopeRef.current.busy()) return; const slice = sliceSegments.find((item) => item.id === id); - if (!slice || slice.deliveryDisposition === "discard" || slice.name === name) return; - const next = sliceSegments.map((item) => (item.id === id ? { ...item, name } : item)); + if ( + !slice || + slice.deliveryDisposition === "discard" || + (slice.name === name && JSON.stringify(slice.dictionary) === JSON.stringify(dictionary)) + ) + return; + const next = sliceSegments.map((item) => (item.id === id ? { ...item, name, dictionary } : item)); const before = sliceEditSnapshot(id); editHistoryRef.current.record({ before, after: { ...before, slices: next } }); setSliceSegments(next); diff --git a/src/types.ts b/src/types.ts index 6c277f28..7de4fc35 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1904,7 +1904,17 @@ export interface ObjectStorageUploadSessionSummary { pendingCount: number; } +export interface DictionarySelection { + id: string; + kind: "action" | "slice" | "discard_reason"; + version: number; + code: string; + name: string; + tags: string[]; +} + export interface SemanticLabelDefinition { + dictionary?: DictionarySelection; id: string; name: string; description?: string; @@ -1913,6 +1923,7 @@ export interface SemanticLabelDefinition { } export interface AnnotationLabelRecord { + tags?: string[]; kind?: "action" | "slice" | "discard_reason"; requiresNote?: boolean; id: string; @@ -1928,6 +1939,7 @@ export interface AnnotationLabelRecord { version: number; updatedAt: string; history?: Array<{ + tags?: string[]; version: number; code: string; name: string; @@ -1985,6 +1997,8 @@ export interface AnnotationTemplate { } export interface AnnotationSegment { + clearDictionary?: boolean; + dictionary?: DictionarySelection; status?: "pending" | "described" | "invalid"; id: string; labelId: string; @@ -2000,6 +2014,8 @@ export interface AnnotationSegment { } export interface SliceSegment { + dictionary?: DictionarySelection; + discardReasonDictionary?: DictionarySelection; deliveryDisposition?: "keep" | "discard"; discardReason?: string; discardReasonLabelId?: string; diff --git a/tests/backend/export_delivery_v4_fixture.py b/tests/backend/export_delivery_v4_fixture.py new file mode 100644 index 00000000..ed060f1f --- /dev/null +++ b/tests/backend/export_delivery_v4_fixture.py @@ -0,0 +1,39 @@ +"""Synthetic v4 examples from the actual package entry point, with private audit kept separately.""" +import argparse +import hashlib +import json +import shutil +from pathlib import Path + +from backend.operators.builtin.package import operator as package +from tests.backend.test_delivery_v4_package import NODE, ACTION, prepare +from tests.backend.test_part_index_package import helpers + + +def export(destination): + destination.mkdir(parents=True, exist_ok=True) + for part in ("part0001", "part0002"): + work=destination/("work-"+part) + work.mkdir() + inputs=prepare(work,[{"id":"blank","description":"","clipStartMs":0,"clipEndMs":500},{"id":"pick","description":"拿起杯子","dictionary":ACTION,"clipStartMs":500,"clipEndMs":1000}]) + inputs[1]["data"]["capture"].update(captureId="capture-A/"+part,prefix="incoming/capture-A/"+part) + manifest_path=Path(inputs[0]["path"])/"revision-manifest.json" + m=json.loads(manifest_path.read_text());m["runId"]="source-run-"+part + if part=="part0002": + m["items"][0]["deliveryDisposition"]="discard" + m["items"][0]["discardEvidence"]=m["items"][1]["discardEvidence"] + manifest_path.write_text(json.dumps(m)) + inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"]=hashlib.sha256(manifest_path.read_bytes()).hexdigest() + artifacts=package.run("package-"+part,"package",work/"out",NODE,inputs,helpers()) + shutil.copytree(artifacts[0]["path"],destination/part) + audit=destination/"internal"/part + audit.mkdir(parents=True) + shutil.copy2(artifacts[1]["path"],audit/"review-history.json") + shutil.rmtree(work) + (destination/"meta_capture-A.json").write_text('{"synthetic":true,"parts":["part0001","part0002"]}\n') + + +if __name__=="__main__": + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument("output",type=Path) + export(parser.parse_args().output) diff --git a/tests/backend/test_delivery_v4_package.py b/tests/backend/test_delivery_v4_package.py new file mode 100644 index 00000000..fafa7f1e --- /dev/null +++ b/tests/backend/test_delivery_v4_package.py @@ -0,0 +1,101 @@ +"""Final classifications and private review history are different outputs.""" +import hashlib +import json +from pathlib import Path + +import pytest +from backend.operators.builtin.package import operator as package +from tests.backend.test_delivery_v3_package import inputs_for +from tests.backend.test_part_index_package import helpers +from tests.backend.test_slice_discard_package import accepted_inputs, fixture + +NODE = { + "outputs": [{"id": n, "name": n} for n in ("deliveryDirectory", "deliverySummary", "deliveryAudit")], + "hyperparams": [{"key": k, "value": v} for k, v in { + "PART_INDEX_DELIVERY": "true", "DELIVERY_JSON_V4": "true", "PROCESSING_PROFILE": "raw_human" + }.items()], +} +BASE = {"id": "slice-word", "kind": "slice", "version": 2, "code": "base", "name": "物品操作", "tags": ["商品", "手部"]} +ACTION = {"id": "action-word", "kind": "action", "version": 3, "code": "pick", "name": "拿取", "tags": ["抓取"]} + + +def prepare(tmp_path, actions): + inputs = inputs_for(tmp_path, actions) + root = Path(inputs[0]["path"]) + sync = root / "slices/slice-000001/synchronized-slice.json" + doc = json.loads(sync.read_text()) + doc["dictionary"] = BASE + sync.write_text(json.dumps(doc)) + manifest_path = root / "revision-manifest.json" + m = json.loads(manifest_path.read_text()) + m["runId"] = "source-run" + m["items"][0]["sliceName"] = BASE["name"] + reason = m["items"][1]["discardEvidence"] + reason.update(reasonLabelId="discard-word", reasonVersion=4, reasonName="遮挡", reasonNote="持续遮挡", at="2026-10-20T00:00:00Z", dictionary={"id":"discard-word","kind":"discard_reason","version":4,"code":"occluded","name":"遮挡","tags":["不可用"]}) + manifest_path.write_text(json.dumps(m)) + inputs[1]["data"]["acceptedMaterializations"][0]["manifestSha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + inputs[1]["data"]["qualityReviews"] = [{"reviewRound":0,"decision":"rejected","comment":"PRIVATE_REWORK"},{"reviewRound":1,"decision":"approved","confirmations":[{"opinionId":"old","resolved":True}]}] + inputs[1]["data"]["trace"] = [{"humanItemId":"item","reworkBaselines":[{"round":1,"comment":"PRIVATE_BASELINE"}],"reworkHandling":[{"round":1}],"events":[{"type":"rework_submitted"}]}] + return inputs + + +def read_outputs(tmp_path, inputs, out="out"): + result = package.run("r", "p", tmp_path / out, NODE, inputs, helpers()) + assert len(result) == 2 + directory = Path(result[0]["path"]) + assert {p.name for p in directory.iterdir()} == {"part.json"} + raw = (directory / "part.json").read_bytes() + audit = json.loads(Path(result[1]["path"]).read_text()) + assert result[1]["portId"] == "deliveryAudit" + assert Path(result[1]["path"]).parent != directory + assert audit["partSha256"] == hashlib.sha256(raw).hexdigest() + assert audit["revision"] == json.loads(raw)["revision"] + return json.loads(raw), audit, raw + + +@pytest.mark.parametrize("actions,kinds,status", [ + ([{"description":"", "status":"pending"}], ["gap_placeholder"], "empty_confirmed"), + ([{"description":"拿起杯子", "dictionary":ACTION},{"description":""},{"description":"无效", "isInvalid":True}], ["annotated","gap_placeholder","invalid_marked"], "described"), + ([{"description":"无效", "isInvalid":True}], ["invalid_marked"], "invalid_only"), +]) +def test_v4_freezes_classification_without_exporting_review_process(tmp_path, actions, kinds, status): + inputs = prepare(tmp_path, actions) + doc, audit, raw = read_outputs(tmp_path, inputs) + first = doc["slices"][0] + assert doc["schemaVersion"] == "droboticflow.delivery.part-index/v4" + assert first["classification"]["dictionary"] == BASE + assert first["classification"]["descriptionStatus"] == status + assert [a["contentKind"] for a in first["segments"]] == kinds + if kinds[0] == "annotated": assert first["segments"][0]["dictionary"] == ACTION + discard = doc["discardedSlices"][0]["classification"]["discardEvidence"] + assert discard["note"] == "持续遮挡" and discard["dictionary"]["tags"] == ["不可用"] + assert discard["reasonVersion"] == 4 + for key in ("acceptance","reviewHistory","lineage"): assert key not in doc + for text in (b"PRIVATE_REWORK",b"PRIVATE_BASELINE",b"reviewerUserId",b"sampling",b"discardedAt",b"confirmedByUserId"): + assert text not in raw + assert audit["qualityReviews"][0]["comment"] == "PRIVATE_REWORK" + assert audit["trace"][0]["reworkBaselines"] + assert audit["lineage"]["sourceRunId"] == "source-run" + assert read_outputs(tmp_path, inputs, "retry")[2] == raw + # Intermediate audit-only changes cannot change final business bytes. + inputs[1]["data"]["qualityReviews"][0]["comment"] = "OTHER_REVIEW_NOTE" + assert read_outputs(tmp_path, inputs, "audit-change")[2] == raw + + +def test_v4_all_discard_outputs_classification_and_legacy_stays_unchanged(tmp_path): + inputs = fixture(tmp_path,["discard","discard"]) + inputs[1]["data"].update(capture={"captureId":"capture-A/part0001","prefix":"incoming/capture-A/part0001"},trace=[]) + inputs[1]["data"]["finalAcceptance"]["acceptedAt"] = "2026-10-20T01:00:00Z" + doc, _, _ = read_outputs(tmp_path, inputs) + assert doc["slices"] == [] and len(doc["discardedSlices"]) == 2 + assert doc["deliveryClassification"]["discardedSliceCount"] == 2 + legacy={**NODE,"hyperparams":[p for p in NODE["hyperparams"] if p["key"]!="DELIVERY_JSON_V4"]} + result=package.run("r","p",tmp_path/"legacy",legacy,inputs,helpers()) + assert len(result)==1 and result[0]["data"]["outcome"]=="no_deliverables" + + +def test_v4_rejects_unaccepted_or_wrong_revision(tmp_path): + inputs = prepare(tmp_path,[]) + inputs[1]["data"]["acceptedMaterializations"][0]["revision"] = 999 + with pytest.raises(RuntimeError,match="not pinned"): + read_outputs(tmp_path, inputs) diff --git a/tests/browser/dictionary-classification-tags.mjs b/tests/browser/dictionary-classification-tags.mjs new file mode 100644 index 00000000..5d799bba --- /dev/null +++ b/tests/browser/dictionary-classification-tags.mjs @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import { chromium } from "playwright"; + +const base = process.env.DICTIONARY_TEST_URL || "http://127.0.0.1:5197"; +const out = process.env.DICTIONARY_TEST_OUTPUT || "local-data/verification/delivery-classification-20261021/browser"; +await fs.mkdir(out, { recursive: true }); +const browser = await chromium.launch({ + executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + headless: true, +}); +try { + const page = await browser.newPage({ viewport: { width: 1365, height: 900 } }); + page.setDefaultTimeout(60000); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + let label = { + id: "base", + name: "物品操作", + kind: "slice", + code: "base", + version: 2, + tags: ["商品"], + status: "active", + }; + let patch; + await page.route("**/api/**", async (route) => { + const request = route.request(), + path = new URL(request.url()).pathname; + if (path === "/api/annotation-labels/base" && request.method() === "PATCH") { + patch = request.postDataJSON(); + label = { ...label, ...patch, version: label.version + 1 }; + return route.fulfill({ json: label }); + } + if (path === "/api/annotation-labels") return route.fulfill({ json: { items: [label] } }); + if (path === "/api/projects/project/slice-labels") + return route.fulfill({ json: { labelIds: [label.id], items: [label] } }); + return route.fulfill({ status: 404, json: { error: "unexpected route" } }); + }); + await page.goto(base + "/tests/browser/fixtures/dictionary-classification-tags.html", { waitUntil: "commit" }); + await page.getByRole("button", { name: "编辑切片名称", exact: true }).click(); + await page.getByRole("textbox", { name: "分类标签" }).fill("商品,手部, 商品"); + await page.getByRole("button", { name: "保存修改", exact: true }).click(); + await page.getByText("手部", { exact: true }).waitFor(); + assert.deepEqual(patch.tags, ["商品", "手部"]); + assert.equal(label.version, 3); + await page.getByRole("button", { name: "从切片字典选择", exact: true }).click(); + await page.getByRole("button", { name: "物品操作", exact: true }).click(); + const selected = await page.evaluate(() => window.selected); + assert.equal(selected.dictionary.id, "base"); + assert.equal(selected.dictionary.version, 2); // Picker represents the version actually shown, not the later edit. + assert.deepEqual(selected.dictionary.tags, ["商品"]); + assert.deepEqual(errors, []); + await page.screenshot({ path: `${out}/dictionary-tags.png`, fullPage: true }); + await fs.writeFile( + `${out}/result.json`, + JSON.stringify({ passed: true, edited: label, selected, pageErrors: errors }, null, 2), + ); +} finally { + await browser.close(); +} diff --git a/tests/browser/fixtures/dictionary-classification-tags.html b/tests/browser/fixtures/dictionary-classification-tags.html new file mode 100644 index 00000000..46d79ccc --- /dev/null +++ b/tests/browser/fixtures/dictionary-classification-tags.html @@ -0,0 +1,11 @@ + + + + + 字典分类验证 + + +
+ + + diff --git a/tests/browser/fixtures/dictionary-classification-tags.tsx b/tests/browser/fixtures/dictionary-classification-tags.tsx new file mode 100644 index 00000000..dcdee132 --- /dev/null +++ b/tests/browser/fixtures/dictionary-classification-tags.tsx @@ -0,0 +1,34 @@ +import React from "react"; +import { createRoot } from "react-dom/client"; +import "../../../src/index.css"; +import { AnnotationLabelsPanel } from "../../../src/components/AnnotationLabelsPanel"; +import { SliceNameDictionaryPicker } from "../../../src/components/SliceNameDictionaryPicker"; +import type { AnnotationLabelRecord, DictionarySelection } from "../../../src/types"; + +const label: AnnotationLabelRecord = { + id: "base", + name: "物品操作", + kind: "slice", + code: "base", + version: 2, + tags: ["商品"], + status: "active", + organizationId: "fixture", + updatedAt: "2026-10-21T00:00:00Z", +}; +createRoot(document.getElementById("root")!).render( +
+ + { + (window as Window & { selected?: { name: string; dictionary?: DictionarySelection } }).selected = { + name, + dictionary, + }; + }} + /> +
, +); diff --git a/tests/browser/project-dictionary-reuse.mjs b/tests/browser/project-dictionary-reuse.mjs index c3c08dbf..39042aad 100644 --- a/tests/browser/project-dictionary-reuse.mjs +++ b/tests/browser/project-dictionary-reuse.mjs @@ -4,7 +4,10 @@ import fs from "node:fs/promises"; const base = process.env.DICTIONARY_TEST_URL || "http://127.0.0.1:5178", out = process.env.DICTIONARY_TEST_OUTPUT || "verification/dictionary-reuse"; await fs.mkdir(out, { recursive: true }); -const browser = await chromium.launch({ channel: "chrome", headless: true }); +const browser = await chromium.launch({ + ...(process.env.BROWSER_EXECUTABLE ? { executablePath: process.env.BROWSER_EXECUTABLE } : { channel: "chrome" }), + headless: true, +}); try { for (const kind of ["slice", "action", "discard_reason"]) { const page = await browser.newPage({ viewport: { width: 1280, height: 900 } }); @@ -54,7 +57,7 @@ try { ); await page.goto(base + "/__dictionary_reuse"); assert.equal(await page.title(), "项目字典复用验证"); - const input = page.getByRole("textbox"), + const input = page.getByRole("textbox").first(), add = page.getByRole("button", { name: kind === "slice" ? "添加名称" : kind === "discard_reason" ? "添加原因" : "添加描述", exact: true, diff --git a/tests/frontend/project-ui-regressions.test.mjs b/tests/frontend/project-ui-regressions.test.mjs index 1d7f05b7..d65ac09f 100644 --- a/tests/frontend/project-ui-regressions.test.mjs +++ b/tests/frontend/project-ui-regressions.test.mjs @@ -28,10 +28,12 @@ test("theme utilities follow the application class rather than OS preference", a assert.match(css, /:where\(\.dark, \.dark \*\)/); assert.doesNotMatch(css, /prefers-color-scheme/); }); -test("project dictionary only shows explicitly selected descriptions and never edits shared labels", () => { +test("project dictionary only shows selected descriptions and relinks names while versioning explicit tags", () => { const s = read("components/AnnotationLabelsPanel.tsx"); assert.match(s, /enabled\.includes\(item\.id\)/); - assert.doesNotMatch(s, /method: "PATCH"/); + assert.match(s, /reuseOrCreateDictionaryLabel/); + assert.match(s, /editingLabelId === saved.id \|\| tags.length > 0/); + assert.match(s, /\{ \.\.\.saved, tags \}/); assert.match(s, /await apiRequest/); assert.match(s, /controller\.abort\(\)/); }); diff --git a/tests/frontend/slice-dictionary.test.mjs b/tests/frontend/slice-dictionary.test.mjs index b555284d..141a215d 100644 --- a/tests/frontend/slice-dictionary.test.mjs +++ b/tests/frontend/slice-dictionary.test.mjs @@ -2,15 +2,15 @@ import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; const read = (p) => fs.readFileSync(new URL("../../" + p, import.meta.url), "utf8"); -test("slice dictionary selection only fills the existing name callback", () => { +test("slice dictionary selection carries the displayed classification snapshot without changing boundaries", () => { const picker = read("src/components/SliceNameDictionaryPicker.tsx"); - assert.match(picker, /onSelect\(item.name\)/); + assert.match(picker, /onSelect\(item.name, dictionarySelection\(item\)\)/); assert.doesNotMatch(picker, /apiRequest|\.start\s*=|\.end\s*=/); assert.match(picker, /搜索本项目切片字典/); assert.match(picker, /请联系管理员配置后再确认切片/); assert.doesNotMatch(picker, /可直接输入名称/); const workbench = read("src/components/CaptureAnnotationWorkbench.tsx"); - assert.match(workbench, /onSliceNameChange\?\.\(selectedSlice\.id,\s*name\)/); + assert.match(workbench, /onSliceNameChange\?\.\(selectedSlice\.id,\s*name,\s*dictionary\)/); }); test("dictionary management is separated and stale task results are hidden", () => { assert.match(read("src/pages/ProjectDetail.tsx"), /动作描述字典/); From 1469e1d9e12453f12db967cafc6288e839905f84 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 21 Oct 2026 09:57:40 +0800 Subject: [PATCH 05/54] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Harden=20classifi?= =?UTF-8?q?cation=20saves=20and=20recover=20frozen=20delivery=20selections?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../delivery-classification-20261021/spec.md | 13 +- backend/operators/builtin/package/operator.py | 17 +- docs/delivery-classification-v4.zh-CN.md | 4 +- internal/api/annotation_labels.go | 19 +- internal/api/dictionary_selection_history.go | 80 +++++ internal/api/dictionary_selections.go | 13 +- internal/api/dictionary_selections_test.go | 290 ++++++++++++++++++ internal/api/discard_reason_dictionary.go | 34 ++ internal/api/rework_mongo_http_test.go | 4 + internal/api/slice_dictionary_names.go | 42 ++- internal/api/slice_disposition.go | 42 ++- .../slice_disposition_materialization_test.go | 5 +- internal/api/slice_edit_application.go | 31 +- internal/api/slice_edit_restoration.go | 2 +- internal/deliveryindex/batch.go | 14 +- internal/deliveryindex/classification_test.go | 20 ++ internal/worker/artifact_storage.go | 12 + internal/worker/delivery_audit.go | 3 + internal/worker/delivery_audit_test.go | 25 ++ .../builtin/package/src/operator.py | 17 +- src/components/AnnotationLabelsPanel.tsx | 4 +- src/components/CaptureAnnotationWorkbench.tsx | 32 +- src/lib/invalid-slice-consolidation.js | 4 + src/pages/AnnotationOperations.tsx | 5 + tests/backend/export_delivery_v4_fixture.py | 2 +- .../test_delivery_v4_real_materialization.py | 46 +++ .../dictionary-classification-tags.mjs | 18 +- .../dictionary-classification-vite.config.ts | 13 + .../invalid-slice-consolidation.test.mjs | 15 + 29 files changed, 772 insertions(+), 54 deletions(-) create mode 100644 internal/api/dictionary_selection_history.go create mode 100644 tests/backend/test_delivery_v4_real_materialization.py create mode 100644 tests/browser/dictionary-classification-vite.config.ts diff --git a/.scratch/delivery-classification-20261021/spec.md b/.scratch/delivery-classification-20261021/spec.md index 758f1c50..fe4c60eb 100644 --- a/.scratch/delivery-classification-20261021/spec.md +++ b/.scratch/delivery-classification-20261021/spec.md @@ -1,6 +1,6 @@ # 最终分类交付与内部追溯分离 -Status: claimed +Status: done 2026-10-21 用户要求将两轮总 JSON 核查的问题一起本地修复,并明确选择独立 `tags` 列表、管理员维护。依据为马迪 10-09 至 10-14 的格式对话与本会话补充。原始核查证据在主仓 `local-data/diagnostics/delivery-json-audit-20261021/`。 @@ -20,3 +20,14 @@ Status: claimed 定向 Go/Python/前端回归,隔离本机 Mongo 与真实 router 的保存→返工→复审→验收→原生打包→批次及下载,检查最终分类/标签和完整内部追溯;覆盖失败重试、重启/旧版本读取、字典更新、未授权篡改、全舍弃、缺失历史证据。前端与 Go 全量门禁按既有基线对照;真实浏览器检查字典 tags 管理和选择,Web/Go 构建及生产包启动。 不上传镜像、不部署、不写公司数据、不补写历史交付、不 push/PR。本地完成后以新分支提交。 + + +## 本地完成证据(2026-10-21) + +- 独立分支 `codex/delivery-classification-20261021`,基于 `bc00467d`。本次只做本地代码、合成数据验证及提交;未 push、提 PR、上传镜像、部署、启用工作流或改写历史交付。 +- Go 与前端全量门禁无新增失败,保留原有 70 / 13 个已知失败,基线文件未修改;类型、格式、构建、生成源码一致性及生产页面启动通过。 +- 定向 Python 37 项通过(含真实视频物化→v4 的 5 项验证);一个旧版 Go 物化 opt-in 用例未单独启用,新旧真实媒体链路已由独立用例覆盖。 +- 本机 27018 随机独立 Mongo 库的三轮退回和 API 重启、原生 v4 的正式接口批次发布/失败恢复/重试/下载、停用与移除词条后的直接及面板撤销重做、v5/v6 标签防伪造均通过。测试库执行后清理。 +- 静态复核的 Standards / Spec 两轴发现均已修正并关闭;保留无快照历史的名称事实,不以当前 tags 猜补。未确认临时名称不能绕过最终确认/提交。 +- 完整日志、分离的客户/内部样例和浏览器截图保留在本工作树 `local-data/verification/delivery-classification-20261021/`。客户样例子目录 `customer-samples/partNNNN/` 每份只含 `part.json`;`customer-samples/internal/` 不属于客户包。 +- 新协议默认关闭,线上尚未验证/启用;后续正式发布需要覆盖 API/processor、Worker、package 镜像及内部 artifact 存储,按现有发布授权流程执行。未把旧镜像 release 摘要伪装成新源码摘要。 diff --git a/backend/operators/builtin/package/operator.py b/backend/operators/builtin/package/operator.py index 4ee57b03..6b2cd8a8 100644 --- a/backend/operators/builtin/package/operator.py +++ b/backend/operators/builtin/package/operator.py @@ -846,7 +846,10 @@ def _description_status(actions: list[dict[str, Any]]) -> str: def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None): - """Use the established immutable directory publisher for the two JSON files. + """Use the established immutable publisher for versioned customer JSON. + + V3 retains its historical sibling review file; v4 stores that audit in a + separate internal artifact before publishing the customer directory. Legacy single-file publishers deliberately accept only v1/v2. A directory artifact preserves their ownership/conditional-publication protocol without @@ -927,9 +930,9 @@ def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payl action.pop(key, None) if action.get("dictionary") is None: action.pop("dictionary", None) body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8") - if len(json.dumps(audit, ensure_ascii=False).encode("utf8")) > 8 * 1024 * 1024: - raise RuntimeError("internal delivery audit exceeds 8 MiB; refusing incomplete export") audit["partSha256"] = hashlib.sha256(body).hexdigest() + if len(json.dumps(audit, ensure_ascii=False, indent=2).encode("utf8")) > 8 * 1024 * 1024: + raise RuntimeError("internal delivery audit exceeds 8 MiB; refusing incomplete export") staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v4-", dir=output)) (staging / "part.json").write_bytes(body) # A sibling artifact stays in the internal run, never in the customer tree. @@ -1729,7 +1732,11 @@ def _run_accepted_slice_directory( discarded_slices.append(row) else: kept_slices.append(row) - if disposition_contract: + params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + classification_v4 = _bool(params, "DELIVERY_JSON_V4", False) + if classification_v4 and not coverage_required: + raise RuntimeError("DELIVERY_JSON_V4 requires verified full-video coverage") + if disposition_contract or classification_v4: pins = summary_payload.get("acceptedMaterializations", []) expected_sha = _sha256_file(revision_manifest_path) if summary_payload.get("decision") != "approved" or final_acceptance.get("confirmed") is not True or not any( @@ -1739,7 +1746,6 @@ def _run_accepted_slice_directory( raise RuntimeError("slice dispositions are not pinned to the accepted manifest revision") if int(coverage.get("sliceCount", -1)) != len(all_slices): raise RuntimeError("review coverage slice count mismatch") - params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} if disposition_contract and not kept_slices and not _bool(params, "DELIVERY_JSON_V4", False): if any(p.get("key") == "PROCESSING_PROFILE" and p.get("value") == "raw_human" for p in node_data.get("hyperparams", [])) and (summary_payload.get("processingProfile") != "raw_human" or final_acceptance.get("processingProfile") != "raw_human"): raise RuntimeError("raw-human package requires matching accepted task profile") @@ -1758,7 +1764,6 @@ def _run_accepted_slice_directory( payloads = _payload_tree(_payloads_with_full_pose_manifests(inputs, output, helpers)) headpose = _first_payload(payloads, "droboticflow.headpose/v1") handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1") - params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} if _bool(params, "PART_INDEX_DELIVERY", False): if source_index and not _bool(params, "SOURCE_INDEX_DELIVERY", False): raise RuntimeError("source-index JSON delivery requires explicit SOURCE_INDEX_DELIVERY approval; use directory delivery to preserve physical discard exclusion") diff --git a/docs/delivery-classification-v4.zh-CN.md b/docs/delivery-classification-v4.zh-CN.md index d0af3195..35772284 100644 --- a/docs/delivery-classification-v4.zh-CN.md +++ b/docs/delivery-classification-v4.zh-CN.md @@ -12,8 +12,8 @@ - `discardedSlices[].classification.discardEvidence` 保留业务舍弃原因、原因词条/版本/标签和补充说明。全舍弃 part 也输出总分类 JSON。审核退回原因不放在这里。 - `sourceMetadata.path` 与标定路径相对原采集目录。源索引 `sources` 的 workspace 路径基准显式标记。内部审核文件不在客户目录内,也不从主 JSON 引用。 -内部 `droboticflow.delivery.review-history/v2` JSON 按 capture/part/revision 关联,以 `partSha256` 固定原生主文件字节;包含最终验收、抽检、审核人/批注/轮次/确认项、已有正式退回基线、处理记录和阶段事件。该 artifact 留在内部 Run 存储,不进入最终交付列表、不可变客户批次或客户下载。旧记录没有的追溯不制造补写。追溯超出限制时拒绝交付,不静默截断。 +内部 `droboticflow.delivery.review-history/v2` JSON 按 capture/part/revision 关联,以 `partSha256` 固定原生主文件字节;包含最终验收、抽检、审核人/批注/轮次/确认项、已有正式退回基线、处理记录和阶段事件。该 artifact 留在内部 Run 存储,不进入最终交付列表、不可变客户批次或客户下载。分类选择的服务端快照另外保存在当前任务的恢复记录中,使字典改名、更新或停用后的撤销仍可恢复原版本,客户端不能伪造历史标签。旧记录没有的追溯不制造补写。追溯超出限制时拒绝交付,不静默截断。 -最终分类取通过验收的固定物化修订。返工正式修改分类后必须重新送检、验收;查看或处理意见本身不修改分类。相同名称而字典版本/标签不同的切片不能自动合并成一个分类。 +最终分类取通过验收的固定物化修订;沿用既有“未抽中”验收策略的任务,内部保留明确豁免,不能伪造人工通过日期。返工正式修改分类后必须重新送检、验收;查看或处理意见本身不修改分类。相同名称而字典版本/标签不同的切片不能自动合并成一个分类。 本地可用 `python -m tests.backend.export_delivery_v4_fixture <全新输出目录>` 生成明确标记的合成样例。目录中 `partNNNN/part.json` 是客户主文件;`internal/partNNNN/review-history.json` 是分离的内部示例,不属于客户包。不要把整个验证输出目录直接交付给客户。 diff --git a/internal/api/annotation_labels.go b/internal/api/annotation_labels.go index d0772a52..dbe1c8bc 100644 --- a/internal/api/annotation_labels.go +++ b/internal/api/annotation_labels.go @@ -293,13 +293,16 @@ func (server *server) handleProjectAnnotationLabelsGet(writer http.ResponseWrite if selection == nil { selection = &domain.ProjectAnnotationLabelsRecord{ID: "project-annotation-labels-" + project.ID, OrganizationID: project.OrganizationID, ProjectID: project.ID, LabelIDs: []string{}} } - if projectDictionaryKind(request) != "action" { + { available, e := server.store.ListAnnotationLabels(request.Context(), project.OrganizationID) if e != nil { renderError(writer, 500, e) return } - ids := selection.SliceLabelIDs + ids := selection.LabelIDs + if projectDictionaryKind(request) == "slice" { + ids = selection.SliceLabelIDs + } if projectDictionaryKind(request) == "discard_reason" { ids = selection.DiscardReasonLabelIDs } @@ -316,14 +319,20 @@ func (server *server) handleProjectAnnotationLabelsGet(writer http.ResponseWrite items = append(items, label) } } - response := map[string]any{"projectId": project.ID, "labelIds": ids, "items": items} + response := struct { + domain.ProjectAnnotationLabelsRecord + Items []domain.AnnotationLabelRecord `json:"items"` + AllowManual *bool `json:"allowManual,omitempty"` + }{ProjectAnnotationLabelsRecord: *selection, Items: items} + response.LabelIDs = ids if projectDictionaryKind(request) == "discard_reason" { - response["allowManual"] = selection.OrganizationID == project.OrganizationID && len(ids) == 0 + manual := selection.OrganizationID == project.OrganizationID && len(ids) == 0 + response.AllowManual = &manual } + renderJSON(writer, http.StatusOK, response) return } - renderJSON(writer, http.StatusOK, selection) } func (server *server) handleProjectAnnotationLabelsPut(writer http.ResponseWriter, request *http.Request) { diff --git a/internal/api/dictionary_selection_history.go b/internal/api/dictionary_selection_history.go new file mode 100644 index 00000000..d2d4e017 --- /dev/null +++ b/internal/api/dictionary_selection_history.go @@ -0,0 +1,80 @@ +package api + +import ( + "encoding/json" + "fmt" + + "droboticflow/internal/domain" +) + +const dictionarySelectionHistoryKey = "dictionarySelectionHistory" + +// Only server-resolved selections enter this item-scoped recovery ledger. +// It allows undo after a shared word changes or leaves the project, without +// treating the client's undo snapshot as trustworthy dictionary metadata. +func rememberDictionarySelections(item *domain.HumanItemRecord, selections ...*domain.DictionarySelection) []validationIssue { + if item == nil { + return nil + } + history := decodeMaterializationValue[domain.DictionarySelection](item.Subject.Payload[dictionarySelectionHistoryKey]) + seen := map[string]bool{} + key := func(d domain.DictionarySelection) string { return fmt.Sprintf("%s/%s/%d", d.Kind, d.ID, d.Version) } + for _, d := range history { + seen[key(d)] = true + } + for _, d := range selections { + if d != nil && !seen[key(*d)] { + copy := *d + copy.Tags = append([]string{}, d.Tags...) + history = append(history, copy) + seen[key(copy)] = true + } + } + body, err := json.Marshal(history) + if err != nil || len(body) > 1<<20 { + return []validationIssue{{Field: "dictionary", Message: "分类恢复记录超过限制,请拆分任务后重试"}} + } + if item.Subject.Payload == nil { + item.Subject.Payload = map[string]any{} + } + if len(history) > 0 { + item.Subject.Payload[dictionarySelectionHistoryKey] = history + } + return nil +} +func savedDictionarySelection(item *domain.HumanItemRecord, requested *domain.DictionarySelection, kind string) *domain.DictionarySelection { + if item == nil || requested == nil { + return nil + } + for _, saved := range decodeMaterializationValue[domain.DictionarySelection](item.Subject.Payload[dictionarySelectionHistoryKey]) { + if saved.Kind == kind && saved.ID == requested.ID && saved.Version == requested.Version { + copy := saved + return © + } + } + return nil +} +func sliceDictionarySelections(slices []domain.SliceSegment) []*domain.DictionarySelection { + selections := []*domain.DictionarySelection{} + for _, slice := range slices { + selections = append(selections, slice.Dictionary, slice.DiscardReasonDictionary) + } + return selections +} +func actionDictionarySelections(actions []domain.AnnotationSegment) []*domain.DictionarySelection { + selections := []*domain.DictionarySelection{} + for _, action := range actions { + selections = append(selections, action.Dictionary) + } + return selections +} + +func copyClassificationRecoveryState(target, source *domain.HumanItemRecord) { + for _, key := range []string{dictionarySelectionHistoryKey, "unconfirmedSliceNames"} { + if value, ok := source.Subject.Payload[key]; ok { + target.Subject.Payload[key] = value + } else { + delete(target.Subject.Payload, key) + } + } +} diff --git a/internal/api/dictionary_selections.go b/internal/api/dictionary_selections.go index a554ad97..bd05485d 100644 --- a/internal/api/dictionary_selections.go +++ b/internal/api/dictionary_selections.go @@ -43,7 +43,7 @@ func sameDictionaryVersion(a, b *domain.DictionarySelection) bool { return a != nil && b != nil && a.ID == b.ID && a.Version == b.Version } -func (s *server) canonicalActionDictionaries(ctx context.Context, item *domain.HumanItemRecord, segments []domain.AnnotationSegment) []validationIssue { +func (s *server) canonicalActionDictionaries(ctx context.Context, item *domain.HumanItemRecord, segments []domain.AnnotationSegment, mode ...string) []validationIssue { previous := decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(item)["segments"]) if len(previous) == 0 && item.LegacyTaskID != "" { revisions, err := s.store.ListAnnotationRevisions(ctx, item.OrganizationID, item.LegacyTaskID) @@ -56,6 +56,9 @@ func (s *server) canonicalActionDictionaries(ctx context.Context, item *domain.H } } } + if issues := rememberDictionarySelections(item, actionDictionarySelections(previous)...); len(issues) > 0 { + return issues + } old := map[string]domain.AnnotationSegment{} for _, seg := range previous { old[seg.ID] = seg @@ -69,6 +72,12 @@ func (s *server) canonicalActionDictionaries(ctx context.Context, item *domain.H seg.ClearDictionary = false continue } + if len(mode) > 0 && mode[0] == "restore" && !seg.IsInvalid && strings.TrimSpace(seg.Description) != "" { + if saved := savedDictionarySelection(item, seg.Dictionary, "action"); saved != nil { + seg.Dictionary = saved + continue + } + } if seg.IsInvalid || strings.TrimSpace(seg.Description) == "" { seg.Dictionary = nil continue @@ -93,5 +102,5 @@ func (s *server) canonicalActionDictionaries(ctx context.Context, item *domain.H } seg.Dictionary = label.Selection() } - return nil + return rememberDictionarySelections(item, actionDictionarySelections(segments)...) } diff --git a/internal/api/dictionary_selections_test.go b/internal/api/dictionary_selections_test.go index e642f34d..cfe81685 100644 --- a/internal/api/dictionary_selections_test.go +++ b/internal/api/dictionary_selections_test.go @@ -119,3 +119,293 @@ func TestDictionaryTagsHaveIndependentRevisionAndAdminAuthorization(t *testing.T t.Fatal("tag history lost") } } + +func TestLegacyClassificationDoesNotGuessCurrentTagsAtConfirmation(t *testing.T) { + f := newCombinedDraftFixture(t) + legacy := domain.SliceSegment{ID: "legacy", Name: "物品操作", Start: 0, End: 10} + f.item.Subject.Payload["annotationWorkspace"] = map[string]any{"sliceSegments": []domain.SliceSegment{legacy}} + f.store.UpsertHumanItem(f.ctx, f.item) + f.store.UpsertAnnotationLabel(f.ctx, domain.AnnotationLabelRecord{ID: "current", OrganizationID: f.item.OrganizationID, Kind: "slice", Code: "base", Name: legacy.Name, Tags: []string{"现在新增的标签"}, Version: 9, Status: domain.AnnotationLabelActive}) + f.store.UpsertProjectSliceLabels(f.ctx, domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID, SliceLabelIDs: []string{"current"}}) + server := &server{store: f.store} + slices := []domain.SliceSegment{legacy} + if issues := server.validateProjectSliceNames(f.ctx, &f.item, slices, "confirm"); len(issues) > 0 || slices[0].Dictionary != nil { + t.Fatal("legacy tags guessed at confirm") + } + slices[0].Dictionary = &domain.DictionarySelection{ID: "current", Version: 9} + if issues := server.validateProjectSliceNames(f.ctx, &f.item, slices, "confirm"); len(issues) > 0 || slices[0].Dictionary == nil || slices[0].Dictionary.Tags[0] != "现在新增的标签" { + t.Fatal("explicit new selection did not freeze current tags") + } +} + +func TestSliceEditApplicationCanonicalizesAgainstStoredDictionaryState(t *testing.T) { + for _, foreign := range []bool{false, true} { + t.Run(map[bool]string{false: "forged-tags", true: "foreign-word"}[foreign], func(t *testing.T) { + f := continuousFixture(t) + selection, _ := f.store.GetProjectAnnotationLabels(f.ctx, f.task.ProjectID) + label, _ := f.store.GetAnnotationLabel(f.ctx, selection.SliceLabelIDs[0]) + label.Tags = []string{"服务端标签"} + label.Version = 2 + f.store.UpsertAnnotationLabel(f.ctx, *label) + before := []domain.SliceSegment{{ID: "s", Name: label.Name, Start: 0, End: 10}} + item := f.item + item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "completed", "phase": "annotate"} + item.Subject.Payload["annotationWorkspace"] = map[string]any{"sliceSegments": before, "segments": []domain.AnnotationSegment{}} + f.store.UpsertHumanItem(f.ctx, item) + incoming := *label.Selection() + incoming.Tags = []string{"FORGED"} + if foreign { + incoming.ID = "foreign-label" + } + proposed := append([]domain.SliceSegment(nil), before...) + proposed[0].Dictionary = &incoming + body := sliceEditApplicationInput{ApplicationID: "classification-apply", combinedAnnotationDraftInput: combinedAnnotationDraftInput{Version: item.Version, SliceSegments: proposed}} + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-edit-applications", f.annotator, body) + if foreign { + if r.Code != 400 { + t.Fatalf("foreign classification accepted %d %s", r.Code, r.Body.String()) + } + return + } + if r.Code != 200 { + t.Fatalf("%d %s", r.Code, r.Body.String()) + } + saved, _ := f.store.GetHumanItem(f.ctx, item.ID) + got := decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(saved)["sliceSegments"]) + if got[0].Dictionary == nil || !reflect.DeepEqual(got[0].Dictionary.Tags, label.Tags) { + t.Fatal("client classification snapshot trusted") + } + }) + } +} + +func TestProjectActionDictionaryReturnsVersionedItemsForAnnotator(t *testing.T) { + f := newCombinedDraftFixture(t) + label := domain.AnnotationLabelRecord{ID: "action-word", OrganizationID: f.item.OrganizationID, Code: "pick", Name: "拿取", Kind: "action", Version: 3, Tags: []string{"抓取"}, Status: domain.AnnotationLabelActive} + f.store.UpsertAnnotationLabel(f.ctx, label) + f.store.UpsertProjectAnnotationLabels(f.ctx, domain.ProjectAnnotationLabelsRecord{ID: "selection", ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID, LabelIDs: []string{label.ID}}) + r := f.request("GET", "/api/projects/"+f.task.ProjectID+"/annotation-labels", f.annotator, nil) + var response struct { + ID string `json:"id"` + LabelIDs []string `json:"labelIds"` + Items []domain.AnnotationLabelRecord `json:"items"` + } + if json.Unmarshal(r.Body.Bytes(), &response) != nil || r.Code != 200 || response.ID != "selection" || len(response.LabelIDs) != 1 || len(response.Items) != 1 || response.Items[0].Version != 3 || response.Items[0].Tags[0] != "抓取" { + t.Fatalf("%d %s", r.Code, r.Body.String()) + } +} + +func TestUndoRestoresOnlyServerRememberedClassificationsAfterDictionaryMaintenance(t *testing.T) { + f := continuousFixture(t) + srv := &server{store: f.store} + base := domain.AnnotationLabelRecord{ID: "base-old", OrganizationID: f.item.OrganizationID, Kind: "slice", Code: "base", Name: "旧基础分类", Version: 2, Tags: []string{"原标签"}, Status: domain.AnnotationLabelActive} + action := domain.AnnotationLabelRecord{ID: "action-old", OrganizationID: f.item.OrganizationID, Kind: "action", Code: "pick", Name: "旧动作分类", Version: 3, Tags: []string{"原动作标签"}, Status: domain.AnnotationLabelActive} + f.store.UpsertAnnotationLabel(f.ctx, base) + f.store.UpsertAnnotationLabel(f.ctx, action) + selection := domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID, LabelIDs: []string{action.ID}, SliceLabelIDs: []string{base.ID}} + f.store.UpsertProjectAnnotationLabels(f.ctx, selection) + f.store.UpsertProjectSliceLabels(f.ctx, selection) + oldSlices := []domain.SliceSegment{{ID: "s", Name: base.Name, Start: 0, End: 10, Dictionary: base.Selection()}} + oldActions := []domain.AnnotationSegment{{ID: "a", Description: "原描述", Start: 0, End: 10, Status: "described", Dictionary: action.Selection()}} + item := f.item + if issues := srv.validateProjectSliceNames(f.ctx, &item, oldSlices, "draft"); len(issues) > 0 { + t.Fatal(issues) + } + if issues := srv.canonicalActionDictionaries(f.ctx, &item, oldActions); len(issues) > 0 { + t.Fatal(issues) + } + item.Subject.Payload["sliceWorkflow"] = map[string]any{"phase": "annotate", "status": "completed"} + // Later editing selected a different classification. The word is subsequently + // renamed, retagged, disabled and unlinked by an administrator. + item.Subject.Payload["annotationWorkspace"] = map[string]any{"sliceSegments": []domain.SliceSegment{{ID: "s", Name: "另一个分类", Start: 0, End: 10}}, "segments": []domain.AnnotationSegment{{ID: "a", Description: "修改后描述", Start: 0, End: 10, Status: "described"}}} + f.store.UpsertHumanItem(f.ctx, item) + base.Name = "新版" + base.Tags = []string{"新标签"} + base.Version++ + base.Status = domain.AnnotationLabelInactive + f.store.UpsertAnnotationLabel(f.ctx, base) + action.Version++ + action.Status = domain.AnnotationLabelInactive + f.store.UpsertAnnotationLabel(f.ctx, action) + f.store.UpsertProjectSliceLabels(f.ctx, domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID}) + f.store.UpsertProjectAnnotationLabels(f.ctx, domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID}) + // Simulate serialization and restart; do not use process-local snapshot state. + f.handler = NewHandler(f.store, nil) + oldSlices[0].Dictionary.Tags = []string{"FORGED"} + oldActions[0].Dictionary.Tags = []string{"FORGED"} + body := combinedAnnotationDraftInput{Version: item.Version, SliceSegments: oldSlices, Segments: oldActions} + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-edit-restorations", f.annotator, body) + if r.Code != 200 { + t.Fatalf("undo %d %s", r.Code, r.Body.String()) + } + saved, _ := f.store.GetHumanItem(f.ctx, item.ID) + slices := decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(saved)["sliceSegments"]) + actions := decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(saved)["segments"]) + if slices[0].Dictionary.Version != 2 || slices[0].Dictionary.Tags[0] != "原标签" || actions[0].Dictionary.Version != 3 || actions[0].Dictionary.Tags[0] != "原动作标签" { + t.Fatal("undo lost or forged frozen classification") + } +} + +func TestDirectDispositionCannotPersistForeignOrForgedSliceDictionary(t *testing.T) { + for _, foreign := range []bool{false, true} { + t.Run(map[bool]string{false: "forged-tags", true: "foreign-word"}[foreign], func(t *testing.T) { + f := continuousFixture(t) + body := dispositionFixturePayload(t, f) + selection, _ := f.store.GetProjectAnnotationLabels(f.ctx, f.task.ProjectID) + label, _ := f.store.GetAnnotationLabel(f.ctx, selection.SliceLabelIDs[0]) + label.Tags = []string{"服务端标签"} + label.Version = 2 + f.store.UpsertAnnotationLabel(f.ctx, *label) + body.SliceSegments[0].Name = label.Name + d := *label.Selection() + d.Tags = []string{"FORGED"} + if foreign { + d.ID = "foreign" + } + body.SliceSegments[0].Dictionary = &d + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-dispositions", f.annotator, body) + if foreign { + if r.Code != 400 { + t.Fatalf("foreign dictionary saved %d %s", r.Code, r.Body.String()) + } + return + } + if r.Code != 200 { + t.Fatalf("%d %s", r.Code, r.Body.String()) + } + item, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + saved := decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(item)["sliceSegments"]) + if saved[0].Dictionary == nil || !reflect.DeepEqual(saved[0].Dictionary.Tags, label.Tags) { + t.Fatal("forged tags persisted through direct disposition") + } + }) + } +} + +func TestDispositionUndoRedoPreservesClassificationsAfterDictionaryRemoval(t *testing.T) { + for _, application := range []bool{false, true} { + t.Run(map[bool]string{false: "direct", true: "panel-application"}[application], func(t *testing.T) { + f := continuousFixture(t) + body := dispositionFixturePayload(t, f) + selection, _ := f.store.GetProjectAnnotationLabels(f.ctx, f.task.ProjectID) + base, _ := f.store.GetAnnotationLabel(f.ctx, selection.SliceLabelIDs[0]) + base.Tags = []string{"基础标签"} + base.Version = 2 + f.store.UpsertAnnotationLabel(f.ctx, *base) + reason, _ := f.store.GetAnnotationLabel(f.ctx, body.ReasonLabelID) + reason.Tags = []string{"舍弃标签"} + f.store.UpsertAnnotationLabel(f.ctx, *reason) + for i := range body.SliceSegments { + body.SliceSegments[i].Name = base.Name + body.SliceSegments[i].Dictionary = base.Selection() + } + original := append([]domain.SliceSegment(nil), body.SliceSegments...) + endpoint := "/api/plugins/annotation-qa/tasks/" + f.task.ID + r := f.request("POST", endpoint+"/slice-dispositions", f.annotator, body) + if r.Code != 200 { + t.Fatalf("discard %d %s", r.Code, r.Body.String()) + } + var first struct { + Item domain.HumanItemRecord `json:"item"` + Workspace combinedAnnotationDraftInput `json:"workspace"` + } + json.Unmarshal(r.Body.Bytes(), &first) + base.Name = "新名字" + base.Version++ + base.Tags = []string{"新标签"} + base.Status = domain.AnnotationLabelInactive + f.store.UpsertAnnotationLabel(f.ctx, *base) + reason.Version++ + reason.Status = domain.AnnotationLabelInactive + f.store.UpsertAnnotationLabel(f.ctx, *reason) + f.store.UpsertProjectSliceLabels(f.ctx, domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID}) + f.store.UpsertProjectDiscardReasonLabels(f.ctx, domain.ProjectAnnotationLabelsRecord{ProjectID: f.task.ProjectID, OrganizationID: f.item.OrganizationID}) + item, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + item.Subject.Payload["sliceWorkflow"] = map[string]any{"phase": "annotate", "status": "completed"} + f.store.UpsertHumanItem(f.ctx, *item) + f.handler = NewHandler(f.store, nil) + if application { + input := sliceEditApplicationInput{ApplicationID: "undo-discard", RestoreClassifications: true, combinedAnnotationDraftInput: combinedAnnotationDraftInput{Version: item.Version, SliceSegments: original}, Dispositions: []sliceDispositionInput{{SliceID: body.SliceID, Disposition: "keep"}}} + r = f.request("POST", endpoint+"/slice-edit-applications", f.annotator, input) + } else { + undo := body + undo.combinedAnnotationDraftInput = first.Workspace + undo.Version = item.Version + undo.Disposition = "keep" + undo.RestoreClassifications = true + undo.RestorePreviousName = true + undo.RestoreNameFromVersion = first.Item.Version + r = f.request("POST", endpoint+"/slice-dispositions", f.annotator, undo) + } + if r.Code != 200 { + t.Fatalf("undo %d %s", r.Code, r.Body.String()) + } + var restored struct { + Item domain.HumanItemRecord `json:"item"` + Workspace combinedAnnotationDraftInput `json:"workspace"` + } + json.Unmarshal(r.Body.Bytes(), &restored) + d := restored.Workspace.SliceSegments[1].Dictionary + if d == nil || d.Version != 2 || d.Tags[0] != "基础标签" { + t.Fatal("undo lost frozen base category") + } + if application { + input := sliceEditApplicationInput{ApplicationID: "redo-discard", RestoreClassifications: true, combinedAnnotationDraftInput: combinedAnnotationDraftInput{Version: restored.Item.Version, SliceSegments: first.Workspace.SliceSegments}, Dispositions: []sliceDispositionInput{{SliceID: body.SliceID, Disposition: "discard", ReasonLabelID: body.ReasonLabelID, ReasonVersion: body.ReasonVersion}}} + r = f.request("POST", endpoint+"/slice-edit-applications", f.annotator, input) + } else { + redo := body + redo.combinedAnnotationDraftInput = restored.Workspace + redo.Version = restored.Item.Version + redo.RestoreClassifications = true + r = f.request("POST", endpoint+"/slice-dispositions", f.annotator, redo) + } + if r.Code != 200 { + t.Fatalf("redo %d %s", r.Code, r.Body.String()) + } + var redone struct { + Workspace combinedAnnotationDraftInput `json:"workspace"` + } + json.Unmarshal(r.Body.Bytes(), &redone) + d = redone.Workspace.SliceSegments[1].DiscardReasonDictionary + if d == nil || d.Version != 1 || d.Tags[0] != "舍弃标签" { + t.Fatal("redo lost frozen discard reason") + } + }) + } +} + +func TestUnconfirmedRestoredNamesCannotBecomeFinalClassification(t *testing.T) { + f := newCombinedDraftFixture(t) + srv := &server{store: f.store} + slices := []domain.SliceSegment{{ID: "s", Name: "未选择字典的临时名称", Start: 0, End: 10}} + if issues := srv.validateProjectSliceNames(f.ctx, &f.item, slices, "restore"); len(issues) > 0 { + t.Fatal(issues) + } + f.item.Subject.Payload["annotationWorkspace"] = map[string]any{"sliceSegments": slices} + for _, mode := range []string{"confirm", "submit"} { + if issues := srv.validateProjectSliceNames(f.ctx, &f.item, slices, mode); len(issues) == 0 { + t.Fatalf("unfinished name bypassed %s", mode) + } + } +} + +func TestLegacyDispositionCannotSeedDictionaryRecoveryHistory(t *testing.T) { + f := newCombinedDraftFixture(t) + f.task.TemplateVersion = 5 + f.store.UpsertAnnotationTask(f.ctx, f.task) + body := dispositionFixturePayload(t, f) + body.Segments[0].Handedness = "both" + body.SliceSegments[0].Name = "原有名称" + body.SliceSegments[0].TaskLabelID = "legacy-task-label" + body.SliceSegments[0].TaskLabel = "原有任务分类" + body.SliceSegments[0].Dictionary = &domain.DictionarySelection{ID: "FORGED", Kind: "action", Version: 9, Name: "伪造分类", Code: "forged", Tags: []string{"FORGED"}} + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-dispositions", f.annotator, body) + if r.Code != 200 { + t.Fatalf("legacy save %d %s", r.Code, r.Body.String()) + } + saved, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + slices := decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(saved)["sliceSegments"]) + if slices[0].Dictionary != nil || slices[0].Name != "原有名称" || slices[0].TaskLabelID != "legacy-task-label" || savedDictionarySelection(saved, &domain.DictionarySelection{ID: "FORGED", Version: 9}, "action") != nil { + t.Fatal("legacy task seeded a dictionary snapshot or lost legacy labels") + } +} diff --git a/internal/api/discard_reason_dictionary.go b/internal/api/discard_reason_dictionary.go index b2ffe81a..a293d323 100644 --- a/internal/api/discard_reason_dictionary.go +++ b/internal/api/discard_reason_dictionary.go @@ -11,6 +11,19 @@ func (s *server) resolveDiscardReason(ctx context.Context, item *domain.HumanIte issue := func(message string) (*domain.AnnotationLabelRecord, []validationIssue) { return nil, []validationIssue{{Field: "reasonLabelId", Code: "discard_reason_dictionary_required", Message: message}} } + if body.RestoreClassifications { + if prior := historicalDiscardReason(item, body); prior != nil { + if prior.DiscardReasonDictionary != nil { + d := prior.DiscardReasonDictionary + return &domain.AnnotationLabelRecord{ID: d.ID, Kind: d.Kind, Code: d.Code, Name: d.Name, Version: d.Version, Tags: append([]string{}, d.Tags...), OrganizationID: item.OrganizationID}, nil + } + if prior.DiscardReasonLabelID == "" { + return nil, nil + } + return &domain.AnnotationLabelRecord{ID: prior.DiscardReasonLabelID, Kind: "discard_reason", Name: prior.DiscardReasonName, Version: prior.DiscardReasonVersion, OrganizationID: item.OrganizationID}, nil + } + return issue("未找到当前任务已保存的舍弃原因,无法恢复该历史选择") + } selection, err := s.store.GetProjectAnnotationLabels(ctx, item.ProjectID) if err != nil { return issue("舍弃原因字典加载失败,请重试") @@ -75,3 +88,24 @@ func describeSliceNameIssues(issues []validationIssue, slices []domain.SliceSegm func discardReasonEvidence(slice domain.SliceSegment) map[string]any { return map[string]any{"dictionary": slice.DiscardReasonDictionary, "reason": slice.DiscardReason, "reasonLabelId": slice.DiscardReasonLabelID, "reasonVersion": slice.DiscardReasonVersion, "reasonName": slice.DiscardReasonName, "reasonNote": slice.DiscardReasonNote, "at": slice.DiscardedAt, "previousAnnotations": slice.DiscardedAnnotations} } + +// Restore only a reason/note combination actually recorded for this slice; +// administrator changes never let a client invent historical reason metadata. +func historicalDiscardReason(item *domain.HumanItemRecord, body sliceDispositionInput) *domain.SliceSegment { + history, _ := normalizeStructuredValue(item.Subject.Payload["sliceDispositionHistory"]).([]any) + for i := len(history) - 1; i >= 0; i-- { + entry := mapValue(history[i]) + if stringValue(entry["sliceId"]) != body.SliceID { + continue + } + after := decodeMaterializationValue[domain.SliceSegment]([]any{entry["after"]}) + if len(after) != 1 { + continue + } + saved := after[0] + if saved.IsDiscarded() && saved.DiscardReasonLabelID == body.ReasonLabelID && saved.DiscardReasonVersion == body.ReasonVersion && saved.DiscardReasonNote == strings.TrimSpace(body.ReasonNote) && (saved.DiscardReasonLabelID != "" || saved.DiscardReason == strings.TrimSpace(body.Reason)) { + return &saved + } + } + return nil +} diff --git a/internal/api/rework_mongo_http_test.go b/internal/api/rework_mongo_http_test.go index fe09bbe3..36bedbcf 100644 --- a/internal/api/rework_mongo_http_test.go +++ b/internal/api/rework_mongo_http_test.go @@ -89,6 +89,7 @@ func TestReworkMongoHTTPRepeatedReturnAcrossRestart(t *testing.T) { batch, e := f.store.GetHumanBatch(ctx, f.item.BatchID) must(e) seed("humanBatches", batch) + f.item.Subject.Payload[dictionarySelectionHistoryKey] = []domain.DictionarySelection{{ID: "historical-base", Kind: "slice", Version: 2, Code: "sort", Name: "物品操作", Tags: []string{"商品"}}} seed("humanItems", f.item) seed("annotationTasks", f.task) slices, e := f.store.ListSliceRevisions(ctx, f.item.OrganizationID, f.task.ID) @@ -136,6 +137,9 @@ func TestReworkMongoHTTPRepeatedReturnAcrossRestart(t *testing.T) { if w.Code != 200 || len(out.Items) != 1 || out.Items[0].AssignedUserID != f.annotator.ID { t.Fatalf("persisted return missing from author pool: %d %s", w.Code, w.Body.String()) } + if snap := savedDictionarySelection(&out.Items[0], &domain.DictionarySelection{ID: "historical-base", Version: 2}, "slice"); snap == nil || snap.Tags[0] != "商品" { + t.Fatal("classification recovery ledger lost across Mongo restart") + } baseline, e := ds.GetReworkBaseline(ctx, store.ReworkBaselineID(item.ID, out.Items[0].ReviewRound)) must(e) if baseline == nil || len(baseline.Slices) == 0 || baseline.Slices[0].Dictionary == nil || baseline.Slices[0].Dictionary.Version != 2 || baseline.Slices[0].Dictionary.Tags[0] != "商品" { diff --git a/internal/api/slice_dictionary_names.go b/internal/api/slice_dictionary_names.go index 41a5d7d7..de02c6b2 100644 --- a/internal/api/slice_dictionary_names.go +++ b/internal/api/slice_dictionary_names.go @@ -18,6 +18,11 @@ func (s *server) validateProjectSliceNames(ctx context.Context, item *domain.Hum return []validationIssue{{Field: "sliceSegments", Message: "无法读取切片字典规则"}} } if task == nil || task.TemplateVersion < 6 { + // Older templates use taskLabelId/taskLabel; they cannot introduce the + // independent v6 slice dictionary field, including through undo/discard. + for i := range slices { + slices[i].Dictionary = nil + } return nil } previous := decodeMaterializationValue[domain.SliceSegment](annotationWorkspacePayload(item)["sliceSegments"]) @@ -32,24 +37,47 @@ func (s *server) validateProjectSliceNames(ctx context.Context, item *domain.Hum } } } + if issues := rememberDictionarySelections(item, sliceDictionarySelections(previous)...); len(issues) > 0 { + return issues + } old := map[string]domain.SliceSegment{} for _, p := range previous { old[p.ID] = p } + unconfirmed := mapValue(item.Subject.Payload["unconfirmedSliceNames"]) + if unconfirmed == nil { + unconfirmed = map[string]any{} + } var allowed map[string]domain.AnnotationLabelRecord for i := range slices { c := &slices[i] p, exists := old[c.ID] + if mode == "restore" && c.Dictionary != nil { + if saved := savedDictionarySelection(item, c.Dictionary, "slice"); saved != nil && c.Name == saved.Name { + c.Dictionary = saved + delete(unconfirmed, c.ID) + continue + } + } if c.IsDiscarded() { c.Dictionary = nil + delete(unconfirmed, c.ID) continue } - if exists && p.Name == c.Name && (c.Dictionary == nil || sameDictionaryVersion(p.Dictionary, c.Dictionary)) && (mode != "confirm" || p.Dictionary != nil) { + unchanged := exists && p.Name == c.Name && (c.Dictionary == nil || sameDictionaryVersion(p.Dictionary, c.Dictionary)) + finalizing := mode == "confirm" || mode == "submit" + pendingName := p.Dictionary == nil && c.Name != "" && stringValue(unconfirmed[c.ID]) == c.Name + needsNameCheck := (mode == "confirm" && p.Dictionary == nil) || (mode == "submit" && strings.TrimSpace(c.Name) == "") || (finalizing && pendingName) + if unchanged && !needsNameCheck { c.Dictionary = p.Dictionary continue } + if (mode == "restore" && c.Dictionary == nil) || (mode == "draft" && strings.TrimSpace(c.Name) == "") { c.Dictionary = nil + if c.Name != "" { + unconfirmed[c.ID] = c.Name + } continue } if allowed == nil { @@ -68,7 +96,15 @@ func (s *server) validateProjectSliceNames(ctx context.Context, item *domain.Hum if chosen == nil { return []validationIssue{{Field: fmt.Sprintf("sliceSegments.%d.name", i), Code: "slice_dictionary_name_required", Message: "请从本项目切片字典选择名称;词条已更新时请重新选择"}} } - c.Dictionary = chosen + if !(exists && p.Name == c.Name && p.Dictionary == nil && c.Dictionary == nil && stringValue(unconfirmed[c.ID]) != c.Name) { + c.Dictionary = chosen + } + delete(unconfirmed, c.ID) } - return nil + if len(unconfirmed) > 0 { + item.Subject.Payload["unconfirmedSliceNames"] = unconfirmed + } else { + delete(item.Subject.Payload, "unconfirmedSliceNames") + } + return rememberDictionarySelections(item, sliceDictionarySelections(slices)...) } diff --git a/internal/api/slice_disposition.go b/internal/api/slice_disposition.go index c3fef44c..6f740b38 100644 --- a/internal/api/slice_disposition.go +++ b/internal/api/slice_disposition.go @@ -346,6 +346,7 @@ func restoredDiscardAnnotations(slice domain.SliceSegment) []domain.AnnotationSe type sliceDispositionInput struct { combinedAnnotationDraftInput + RestoreClassifications bool `json:"restoreClassifications,omitempty"` SliceID string `json:"sliceId"` Disposition string `json:"disposition"` Reason string `json:"reason"` @@ -358,6 +359,13 @@ type sliceDispositionInput struct { } func sliceNameBeforeDiscard(item *domain.HumanItemRecord, sliceID string, version int) (string, bool) { + prior := sliceBeforeDiscard(item, sliceID, version) + if prior == nil { + return "", false + } + return prior.Name, true +} +func sliceBeforeDiscard(item *domain.HumanItemRecord, sliceID string, version int) *domain.SliceSegment { history, _ := normalizeStructuredValue(item.Subject.Payload["sliceDispositionHistory"]).([]any) resets, _ := normalizeStructuredValue(item.Subject.Payload["sliceResetHistory"]).([]any) resetVersion := 0 @@ -376,10 +384,11 @@ func sliceNameBeforeDiscard(item *domain.HumanItemRecord, sliceID string, versio before := decodeMaterializationValue[domain.SliceSegment]([]any{entry["before"]}) after := decodeMaterializationValue[domain.SliceSegment]([]any{entry["after"]}) if len(before) == 1 && len(after) == 1 && !before[0].IsDiscarded() && after[0].IsDiscarded() { - return before[0].Name, true + copy := before[0] + return © } } - return "", false + return nil } func (s *server) handleSliceDisposition(w http.ResponseWriter, r *http.Request) { @@ -469,6 +478,7 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, } slices, issues := s.canonicalSliceDispositionState(r.Context(), item, body.normalizedSliceSegments(), body.normalizedOperations()) + issues = append(issues, s.validateProjectSliceNames(r.Context(), item, slices, "restore")...) if task.SourceIsFullVideo { timeline, e := s.annotationFrameTimelineForTask(r.Context(), *task, "left") if e != nil { @@ -481,7 +491,7 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, } issues = append(issues, validateSliceSegments(slices, *task)...) issues = append(issues, validateCompleteSliceCoverage(slices, *task)...) - issues = append(issues, s.canonicalActionDictionaries(r.Context(), item, body.Segments)...) + issues = append(issues, s.canonicalActionDictionaries(r.Context(), item, body.Segments, dispositionDictionaryMode(body))...) issues = append(issues, validateDraftCombinedAnnotationEvents(body.Segments, *task)...) idx := -1 for i, v := range slices { @@ -538,6 +548,7 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, if body.Disposition == "discard" { next.DeliveryDisposition = "discard" next.Name, next.TaskLabelID, next.TaskLabel = "", "", "" + next.Dictionary = nil next.DiscardReason = body.Reason next.DiscardReasonLabelID = "" next.DiscardReasonVersion = 0 @@ -548,6 +559,9 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, next.DiscardReasonVersion = reasonLabel.Version next.DiscardReasonName = reasonLabel.Name next.DiscardReasonDictionary = reasonLabel.Selection() + if body.RestoreClassifications { + next.DiscardReasonDictionary = savedDictionarySelection(item, &domain.DictionarySelection{ID: reasonLabel.ID, Version: reasonLabel.Version}, "discard_reason") + } } next.DiscardReasonNote = body.ReasonNote next.DiscardedByUserID = actorID @@ -626,6 +640,12 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, // Precise undo targets a server-recorded discard, even after later // undo/redo writes. Ordinary restore deliberately remains unnamed. next.Name = restoredName + next.Dictionary = nil + if body.RestorePreviousName || body.RestoreNameFromVersion != 0 { + if prior := sliceBeforeDiscard(item, before.ID, body.RestoreNameFromVersion); prior != nil { + next.Dictionary = prior.Dictionary + } + } } next.DiscardReason = "" next.DiscardReasonLabelID = "" @@ -643,13 +663,20 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, next = slices[idx] // Disposition is an incremental edit, not confirmation of the whole slicing // plan. Other slices may still have draft names; confirmation validates them. - issues = validateDraftCombinedAnnotationEvents(segments, *task) + issues = s.validateProjectSliceNames(r.Context(), item, slices, "restore") + issues = append(issues, s.canonicalActionDictionaries(r.Context(), item, segments, "restore")...) + issues = append(issues, validateDraftCombinedAnnotationEvents(segments, *task)...) + if len(issues) > 0 { renderValidationError(w, describeSliceNameIssues(issues, slices)...) return nil, domain.SliceSegment{}, false } statuses := body.normalizedStatuses() statuses[next.ID] = domain.SliceAnnotationPending + if issues := rememberDictionarySelections(item, sliceDictionarySelections(slices)...); len(issues) > 0 { + renderValidationError(w, issues...) + return nil, domain.SliceSegment{}, false + } workspace := map[string]any{"sliceSegments": slices, "segments": segments, "statuses": statuses, "operations": body.normalizedOperations(), "userId": actorID, "savedAt": nowISO()} if item.Subject.Payload == nil { item.Subject.Payload = map[string]any{} @@ -670,3 +697,10 @@ func (s *server) prepareSliceDisposition(w http.ResponseWriter, r *http.Request, } return workspace, next, true } + +func dispositionDictionaryMode(body sliceDispositionInput) string { + if body.RestoreClassifications { + return "restore" + } + return "draft" +} diff --git a/internal/api/slice_disposition_materialization_test.go b/internal/api/slice_disposition_materialization_test.go index 9f402498..28fccc8d 100644 --- a/internal/api/slice_disposition_materialization_test.go +++ b/internal/api/slice_disposition_materialization_test.go @@ -82,11 +82,14 @@ func TestSliceDiscardMaterialization60Seconds(t *testing.T) { for i := 0; i < 3; i++ { start, end := float64(i*20), float64((i+1)*20) sl := domain.SliceSegment{ID: fmt.Sprintf("segment-%d", i), Name: fmt.Sprintf("片段 %d", i+1), Start: start, End: end} + sl.Dictionary = &domain.DictionarySelection{ID: fmt.Sprintf("base-%d", i), Kind: "slice", Version: 2, Code: fmt.Sprintf("class-%d", i), Name: sl.Name, Tags: []string{"基础分类", "商品"}} if mode == "discard" || (mode == "mixed" && i == 1) { sl.DeliveryDisposition = "discard" sl.DiscardReason = "审查测试:画面被遮挡" sl.DiscardReasonLabelID = "reason-test" sl.DiscardReasonVersion = 3 + sl.Dictionary = nil + sl.DiscardReasonDictionary = &domain.DictionarySelection{ID: "reason-test", Kind: "discard_reason", Version: 3, Code: "occluded", Name: "审查测试", Tags: []string{"不可用"}} sl.DiscardReasonName = "审查测试" sl.DiscardReasonNote = "画面被遮挡" sl.DiscardedAt = "2026-09-16T00:00:00Z" @@ -94,7 +97,7 @@ func TestSliceDiscardMaterialization60Seconds(t *testing.T) { } else if mode == "partial" && i == 0 { segments = append(segments, domain.AnnotationSegment{ID: "partial", Start: 5, End: 10, Description: "仅描述这五秒", Source: "human"}) } else if mode != "blank" && mode != "partial" { - segments = append(segments, domain.AnnotationSegment{ID: fmt.Sprintf("action-%d", i), Start: start, End: end, Description: "测试动作", Source: "human"}) + segments = append(segments, domain.AnnotationSegment{ID: fmt.Sprintf("action-%d", i), Start: start, End: end, Description: "测试动作", Dictionary: &domain.DictionarySelection{ID: "action-test", Kind: "action", Version: 4, Code: "pick", Name: "拿取", Tags: []string{"抓取"}}, Source: "human"}) } slices = append(slices, sl) } diff --git a/internal/api/slice_edit_application.go b/internal/api/slice_edit_application.go index fbb74401..b745faca 100644 --- a/internal/api/slice_edit_application.go +++ b/internal/api/slice_edit_application.go @@ -9,8 +9,9 @@ import ( type sliceEditApplicationInput struct { combinedAnnotationDraftInput - ApplicationID string `json:"applicationId"` - Dispositions []sliceDispositionInput `json:"dispositions"` + RestoreClassifications bool `json:"restoreClassifications,omitempty"` + ApplicationID string `json:"applicationId"` + Dispositions []sliceDispositionInput `json:"dispositions"` } // A panel application is one CAS, including explicit disposition commands and @@ -71,6 +72,7 @@ func (s *server) handleSliceEditApplication(w http.ResponseWriter, r *http.Reque } commands := map[string]sliceDispositionInput{} for _, command := range body.Dispositions { + command.RestoreClassifications = body.RestoreClassifications if command.SliceID == "" || commands[command.SliceID].SliceID != "" || (command.Disposition != "keep" && command.Disposition != "discard") { renderValidationError(w, validationIssue{Field: "dispositions", Message: "切片用途操作无效或重复"}) return @@ -81,6 +83,7 @@ func (s *server) handleSliceEditApplication(w http.ResponseWriter, r *http.Reque // Undo can first restore a discarded child before rejoining it. The restore // remains an explicit, audited command in the same atomic application. for _, command := range body.Dispositions { + command.RestoreClassifications = body.RestoreClassifications if !ids[command.SliceID] { if command.Disposition != "keep" { renderValidationError(w, validationIssue{Field: "dispositions", Message: "舍弃操作必须保留目标切片"}) @@ -143,10 +146,15 @@ func (s *server) handleSliceEditApplication(w http.ResponseWriter, r *http.Reque } } baseline[i] = sliceEditPolicyBaseline(candidate, old) + if command := commands[candidate.ID]; command.Disposition == "discard" && !old.IsDiscarded() { + baseline[i].Name = old.Name + baseline[i].Dictionary = old.Dictionary + } } } slices, issues := s.canonicalSliceDispositionState(r.Context(), &item, baseline, body.normalizedOperations()) + issues = append(issues, s.validateProjectSliceNames(r.Context(), stored, slices, applicationDictionaryMode(body))...) issues = append(issues, s.validateSliceEditFrameBounds(r, task, &item, slices, body.Segments)...) issues = append(issues, validateSliceSegments(slices, *task)...) issues = append(issues, validateCompleteSliceCoverage(slices, *task)...) @@ -194,10 +202,16 @@ func (s *server) handleSliceEditApplication(w http.ResponseWriter, r *http.Reque segmentsForApplication = retained } } + if issues := s.canonicalActionDictionaries(r.Context(), stored, segmentsForApplication, applicationDictionaryMode(body)); len(issues) > 0 { + renderValidationError(w, issues...) + return + } + copyClassificationRecoveryState(&item, stored) workspace := map[string]any{"sliceSegments": slices, "segments": segmentsForApplication, "statuses": body.normalizedStatuses(), "operations": body.normalizedOperations(), "userId": principal.User.ID, "savedAt": nowISO()} item.Subject.Payload["annotationWorkspace"] = workspace delete(item.Subject.Payload, "annotationDraft") for _, command := range body.Dispositions { + command.RestoreClassifications = body.RestoreClassifications if ids[command.SliceID] { command.combinedAnnotationDraftInput = workspaceSliceEditInput(workspace) var prepared bool @@ -243,8 +257,9 @@ func (s *server) handleSliceEditApplication(w http.ResponseWriter, r *http.Reque } } } - issues = s.validateProjectSliceNames(r.Context(), &item, slices, "draft") - issues = append(issues, s.canonicalActionDictionaries(r.Context(), stored, segments)...) + issues = s.validateProjectSliceNames(r.Context(), stored, slices, applicationDictionaryMode(body)) + issues = append(issues, s.canonicalActionDictionaries(r.Context(), stored, segments, applicationDictionaryMode(body))...) + copyClassificationRecoveryState(&item, stored) issues = append(issues, s.validateSliceEditFrameBounds(r, task, &item, slices, segments)...) issues = append(issues, validateDraftCombinedAnnotationEvents(segments, *task)...) issues = append(issues, validateDraftFanyuV5CombinedSemanticSegments(segments, slices, annotationTemplateWithTaskSnapshot(template, task))...) @@ -267,6 +282,7 @@ func (s *server) handleSliceEditApplication(w http.ResponseWriter, r *http.Reque item.Subject.Payload["annotationWorkspace"] = workspace // Recheck dictionary versions immediately before the single commit. for _, command := range body.Dispositions { + command.RestoreClassifications = body.RestoreClassifications if command.Disposition == "discard" { if _, issues := s.resolveDiscardReason(r.Context(), &item, command); len(issues) > 0 { renderValidationError(w, issues...) @@ -347,3 +363,10 @@ func (s *server) validateSliceEditFrameBounds(r *http.Request, task *domain.Anno } return issues } + +func applicationDictionaryMode(body sliceEditApplicationInput) string { + if body.RestoreClassifications { + return "restore" + } + return "draft" +} diff --git a/internal/api/slice_edit_restoration.go b/internal/api/slice_edit_restoration.go index 5e1f33c4..8a80c146 100644 --- a/internal/api/slice_edit_restoration.go +++ b/internal/api/slice_edit_restoration.go @@ -123,7 +123,7 @@ func (s *server) handleSliceEditRestoration(w http.ResponseWriter, r *http.Reque issues = append(issues, validateDraftCombinedAnnotationEvents(body.Segments, *task)...) issues = append(issues, validateDraftStatuses(body.normalizedStatuses(), slices)...) issues = append(issues, s.validateProjectSliceNames(r.Context(), item, slices, "restore")...) - issues = append(issues, s.canonicalActionDictionaries(r.Context(), item, body.Segments)...) + issues = append(issues, s.canonicalActionDictionaries(r.Context(), item, body.Segments, "restore")...) template, err := s.store.GetAnnotationTemplate(r.Context(), task.TemplateID) if err != nil { renderError(w, 500, err) diff --git a/internal/deliveryindex/batch.go b/internal/deliveryindex/batch.go index 6ce33db1..f7fe62e7 100644 --- a/internal/deliveryindex/batch.go +++ b/internal/deliveryindex/batch.go @@ -135,7 +135,7 @@ func checkPart(p part) (Summary, error) { if (p.Schema != PartSchema && p.Schema != ClassificationPartSchema) || !safeName(p.Capture.ID) || !partName.MatchString(p.PartID) || p.Revision < 1 || !p.Acceptance.Final.Confirmed || len(p.Acceptance.Reviews) != 0 { return s, fmt.Errorf("part must pin an accepted native v3 revision without embedded review history") } - if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil { + if _, e := time.Parse(time.RFC3339Nano, p.Acceptance.Final.AcceptedAt); e != nil && p.Schema != ClassificationPartSchema { return s, fmt.Errorf("missing acceptance timestamp") } if p.SourceMetadata.Path != "meta_"+p.Capture.ID+".json" || path.Base(p.Capture.Prefix) != p.Capture.ID || path.Clean(p.Capture.Prefix) != p.Capture.Prefix || strings.HasPrefix(p.Capture.Prefix, "/") || strings.Contains(p.Capture.Prefix, "../") { @@ -427,8 +427,10 @@ type classificationHistory struct { PartSHA string `json:"partSha256"` Acceptance struct { Final struct { - Confirmed bool `json:"confirmed"` - AcceptedAt string `json:"acceptedAt"` + Confirmed bool `json:"confirmed"` + AcceptedAt string `json:"acceptedAt"` + Sampled *bool `json:"sampled"` + SamplingDecision string `json:"samplingDecision"` } `json:"finalAcceptance"` } `json:"acceptance"` } @@ -449,7 +451,11 @@ func classificationAudit(src Source) (classificationHistory, error) { return audit, fmt.Errorf("internal audit does not pin the accepted classification") } if _, err := time.Parse(time.RFC3339Nano, audit.Acceptance.Final.AcceptedAt); err != nil { - return audit, fmt.Errorf("missing internal acceptance timestamp") + final := audit.Acceptance.Final + exempt := final.AcceptedAt == "" && final.Sampled != nil && !*final.Sampled && final.SamplingDecision == "not_selected" + if !exempt { + return audit, fmt.Errorf("missing internal acceptance timestamp or explicit sampling exemption") + } } return audit, nil } diff --git a/internal/deliveryindex/classification_test.go b/internal/deliveryindex/classification_test.go index 365a021f..3599574c 100644 --- a/internal/deliveryindex/classification_test.go +++ b/internal/deliveryindex/classification_test.go @@ -54,3 +54,23 @@ func TestClassificationBatchRejectsMixedVersions(t *testing.T) { t.Fatal("mixed versions accepted") } } + +func TestClassificationBatchAcceptsExplicitSamplingExemptionWithoutInventingHumanDate(t *testing.T) { + src := classificationFixture(false) + var audit map[string]any + json.Unmarshal(src.History, &audit) + final := audit["acceptance"].(map[string]any)["finalAcceptance"].(map[string]any) + delete(final, "acceptedAt") + final["sampled"] = false + final["samplingDecision"] = "not_selected" + src.History = encode(audit) + out := build(t, src) + if bytes.Contains(out.Index, []byte("acceptedAt")) { + t.Fatal("invented human acceptance date") + } + final["samplingDecision"] = "selected" + src.History = encode(audit) + if _, err := Build("project-A", "项目A", "delivery/项目A__project-A", []Source{src}); err == nil { + t.Fatal("unverified human acceptance accepted") + } +} diff --git a/internal/worker/artifact_storage.go b/internal/worker/artifact_storage.go index 51e888c9..6f638788 100644 --- a/internal/worker/artifact_storage.go +++ b/internal/worker/artifact_storage.go @@ -324,7 +324,19 @@ func (runtime *Runtime) normalizeOutputArtifacts(ctx context.Context, workflow * // the exact already-uploaded object refs instead of copying every byte a // second time. sharedDirectoryFileRefs := map[string]domain.ArtifactReference{} + order := make([]int, 0, len(artifacts)) for index, artifact := range artifacts { + if domain.IsInternalDeliveryAudit(artifact) { + order = append(order, index) + } + } + for index, artifact := range artifacts { + if !domain.IsInternalDeliveryAudit(artifact) { + order = append(order, index) + } + } + for _, index := range order { + artifact := artifacts[index] result[index] = artifact if artifact.Payload != nil && artifact.Payload.Mode == domain.ArtifactTransferModeRef { if err := domain.ValidateArtifactPayload(*artifact.Payload, artifact.Type); err != nil { diff --git a/internal/worker/delivery_audit.go b/internal/worker/delivery_audit.go index ae4b9ff5..2ada0247 100644 --- a/internal/worker/delivery_audit.go +++ b/internal/worker/delivery_audit.go @@ -93,6 +93,9 @@ func (runtime *Runtime) classificationDeliveryEnabled(ctx context.Context, workf // inline transport limit with a multi-round audit. Customer publication is a // different path and never sees this source artifact. func (runtime *Runtime) persistClassificationSummary(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, partitionID string, artifact domain.ExecutionArtifact, body []byte) (domain.ExecutionArtifact, error) { + if len(body) > 8<<20 { + return artifact, fmt.Errorf("acceptance audit summary exceeds 8 MiB") + } if len(body) <= domain.MaxInlinePayloadBytes && !structuredJSONRequiresReference(artifact, artifact.Data) { return artifact, nil } diff --git a/internal/worker/delivery_audit_test.go b/internal/worker/delivery_audit_test.go index e77576c8..470ef608 100644 --- a/internal/worker/delivery_audit_test.go +++ b/internal/worker/delivery_audit_test.go @@ -3,6 +3,8 @@ package worker import ( "context" "encoding/json" + "os" + "path/filepath" "strings" "testing" @@ -71,3 +73,26 @@ func TestClassificationAuditTraceRetainsFormalReworkAndScopedEvidence(t *testing t.Fatal("compacted audit no longer excluded from final export") } } + +func TestInternalAuditFailurePreventsCustomerDirectoryPublication(t *testing.T) { + ctx := context.Background() + ds := store.NewMemoryStore() + p := &deliveryPublisher{} + runtime := &Runtime{store: ds, artifactPublisher: p} + connection := domain.StorageConnectionRecord{ID: "audit-store", OrganizationID: "org", Provider: domain.StorageProviderBOS, Bucket: "b"} + ds.UpsertStorageConnection(ctx, connection) + workflow := &domain.WorkflowRecord{ID: "wf", OrganizationID: "org", DefinitionState: domain.WorkflowDefinitionStateReleased, ArtifactStorage: &domain.ArtifactStoragePolicy{ConnectionID: connection.ID, Bucket: connection.Bucket, PrefixTemplate: "artifacts/{runId}/{nodeId}"}} + run := &domain.RunRecord{ID: "run-audit-first", OrganizationID: "org", WorkflowID: "wf"} + ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture", OrganizationID: "org", RunID: run.ID, Delivery: &domain.CaptureDeliveryTarget{ConnectionID: connection.ID, Bucket: connection.Bucket, Prefix: "deliveries/项目__project/capture/part0001"}}) + directory := t.TempDir() + if err := os.WriteFile(filepath.Join(directory, "part.json"), []byte(`{"classification":"synthetic"}`), 0600); err != nil { + t.Fatal(err) + } + artifacts := []domain.ExecutionArtifact{{ID: "customer", RunID: run.ID, Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: directory}, {ID: "audit", RunID: run.ID, PortID: "deliveryAudit", Type: domain.PortTypeJSON, Path: filepath.Join(directory, "missing-internal-audit.json"), Data: map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2"}}} + if _, err := runtime.normalizeOutputArtifacts(ctx, workflow, run, "package", artifacts); err == nil { + t.Fatal("missing audit accepted") + } + if len(p.published) != 0 || len(p.puts) != 0 { + t.Fatal("customer directory became visible before audit persisted") + } +} diff --git a/operator-repos/builtin/package/src/operator.py b/operator-repos/builtin/package/src/operator.py index 4ee57b03..6b2cd8a8 100644 --- a/operator-repos/builtin/package/src/operator.py +++ b/operator-repos/builtin/package/src/operator.py @@ -846,7 +846,10 @@ def _description_status(actions: list[dict[str, Any]]) -> str: def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payload, internal_audit=None): - """Use the established immutable directory publisher for the two JSON files. + """Use the established immutable publisher for versioned customer JSON. + + V3 retains its historical sibling review file; v4 stores that audit in a + separate internal artifact before publishing the customer directory. Legacy single-file publishers deliberately accept only v1/v2. A directory artifact preserves their ownership/conditional-publication protocol without @@ -927,9 +930,9 @@ def _emit_part_index_directory(run_id, node_id, output, node_data, helpers, payl action.pop(key, None) if action.get("dictionary") is None: action.pop("dictionary", None) body = (json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf8") - if len(json.dumps(audit, ensure_ascii=False).encode("utf8")) > 8 * 1024 * 1024: - raise RuntimeError("internal delivery audit exceeds 8 MiB; refusing incomplete export") audit["partSha256"] = hashlib.sha256(body).hexdigest() + if len(json.dumps(audit, ensure_ascii=False, indent=2).encode("utf8")) > 8 * 1024 * 1024: + raise RuntimeError("internal delivery audit exceeds 8 MiB; refusing incomplete export") staging = pathlib.Path(tempfile.mkdtemp(prefix="part-index-v4-", dir=output)) (staging / "part.json").write_bytes(body) # A sibling artifact stays in the internal run, never in the customer tree. @@ -1729,7 +1732,11 @@ def _run_accepted_slice_directory( discarded_slices.append(row) else: kept_slices.append(row) - if disposition_contract: + params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} + classification_v4 = _bool(params, "DELIVERY_JSON_V4", False) + if classification_v4 and not coverage_required: + raise RuntimeError("DELIVERY_JSON_V4 requires verified full-video coverage") + if disposition_contract or classification_v4: pins = summary_payload.get("acceptedMaterializations", []) expected_sha = _sha256_file(revision_manifest_path) if summary_payload.get("decision") != "approved" or final_acceptance.get("confirmed") is not True or not any( @@ -1739,7 +1746,6 @@ def _run_accepted_slice_directory( raise RuntimeError("slice dispositions are not pinned to the accepted manifest revision") if int(coverage.get("sliceCount", -1)) != len(all_slices): raise RuntimeError("review coverage slice count mismatch") - params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} if disposition_contract and not kept_slices and not _bool(params, "DELIVERY_JSON_V4", False): if any(p.get("key") == "PROCESSING_PROFILE" and p.get("value") == "raw_human" for p in node_data.get("hyperparams", [])) and (summary_payload.get("processingProfile") != "raw_human" or final_acceptance.get("processingProfile") != "raw_human"): raise RuntimeError("raw-human package requires matching accepted task profile") @@ -1758,7 +1764,6 @@ def _run_accepted_slice_directory( payloads = _payload_tree(_payloads_with_full_pose_manifests(inputs, output, helpers)) headpose = _first_payload(payloads, "droboticflow.headpose/v1") handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1") - params = helpers["hyperparams"](node_data) if callable(helpers.get("hyperparams")) else {} if _bool(params, "PART_INDEX_DELIVERY", False): if source_index and not _bool(params, "SOURCE_INDEX_DELIVERY", False): raise RuntimeError("source-index JSON delivery requires explicit SOURCE_INDEX_DELIVERY approval; use directory delivery to preserve physical discard exclusion") diff --git a/src/components/AnnotationLabelsPanel.tsx b/src/components/AnnotationLabelsPanel.tsx index f0f1470b..e04764a1 100644 --- a/src/components/AnnotationLabelsPanel.tsx +++ b/src/components/AnnotationLabelsPanel.tsx @@ -579,9 +579,7 @@ export function AnnotationLabelsPanel({
)}
-

- 支持导出 CSV、批量导入与勾选批量删除;编辑或删除只影响本项目的可选词条,不影响其他项目及已保存的标注、审核记录。 -

+

名称增删只影响本项目;标签随字典词条共享,已保存的分类保持原版本。

); } diff --git a/src/components/CaptureAnnotationWorkbench.tsx b/src/components/CaptureAnnotationWorkbench.tsx index ec4c73bd..4fe747be 100644 --- a/src/components/CaptureAnnotationWorkbench.tsx +++ b/src/components/CaptureAnnotationWorkbench.tsx @@ -155,6 +155,9 @@ type Props = ReviewViewingProps & { slices: SliceSegment[]; segments: AnnotationSegment[]; labels?: SemanticLabelDefinition[]; + actionDictionaryLoading?: boolean; + actionDictionaryError?: string; + onRefreshActionDictionary?: () => void; discardReasons?: AnnotationLabelRecord[]; discardReasonsLoading?: boolean; discardReasonsAllowManual?: boolean; @@ -342,6 +345,9 @@ export function CaptureAnnotationWorkbench({ slices, segments: providedSegments, labels = [], + actionDictionaryLoading = false, + actionDictionaryError = "", + onRefreshActionDictionary, discardReasons = [], discardReasonsLoading = false, discardReasonsAllowManual = false, @@ -1988,6 +1994,19 @@ export function CaptureAnnotationWorkbench({ setEditingAnnotation((current) => (current ? { ...current, ...patch } : current)); }; + const actionDictionaryStatus = actionDictionaryLoading ? ( +

正在加载动作字典…

+ ) : actionDictionaryError ? ( +
+ {userErrorMessage(actionDictionaryError)} + +
+ ) : !labels.length ? ( +

暂无字典词条,可直接填写描述。

+ ) : undefined; + const chooseTemplate = (definition: SemanticLabelDefinition) => { updateEditingAnnotation({ labelId: "", @@ -5701,7 +5720,7 @@ export function CaptureAnnotationWorkbench({ text: item.description || item.name, searchText: `${item.name} ${item.description || ""}`, }))} - status={!labels.length ?

暂无字典词条,可直接填写描述。

: undefined} + status={actionDictionaryStatus} onSelect={(id) => { const item = labels.find((item) => item.id === id); if (item) chooseTemplate(item); @@ -5978,7 +5997,10 @@ export function CaptureAnnotationWorkbench({ disabled={isInvalid || saving} title="从字典选择(F / D,填写文本时不触发)" aria-keyshortcuts="F D" - onClick={() => setShowDictionary((value) => !value)} + onClick={() => { + if (!showDictionary) onRefreshActionDictionary?.(); + setShowDictionary((value) => !value); + }} className="inline-flex items-center gap-1.5 rounded border border-slate-300 px-2 py-1 text-xs text-slate-700 dark:border-slate-600 dark:text-gray-300 disabled:opacity-40 disabled:cursor-not-allowed" > @@ -6003,11 +6025,7 @@ export function CaptureAnnotationWorkbench({

↑ / ↓ 选择 · Enter 填入 · Esc 关闭

- {!labels.length && ( -

- 暂无字典词条,可直接填写描述。 -

- )} + {actionDictionaryStatus} {labels.map((item) => ( diff --git a/src/lib/slice-edit-drag-quick-path.js b/src/lib/slice-edit-drag-quick-path.js index 654cb713..899c6560 100644 --- a/src/lib/slice-edit-drag-quick-path.js +++ b/src/lib/slice-edit-drag-quick-path.js @@ -1,3 +1,4 @@ +import { previewSliceEditDispositions } from "./slice-edit-dispositions.js"; // 拖动快路径释放决策(票 05,spec §2 R2-Q3「拖动确认时机」):拖动边界手柄松手(或 // 悬停+拆分等结构快捷入口落点)时,把「基线结构 → 预览结构」交给 03 引擎——clean // 则直接给出可落库产物(快路径保持快:不进面板、不按 Enter;贴边无效动作的自动跟随 @@ -32,11 +33,16 @@ export function planSliceEditDragRelease({ baseSlices, pendingSlices, actions }) * 「动作完整落在恰好一个切片」(ADR 0007 / T3 钉死)。逐片无缝铺满刻意不进自检—— * 标注相草稿允许语义留空,铺满是提交守卫而非结构改动守卫。 * @param {{ actions: Array, slices: Array }} resolution resolveSliceEditConflicts 产物 + * @param {{baseSlices?: Array}} options 基线切片;检查归档后的活动动作,但原始状态仍须合法。 + * 服务端接收原始分割动作,在原子应用中维护归档;此处预览不代替服务端校验。 * @returns {Array} issues;空数组 = 通过 */ -export function checkSliceEditResolution(resolution) { +export function checkSliceEditResolution(resolution, { baseSlices } = {}) { + const preview = baseSlices + ? previewSliceEditDispositions(baseSlices, resolution.slices, resolution.actions, { resetRestoredActions: true }) + : resolution; return [ ...validateActionStatuses(resolution.actions), - ...assertActionsCoveredByExactlyOneSlice(resolution.actions, resolution.slices), + ...assertActionsCoveredByExactlyOneSlice(preview.actions, preview.slices), ]; } diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index a462f3b3..7a7ba530 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -152,6 +152,7 @@ import { autosaveSegmentsWithDrafts, sliceEditSessionFlushEdge, workbenchDraftsFingerprint, + sliceEditActionsWithDrafts, } from "../lib/annotation-autosave.js"; import { useAnnotationAutosave } from "../hooks/useAnnotationAutosave"; import { getActionErrorMessage, recoverableGatewayStatus } from "../lib/action-errors.js"; @@ -229,10 +230,14 @@ type SliceEditSnapshot = { // 落定时整段手势 + 处置合并为一条可撤销历史(与旧处置卡同款承诺)。 // 票 06 起 slice 相提交式挪动(←/→ 与 Shift+←/→)同走此管线(单一处置入口)。 type SliceEditQuickConflict = { + taskId: string; entries: SliceEditConflictEntry[]; autoFollows?: SliceEditAutoFollow[]; baseSlices: SliceSegment[]; baseSegments: AnnotationSegment[]; + actions: AnnotationSegment[]; + draftsFingerprint: string; + hasUnfinishedDrafts: boolean; pendingSlices: SliceSegment[]; operation: Omit; anchorTime: number; @@ -814,6 +819,8 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const sliceDragEditRef = useRef<{ before: SliceEditSnapshot; after?: SliceEditSnapshot } | null>(null); // 票 05:结构快捷入口的冲突清单待决集(03 引擎条目);非空即冻结其它边界改动。 const [sliceEditQuickConflict, setSliceEditQuickConflict] = useState(null); + const sliceEditQuickApplyingRef = useRef(null); + const [sliceEditAppliedRevision, setSliceEditAppliedRevision] = useState(0); const [historyReveal, setHistoryReveal] = useState<{ sliceId: string; time: number; @@ -2895,7 +2902,9 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: combinedSubmissionLocked || annotationMutationScopeRef.current.busy() || sliceEditSessionOpen || - Boolean(sliceEditQuickConflict), + Boolean(sliceEditQuickConflict) || + Boolean(sliceDragEditRef.current) || + Boolean(sliceEditQuickApplyingRef.current), buildSaveContext: () => { const item = detailRef.current?.humanItem; if (!item || !detailRef.current?.task) return null; @@ -3213,6 +3222,17 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: location: { sliceId, time: time ?? sliceSegments.find((slice) => slice.id === sliceId)?.start ?? 0 }, }); + const quickSliceEditActionSnapshot = (baseSegments: AnnotationSegment[]) => { + const prepared = sliceEditActionsWithDrafts(baseSegments, workbenchDrafts); + return { + taskId: detail?.task.id || "", + baseSegments, + actions: prepared.actions as AnnotationSegment[], + draftsFingerprint: workbenchDraftsFingerprint(workbenchDrafts), + hasUnfinishedDrafts: prepared.unresolved.length > 0, + }; + }; + const beginSliceBoundaryEdit = (sliceId: string) => { if (!annotationAuthoringRef.current) return; // 冲突清单开着时不接受新的边界拖动:一次只决一件事(待决快照不被打穿)。 @@ -3224,7 +3244,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: // 状态白名单 +「动作完整落在恰好一个切片」(与 04 面板 Enter 同一管线);有冲突弹 // 04 同款冲突清单(同一键盘模型),Enter 落定、Esc 整体回退本次拖动(R1-Q4 不支持 // 部分应用)。旧三选卡不再承接拖动释放(06 票删除旧卡与非拖动调用方)。 - const finishSliceBoundaryEdit = () => { + const finishSliceBoundaryEdit = async () => { const edit = sliceDragEditRef.current; sliceDragEditRef.current = null; if (!annotationAuthoringRef.current) return; @@ -3238,7 +3258,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const prompt: SliceEditQuickConflict = { entries: [], baseSlices: edit.before.slices, - baseSegments: edit.before.segments, + ...quickSliceEditActionSnapshot(edit.before.segments), pendingSlices: sliceSegments, operation: { type: "adjust", @@ -3252,7 +3272,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const plan = planSliceEditDragRelease({ baseSlices: prompt.baseSlices, pendingSlices: prompt.pendingSlices, - actions: prompt.baseSegments, + actions: prompt.actions, }); if (plan.kind === "conflict") { sliceDragEditRef.current = edit; // 清单落定时继续并入这条拖动编辑的历史(Esc 回退基线同源) @@ -3263,13 +3283,13 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: }); return; } - // 无冲突:立即生效(快路径保持快)。暂不可落库(保存锁等)或自检失败时按旧快 - // 路径收尾——几何停在释放位、手势照常入历史,绝不留半途结构;用户可重拖或撤销。 - sliceDragEditRef.current = edit; // applySliceChange 以拖动前快照为历史基线(一条可撤销历史) - const outcome = applySliceEditQuickResolution(prompt, plan.resolution); + // 无冲突也等待服务端原子保存;失败保留清单供重试或取消。 + sliceDragEditRef.current = edit; // 持久化历史以拖动前快照为基线 + const outcome = await applySliceEditQuickResolution(prompt, plan.resolution); + if (detailRef.current?.task.id !== prompt.taskId) return; sliceDragEditRef.current = null; if (outcome === "applied") return; - recordSliceDragEdit(edit); + setSliceEditQuickConflict(prompt); return; } recordSliceDragEdit(edit); @@ -3336,22 +3356,27 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: }; // —— 票 05(spec §2 R2-Q3):结构快捷入口的冲突清单落定/回退 —— - // 清单 Enter 与无冲突快路径共用的落库原语:解析产物过 Go 守卫前端镜像自检 → - // applySliceChange 原子落库(拖动来源以 sliceDragEditRef 的拖动前快照为历史基线, - // 整段手势 + 处置合并为一条可撤销历史)。返回 "applied" = 已收尾(含无变化早退, - // 几何已在预览位、手势照常入历史);"blocked" = 暂不可落库或期间外部改动——调用 - // 方保留清单供重试或 Esc,不写任何东西。 - const applySliceEditQuickResolution = ( + // 清单与无冲突快捷操作共享服务端原子应用:归档预览自检后发送原始动作, + // 等待权威工作区确认。失败保留清单与选择;拖动前快照作为一条持久化撤销历史。 + const applySliceEditQuickResolution = async ( prompt: SliceEditQuickConflict, resolution: { slices: SliceSegment[]; actions: AnnotationSegment[] }, ) => { - if (!annotationAuthoringRef.current) return "blocked"; - if (segments !== prompt.baseSegments) { - // 清单期间工作区被其它编辑改动:宁可拦下也不覆盖(旧处置卡同款承诺)。 + if ( + !annotationAuthoringRef.current || + sliceEditQuickApplyingRef.current || + detailRef.current?.task.id !== prompt.taskId + ) + return "blocked"; + if (segments !== prompt.baseSegments || prompt.draftsFingerprint !== workbenchDraftsFingerprint(workbenchDrafts)) { setError("处置期间工作区发生了其它修改,请按 Esc 关闭清单后重试。"); return "blocked"; } - const issues = checkSliceEditResolution(resolution); + if (prompt.hasUnfinishedDrafts) { + setError("仍有描述尚未确认结束点,请按 Esc 返回完成区间后再调整切片;草稿会保留。"); + return "blocked"; + } + const issues = checkSliceEditResolution(resolution, { baseSlices: prompt.baseSlices }); if (issues.length) { setError(`切片改动未通过提交自检:${issues[0].message};请按 Esc 取消本次改动。`); return "blocked"; @@ -3360,13 +3385,51 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: setError("当前状态暂不能应用切片改动,请稍后重试,或按 Esc 取消。"); return "blocked"; } - const changed = applySliceChange( - resolution.slices as SliceSegment[], - prompt.operation, - resolution.actions as AnnotationSegment[], - ); - if (prompt.dragEdit) recordSliceDragEdit(prompt.dragEdit); - return "applied"; // changed=false 为「无变化」早退:以当前状态收尾这条拖动历史 + const before = prompt.dragEdit?.before ?? { + ...sliceEditSnapshot(prompt.focusSliceId, prompt.anchorTime), + slices: prompt.baseSlices, + segments: prompt.baseSegments, + }; + let nextActions = resolution.actions; + // Preserve the existing cancel-cut rule: merged regions return to pending. + if (prompt.operation.type === "merge" || prompt.operation.type === "delete") { + const changed = resolution.slices.filter( + (slice) => + slice.sourceIds?.length && + prompt.baseSlices.some((old) => old.id === slice.id && (old.start !== slice.start || old.end !== slice.end)), + ); + nextActions = nextActions.filter( + (action) => !changed.some((slice) => action.start < slice.end && action.end > slice.start), + ); + } + if ( + JSON.stringify(resolution.slices) === JSON.stringify(prompt.baseSlices) && + JSON.stringify(nextActions) === JSON.stringify(prompt.baseSegments) + ) { + if (prompt.dragEdit) setSliceOperations(prompt.dragEdit.before.operations); + return "applied"; + } + const operations = [...before.operations, { ...prompt.operation, before: before.slices, after: resolution.slices }]; + const taskId = detail?.task.id; + sliceEditQuickApplyingRef.current = prompt; + try { + // Send raw split pieces, not the client archive preview. The server owns + // canonical archives, CAS, audit and recovery for every entry point. + const workspace = await persistSliceEditApplication( + resolution.slices, + nextActions, + prompt.focusSliceId, + true, + operations, + before, + ); + if (!workspace || detailRef.current?.task.id !== taskId) return "blocked"; + setWorkbenchDrafts([]); + setSliceEditAppliedRevision((revision) => revision + 1); + return "applied"; + } finally { + if (sliceEditQuickApplyingRef.current === prompt) sliceEditQuickApplyingRef.current = null; + } }; // 票 05:传给工作台的清单条目视图。useMemo 固定引用——工作台的播种 effect 依赖 @@ -3388,7 +3451,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: // 冲突清单 Enter(R2-Q2):未决条目按默认落定(03 引擎 resolver 语义),产物过自检 // 后原子落库;拖动来源整段手势 + 处置合并为一条可撤销历史。失败保留清单(重试/Esc)。 // 成功时返回焦点切片 id 供工作台选中(拆分来源为新片;拖动来源为被拖片)。 - const handleSliceEditQuickConflictApply = (decisions: Partial>) => { + const handleSliceEditQuickConflictApply = async (decisions: Partial>) => { if (!annotationAuthoringRef.current) return; const prompt = sliceEditQuickConflict; if (!prompt) return; @@ -3396,14 +3459,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const resolveInput = { baseSlices: prompt.baseSlices, pendingSlices: prompt.pendingSlices, - actions: prompt.baseSegments, + actions: prompt.actions, choices: decisions, }; const resolution = resolveSliceEditConflicts(resolveInput) as { slices: SliceSegment[]; actions: AnnotationSegment[]; }; - if (applySliceEditQuickResolution(prompt, resolution) !== "applied") return; + if ((await applySliceEditQuickResolution(prompt, resolution)) !== "applied") return; sliceDragEditRef.current = null; setSliceEditQuickConflict(null); setError(""); @@ -3413,6 +3476,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: // 冲突清单 Esc(R2-Q3/R1-Q4):拖动来源整段回退到拖动前快照(边界回原位、不留 // 历史);拆分等未落库来源直接丢弃(改动从未发生)。 const handleSliceEditQuickConflictCancel = () => { + if (sliceEditQuickApplyingRef.current || saving) return; const prompt = sliceEditQuickConflict; if (!prompt) return; setSliceEditQuickConflict(null); @@ -3439,7 +3503,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: } }; - const adjustSlice = (id: string, field: "start" | "end", value: number) => { + const adjustSlice = async (id: string, field: "start" | "end", value: number) => { if (!annotationAuthoringRef.current) return; // 冲突清单开着时冻结其它边界改动,防止待决快照被打穿。 if (sliceEditQuickConflict) return; @@ -3481,7 +3545,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const prompt: SliceEditQuickConflict = { entries: [], baseSlices: sliceSegments, - baseSegments: segments, + ...quickSliceEditActionSnapshot(segments), pendingSlices: next, operation: { type: "adjust", segmentIds: affected, at: value }, anchorTime: value, @@ -3491,7 +3555,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const plan = planSliceEditDragRelease({ baseSlices: prompt.baseSlices, pendingSlices: prompt.pendingSlices, - actions: prompt.baseSegments, + actions: prompt.actions, }); if (plan.kind === "conflict") { setSliceEditQuickConflict({ @@ -3501,10 +3565,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: }); return; } - applySliceEditQuickResolution(prompt, plan.resolution); + if ( + (await applySliceEditQuickResolution(prompt, plan.resolution)) !== "applied" && + detailRef.current?.task.id === prompt.taskId + ) + setSliceEditQuickConflict(prompt); }; - const splitSlice = (index: number, requestedAt = annotationPlayheadRef.current) => { + const splitSlice = async (index: number, requestedAt = annotationPlayheadRef.current) => { if (!annotationAuthoringRef.current) return; const current = sliceSegments[index]; if (!current) return; @@ -3533,7 +3601,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const prompt: SliceEditQuickConflict = { entries: [], baseSlices: sliceSegments, - baseSegments: segments, + ...quickSliceEditActionSnapshot(segments), pendingSlices: split.slices, operation: { type: "split", segmentIds: [current.id], at }, anchorTime: at, @@ -3543,7 +3611,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const plan = planSliceEditDragRelease({ baseSlices: prompt.baseSlices, pendingSlices: prompt.pendingSlices, - actions: prompt.baseSegments, + actions: prompt.actions, }); if (plan.kind === "conflict") { setSliceEditQuickConflict({ @@ -3553,7 +3621,9 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: }); return; // 未应用:不选中新片(Enter 落定后经回调返回焦点片) } - return applySliceEditQuickResolution(prompt, plan.resolution) === "applied" ? split.selectedSliceId : undefined; + if ((await applySliceEditQuickResolution(prompt, plan.resolution)) === "applied") return split.selectedSliceId; + if (detailRef.current?.task.id === prompt.taskId) setSliceEditQuickConflict(prompt); + return undefined; }; const mergeSlice = (index: number) => { @@ -3565,7 +3635,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: if (merged) applySliceChange(merged.slices, { type: "merge", segmentIds: [current.id, nextItem.id] }); }; - const deleteSlice = (index: number) => { + const deleteSlice = async (index: number) => { if (!annotationAuthoringRef.current) return; const target = sliceSegments[index]; const merged = target ? cancelSlicePartition(sliceSegments, index) : null; @@ -3576,7 +3646,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const prompt: SliceEditQuickConflict = { entries: [], baseSlices: sliceSegments, - baseSegments: segments, + ...quickSliceEditActionSnapshot(segments), pendingSlices: merged.slices, operation: { type: "delete", segmentIds: [target.id] }, anchorTime: target.start, @@ -3586,7 +3656,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const plan = planSliceEditDragRelease({ baseSlices: prompt.baseSlices, pendingSlices: prompt.pendingSlices, - actions: prompt.baseSegments, + actions: prompt.actions, }); if (plan.kind === "conflict") { setSliceEditQuickConflict({ @@ -3596,7 +3666,11 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: }); return; } - applySliceEditQuickResolution(prompt, plan.resolution); + if ( + (await applySliceEditQuickResolution(prompt, plan.resolution)) !== "applied" && + detailRef.current?.task.id === prompt.taskId + ) + setSliceEditQuickConflict(prompt); }; const updateSliceName = (id: string, name: string) => { @@ -3727,11 +3801,12 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: focusSliceId: string, recordHistory = true, restoredOperations = sliceOperations, + historyBefore?: SliceEditSnapshot, ) => { if (!annotationAuthoringRef.current) return null; if (!detail?.humanItem || !selectedClaimedByMe || combinedSubmissionLocked || !exactFrameEditing) return null; const taskId = detail.task.id; - const before = sliceEditSnapshot(focusSliceId); + const before = historyBefore ?? sliceEditSnapshot(focusSliceId); const token = await beginAnnotationMutation(taskId); if (!token) return null; const applicationId = globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`; @@ -3742,7 +3817,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: segments: nextSegments, operations: restoredOperations, statuses: deriveSliceStatuses(nextSlices, nextSegments, sliceStatuses) as typeof sliceStatuses, - dispositions: sliceEditDispositionCommands(sliceSegments, nextSlices, { + dispositions: sliceEditDispositionCommands(before.slices, nextSlices, { resetRestoredActions: recordHistory, restoredActions: nextSegments, }), @@ -7642,6 +7717,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: sliceEditQuickConflict={sliceEditQuickConflictView} onSliceEditQuickConflictApply={handleSliceEditQuickConflictApply} onSliceEditQuickConflictCancel={handleSliceEditQuickConflictCancel} + sliceEditAppliedRevision={sliceEditAppliedRevision} onSliceEditSessionChange={setSliceEditSessionOpen} onApplySliceEdit={async (nextSlices, nextSegments, focusSliceId) => Boolean(await persistSliceEditApplication(nextSlices, nextSegments, focusSliceId)) diff --git a/tests/browser/rework-boundary-replica.mjs b/tests/browser/rework-boundary-replica.mjs new file mode 100644 index 00000000..de8b596c --- /dev/null +++ b/tests/browser/rework-boundary-replica.mjs @@ -0,0 +1,211 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { chromium } from "playwright"; +import { createServer } from "vite"; +const api = process.env.REWORK_BOUNDARY_API; +assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); +const out = path.resolve("local-data/test-results/rework-boundary"); +fs.mkdirSync(out, { recursive: true }); +const server = await createServer({ + server: { host: "127.0.0.1", port: 5202, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, +}); +let browser, + page, + fault = "", + releaseSave; +const report = { cases: [], pageErrors: [], writes: [] }; +const taskURL = + "http://127.0.0.1:5202/plugins/annotation-qa/workflows/all/work?projectId=draft-project&taskId=draft-task"; +const read = async () => (await (await fetch(api + "/__boundary/state")).json()).subject.payload.annotationWorkspace; +const waitFor = async (check) => { + for (let n = 0; n < 80; n++) { + if (await check()) return; + await new Promise((r) => setTimeout(r, 50)); + } + assert.fail("server state did not settle"); +}; +try { + await server.listen(); + browser = await chromium.launch({ + executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + headless: true, + args: ["--no-proxy-server"], + }); + const ctx = await browser.newContext({ viewport: { width: 1600, height: 1100 }, serviceWorkers: "block" }); + page = await ctx.newPage(); + page.setDefaultTimeout(12000); + page.on("pageerror", (e) => report.pageErrors.push(e.message)); + page.on("request", (r) => { + if (!["GET", "HEAD"].includes(r.method())) + report.writes.push({ method: r.method(), path: new URL(r.url()).pathname }); + }); + await page.route("**/*", async (route) => { + const req = route.request(), + u = new URL(req.url()); + if (u.hostname !== "127.0.0.1") return route.abort(); + if (u.pathname.includes("/worker-api/")) return route.fulfill({ json: { status: "ok" } }); + if (req.method() === "POST" && u.pathname.endsWith("/slice-edit-applications")) { + const mode = fault; + fault = ""; + if (mode === "reject") return route.fulfill({ status: 503, json: { error: "合成保存失败,请重试" } }); + if (mode === "delay") + await new Promise((resolve) => { + releaseSave = resolve; + }); + if (mode === "lost") { + await route.fetch(); + return route.abort("failed"); + } + } + return route.continue(); + }); + const load = async () => { + await page.goto(taskURL, { waitUntil: "domcontentloaded" }); + await page.getByRole("button", { name: "继续工作", exact: true }).click(); + await page + .getByTestId("capture-annotation-workbench") + .getByRole("button", { name: "拖动切片边界", exact: true }) + .first() + .waitFor(); + }; + const reset = async (side = "left", status = "described", round = 1) => { + assert.equal((await fetch(`${api}/__boundary/reset?side=${side}&status=${status}&round=${round}`)).status, 200); + await load(); + }; + const list = () => page.getByTestId("slice-edit-conflict-list"); + const drag = async (side = "left") => { + const handle = page + .getByTestId("capture-annotation-workbench") + .getByRole("button", { name: "拖动切片边界", exact: true }) + .first(); + const box = await handle.boundingBox(); + await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + await page.mouse.down(); + await page.mouse.move(box.x + box.width / 2 + (side === "left" ? 100 : -100), box.y + box.height / 2, { steps: 8 }); + await page.mouse.up(); + await list().waitFor(); + }; + const confirm = async (keyboard = false) => { + if (keyboard) { + await list().click({ position: { x: 15, y: 15 } }); + await page.keyboard.press("Enter"); + } else await list().getByTestId("slice-edit-conflict-apply").click(); + await list().waitFor({ state: "hidden" }); + }; + const assertSaved = async (side, status) => { + const w = await read(), + k = w.sliceSegments.find((s) => s.id === "k"), + d = w.sliceSegments.find((s) => s.id === "d"); + assert.equal(w.segments.length, 1); + assert.equal(w.segments[0].status, status); + assert.equal(w.segments[0].description || "", status === "described" ? "合成有效动作" : ""); + assert.ok(!w.segments[0].isInvalid); + assert.equal(d.discardedAnnotations.length, 1); + assert.equal(d.discardedAnnotations[0].status, status); + if (side === "left") { + assert.equal(k.start, 6); + assert.equal(d.end, 6); + assert.equal(w.segments[0].start, 6); + } else { + assert.equal(k.end, 4); + assert.equal(d.start, 4); + assert.equal(w.segments[0].end, 4); + } + }; + for (const side of ["left", "right"]) + for (const status of ["pending", "described"]) { + await reset(side, status, status === "pending" ? 0 : 1); + await drag(side); + await confirm(status === "described"); + await assertSaved(side, status); + await load(); + await assertSaved(side, status); + report.cases.push( + `${side} ${status}: ${status === "described" ? "Enter/rework" : "button/first-pass"}, archive and reload`, + ); + } + await reset("left", "invalid", 0); + const cleanHandle = page + .getByTestId("capture-annotation-workbench") + .getByRole("button", { name: "拖动切片边界", exact: true }) + .first(); + const cleanBox = await cleanHandle.boundingBox(); + await page.mouse.move(cleanBox.x + cleanBox.width / 2, cleanBox.y + cleanBox.height / 2); + await page.mouse.down(); + await page.mouse.move(cleanBox.x + cleanBox.width / 2 - 100, cleanBox.y + cleanBox.height / 2, { steps: 8 }); + await page.mouse.up(); + await waitFor(async () => (await read()).sliceSegments.find((s) => s.id === "d").end === 4); + assert.equal(await list().count(), 0); + report.cases.push("conflict-free drag persists without an extra confirmation"); + await reset(); + await drag(); + fault = "reject"; + await list().getByTestId("slice-edit-conflict-apply").click(); + await list().getByRole("alert").filter({ hasText: "合成保存失败" }).waitFor(); + assert.equal((await read()).sliceSegments.find((s) => s.id === "d").end, 5); + await confirm(true); + await assertSaved("left", "described"); + report.cases.push("failed write visible, choices retained, Enter retry"); + await reset(); + await drag(); + fault = "delay"; + const count = () => report.writes.filter((r) => r.path.endsWith("/slice-edit-applications")).length; + const before = count(); + await list().getByTestId("slice-edit-conflict-apply").click(); + await page.getByRole("button", { name: "正在保存…", exact: true }).waitFor(); + assert.equal(await list().getByRole("button", { name: "Esc 取消", exact: true }).isDisabled(), true); + await page.keyboard.press("Enter"); + await page.keyboard.press("Escape"); + assert.equal(count(), before + 1); + assert.equal(await list().isVisible(), true); + await waitFor(() => Boolean(releaseSave)); + releaseSave(); + releaseSave = null; + await list().waitFor({ state: "hidden" }); + await assertSaved("left", "described"); + report.cases.push("pending save blocks repeat Enter and cancellation"); + await reset(); + await drag(); + fault = "lost"; + await confirm(true); + await assertSaved("left", "described"); + report.cases.push("lost response reconciled against authoritative API"); + await page.getByRole("button", { name: "撤销", exact: true }).click(); + await waitFor(async () => (await read()).sliceSegments.find((s) => s.id === "d").end === 5); + let restored = await read(); + assert.equal(restored.segments.length, 1); + assert.equal(restored.segments[0].start, 5); + assert.equal(restored.segments[0].description, "合成有效动作"); + // Existing history intentionally throttles accepted commands to one per second. + await page.waitForTimeout(1100); + await page.getByRole("button", { name: "重做", exact: true }).click(); + await waitFor(async () => (await read()).sliceSegments.find((s) => s.id === "d").end === 6); + await assertSaved("left", "described"); + report.cases.push("persisted undo/redo restores original action and archive"); + await reset(); + await drag(); + await page.keyboard.press("Escape"); + await list().waitFor({ state: "hidden" }); + assert.equal((await read()).sliceSegments.find((s) => s.id === "d").end, 5); + report.cases.push("Escape cancels preview without server write"); + assert.deepEqual(report.pageErrors, []); + console.log(JSON.stringify(report)); +} catch (e) { + if (page) { + await page.screenshot({ path: path.join(out, "failure.png"), fullPage: true }).catch(() => {}); + fs.writeFileSync( + path.join(out, "failure-body.txt"), + await page + .locator("body") + .innerText() + .catch(() => ""), + ); + } + throw e; +} finally { + releaseSave?.(); + fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2)); + await browser?.close(); + await server.close(); +} diff --git a/tests/frontend/capture-annotation-workbench.test.mjs b/tests/frontend/capture-annotation-workbench.test.mjs index ed1c8925..8b14b915 100644 --- a/tests/frontend/capture-annotation-workbench.test.mjs +++ b/tests/frontend/capture-annotation-workbench.test.mjs @@ -173,7 +173,7 @@ test("slice reconfig panel: E opens the preview panel with Enter/Esc, locks bypa workbench.indexOf("const applySliceEditPanel ="), ); assert.match(commit, /resolveSliceEditConflicts\(/); - assert.match(commit, /assertActionsCoveredByExactlyOneSlice/); + assert.match(commit, /checkSliceEditResolution\(resolution, \{ baseSlices: panel\.snapshot\.slices \}\)/); assert.match(commit, /onApplySliceEdit\?\.\(/, "应用经页面侧原子落库回调"); // 面板与清单组件、常显提示、红块高亮、播放族放行(seam ③ 表 + 分发)。 assert.match(sliceEditPanel, /data-testid="slice-edit-panel-hint"/); @@ -236,7 +236,7 @@ test("drag quick path: releases route through the 03 engine into the 04 conflict ); assert.ok(quickHelpers.indexOf("const applySliceEditQuickResolution =") >= 0, "落库原语存在"); assert.match(quickHelpers, /checkSliceEditResolution\(/, "落库前 Go 守卫前端镜像自检(T3)"); - assert.match(quickHelpers, /applySliceChange\(/, "原子落库(一条可撤销历史)"); + assert.match(quickHelpers, /await persistSliceEditApplication\(/, "服务端原子归档及一条可撤销历史"); assert.match(quickHelpers, /segments !== prompt\.baseSegments/, "清单期间外部改动宁可拦下"); assert.match(quickHelpers, /resolveSliceEditConflicts\(/, "清单 Enter 未决按默认落定"); const cancel = quickHelpers.slice( diff --git a/tests/frontend/rework-boundary.test.mjs b/tests/frontend/rework-boundary.test.mjs new file mode 100644 index 00000000..1f3b6cff --- /dev/null +++ b/tests/frontend/rework-boundary.test.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { resolveSliceEditConflicts } from "../../src/lib/slice-edit-conflict.js"; +import { checkSliceEditResolution } from "../../src/lib/slice-edit-drag-quick-path.js"; +import { previewSliceEditDispositions } from "../../src/lib/slice-edit-dispositions.js"; + +for (const side of ["left", "right"]) + for (const status of ["pending", "described"]) { + test(`default truncate archives ${side} discard expansion and preserves ${status} kept classification`, () => { + const d = { id: "d", start: 0, end: 5, deliveryDisposition: "discard" }; + const k = { id: "k", start: 5, end: 10 }; + if (side === "right") { + d.start = 5; + d.end = 10; + k.start = 0; + k.end = 5; + } + const baseSlices = side === "left" ? [d, k] : [k, d]; + const boundary = side === "left" ? 6 : 4; + const pendingSlices = baseSlices.map((s, i) => ({ + ...s, + ...(i === 0 ? { end: boundary } : { start: boundary }), + })); + const dictionary = { + id: "frozen", + kind: "action", + version: 3, + name: "Original classification", + tags: ["frozen-tag"], + }; + const action = { + id: "a", + start: k.start, + end: k.end, + status, + description: status === "described" ? "Original description" : "", + labelId: "frozen-label", + label: "Original classification", + phrases: ["frozen-phrase"], + source: "manual", + dictionary, + }; + const result = resolveSliceEditConflicts({ baseSlices, pendingSlices, actions: [action], choices: {} }); + assert.deepEqual(checkSliceEditResolution(result, { baseSlices }), []); + const preview = previewSliceEditDispositions(baseSlices, result.slices, result.actions); + assert.equal(preview.actions.length, 1); + assert.equal(preview.actions[0].status, status); + assert.equal(preview.actions[0].description, action.description); + assert.equal(preview.actions[0].labelId, action.labelId); + assert.equal(preview.actions[0].label, action.label); + assert.deepEqual(preview.actions[0].phrases, action.phrases); + assert.equal(preview.actions[0].source, action.source); + assert.deepEqual(preview.actions[0].dictionary, dictionary); + assert.equal(preview.actions[0].isInvalid, undefined); + const archived = preview.slices.find((s) => s.id === "d").discardedAnnotations; + assert.equal(archived.length, 1); + assert.deepEqual(archived[0].dictionary, dictionary); + }); + } +test("invalid source states cannot be hidden in the discard preview", () => { + const baseSlices = [ + { id: "d", start: 0, end: 5, deliveryDisposition: "discard" }, + { id: "k", start: 5, end: 10 }, + ]; + const slices = [ + { ...baseSlices[0], end: 6 }, + { ...baseSlices[1], start: 6 }, + ]; + const issues = checkSliceEditResolution( + { slices, actions: [{ id: "a", start: 5, end: 6, status: "unsupported" }] }, + { baseSlices }, + ); + assert.ok(issues.some((i) => i.code === "action_annotation_incomplete")); +}); From 867ca698356ee4e3a56c8bed13b3e921479566cc Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 21 Oct 2026 14:43:36 +0800 Subject: [PATCH 10/54] =?UTF-8?q?=F0=9F=90=9B=20Keep=20initial=20slicing?= =?UTF-8?q?=20conflicts=20visible=20and=20repair=20live=20rework=20feedbac?= =?UTF-8?q?k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice-phase quick edits with pending actions were canceled before history recording because conflict rendering and teardown were annotation-only. Preserve the actions and use the existing atomic application and restoration flow in both editing phases. Compare complete live editor drafts without derived completion status, and keep discarded-slice evidence and opinions visible. --- .scratch/cheng-feedback-20261021/spec.md | 12 ++ ...ework-emphasis-replaces-feedback-editor.md | 8 + ...-inplace-slice-editing-replaces-reslice.md | 6 + docs/verification/cheng-feedback-20261021.md | 19 ++ internal/api/slice_phase_browser_test.go | 154 ++++++++++++++ src/components/CaptureAnnotationWorkbench.tsx | 40 ++-- src/components/SliceEditConflictList.tsx | 6 + src/lib/annotation-autosave.js | 24 ++- src/lib/rework-emphasis.js | 10 +- src/pages/AnnotationOperations.tsx | 9 +- tests/browser/slice-phase-replica.mjs | 189 ++++++++++++++++++ .../capture-annotation-workbench.test.mjs | 4 +- tests/frontend/e-open-jk-unlock.test.mjs | 2 +- tests/frontend/rework-live-feedback.test.mjs | 55 +++++ tests/frontend/rework-review-layout.test.mjs | 12 +- 15 files changed, 518 insertions(+), 32 deletions(-) create mode 100644 .scratch/cheng-feedback-20261021/spec.md create mode 100644 docs/verification/cheng-feedback-20261021.md create mode 100644 internal/api/slice_phase_browser_test.go create mode 100644 tests/browser/slice-phase-replica.mjs create mode 100644 tests/frontend/rework-live-feedback.test.mjs diff --git a/.scratch/cheng-feedback-20261021/spec.md b/.scratch/cheng-feedback-20261021/spec.md new file mode 100644 index 00000000..119c7113 --- /dev/null +++ b/.scratch/cheng-feedback-20261021/spec.md @@ -0,0 +1,12 @@ +# Repair initial slicing and rework feedback +Status: resolved + +Base: local rework boundary repair 4d12d8cb, based on deployed fab5be57. + +- Initial slice quick conflicts must be visible and confirmable/cancellable by the same keys and buttons as annotation conflicts. Preserve pending, described and invalid records; no customer migration. +- Successful C split, drag and delete must survive save/reload and support same-session undo/redo, respecting the existing one-second acceptance window and stage boundaries. +- Live complete editor drafts update feedback immediately, preserving classification fields. Derived slice status alone must not clear opinions. +- Discarded slices retain evidence and a right-side opinion rail without an action editor. Explicit whole-action invalidation shows its impact. +- Review remains read-only; E remains annotation-only. No production mutation or deployment in this task. + +Evidence: [verification](../../docs/verification/cheng-feedback-20261021.md). diff --git a/docs/adr/0006-rework-emphasis-replaces-feedback-editor.md b/docs/adr/0006-rework-emphasis-replaces-feedback-editor.md index a5e5139a..575e2371 100644 --- a/docs/adr/0006-rework-emphasis-replaces-feedback-editor.md +++ b/docs/adr/0006-rework-emphasis-replaces-feedback-editor.md @@ -22,6 +22,14 @@ - 复审新增“本轮改动”清单,覆盖全部切片与动作,包括原先没有退回意见的位置。它比较本轮冻结基线与当前实际送检修订,展示净改动及前后内容;不使用客户端操作日志、不把自动保存草稿混入正式结果,不修改冻结基线和正式审查记录。标注员侧显示当前工作区的同一比较结果。 - 缺少旧基线快照或匹配的送检修订时显示“无法确认全部改动”,不以空数组推断全部删除或没有改动。无需数据库迁移;旧结构计数摘要和旧格式退回兼容路径保留。 +## 2026-10-21:实时草稿与派生状态比较修正 + +标注员侧强调层合入区间完整的编辑器草稿,保留分类与左右手等业务字段;未完成终点选择不参与比较,自动保存载荷的字段清洗不用于展示比较。撤回到原文恢复未修改状态。 + +切片 `annotationStatus` 是派生完成状态,单独变化不代表已按意见修改;旧记录省略用途与显式 keep 等价。名称、分类、边界、用途、舍弃原因及片内动作仍参与比较。灰态继续只表示已修改待复审,不替代正式判定。 + +选择舍弃切片时仍显示只读舍弃证据与返工意见栏;不提供直接修改归档动作的编辑器。上述均为展示修正,不迁移冻结基线、正式审查记录或客户工作区。 + ## 背景 B02 上线的返工模式给标注员两个新面板:意见列表上方的"处理反馈"文字编辑器(每条意见写回应,8 秒空闲自动保存)与"上下对比"面板。客户实测后提出三点不满: diff --git a/docs/adr/0007-inplace-slice-editing-replaces-reslice.md b/docs/adr/0007-inplace-slice-editing-replaces-reslice.md index 42032e2f..f0283141 100644 --- a/docs/adr/0007-inplace-slice-editing-replaces-reslice.md +++ b/docs/adr/0007-inplace-slice-editing-replaces-reslice.md @@ -25,6 +25,12 @@ - Enter 和按钮均等待权威保存结果;保存中禁用重复应用、处置选择及取消,错误在清单内显示并保留选择。响应丢失时通过原应用编号回读确认。成功后清理已合入保存的编辑器草稿,避免用旧几何覆盖新工作区。 - 不批量修改已保存、送检或交付数据;现有权限、阶段锁、历史数据恢复接口保持有效。 +## 2026-10-21:初始切片快捷冲突兼容 + +初始切片阶段保存、恢复的工作区可能已有连续待标注动作。切分、拖边和删除遇到这些动作时,与逐片标注阶段一样显示冲突清单,Enter/按钮确认或 Esc 取消;不再因阶段不为 annotate 静默取消。E 重设面板仍限定逐片标注,质检、验收、只读及后台处理阶段不开放编辑。 + +不按描述为空删除动作,也不批量改写历史草稿。既有自动补齐、空白动作、分类、舍弃证据与版本校验保持原义。成功操作按原子应用记录撤销,失败保留选择及明确错误;原有会话、阶段边界和一秒防重复规则保留。 + ## 背景 返修中要改切片结构只能"重新切片":清空全部动作标注、整单退回切片阶段,代价高且不可审计(旧文案"请先返回重新切片"即两把阶段锁的报错)。同时旧 D2 规则在拖动边界穿过动作时自动拆两段标"待核对"——该状态不在服务端白名单(pending/described/invalid)内,不处置则保存被堵死。本决策来自 24 项已拍板裁决(Q1:a–Q8:a、R2-Q1–Q8、R3-1–R3-4)。 diff --git a/docs/verification/cheng-feedback-20261021.md b/docs/verification/cheng-feedback-20261021.md new file mode 100644 index 00000000..9cd58a7c --- /dev/null +++ b/docs/verification/cheng-feedback-20261021.md @@ -0,0 +1,19 @@ +# Platform feedback repairs — 2026-10-21 + +Local branch `codex/rework-boundary-20261021`, based on deployed `fab5be57` plus local boundary repair `4d12d8cb`. No push, PR, cluster deployment or customer data writes. + +## Cause and compatibility + +Initial slicing can recover saved pending actions. The quick edit planner correctly detects action conflicts, but Workbench previously rendered and accepted those conflicts only in annotation phase and silently canceled them in slice phase. Thus attempted edits never entered history. Preserve all existing action records and make the stage contract consistent; keep the server's archive, CAS and restoration paths. No schema, workflow, formal decision or delivery format changes. + +Feedback previously omitted complete editor drafts and compared derived slice annotationStatus. It now includes those drafts while preserving display classification/handedness, excludes incomplete end selections, ignores derived slice status and normalizes implicit keep. Discarded slices retain evidence and the opinion rail in the right sidebar. Selecting whole-action invalidation explicitly warns that both sides are affected. + +## Regression evidence + +- Red before repair: `TestSlicePhaseBrowserReplica` failed `pending actions must show a visible conflict` (0 vs 1); the derived-status unit regression failed uncleared count (1 vs 2). +- `REWORK_BOUNDARY_BROWSER_TEST=1 go test ./internal/api -run '^TestSlicePhaseBrowserReplica$' -count=1 -v`: real AnnotationOperations page, real HTTP handlers, isolated MemoryStore and synthetic H.264 media. Covers empty control, saved pending actions and a newer owner-matched autosave draft over an empty confirmed workspace, C/Enter/save/undo/redo/reload, pending drag/delete undo/redo, actual draft editing and reversal, false-gray prevention, discarded evidence/opinions and sidebar geometry. +- `REWORK_BOUNDARY_BROWSER_TEST=1 go test ./internal/api -run '^TestReworkBoundaryBrowserReplica$' -count=1 -v`: existing discard-boundary, Enter/button, failure/retry, lost-response reconciliation, busy/cancel and persisted undo/redo scenarios remain passing. +- 66 targeted frontend tests and targeted Go slice application, restoration, rework and draft tests pass; TypeScript and web build pass. Production bundle boot has no runtime errors. +- Full frontend gate: 13 existing failures, all in the unchanged known baseline; no new failures. Four source-shape assertions were updated for the new phase contract while retaining review permission assertions. Baseline entries were not edited. + +Browser is local Edge/Chromium, not a claim of testing the customer's exact Chrome installation. No production browser/client acceptance or historical result repair is claimed. Old mistaken invalidation results require individual evidence review; they are not guessed or overwritten. Source diagnosis evidence remains under untracked `local-data/diagnostics/chengxiaoming-feedback-20261021/` in the working root. diff --git a/internal/api/slice_phase_browser_test.go b/internal/api/slice_phase_browser_test.go new file mode 100644 index 00000000..a60b92f0 --- /dev/null +++ b/internal/api/slice_phase_browser_test.go @@ -0,0 +1,154 @@ +package api + +import ( + "context" + "droboticflow/internal/config" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +// Actual page -> actual HTTP handler -> isolated memory store. Neither a +// production endpoint nor the local snapshot database is reachable here. +func TestSlicePhaseBrowserReplica(t *testing.T) { + if os.Getenv("REWORK_BOUNDARY_BROWSER_TEST") != "1" { + t.Skip("local browser opt-in") + } + f := continuousFixture(t) + reason := dispositionFixturePayload(t, f) + if err := f.store.UpsertOrganizationPlugin(f.ctx, domain.OrganizationPluginRecord{OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", Enabled: true, Config: defaultAnnotationPluginConfig()}); err != nil { + t.Fatal(err) + } + reset := func(status, shape string) error { + before := []domain.SliceSegment{{ID: "first", Start: 0, End: 10}} + item := f.item + item.ReviewRound = 0 + actions := []domain.AnnotationSegment{{ID: "pending-first", Start: 0, End: 10, Status: "pending"}} + if status == "empty" { + actions = []domain.AnnotationSegment{} + } + item.Subject.Payload = map[string]any{"sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}, "annotationWorkspace": map[string]any{"userId": f.annotator.ID, "savedAt": nowISO(), "sliceSegments": before, "segments": actions, "statuses": map[string]string{"first": "pending"}}} + if status == "draft" { + item.Subject.Payload["annotationDraft"] = map[string]any{"userId": f.annotator.ID, "savedAt": time.Now().Add(time.Second).UTC().Format(time.RFC3339Nano), "savedVersion": item.Version, "sliceSegments": before, "segments": actions, "statuses": map[string]string{"first": "pending"}} + item.Subject.Payload["annotationWorkspace"].(map[string]any)["segments"] = []domain.AnnotationSegment{} + } + + if shape == "two" { + before = []domain.SliceSegment{{ID: "first", Name: "测试切片", Start: 0, End: 0.2}, {ID: "second", Start: 0.2, End: 10}} + actions = []domain.AnnotationSegment{{ID: "a", Start: 0, End: 0.2, Status: "pending"}, {ID: "b", Start: 0.2, End: 10, Status: "pending"}} + item.Subject.Payload["annotationWorkspace"].(map[string]any)["sliceSegments"] = before + item.Subject.Payload["annotationWorkspace"].(map[string]any)["segments"] = actions + } + if shape == "rework" { + before = []domain.SliceSegment{{ID: "first", Name: "测试切片", Start: 0, End: 5, AnnotationStatus: "pending"}, {ID: "discard", Start: 5, End: 10, DeliveryDisposition: "discard", DiscardReason: "相机被遮挡", DiscardReasonLabelID: reason.ReasonLabelID, DiscardReasonVersion: reason.ReasonVersion}} + actions = []domain.AnnotationSegment{{ID: "a", Start: 0, End: 5, Status: "described", Description: "原描述"}} + item.ReviewRound = 1 + item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "completed", "phase": "annotate"} + item.Subject.Payload["annotationWorkspace"].(map[string]any)["sliceSegments"] = before + item.Subject.Payload["annotationWorkspace"].(map[string]any)["segments"] = actions + frozen := append([]domain.SliceSegment(nil), before...) + frozen[0].AnnotationStatus = "labeled" + if err := f.store.UpsertReworkBaseline(f.ctx, domain.ReworkBaselineRecord{ID: store.ReworkBaselineID(item.ID, 1), OrganizationID: item.OrganizationID, ProjectID: item.ProjectID, ItemID: item.ID, Round: 1, StageID: domain.FanyuV58StageAnnotationCombined, Slices: frozen, Actions: actions, Problems: []domain.ReviewProblem{{ID: "p-slice", Scope: "slice", SliceID: "first", Text: "核对切片"}, {ID: "p-action", Scope: "action", SliceID: "first", ActionID: "a", Text: "核对描述"}}}); err != nil { + return err + } + } + + return f.store.UpsertHumanItem(f.ctx, item) + } + root := t.TempDir() + f.task.SourceVideo = "/api/runs/" + f.task.RunID + "/artifacts/left-video/content" + f.task.RightSourceVideo = "/api/runs/" + f.task.RunID + "/artifacts/right-video/content" + f.task.NodeID = "human" + f.task.ArtifactID = "left-video" + f.task.RightArtifactID = "right-video" + f.task.SourceIsFullVideo = true + f.task.SourceDurationSeconds = 10 + f.task.CoverageEndSecond = 10 + pts := "timestamp_us\n" + for i := 0; i < 100; i++ { + pts += fmt.Sprintf("%d\n", 1000000+i*100000) + } + bindings := []domain.NodeInputBinding{} + for _, side := range []string{"left", "right"} { + video := filepath.Join(root, side+".mp4") + if out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "color=c=navy:s=160x90:r=10:d=10", "-c:v", "libx264", "-pix_fmt", "yuv420p", video).CombinedOutput(); err != nil { + t.Fatalf("video: %v: %s", err, out) + } + timestamps := filepath.Join(root, side+"_pts.csv") + if err := os.WriteFile(timestamps, []byte(pts), 0600); err != nil { + t.Fatal(err) + } + port := "fullLeftVideos" + if side == "right" { + port = "fullRightVideos" + } + bindings = append(bindings, domain.NodeInputBinding{TargetPortID: port, Artifacts: []domain.ExecutionArtifact{{ID: side + "-video", Name: side + ".mp4", Path: video, Data: map[string]any{"collectionId": "capture"}}}}, domain.NodeInputBinding{TargetPortID: side + "Timestamps", Artifacts: []domain.ExecutionArtifact{{ID: side + "-pts", Name: side + "_pts.csv", Path: timestamps, Data: map[string]any{"collectionId": "capture"}}}}) + } + imu := "timestamp_us,x,y,z,type\n" + for i := 0; i < 100; i++ { + stamp := 1000000 + i*100000 + imu += fmt.Sprintf("%d,1,2,3,accel\n%d,4,5,6,gyro\n", stamp, stamp) + } + imuPath := filepath.Join(root, "imu.csv") + if err := os.WriteFile(imuPath, []byte(imu), 0600); err != nil { + t.Fatal(err) + } + bindings = append(bindings, domain.NodeInputBinding{TargetPortID: "imu", Artifacts: []domain.ExecutionArtifact{{ID: "imu", Name: "imu.csv", Path: imuPath, Data: map[string]any{"collectionId": "capture"}}}}) + if err := f.store.UpsertRun(f.ctx, domain.RunRecord{ID: f.task.RunID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, WorkflowID: f.task.WorkflowID, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: f.task.NodeID, InputBindings: bindings}}}); err != nil { + t.Fatal(err) + } + if err := f.store.UpsertAnnotationTask(f.ctx, f.task); err != nil { + t.Fatal(err) + } + if err := reset("pending", ""); err != nil { + t.Fatal(err) + } + handler := NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: root}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/auth/me" { + renderJSON(w, 200, map[string]any{"user": sanitizeUser(&f.annotator), "organization": domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}}) + return + } + if r.URL.Path == "/__boundary/reset" { + if err := reset(r.URL.Query().Get("status"), r.URL.Query().Get("shape")); err != nil { + renderError(w, 500, err) + return + } + renderJSON(w, 200, map[string]any{"ok": true}) + return + } + if r.URL.Path == "/__boundary/state" { + item, _ := f.store.GetHumanItem(r.Context(), f.item.ID) + renderJSON(w, 200, item) + return + } + if r.Method != "GET" && r.Method != "HEAD" && !strings.HasSuffix(r.URL.Path, "/slice-edit-applications") && !strings.HasSuffix(r.URL.Path, "/slice-edit-restorations") && !strings.HasSuffix(r.URL.Path, "/drafts") { + renderJSON(w, 403, map[string]string{"error": "fixture write not permitted"}) + return + } + r.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "boundary-local-csrf"}) + r.Header.Set(csrfHeaderName, "boundary-local-csrf") + r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: f.annotator, Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}})) + handler.ServeHTTP(w, r) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "node", "tests/browser/slice-phase-replica.mjs") + cmd.Dir = "../.." + cmd.Env = append(os.Environ(), "REWORK_BOUNDARY_API="+server.URL) + out, err := cmd.CombinedOutput() + t.Log(string(out)) + if err != nil { + t.Fatal(fmt.Errorf("browser: %w", err)) + } +} diff --git a/src/components/CaptureAnnotationWorkbench.tsx b/src/components/CaptureAnnotationWorkbench.tsx index c2620a55..e867c014 100644 --- a/src/components/CaptureAnnotationWorkbench.tsx +++ b/src/components/CaptureAnnotationWorkbench.tsx @@ -745,24 +745,26 @@ export function CaptureAnnotationWorkbench({ setDispositionError(""); } }, [selectedSlice?.id, dispositionOpen, dispositionDraft.sliceId]); - // 切片重设面板只在 annotate 相的可编辑工作台内存在;阶段流转/只读化时整体取消 - // (预览本就不落库,丢弃即回退)。不随 slices.length < 2 退出:面板内合并到单切片 - // 合法(拆分可再补回),守卫只在进入时拦。 + // E-panel belongs to annotation; quick structural edits are supported in + // both editing phases. Never silently cancel a slice-phase conflict. + const conflictPhaseAllowed = phase === "annotate" || (phase === "slice" && sliceEditConflictList?.source === "quick"); + const showSliceEditConflict = Boolean( + sliceEditConflictList && conflictPhaseAllowed && !readOnly && !isQualityReview && !mediaOnly, + ); useEffect(() => { - if ( - (sliceEditPanel || sliceEditConflictList) && - (phase !== "annotate" || readOnly || isQualityReview || mediaOnly) - ) { - // 票 05:快路径清单随阶段/只读化一并取消——页面回退拖动几何(或丢弃未落库 - // 改动),不留悬空待决结构。 - if (sliceEditConflictList?.source === "quick") onSliceEditQuickConflictCancel?.(); + const unavailable = readOnly || isQualityReview || mediaOnly; + if (sliceEditPanel && (phase !== "annotate" || unavailable)) { setSliceEditPanel(null); - setSliceEditConflictList(null); setSliceEditDiscardDraft(null); } + if (sliceEditConflictList && (!conflictPhaseAllowed || unavailable)) { + if (sliceEditConflictList.source === "quick") onSliceEditQuickConflictCancel?.(); + setSliceEditConflictList(null); + } }, [ sliceEditPanel, sliceEditConflictList, + conflictPhaseAllowed, phase, readOnly, isQualityReview, @@ -795,8 +797,7 @@ export function CaptureAnnotationWorkbench({ const playheadSliceIndex = sliceIndexAtPlayhead(slices, currentTime); const playheadSlice = playheadSliceIndex >= 0 ? slices[playheadSliceIndex] : null; const selectedDiscarded = selectedSlice.deliveryDisposition === "discard"; - const showDispositionPanel = - (phase === "slice" || readOnly || isQualityReview || dispositionOpen) && (selectedDiscarded || dispositionOpen); + const showDispositionPanel = selectedDiscarded || dispositionOpen; const canEditDisposition = phase === "slice" && !readOnly && !isQualityReview && !!onSliceDispositionChange; const sliceNameIssue = sliceDictionaryNameIssue( slices, @@ -3131,7 +3132,7 @@ export function CaptureAnnotationWorkbench({ // 冲突清单(票 04,R2-Q2):清单开着时全权接管——↑/↓ 条目、1/2/3 处置、 // Enter 按默认应用、Esc 取消整个待应用改动集;播放族 "pass" 放行给全局表 // (视频是参照物),J/K "locked" 给提示,其余键整体吞掉(面板键不生效)。 - if (sliceEditConflictList && phase === "annotate") { + if (sliceEditConflictList && showSliceEditConflict) { if (saving || quickConflictApplyingRef.current) { event.preventDefault(); return; @@ -3639,7 +3640,7 @@ export function CaptureAnnotationWorkbench({ - {sliceEditConflictList && phase === "annotate" ? ( + {sliceEditConflictList && showSliceEditConflict ? ( 冲突清单已打开 · 以下为清单键位,Enter 应用或 Esc 取消后恢复全局键位 @@ -3652,7 +3653,7 @@ export function CaptureAnnotationWorkbench({ ) : null} - {(sliceEditConflictList && phase === "annotate" + {(sliceEditConflictList && showSliceEditConflict ? sliceEditConflictListShortcutHelp : sliceEditPanel && phase === "annotate" ? sliceEditPanelShortcutHelp @@ -5194,6 +5195,11 @@ export function CaptureAnnotationWorkbench({ )} )} + {!mediaOnly && phase === "annotate" && selectedDiscarded && reworkRail ? ( +
+ {reworkRail} +
+ ) : null} {!mediaOnly && phase === "annotate" && !selectedDiscarded ? (
@@ -6627,7 +6633,7 @@ export function CaptureAnnotationWorkbench({ ); })() : null} - {sliceEditConflictList && phase === "annotate" ? ( + {sliceEditConflictList && showSliceEditConflict ? ( !(entry.actionId in decisions)).length; const autoFollowHint = conflictAutoFollowHint(autoFollows); const changes = sliceEditChangeSummary(beforeSlices, afterSlices); + const invalidating = entries.filter((entry) => decisions[entry.actionId] === "invalidate").length; return (
+ {invalidating > 0 ? ( +

+ 已选择将 {invalidating} 个动作整体判无效,包含新边界两侧;如需保留正常侧描述,请选择“截断”。应用后可撤销。 +

+ ) : null} {autoFollowHint ? (
{ if (typeof segment.id === "string" && segment.id) positions.set(segment.id, index); @@ -78,7 +78,7 @@ export function autosaveSegmentsWithDrafts(segments = [], drafts = []) { if (!annotation) continue; const { start, end } = annotation; if (typeof start !== "number" || typeof end !== "number" || !(end > start)) continue; - const value = segmentForDraftPayload(annotation); + const value = prepare(annotation); // Existing editors and newly split actions both retain the action ID. // Appending would duplicate the ID and overlap the entire original interval. const index = positions.get(annotation.id); @@ -91,10 +91,14 @@ export function autosaveSegmentsWithDrafts(segments = [], drafts = []) { return merged; } +export function autosaveSegmentsWithDrafts(segments = [], drafts = []) { + return mergeDrafts(segments, drafts, segmentForDraftPayload); +} + /** Freeze complete editor drafts into the same workspace transaction as a * slice edit. Incomplete end-point selections remain owned by their editor. * This does not submit the task or create a formal annotation revision. */ -export function sliceEditActionsWithDrafts(segments = [], drafts = []) { +function completeActionsWithDrafts(segments, drafts, prepare) { const unresolved = []; const complete = []; for (const draft of drafts) { @@ -116,5 +120,15 @@ export function sliceEditActionsWithDrafts(segments = [], drafts = []) { annotation: { ...action, description, status: invalid ? "invalid" : description ? "described" : "pending" }, }); } - return { actions: autosaveSegmentsWithDrafts(segments, complete), unresolved }; + return { actions: mergeDrafts(segments, complete, prepare), unresolved }; +} + +export function sliceEditActionsWithDrafts(segments = [], drafts = []) { + return completeActionsWithDrafts(segments, drafts, segmentForDraftPayload); +} + +// Feedback compares business content, including classification and handedness. +// Payload serialization intentionally strips display fields; a live view must not. +export function reworkActionsWithDrafts(segments = [], drafts = []) { + return completeActionsWithDrafts(segments, drafts, (action) => ({ ...action })); } diff --git a/src/lib/rework-emphasis.js b/src/lib/rework-emphasis.js index ab1a4bc8..7989c265 100644 --- a/src/lib/rework-emphasis.js +++ b/src/lib/rework-emphasis.js @@ -4,7 +4,7 @@ // // 判定规则(与用户确认的设计一致): // - 红(未处理):意见目标在当前工作区里仍与退回基线一致(未改动)。 -// - 灰(已处理):目标被改动(切片名称/分类标签/边界/弃用状态/标注状态、动作 +// - 灰(已处理):目标被改动(切片名称/分类标签/边界/弃用状态、动作 // 文字/类别/左右手/边界/状态任一变化),或目标被合并/删除后找不到承接块; // 或标注员一键标记"不改了"(skipped,单独状态,复审可见)。 // - 跟块(reattach)扩展到切片结构变化(票 04,spec §3.3 / Q5:a): @@ -30,9 +30,11 @@ function sliceKey(slice) { round3(slice.start), round3(slice.end), slice.taskLabelId || "", - slice.deliveryDisposition || "", - slice.discardReason || slice.discardReasonLabelId || "", - slice.annotationStatus || "", + slice.deliveryDisposition || "keep", + slice.discardReason || "", + slice.discardReasonLabelId || "", + slice.discardReasonVersion || 0, + slice.discardReasonNote || "", ].join("|"); } diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 7a7ba530..1abf4870 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -152,6 +152,7 @@ import { autosaveSegmentsWithDrafts, sliceEditSessionFlushEdge, workbenchDraftsFingerprint, + reworkActionsWithDrafts, sliceEditActionsWithDrafts, } from "../lib/annotation-autosave.js"; import { useAnnotationAutosave } from "../hooks/useAnnotationAutosave"; @@ -2246,7 +2247,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: }; const [reworkSeekRequest, setReworkSeekRequest] = useState<{ time: number; revision: number } | null>(null); const reworkSeekRevisionRef = useRef(0); - // 返工强调层:标注员返工视图按"退回基线 vs 已保存工作区"计算红/灰(多轮,最新轮红)。 + // 返工强调层包含区间完整的编辑器草稿;未完成终点选择不推定已修改。 const reworkEmphasisEligible = Boolean( detail?.humanItem && detail.humanItem.currentStageId === "annotation_combined" && @@ -2255,12 +2256,16 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: ["claimed", "rework", "in_progress"].includes(detail.humanItem.status) && workspaceMode === "annotate", ); + const reworkLiveActions = useMemo( + () => reworkActionsWithDrafts(segments, workbenchDrafts).actions, + [segments, workbenchDrafts], + ); const reworkEmphasisData = useReworkEmphasis( detail?.humanItem?.id || "", detail?.humanItem?.reviewRound || 0, reworkEmphasisEligible, sliceSegments, - segments, + reworkLiveActions, ); const reworkEmphasis = reworkEmphasisEligible ? reworkEmphasisData.emphasis : null; // QC 返修接收视图(票 08/09):强调层输入 = 已保存工作区(ADR 0006:31),与标注员侧 diff --git a/tests/browser/slice-phase-replica.mjs b/tests/browser/slice-phase-replica.mjs new file mode 100644 index 00000000..7302feab --- /dev/null +++ b/tests/browser/slice-phase-replica.mjs @@ -0,0 +1,189 @@ +import fs from "node:fs"; +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +const source = process.cwd(); +const require = createRequire(source + "/package.json"), + { createServer } = require("vite"), + { chromium } = require("playwright"); +const api = process.env.REWORK_BOUNDARY_API, + root = new URL("../../local-data/verification/slice-phase/", import.meta.url), + report = { cases: [], errors: [], keys: [], writes: [] }; +assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); +fs.mkdirSync(root, { recursive: true }); +const server = await createServer({ + root: source, + server: { host: "127.0.0.1", port: 5204, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, +}); +let browser, page; +try { + await server.listen(); + browser = await chromium.launch({ + executablePath: "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + headless: true, + args: ["--no-proxy-server"], + }); + page = await browser.newPage({ viewport: { width: 1600, height: 1100 } }); + page.setDefaultTimeout(10000); + page.on("pageerror", (e) => report.errors.push(e.message)); + page.on("request", (r) => { + if (!["GET", "HEAD"].includes(r.method())) report.writes.push(new URL(r.url()).pathname); + }); + await page.route("**/*", (r) => { + const u = new URL(r.request().url()); + if (u.hostname !== "127.0.0.1") return r.abort(); + if (u.pathname.includes("/worker-api/")) return r.fulfill({ json: { status: "ok" } }); + return r.continue(); + }); + for (const status of ["empty", "pending", "draft"]) { + await fetch(api + "/__boundary/reset?status=" + status); + await page.goto( + "http://127.0.0.1:5204/plugins/annotation-qa/workflows/all/work?projectId=draft-project&taskId=draft-task", + ); + await page.getByRole("button", { name: "继续工作", exact: true }).click(); + const bench = page.getByTestId("capture-annotation-workbench"), + del = bench.getByRole("button", { name: "删除切片", exact: true }); + await del.first().waitFor(); + assert.equal(await del.count(), 1); + await page.evaluate(() => { + window.keyProbe = []; + document.addEventListener("keydown", (e) => + setTimeout( + () => + window.keyProbe.push({ + key: e.key, + code: e.code, + isComposing: e.isComposing, + defaultPrevented: e.defaultPrevented, + }), + 0, + ), + ); + }); + const slider = bench.locator("input[type=range]").first(), + box = await slider.boundingBox(); + await page.mouse.click(box.x + box.width * 0.15, box.y + box.height / 2); + await page.waitForTimeout(300); + await bench.focus(); + await page.keyboard.press("c"); + await page.waitForTimeout(500); + const result = { + status, + afterC: await del.count(), + visibleConflictLists: await page.getByTestId("slice-edit-conflict-list").count(), + receivedKeys: await page.evaluate(() => window.keyProbe), + text: await bench.innerText(), + }; + report.cases.push({ ...result, text: undefined }); + if (status === "empty") assert.equal(result.afterC, 2, "no actions: C must split"); + else { + assert.equal(result.visibleConflictLists, 1, "pending actions must show a visible conflict"); + await bench.focus(); + await page.keyboard.press("Enter"); + await del.nth(1).waitFor(); + assert.equal(await del.count(), 2); + await page.getByRole("button", { name: "撤销", exact: true }).click(); + await page.waitForTimeout(500); + assert.equal(await del.count(), 1); + await page.waitForTimeout(1100); + await page.getByRole("button", { name: "重做", exact: true }).click(); + await del.nth(1).waitFor(); + assert.equal(await del.count(), 2); + await page.reload(); + await page.getByRole("button", { name: "继续工作", exact: true }).click(); + await del.nth(1).waitFor(); + assert.equal(await del.count(), 2); + } + } + + const taskURL = + "http://127.0.0.1:5204/plugins/annotation-qa/workflows/all/work?projectId=draft-project&taskId=draft-task"; + const state = async () => (await (await fetch(api + "/__boundary/state")).json()).subject.payload.annotationWorkspace; + const load = async (shape) => { + await fetch(api + "/__boundary/reset?status=pending&shape=" + shape); + await page.goto(taskURL); + await page.getByRole("button", { name: "继续工作", exact: true }).click(); + await page.getByTestId("capture-annotation-workbench").waitFor(); + }; + const bench = page.getByTestId("capture-annotation-workbench"); + const labels = () => bench.getByRole("combobox").first().locator("option").allTextContents(); + const settle = async () => { + const list = page.getByTestId("slice-edit-conflict-list"); + if (await list.count()) await list.getByRole("button", { name: /Enter 应用/ }).click(); + await page.waitForTimeout(500); + }; + await load("two"); + await bench.getByRole("button", { name: "拖动切片边界", exact: true }).first().waitFor(); + const before = await labels(); + const handle = await bench.getByRole("button", { name: "拖动切片边界", exact: true }).first().boundingBox(); + await page.mouse.move(handle.x + handle.width / 2, handle.y + handle.height / 2); + await page.mouse.down(); + await page.mouse.move(handle.x + handle.width / 2 + 100, handle.y + handle.height / 2, { steps: 8 }); + await page.mouse.up(); + await page.waitForTimeout(200); + await settle(); + const after = await labels(); + assert.notDeepEqual(after, before, "pending actions: drag must apply after confirmation"); + await page.getByRole("button", { name: "撤销", exact: true }).click(); + await page.waitForTimeout(500); + assert.deepEqual(await labels(), before); + await page.waitForTimeout(1100); + await page.getByRole("button", { name: "重做", exact: true }).click(); + await page.waitForTimeout(500); + assert.deepEqual(await labels(), after); + assert.equal((await state()).segments.length >= 2, true); + report.cases.push({ scenario: "pending-drag-undo-redo" }); + await page.waitForTimeout(1100); + await bench.getByRole("button", { name: "删除切片", exact: true }).first().click(); + await page.waitForTimeout(200); + await settle(); + assert.equal(await bench.getByRole("button", { name: "删除切片", exact: true }).count(), 1); + await page.getByRole("button", { name: "撤销", exact: true }).click(); + await page.waitForTimeout(500); + assert.deepEqual(await labels(), after); + await page.waitForTimeout(1100); + await page.getByRole("button", { name: "重做", exact: true }).click(); + await page.waitForTimeout(500); + assert.equal(await bench.getByRole("button", { name: "删除切片", exact: true }).count(), 1); + report.cases.push({ scenario: "pending-delete-undo-redo" }); + await load("rework"); + const rail = page.getByTestId("rework-opinion-rail"); + await rail.waitFor(); + assert.equal( + await rail.locator('[data-rework-state="red"]').count(), + 2, + "derived slice status must stay red on load", + ); + const block = bench.locator("[data-annotation-block]").first(); + await block.click(); + const editor = bench.getByRole("textbox", { name: "动作描述", exact: true }); + await editor.waitFor(); + await editor.fill("更正描述"); + await page.waitForTimeout(300); + assert.equal( + await rail.locator('[data-rework-state="gray"]').count(), + 2, + "unsaved editor draft updates opinions immediately", + ); + await editor.fill("原描述"); + await page.waitForTimeout(300); + assert.equal(await rail.locator('[data-rework-state="red"]').count(), 2, "reverted editor draft restores red"); + await bench.getByRole("combobox").first().selectOption("discard"); + await page.getByTestId("slice-disposition-panel").waitFor(); + assert.equal(await rail.count(), 1); + assert.equal(await page.getByTestId("discard-review-sidebar").count(), 1); + assert.equal(await page.getByTestId("annotation-description-editor").count(), 0); + const side = await page.getByTestId("discard-review-sidebar").boundingBox(); + const video = await page.getByTestId("capture-video-panel").boundingBox(); + assert.ok(side.x >= video.x + video.width - 2, "discard opinions remain in right sidebar"); + report.cases.push({ scenario: "live-draft-status-and-discard-evidence" }); + assert.deepEqual(report.errors, []); + console.log(JSON.stringify(report)); +} catch (e) { + report.failure = e.message; + if (page) fs.writeFileSync(new URL("failure.txt", root), await page.locator("body").innerText()); + throw e; +} finally { + fs.writeFileSync(new URL("c-page-report.json", root), JSON.stringify(report, null, 2)); + await browser?.close(); + await server.close(); +} diff --git a/tests/frontend/capture-annotation-workbench.test.mjs b/tests/frontend/capture-annotation-workbench.test.mjs index 8b14b915..53e2dba2 100644 --- a/tests/frontend/capture-annotation-workbench.test.mjs +++ b/tests/frontend/capture-annotation-workbench.test.mjs @@ -189,7 +189,7 @@ test("slice reconfig panel: E opens the preview panel with Enter/Esc, locks bypa // rua 04(D6):J/K 在面板打开期间放行给返工跳转分支——面板分支不再有 locked // 处理;冲突清单分支保留 locked + 提示(动过开始/结束时间必须先决断)。 const conflictDispatch = workbench.slice( - workbench.indexOf('if (sliceEditConflictList && phase === "annotate") {'), + workbench.indexOf("if (sliceEditConflictList && showSliceEditConflict) {"), workbench.indexOf('else if (sliceEditPanel && phase === "annotate") {'), ); assert.match(conflictDispatch, /command === "locked"[\s\S]{0,80}notifySliceEditLocked/, "冲突清单 J/K 锁定有提示"); @@ -296,7 +296,7 @@ test("drag quick path: releases route through the 03 engine into the 04 conflict assert.match(seed, /list\?\.source === "quick" \? null : list/, "条目清空只收 quick 来源"); assert.match( workbench, - /sliceEditConflictList\?\.source === "quick"\) onSliceEditQuickConflictCancel\?\.\(\)/, + /sliceEditConflictList\.source === "quick"\) onSliceEditQuickConflictCancel\?\.\(\)/, "阶段/只读化经页面回退", ); const dispatchApply = workbench.slice( diff --git a/tests/frontend/e-open-jk-unlock.test.mjs b/tests/frontend/e-open-jk-unlock.test.mjs index 53b64b42..6bbdda56 100644 --- a/tests/frontend/e-open-jk-unlock.test.mjs +++ b/tests/frontend/e-open-jk-unlock.test.mjs @@ -30,7 +30,7 @@ const collapsePanel = workbench.slice( workbench.indexOf("const sliceEditPanelOpen ="), ); const conflictDispatch = workbench.slice( - workbench.indexOf('if (sliceEditConflictList && phase === "annotate") {'), + workbench.indexOf("if (sliceEditConflictList && showSliceEditConflict) {"), workbench.indexOf('else if (sliceEditPanel && phase === "annotate") {'), ); const panelDispatch = workbench.slice( diff --git a/tests/frontend/rework-live-feedback.test.mjs b/tests/frontend/rework-live-feedback.test.mjs new file mode 100644 index 00000000..eac133a7 --- /dev/null +++ b/tests/frontend/rework-live-feedback.test.mjs @@ -0,0 +1,55 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { computeReworkEmphasis } from "../../src/lib/rework-emphasis.js"; +import { reworkActionsWithDrafts } from "../../src/lib/annotation-autosave.js"; +const slices = [{ id: "s", name: "清洗清洁", start: 0, end: 10, annotationStatus: "labeled" }]; +const actions = [{ id: "a", start: 0, end: 10, description: "原描述", status: "described", labelId: "category" }]; +const baseline = { + round: 1, + slices, + actions, + problems: [ + { id: "slice", scope: "slice", sliceId: "s", text: "核对" }, + { id: "action", scope: "action", sliceId: "s", actionId: "a", text: "修改描述" }, + ], +}; +const emphasis = (currentSlices = slices, currentActions = actions) => + computeReworkEmphasis({ baselines: [baseline], currentSlices, currentActions }); +test("derived slice status and legacy keep defaults never imply a business edit", () => { + const result = emphasis([{ ...slices[0], annotationStatus: "pending", deliveryDisposition: "keep" }]); + assert.equal(result.unclearedCount, 2); + assert.equal(result.clearedCount, 0); + assert.equal(result.changes.items.length, 0); +}); +test("complete editor drafts update both action and slice feedback; reverting restores red", () => { + const draft = { sliceId: "s", annotation: { ...actions[0], description: "更正描述" }, selectingEnd: false }; + const live = reworkActionsWithDrafts(actions, [draft]).actions; + const changed = emphasis(slices, live); + assert.equal(changed.clearedCount, 2); + assert.equal(emphasis().unclearedCount, 2); + const incomplete = reworkActionsWithDrafts(actions, [{ ...draft, selectingEnd: true }]).actions; + assert.equal(emphasis(slices, incomplete).unclearedCount, 2); +}); +test("dictionary changes and explicit discard remain business edits", () => { + const discarded = { + ...slices[0], + deliveryDisposition: "discard", + discardReason: "遮挡", + discardReasonLabelId: "r", + discardReasonVersion: 1, + discardReasonNote: "旧备注", + }; + const result = emphasis([{ ...slices[0], taskLabelId: "new-category" }]); + assert.equal(result.sliceStates.get("s").gray.length, 1); + assert.equal(emphasis([discarded]).sliceStates.get("s").gray.length, 2); +}); + +test("live comparison preserves classification display fields without drafts", () => { + const decorated = actions.map((a) => ({ ...a, handedness: "left", label: "动作类别" })); + assert.deepEqual(reworkActionsWithDrafts(decorated, []).actions, decorated); + const live = reworkActionsWithDrafts(decorated, [ + { annotation: { ...decorated[0], description: "更正" }, selectingEnd: false }, + ]).actions; + assert.equal(live[0].handedness, "left"); + assert.equal(live[0].label, "动作类别"); +}); diff --git a/tests/frontend/rework-review-layout.test.mjs b/tests/frontend/rework-review-layout.test.mjs index 443d2fdc..58a78178 100644 --- a/tests/frontend/rework-review-layout.test.mjs +++ b/tests/frontend/rework-review-layout.test.mjs @@ -54,9 +54,19 @@ test("spec §5: QC review stays read-only (no E panel, no boundary-drag entry)", const workbench = read("src/components/CaptureAnnotationWorkbench.tsx"); assert.match( workbench, - /\(sliceEditPanel \|\| sliceEditConflictList\) &&\s*\(phase !== "annotate" \|\| readOnly \|\| isQualityReview/, + /const unavailable = readOnly \|\| isQualityReview \|\| mediaOnly;[\s\S]*?sliceEditPanel && \(phase !== "annotate" \|\| unavailable\)/, "E 面板在质检视图禁用(票 06 后经面板 teardown 效应)", ); + assert.match( + workbench, + /sliceEditConflictList && \(!conflictPhaseAllowed \|\| unavailable\)/, + "review also tears down quick conflicts", + ); + assert.match( + workbench, + /conflictPhaseAllowed && !readOnly && !isQualityReview && !mediaOnly/, + "review cannot render or dispatch a structural conflict", + ); assert.match( workbench, /selectedBoundary && phase === "slice" && !readOnly && !isQualityReview/, From 2876a14d4a4f0a15f3d6f0f9b72461237c1b9cc6 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 21 Oct 2026 15:31:05 +0800 Subject: [PATCH 11/54] =?UTF-8?q?=F0=9F=90=9B=20Separate=20administrative?= =?UTF-8?q?=20pool=20totals=20from=20bounded=20cursor=20pages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Administrator counters were derived from the stage-filtered first human-item page, while annotators used an aggregate. Add authorized whole-pool counters with a five-second bounded cache, page human items before joining task summaries, and search older metadata on the server. Cancel stale reads and preserve existing non-admin scope behavior. Cover permissions, mixed stages, real Mongo parity, old-page search, categories, pagination, stale responses and claim invalidation. --- .scratch/admin-pool-counts-20261021/spec.md | 12 + docs/annotation-admin-task-pool.zh-CN.md | 13 + .../admin-pool-counts-20261021.md | 33 ++ internal/api/admin_annotation_pool.go | 153 +++++++ .../api/admin_annotation_pool_browser_test.go | 70 ++++ internal/api/admin_annotation_pool_test.go | 146 +++++++ internal/api/human_display.go | 23 +- internal/api/routes_annotation.go | 1 + internal/api/server.go | 5 +- internal/domain/business_display.go | 34 ++ internal/domain/business_display_test.go | 13 + internal/store/admin_annotation_pool.go | 380 ++++++++++++++++++ internal/store/admin_annotation_pool_test.go | 166 ++++++++ internal/store/store.go | 2 + src/pages/AnnotationOperations.tsx | 177 ++++++-- src/services/api/annotation-operations.ts | 23 ++ .../browser/admin-annotation-pool-replica.mjs | 114 ++++++ 17 files changed, 1312 insertions(+), 53 deletions(-) create mode 100644 .scratch/admin-pool-counts-20261021/spec.md create mode 100644 docs/annotation-admin-task-pool.zh-CN.md create mode 100644 docs/verification/admin-pool-counts-20261021.md create mode 100644 internal/api/admin_annotation_pool.go create mode 100644 internal/api/admin_annotation_pool_browser_test.go create mode 100644 internal/api/admin_annotation_pool_test.go create mode 100644 internal/store/admin_annotation_pool.go create mode 100644 internal/store/admin_annotation_pool_test.go create mode 100644 tests/browser/admin-annotation-pool-replica.mjs diff --git a/.scratch/admin-pool-counts-20261021/spec.md b/.scratch/admin-pool-counts-20261021/spec.md new file mode 100644 index 00000000..65d25a8b --- /dev/null +++ b/.scratch/admin-pool-counts-20261021/spec.md @@ -0,0 +1,12 @@ +# Correct administrative annotation pool totals and paging +Status: resolved + +Base: 867ca698 (contains the prior local editing repairs). Local only. + +The annotation workspace administrator badges must count all authorized human items, independent of the current list page, text search and status category. Project/workflow/run/Capture scope still applies. Lists must page human items, then join tasks by IDs; searching must include older items beyond the former first 500. Never fetch all business payloads into the browser, change customer records, widen annotator visibility or modify claim permissions. + +Use an admin-only read endpoint, compact server-side counts and bounded cursor pages. Keep existing non-admin/review behavior and old APIs for compatibility. Test >500 items, mixed stages, orphan/cross-org batches, permissions, claim changes, old-page search, category filtering, stale page responses and pagination with stable totals. No production deployment in this task. + +## Answer + +Implemented admin-only totals with a 5-second bounded actor/scope cache; list pages follow human item cursors and join only page task IDs. Server-side Capture/status search sees older items; abandoned searches are cancelled and stale responses fenced. Existing role paths and APIs remain. See docs/annotation-admin-task-pool.zh-CN.md and docs/verification/admin-pool-counts-20261021.md. diff --git a/docs/annotation-admin-task-pool.zh-CN.md b/docs/annotation-admin-task-pool.zh-CN.md new file mode 100644 index 00000000..6cf15396 --- /dev/null +++ b/docs/annotation-admin-task-pool.zh-CN.md @@ -0,0 +1,13 @@ +# 管理员切分与标注任务池 + +2026-10-21 本地修复。适用于平台管理员的“切分与语义标注”队列;标注员随机领取、个人缓冲池及质检/验收原入口保持原合同。 + +“可领取/我的任务/全部任务”统计当前权限与项目、工作流、执行、Capture 范围内的完整人工任务池,不再把首屏或已加载列表条数当作总数。搜索词和状态分类只筛选下方列表,不改变这些总数。管理员“全部任务”沿用当前环节范围,包含标注及已送检质检任务,不是全部历史已交付任务。 + +新增只读接口 `GET /api/plugins/annotation-qa/admin-pool`,只有组织管理员/平台管理员可用,组织范围由服务端身份决定。有效批次及阶段校验保留;非管理员不能通过传入 pool、用户或组织参数取得公共任务明细。旧接口保留,已有浏览器和在途任务无需迁移。 + +列表默认且最多每页 200 条,以人工任务的 `(createdAt,id)` 游标继续,任务摘要只关联这一页的 ID。“显示更多”读取下一页人工任务,而不是继续拉取独立的最新任务窗口。搜索及状态分类在服务端处理,能够找到第 500 条以后的任务;旧 Capture 名称仍通过原有元数据提取规则兼容。切换项目、列表或搜索后,不接受旧请求迟到的内容。 + +总数按批次/阶段聚合紧凑计数,再用既有批次权限投影校验;不读取动作正文或草稿来计数。总数使用有身份和范围隔离、容量上限的 5 秒进程缓存,当前进程的 API 写入会使缓存失效;其他副本与后台写入最多延迟到缓存到期。列表不缓存。搜索输入合并 350 毫秒,新搜索与范围切换取消过期请求;旧任务搜索在服务端流式读取投影元数据,查询和范围统计受 8 秒期限限制,不向浏览器传整池,也不把失败当成零条。计数与页面是相邻的只读操作,并非冻结生产时刻的结算快照。 + +不新增统计集合、数据库字段或索引,不改领取、标注、返工、审核及交付结果。生产环境仍需单独发布与验收。 diff --git a/docs/verification/admin-pool-counts-20261021.md b/docs/verification/admin-pool-counts-20261021.md new file mode 100644 index 00000000..f93aa909 --- /dev/null +++ b/docs/verification/admin-pool-counts-20261021.md @@ -0,0 +1,33 @@ +# Administrator annotation pool totals — local verification + +Base `867ca698`, branch `codex/rework-boundary-20261021`. Diagnosis captured deployed `fab5be57`: administrator badges were the stage-filtered first 500 human records while annotator blind claim used an aggregate. The original metadata replay produced 495 vs 14902; screenshots were 496 vs 14911 at an earlier time. + +## Regression and implementation + +`TestAdminAnnotationPoolTotalsAndOldPageSearch` first failed with 404 before adding the read endpoint. It now covers 610 added mixed-stage items, stable whole-pool counts across cursors, a search hit older than the previous 500-row window, and denial to annotators. Permission/cache/error tests cover role changes before cache access, normalized count keys independent of page/list filters, bounded page limits, malformed inputs, 5-second expiry and a failed aggregate returning an error rather than false zero. + +The real page/HTTP fixture uses MemoryStore and synthetic records only: + +``` +ADMIN_POOL_BROWSER_TEST=1 go test ./internal/api -run 'TestAdminAnnotationPool' -count=1 -v +``` + +Covers 200→400 loaded cards with unchanged 606 available/611 all totals; search for CAPTURE_0001; status category returning zero without changing badges; all-pool review filtering; delayed old search response ignored; real local claim invalidating cached count. No page errors or unexpected browser writes. + +Actual Mongo parity and timings use a new, unique synthetic database on local Mongo 27018, cleaned afterward: + +``` +POOL_LOCAL_MONGO_URI=mongodb://127.0.0.1:27018 go test ./internal/store -run '^TestAdminAnnotationPoolMongoParity$' -count=1 -v +``` + +1500 synthetic records in separate batches cover mixed stages/assignees, orphan and cross-organization rows, full old-page searches, literal rather than regex search, cursor de-duplication, persisted workflow categories, legacy Capture names obtained through Capture records, and absence of heavy annotationWorkspace on list responses. Timings are local store measurements, not claims about production latency. + +Legacy API compatibility, shared Capture extraction, existing dashboard-cache defaults, annotator/review editing browser flows, TypeScript, build, production-bundle boot, and full frontend baseline gate are checked separately. No baseline entries are edited. Production data, workflow, indexes, Worker and delivery JSON are not changed in this task; no push, PR or deployment is included. + +## Final evidence + +Store-level measurements with 1500 separate batches: counts 42.8 ms, first page 29.4 ms, old-record search 486.6 ms; these are synthetic local observations only. No new production index or statistical collection is required. + +One combined browser run lost a fixture work session during the draft C/undo/redo scenario; there were no page errors. The same test passed unchanged in an isolated rerun, including the newer-draft recovery path. This is recorded separately rather than erased from the results or accepted by changing test expectations. The administrator browser flow and existing boundary browser passed in that combined run. Logs remain in untracked local test results and `/tmp/admin-pool-all-browser.log` / `/tmp/admin-pool-slice-isolated.log` for this session. + +The final combined administrator + slice-phase browser rerun passed with the newer-draft case, and no runtime errors. Final frontend gate retains only 13 known baseline failures; TypeScript, Go build, web build and production boot pass. Old non-admin project scope/dependency behavior is preserved, so local annotator filters do not trigger the new administrator queries. diff --git a/internal/api/admin_annotation_pool.go b/internal/api/admin_annotation_pool.go new file mode 100644 index 00000000..d40fac8d --- /dev/null +++ b/internal/api/admin_annotation_pool.go @@ -0,0 +1,153 @@ +package api + +import ( + "droboticflow/internal/domain" + "droboticflow/internal/store" + "encoding/json" + "net/http" + "net/url" + "strings" + "time" +) + +func (s *server) adminAnnotationPoolQuery(w http.ResponseWriter, r *http.Request) (store.AdminAnnotationPoolQuery, bool) { + p, _ := currentPrincipal(r) + if !annotationManager(p) { + renderJSON(w, 403, map[string]string{"error": "administrator annotation pool required"}) + return store.AdminAnnotationPoolQuery{}, false + } + v := r.URL.Query() + q := store.AdminAnnotationPoolQuery{ProjectID: v.Get("projectId"), WorkflowID: v.Get("workflowId"), RunID: v.Get("runId"), NodeID: v.Get("nodeId"), ArtifactID: v.Get("artifactId"), Capture: strings.TrimSpace(v.Get("capture")), Search: strings.TrimSpace(v.Get("search")), Category: v.Get("category"), Pool: v.Get("pool"), UserID: p.User.ID} + if q.Pool == "" { + q.Pool = "available" + } + if q.Pool != "available" && q.Pool != "mine" && q.Pool != "all" { + renderValidationError(w, validationIssue{Field: "pool", Message: "pool must be available, mine, or all"}) + return q, false + } + if len(q.Search) > 256 || len(q.Capture) > 1024 { + renderValidationError(w, validationIssue{Field: "search", Message: "search is too long"}) + return q, false + } + switch q.Category { + case "", "all", "available", "pending_slice", "slicing", "pending_annotation", "rework", "pending_quality", "rework_review", "pending_acceptance", "accepted", "submitting", "submission_conflicted": + default: + renderValidationError(w, validationIssue{Field: "category", Message: "unknown task category"}) + return q, false + } + limit, issue := parseDatabaseBrowserPageQuery(v.Get("limit"), "limit", 200, 1, 200) + if issue != nil { + renderValidationError(w, *issue) + return q, false + } + q.Limit = limit + cursor, err := decodeHumanItemCursor(v.Get("cursor")) + if err != nil { + renderValidationError(w, validationIssue{Field: "cursor", Message: "cursor must be a valid opaque human item cursor"}) + return q, false + } + q.Cursor = cursor + return q, true +} + +// Counts use the existing bounded per-actor cache; list criteria and cursor are +// deliberately excluded from its key. Permission is checked before cache hits. +func (s *server) adminAnnotationCounts(w http.ResponseWriter, r *http.Request, q store.AdminAnnotationPoolQuery) (*store.AdminAnnotationPoolCounts, bool) { + countRequest := r.Clone(r.Context()) + copyURL := *r.URL + countRequest.URL = ©URL + values := url.Values{} + for _, key := range []string{"projectId", "workflowId", "runId", "nodeId", "artifactId", "capture"} { + if value := r.URL.Query().Get(key); value != "" { + values.Set(key, value) + } + } + countRequest.URL.RawQuery = values.Encode() + key := s.dashboardCacheKey("admin-annotation-pool-counts", countRequest) + if entry := s.dashboardResponses.get(key); entry != nil { + var counts store.AdminAnnotationPoolCounts + if json.Unmarshal(entry.body, &counts) == nil { + return &counts, true + } + } + counts, err := s.store.CountAdminAnnotationPools(r.Context(), s.requestOrganizationID(r), q) + if err != nil { + renderError(w, 500, err) + return nil, false + } + body, err := json.Marshal(counts) + if err != nil { + renderError(w, 500, err) + return nil, false + } + // Five seconds bounds cross-pod/background changes. Existing API write + // invalidation clears these entries immediately on this process. + s.dashboardResponses.setTTL(key, 200, "application/json", body, 5*time.Second) + return &counts, true +} + +func (s *server) handleAdminAnnotationPool(w http.ResponseWriter, r *http.Request) { + q, ok := s.adminAnnotationPoolQuery(w, r) + if !ok { + return + } + counts, ok := s.adminAnnotationCounts(w, r, q) + if !ok { + return + } + size := q.Limit + q.Limit++ + items, err := s.store.ListAdminAnnotationPool(r.Context(), s.requestOrganizationID(r), q) + if err != nil { + renderError(w, 500, err) + return + } + page := humanItemListPage{Items: items} + if len(items) > size { + page.Items = items[:size] + last := page.Items[len(page.Items)-1] + page.HasMore = true + page.NextCursor = encodeHumanItemCursor(&domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}) + } + resolver := newHumanDisplayResolver(s) + for i := range page.Items { + if err := resolver.enrichItem(r.Context(), &page.Items[i]); err != nil { + renderError(w, 500, err) + return + } + } + // Join only the page's task IDs, never an independent newest-first window. + tasks := []annotationTaskListRecord{} + ids := []string{} + seen := map[string]bool{} + for _, item := range page.Items { + if item.LegacyTaskID != "" && !seen[item.LegacyTaskID] { + ids = append(ids, item.LegacyTaskID) + seen[item.LegacyTaskID] = true + } + } + if len(ids) > 0 { + ctx, err := s.withHumanListReads(r.Context(), s.requestOrganizationID(r), q.ProjectID, page.Items) + if err != nil { + renderError(w, 500, err) + return + } + request := r.Clone(ctx) + u := *r.URL + request.URL = &u + v := url.Values{"ids": {strings.Join(ids, ",")}} + if q.ProjectID != "" { + v.Set("projectId", q.ProjectID) + } + request.URL.RawQuery = v.Encode() + joined := s.readAnnotationTasksList(w, request) + if joined == nil { + return + } + tasks = joined.Items + } + result := page.response() + result["counts"] = counts + result["tasks"] = tasks + renderJSON(w, 200, result) +} diff --git a/internal/api/admin_annotation_pool_browser_test.go b/internal/api/admin_annotation_pool_browser_test.go new file mode 100644 index 00000000..28937bde --- /dev/null +++ b/internal/api/admin_annotation_pool_browser_test.go @@ -0,0 +1,70 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "testing" + "time" +) + +func TestAdminAnnotationPoolBrowserReplica(t *testing.T) { + if os.Getenv("ADMIN_POOL_BROWSER_TEST") != "1" { + t.Skip("local browser opt-in") + } + f := randomClaimFixture(t, 610) + admin := domain.UserRecord{ID: "pool-admin", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive} + _ = f.store.UpsertUser(f.ctx, admin) + _ = f.store.UpsertOrganizationPlugin(f.ctx, domain.OrganizationPluginRecord{OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", Enabled: true, Config: defaultAnnotationPluginConfig()}) + for n := 0; n < 610; n++ { + item, _ := f.store.GetHumanItem(f.ctx, fmt.Sprintf("random-item-%d", n)) + task, _ := f.store.GetAnnotationTask(f.ctx, item.LegacyTaskID) + stamp := fmt.Sprintf("2026-10-21T00:%02d:%02dZ", n/60, n%60) + item.CreatedAt = stamp + task.CreatedAt = stamp + item.Subject.ItemKey = fmt.Sprintf("CAPTURE_%04d_camera_left_part0001.mp4", n) + task.ArtifactID = item.Subject.ItemKey + task.SourceVideo = item.Subject.ItemKey + item.Subject.Payload = map[string]any{"sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}} + if n >= 606 { + item.CurrentStageID = "quality_review" + task.Status = domain.AnnotationTaskStatusPendingQuality + } + _ = f.store.UpsertHumanItem(f.ctx, *item) + _ = f.store.UpsertAnnotationTask(f.ctx, *task) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/auth/me" { + renderJSON(w, 200, map[string]any{"user": sanitizeUser(&admin), "organization": domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}}) + return + } + if r.URL.Path == "/__fixture/claim" { + result := f.request(http.MethodPost, "/api/plugins/annotation-qa/random-claims", f.annotator, map[string]any{"projectId": f.task.ProjectID, "requestId": "admin-pool-fixture-claim-01"}) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(result.Code) + _, _ = w.Write(result.Body.Bytes()) + return + } + if r.Method != "GET" && r.Method != "HEAD" { + renderJSON(w, 403, map[string]string{"error": "fixture read-only"}) + return + } + r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: admin, Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}})) + f.handler.ServeHTTP(w, r) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "node", "tests/browser/admin-annotation-pool-replica.mjs") + cmd.Dir = "../.." + cmd.Env = append(os.Environ(), "ADMIN_POOL_API="+server.URL) + out, err := cmd.CombinedOutput() + t.Log(string(out)) + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/api/admin_annotation_pool_test.go b/internal/api/admin_annotation_pool_test.go new file mode 100644 index 00000000..a4a5d101 --- /dev/null +++ b/internal/api/admin_annotation_pool_test.go @@ -0,0 +1,146 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/store" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "testing" + "time" +) + +func TestAdminAnnotationPoolTotalsAndOldPageSearch(t *testing.T) { + f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticAnnotation, domain.HumanItemReady, "", 0) + f.batch.Stages = append(f.batch.Stages, domain.HumanStageDefinition{ID: "quality_review", Kind: domain.HumanStageReview, RequiredRole: "quality_reviewer"}) + if err := f.store.UpsertHumanBatch(f.ctx, f.batch); err != nil { + t.Fatal(err) + } + admin := domain.UserRecord{ID: "pool-admin", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive} + for i := 0; i < 610; i++ { + item := f.item + item.ID = fmt.Sprintf("pool-%04d", i) + item.LegacyTaskID = fmt.Sprintf("pool-task-%04d", i) + item.CreatedAt = fmt.Sprintf("2026-10-21T00:%02d:%02dZ", i/60, i%60) + item.Subject.ItemKey = fmt.Sprintf("CAPTURE_%04d_camera_left_part0001.mp4", i) + if i >= 606 { + item.CurrentStageID = "quality_review" + } + if e := f.store.UpsertHumanItem(f.ctx, item); e != nil { + t.Fatal(e) + } + } + base := "/api/plugins/annotation-qa/admin-pool?projectId=" + f.task.ProjectID + read := func(path string) map[string]json.RawMessage { + t.Helper() + r := f.request(http.MethodGet, path, admin, nil) + if r.Code != 200 { + t.Fatalf("%d %s", r.Code, r.Body) + } + var d map[string]json.RawMessage + if e := json.Unmarshal(r.Body.Bytes(), &d); e != nil { + t.Fatal(e) + } + return d + } + first := read(base + "&pool=available&limit=100") + var counts struct{ Available, All, Mine int } + _ = json.Unmarshal(first["counts"], &counts) + if counts.Available != 607 || counts.All != 611 || counts.Mine != 0 { + t.Fatalf("counts %+v", counts) + } + var items []domain.HumanItemRecord + _ = json.Unmarshal(first["items"], &items) + if len(items) != 100 { + t.Fatalf("page=%d", len(items)) + } + var cursor string + _ = json.Unmarshal(first["nextCursor"], &cursor) + if cursor == "" { + t.Fatal("missing cursor") + } + second := read(base + "&pool=available&limit=100&cursor=" + url.QueryEscape(cursor)) + var nextCounts struct{ Available, All, Mine int } + _ = json.Unmarshal(second["counts"], &nextCounts) + if counts != nextCounts { + t.Fatal("paging changed totals") + } + old := read(base + "&pool=available&search=CAPTURE_0001") + _ = json.Unmarshal(old["items"], &items) + if len(items) != 1 || items[0].ID != "pool-0001" { + t.Fatalf("old search %+v", items) + } + _ = json.Unmarshal(old["counts"], &nextCounts) + if counts != nextCounts { + t.Fatal("search changed badges") + } + denied := f.request(http.MethodGet, base+"&pool=all", f.annotator, nil) + if denied.Code != 403 { + t.Fatalf("annotator=%d", denied.Code) + } +} + +type adminPoolProbe struct { + store.Store + counts, pages int + fail bool +} + +func (s *adminPoolProbe) CountAdminAnnotationPools(ctx context.Context, org string, q store.AdminAnnotationPoolQuery) (store.AdminAnnotationPoolCounts, error) { + s.counts++ + if s.fail { + return store.AdminAnnotationPoolCounts{}, errors.New("synthetic unavailable") + } + return s.Store.CountAdminAnnotationPools(ctx, org, q) +} +func (s *adminPoolProbe) ListAdminAnnotationPool(ctx context.Context, org string, q store.AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) { + s.pages++ + if q.Limit > 201 { + panic("unbounded administrator page") + } + return s.Store.ListAdminAnnotationPool(ctx, org, q) +} +func TestAdminAnnotationPoolFreshPermissionsCacheAndFailures(t *testing.T) { + f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticAnnotation, domain.HumanItemReady, "", 0) + probe := &adminPoolProbe{Store: f.store} + now := time.Now() + srv := &server{store: probe, dashboardResponses: dashboardResponseCache{now: func() time.Time { return now }}} + f.handler = http.HandlerFunc(srv.handleAdminAnnotationPool) + admin := domain.UserRecord{ID: "admin-cache", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive} + base := "/api/plugins/annotation-qa/admin-pool?projectId=" + f.task.ProjectID + for _, suffix := range []string{"&pool=available", "&pool=all&search=missing&category=pending_slice", "&pool=mine"} { + r := f.request(http.MethodGet, base+suffix, admin, nil) + if r.Code != 200 { + t.Fatalf("%d %s", r.Code, r.Body) + } + } + if probe.counts != 1 || probe.pages != 3 { + t.Fatalf("counts=%d pages=%d", probe.counts, probe.pages) + } + for _, role := range []domain.UserRole{domain.UserRoleAnnotator, domain.UserRoleQualityReviewer, domain.UserRoleAcceptanceReviewer, domain.UserRoleSupplierManager} { + p := admin + p.Role = role + if r := f.request(http.MethodGet, base, p, nil); r.Code != 403 { + t.Fatalf("cached role %s=%d", role, r.Code) + } + } + for _, suffix := range []string{"&cursor=broken", "&limit=bad", "&pool=reviewed", "&category=unknown"} { + if r := f.request(http.MethodGet, base+suffix, admin, nil); r.Code != 422 && r.Code != 400 { + t.Fatalf("invalid %s=%d", suffix, r.Code) + } + } + // A failed total read must never masquerade as zero, even when the old page + // would fit. Cross-process/background changes become visible after 5 seconds. + now = now.Add(6 * time.Second) + probe.fail = true + r := f.request(http.MethodGet, base, admin, nil) + if r.Code != 500 { + t.Fatalf("failed aggregate=%d %s", r.Code, r.Body) + } + if probe.pages != 3 { + t.Fatal("page fetched after count failure") + } +} diff --git a/internal/api/human_display.go b/internal/api/human_display.go index fd6de5f3..0e9f386c 100644 --- a/internal/api/human_display.go +++ b/internal/api/human_display.go @@ -2,7 +2,6 @@ package api import ( "context" - "fmt" "sort" "strings" @@ -122,27 +121,7 @@ func appendCaptureDirectory(values *[]string, candidate string) { } func humanCaptureDirectoriesFromPayload(value any, values *[]string) { - switch typed := normalizeStructuredValue(value).(type) { - case map[string]any: - for key, item := range typed { - normalizedKey := strings.ToLower(strings.TrimSpace(key)) - switch normalizedKey { - case "capturedirectory", "captureprefix", "capturepath", "sourcedirectory", "sourceprefix", "recordingdirectory": - appendCaptureDirectory(values, domain.CaptureDirectoryFromDirectoryValue(fmt.Sprint(item))) - case "sourcevideo", "artifactid", "clipartifactid": - appendCaptureDirectory(values, domain.CaptureDirectoryFromArtifactValue(fmt.Sprint(item))) - } - humanCaptureDirectoriesFromPayload(item, values) - } - case []any: - for _, item := range typed { - humanCaptureDirectoriesFromPayload(item, values) - } - case []map[string]any: - for _, item := range typed { - humanCaptureDirectoriesFromPayload(item, values) - } - } + domain.AppendCaptureDirectoriesFromPayload(value, values) } func uniqueCaptureDirectories(values []string) []string { diff --git a/internal/api/routes_annotation.go b/internal/api/routes_annotation.go index d0ed4c82..9fbdd5d8 100644 --- a/internal/api/routes_annotation.go +++ b/internal/api/routes_annotation.go @@ -17,6 +17,7 @@ func (server *server) mountAnnotationRoutes(router chi.Router) { router.Get("/api/plugins/annotation-qa/tasks/{id}", server.handleAnnotationTaskGet) router.Get("/api/plugins/annotation-qa/tasks/{id}/frame-timeline", server.handleAnnotationTaskFrameTimelineGet) router.Get("/api/plugins/annotation-qa/sidebar", server.handleAnnotationSidebar) + router.Get("/api/plugins/annotation-qa/admin-pool", server.handleAdminAnnotationPool) // The workload summary is a read-only aggregate over 30 days of worklog // events. The shared 45s per-user response cache (invalidated by any // /api write) keeps the annotation center's initial load from re-running diff --git a/internal/api/server.go b/internal/api/server.go index 32397f8c..af84c38c 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -140,6 +140,9 @@ func (c *dashboardResponseCache) get(key string) *dashboardResponseCacheEntry { } func (c *dashboardResponseCache) set(key string, status int, contentType string, body []byte) { + c.setTTL(key, status, contentType, body, dashboardResponseCacheTTL) +} +func (c *dashboardResponseCache) setTTL(key string, status int, contentType string, body []byte, ttl time.Duration) { c.mu.Lock() defer c.mu.Unlock() if c.entries == nil { @@ -150,7 +153,7 @@ func (c *dashboardResponseCache) set(key string, status int, contentType string, } stored := make([]byte, len(body)) copy(stored, body) - c.entries[key] = dashboardResponseCacheEntry{body: stored, contentType: contentType, status: status, expires: c.now().Add(dashboardResponseCacheTTL)} + c.entries[key] = dashboardResponseCacheEntry{body: stored, contentType: contentType, status: status, expires: c.now().Add(ttl)} } func (c *dashboardResponseCache) invalidateAll() { diff --git a/internal/domain/business_display.go b/internal/domain/business_display.go index 132e0ca7..2c1fdd87 100644 --- a/internal/domain/business_display.go +++ b/internal/domain/business_display.go @@ -1,8 +1,10 @@ package domain import ( + "fmt" "net/url" "path" + "reflect" "regexp" "sort" "strings" @@ -143,3 +145,35 @@ func CaptureDirectoriesFromRun(run RunRecord) []string { } return uniqueSortedCaptureDirectories(values) } + +// AppendCaptureDirectoriesFromPayload shares legacy metadata extraction between +// task display and administrative search. It accepts string-keyed map aliases +// (including BSON maps) and nested slices without depending on a DB driver. +func AppendCaptureDirectoriesFromPayload(value any, values *[]string) { + if value == nil { + return + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Map: + if rv.Type().Key().Kind() != reflect.String { + return + } + iter := rv.MapRange() + for iter.Next() { + key := strings.ToLower(strings.TrimSpace(iter.Key().String())) + v := iter.Value().Interface() + switch key { + case "capturedirectory", "captureprefix", "capturepath", "sourcedirectory", "sourceprefix", "recordingdirectory": + *values = append(*values, CaptureDirectoryFromDirectoryValue(fmt.Sprint(v))) + case "sourcevideo", "artifactid", "clipartifactid": + *values = append(*values, CaptureDirectoryFromArtifactValue(fmt.Sprint(v))) + } + AppendCaptureDirectoriesFromPayload(v, values) + } + case reflect.Slice, reflect.Array: + for n := 0; n < rv.Len(); n++ { + AppendCaptureDirectoriesFromPayload(rv.Index(n).Interface(), values) + } + } +} diff --git a/internal/domain/business_display_test.go b/internal/domain/business_display_test.go index a61c9706..e86fe170 100644 --- a/internal/domain/business_display_test.go +++ b/internal/domain/business_display_test.go @@ -43,3 +43,16 @@ func TestCaptureDirectoriesFromRunKeepsPersistedCaptureDirectoryAfterArtifactsAr t.Fatalf("capture directories = %#v", actual) } } + +func TestCapturePayloadDirectoriesAcceptNestedMapAliases(t *testing.T) { + type legacyMap map[string]any + var values []string + AppendCaptureDirectoriesFromPayload(legacyMap{" records ": []legacyMap{{" captureDirectory ": "bos://bucket/path/客户采集-42"}, {"sourceVideo": "CAPTURE_0001_camera_left_part0001.mp4"}}}, &values) + found := map[string]bool{} + for _, v := range values { + found[v] = true + } + if !found["客户采集-42"] || !found["CAPTURE_0001"] { + t.Fatalf("directories=%v", values) + } +} diff --git a/internal/store/admin_annotation_pool.go b/internal/store/admin_annotation_pool.go new file mode 100644 index 00000000..21614f21 --- /dev/null +++ b/internal/store/admin_annotation_pool.go @@ -0,0 +1,380 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "sort" + "strconv" + "strings" + "time" +) + +// Server-authorized administrative read model. Search/category affect only the +// page; Capture is a scope and also affects the badge totals. No payload writes. +type AdminAnnotationPoolQuery struct { + ProjectID, WorkflowID, RunID, NodeID, ArtifactID string + UserID, Pool, Search, Capture, Category string + Limit int + Cursor *domain.HumanItemCursor +} +type AdminAnnotationPoolCounts struct { + Available int `json:"available"` + Mine int `json:"mine"` + All int `json:"all"` +} + +func adminPoolScope(i domain.HumanItemRecord) bool { + return i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined" || i.CurrentStageID == "quality_review" || (i.CurrentStageID == "split_annotation" && i.Status == domain.HumanItemRework) +} +func adminPoolAvailable(i domain.HumanItemRecord) bool { + return i.AssignedUserID == "" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemRework || i.Status == domain.HumanItemSubmitted) && (i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined") +} +func adminPoolMine(i domain.HumanItemRecord, user string) bool { + return i.AssignedUserID == user || (i.LatestAuthorUserID == user && i.CurrentStageID == "quality_review" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemSubmitted)) +} +func adminPoolSelected(i domain.HumanItemRecord, q AdminAnnotationPoolQuery) bool { + switch q.Pool { + case "available": + return adminPoolAvailable(i) + case "mine": + return adminPoolMine(i, q.UserID) + } + return true +} +func adminPoolBaseQuery(q AdminAnnotationPoolQuery) domain.HumanItemQuery { + return domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: q.ProjectID, WorkflowID: q.WorkflowID, RunID: q.RunID} +} +func adminPoolValidBatch(i domain.HumanItemRecord, b domain.HumanBatchRecord, org string) bool { + if b.ID == "" || !humanOrganizationMatches(b.OrganizationID, org) { + return false + } + for _, st := range b.Stages { + if st.ID == i.CurrentStageID { + return true + } + } + return false +} +func adminPoolScopeFilter(org string, q AdminAnnotationPoolQuery) bson.M { + f := humanItemMongoFilter(org, adminPoolBaseQuery(q)) + f["batchId"] = bson.M{"$ne": ""} + f["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$batchId"}, "string"}}, bson.M{"$eq": bson.A{bson.M{"$type": "$currentStageId"}, "string"}}}} + f["$or"] = bson.A{bson.M{"currentStageId": bson.M{"$in": bson.A{"semantic_annotation", "annotation_combined", "quality_review"}}}, bson.M{"currentStageId": "split_annotation", "status": domain.HumanItemRework}} + return f +} +func adminPoolAvailableFilter() bson.M { + return bson.M{"currentStageId": bson.M{"$in": bson.A{"semantic_annotation", "annotation_combined"}}, "assignedUserId": bson.M{"$in": bson.A{"", nil}}, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}}} +} +func adminPoolMineFilter(user string) bson.M { + return bson.M{"$or": bson.A{bson.M{"assignedUserId": user}, bson.M{"latestAuthorUserId": user, "currentStageId": "quality_review", "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemSubmitted}}}}} +} + +// The same business classification as the workspace, using persisted state. +func adminPoolCategory(i domain.HumanItemRecord, t domain.AnnotationTaskRecord) string { + if i.Status == domain.HumanItemAccepted || t.Status == "accepted" { + return "accepted" + } + if i.PendingCombinedSubmissionID != "" { + prefix := "fanyu-v5-combined-" + i.ID + "-v" + if strings.HasPrefix(i.PendingCombinedSubmissionID, prefix) { + suffix := strings.TrimPrefix(i.PendingCombinedSubmissionID, prefix) + n, err := strconv.ParseInt(suffix, 10, 64) + if err == nil && strings.IndexFunc(suffix, func(r rune) bool { return r < '0' || r > '9' }) < 0 && n >= 0 && n <= 9007199254740991 && int64(i.Version) > n+1 { + return "submission_conflicted" + } + } + return "submitting" + } + if i.ReviewRound > 0 && (i.CurrentStageID == "quality_review" || i.CurrentStageID == "prepackage_acceptance" || i.CurrentStageID == "acceptance") { + return "rework_review" + } + if t.Status == "pending_quality" || i.CurrentStageID == "quality_review" || i.Status == domain.HumanItemSubmitted { + return "pending_quality" + } + if t.Status == "pending_acceptance" || i.CurrentStageID == "acceptance" || i.CurrentStageID == "prepackage_acceptance" { + return "pending_acceptance" + } + if t.Status == "rework" || i.Status == domain.HumanItemRework || (i.ReviewRound > 0 && i.CurrentStageID == "annotation_combined") { + return "rework" + } + workflow := adminPoolMap(i.Subject.Payload["sliceWorkflow"]) + if workflow["status"] == "processing" { + return "slicing" + } + if workflow["status"] == "completed" || workflow["phase"] == "annotate" { + return "pending_annotation" + } + if workflow["status"] == "editing" || workflow["phase"] == "slice" || i.AssignedUserID != "" { + return "pending_slice" + } + return "available" +} + +type adminPoolRow struct { + domain.HumanItemRecord `bson:",inline"` + Tasks []domain.AnnotationTaskRecord `bson:"poolTasks"` + Runs []domain.RunRecord `bson:"poolRuns"` + Captures []domain.CaptureRecord `bson:"poolCaptures"` + Projects []domain.ProjectRecord `bson:"poolProjects"` +} + +func adminPoolMap(v any) map[string]any { + switch x := v.(type) { + case map[string]any: + return x + case bson.M: + return map[string]any(x) + case bson.D: + out := map[string]any{} + for _, e := range x { + out[e.Key] = e.Value + } + return out + } + return nil +} +func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool { + var task domain.AnnotationTaskRecord + if len(row.Tasks) > 0 { + task = row.Tasks[0] + } + if q.NodeID != "" && task.NodeID != q.NodeID { + return false + } + if q.ArtifactID != "" && task.ArtifactID != q.ArtifactID && task.RightArtifactID != q.ArtifactID { + return false + } + if q.Category != "" && q.Category != "all" && adminPoolCategory(row.HumanItemRecord, task) != q.Category { + return false + } + if q.Search == "" && q.Capture == "" { + return true + } + values := []string{} + if len(row.Captures) > 0 { + c := row.Captures[0] + if c.CaptureID != "" { + values = append(values, c.CaptureID) + } else { + values = append(values, domain.CaptureDirectoryFromDirectoryValue(c.Prefix)) + } + } + if len(values) == 0 { + values = append(values, domain.CaptureDirectoryFromArtifactValue(row.Subject.ItemKey)) + domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values) + } + for _, v := range []string{task.ArtifactID, task.SourceVideo} { + values = append(values, domain.CaptureDirectoryFromArtifactValue(v)) + } + if len(row.Runs) > 0 { + values = append(values, domain.CaptureDirectoriesFromRun(row.Runs[0])...) + } + if q.Capture != "" { + found := false + for _, v := range values { + if strings.EqualFold(strings.TrimSpace(v), q.Capture) { + found = true + } + } + if !found { + return false + } + } + if q.Search != "" { + if len(row.Projects) > 0 { + values = append(values, row.Projects[0].Name) + } else if len(row.Runs) > 0 { + values = append(values, row.Runs[0].ProjectName) + } + if !strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search)) { + return false + } + } + return true +} +func (s *MemoryStore) adminPoolRowsLocked(org string, q AdminAnnotationPoolQuery) []adminPoolRow { + rows := []adminPoolRow{} + for _, i := range s.humanItems { + if !humanOrganizationMatches(i.OrganizationID, org) || !humanItemMatches(i, adminPoolBaseQuery(q)) || !adminPoolScope(i) || !adminPoolValidBatch(i, s.humanBatches[i.BatchID], org) { + continue + } + row := adminPoolRow{HumanItemRecord: humanItemListProjectionClone(i, humanItemListPayloadKeysRemoved)} + if t, ok := s.annotationTasks[i.LegacyTaskID]; ok && humanOrganizationMatches(t.OrganizationID, org) { + row.Tasks = []domain.AnnotationTaskRecord{t} + } + if r, ok := s.runs[i.RunID]; ok && humanOrganizationMatches(r.OrganizationID, org) { + row.Runs = []domain.RunRecord{r} + if c, ok := s.captureRecords[r.TaskID]; ok && humanOrganizationMatches(c.OrganizationID, org) { + row.Captures = []domain.CaptureRecord{c} + } + } + if p, ok := s.projects[i.ProjectID]; ok && humanOrganizationMatches(p.OrganizationID, org) { + row.Projects = []domain.ProjectRecord{p} + } + rows = append(rows, row) + } + return rows +} +func (s *MemoryStore) CountAdminAnnotationPools(_ context.Context, org string, q AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) { + s.mu.Lock() + defer s.mu.Unlock() + c := AdminAnnotationPoolCounts{} + q.Search = "" + q.Category = "" + for _, r := range s.adminPoolRowsLocked(org, q) { + if !adminPoolRowMatches(r, q) { + continue + } + c.All++ + if adminPoolAvailable(r.HumanItemRecord) { + c.Available++ + } + if adminPoolMine(r.HumanItemRecord, q.UserID) { + c.Mine++ + } + } + return c, nil +} +func (s *MemoryStore) ListAdminAnnotationPool(_ context.Context, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + rows := s.adminPoolRowsLocked(org, q) + sort.Slice(rows, func(i, j int) bool { + if rows[i].CreatedAt != rows[j].CreatedAt { + return rows[i].CreatedAt > rows[j].CreatedAt + } + return rows[i].ID > rows[j].ID + }) + out := []domain.HumanItemRecord{} + for _, r := range rows { + if adminPoolSelected(r.HumanItemRecord, q) && humanItemAfterCursor(r.HumanItemRecord, q.Cursor) && adminPoolRowMatches(r, q) { + out = append(out, humanItemListProjectionClone(r.HumanItemRecord, humanItemListPayloadKeysRemoved)) + if len(out) >= q.Limit { + break + } + } + } + return out, nil +} + +func adminPoolLookup(from, local, as, org string, projection bson.M) bson.D { + return bson.D{{Key: "$lookup", Value: bson.M{"from": from, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$match": bson.M{"organizationId": normalizedOrganizationID(org)}}, bson.M{"$project": projection}}, "as": as}}} +} +func adminPoolPagePipeline(org string, q AdminAnnotationPoolQuery) mongo.Pipeline { + f := adminPoolScopeFilter(org, q) + conditions := bson.A{f} + if q.Pool == "available" { + conditions = append(conditions, adminPoolAvailableFilter()) + } else if q.Pool == "mine" { + conditions = append(conditions, adminPoolMineFilter(q.UserID)) + } + if q.Cursor != nil { + conditions = append(conditions, bson.M{"$or": bson.A{bson.M{"createdAt": bson.M{"$lt": q.Cursor.CreatedAt}}, bson.M{"createdAt": q.Cursor.CreatedAt, "id": bson.M{"$lt": q.Cursor.ID}}}}) + } + pipeline := mongo.Pipeline{{{Key: "$match", Value: bson.M{"$and": conditions}}}, {{Key: "$sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}}, {{Key: "$project", Value: humanItemListProjection(domain.HumanItemQuery{ListProjection: true})}}} + pipeline = append(pipeline, adminPoolLookup("humanBatches", "batchId", "poolBatches", org, bson.M{"_id": 0, "stages.id": 1}), bson.D{{Key: "$unwind", Value: "$poolBatches"}}, bson.D{{Key: "$match", Value: bson.M{"$expr": bson.M{"$in": bson.A{"$currentStageId", bson.M{"$ifNull": bson.A{"$poolBatches.stages.id", bson.A{}}}}}}}}, bson.D{{Key: "$unset", Value: "poolBatches"}}) + if q.Search != "" || q.Capture != "" || q.Category != "" && q.Category != "all" || q.NodeID != "" || q.ArtifactID != "" { + pipeline = append(pipeline, adminPoolLookup("annotationTasks", "legacyTaskId", "poolTasks", org, bson.M{"_id": 0, "id": 1, "status": 1, "nodeId": 1, "artifactId": 1, "rightArtifactId": 1, "sourceVideo": 1})) + } + if q.Search != "" || q.Capture != "" { + pipeline = append(pipeline, adminPoolLookup("runs", "runId", "poolRuns", org, bson.M{"_id": 0, "id": 1, "taskId": 1, "projectName": 1, "captureDirectories": 1, "inputs.ref": 1, "nodeExecutions.artifacts.id": 1, "nodeExecutions.artifacts.name": 1, "nodeExecutions.artifacts.path": 1, "artifacts.id": 1, "artifacts.name": 1, "artifacts.path": 1}), adminPoolLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org, bson.M{"_id": 0, "captureId": 1, "prefix": 1}), adminPoolLookup("projects", "projectId", "poolProjects", org, bson.M{"_id": 0, "name": 1})) + } + return pipeline +} + +// Stream a narrow metadata cursor; do not materialize the backlog to find an old +// search hit. Unfiltered reads stop after pageSize+1. Context bounds rare scans. +func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q AdminAnnotationPoolQuery, visit func(domain.HumanItemRecord) bool) error { + ctx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + cur, err := s.humanItems.Aggregate(ctx, adminPoolPagePipeline(org, q), options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200)) + if err != nil { + return err + } + defer cur.Close(ctx) + for cur.Next(ctx) { + var row adminPoolRow + if err := cur.Decode(&row); err != nil { + return err + } + if adminPoolRowMatches(row, q) && !visit(row.HumanItemRecord) { + break + } + } + return cur.Err() +} +func (s *MongoStore) ListAdminAnnotationPool(ctx context.Context, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) { + out := []domain.HumanItemRecord{} + err := s.scanAdminAnnotationPool(ctx, org, q, func(i domain.HumanItemRecord) bool { out = append(out, i); return len(out) < q.Limit }) + return out, err +} + +func (s *MongoStore) CountAdminAnnotationPools(ctx context.Context, org string, q AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) { + ctx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + if q.Capture != "" || q.NodeID != "" || q.ArtifactID != "" { + all := q + all.Pool = "all" + all.Search = "" + all.Category = "" + all.Cursor = nil + all.Limit = int(^uint(0) >> 1) + c := AdminAnnotationPoolCounts{} + err := s.scanAdminAnnotationPool(ctx, org, all, func(i domain.HumanItemRecord) bool { + c.All++ + if adminPoolAvailable(i) { + c.Available++ + } + if adminPoolMine(i, q.UserID) { + c.Mine++ + } + return true + }) + return c, err + } + // Group compact routing counters before decoding anything into the API. Same + // projected batch validation used by the existing blind-claim total. + available := bson.M{"$and": bson.A{bson.M{"$in": bson.A{"$currentStageId", bson.A{"semantic_annotation", "annotation_combined"}}}, bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$assignedUserId", ""}}, ""}}, bson.M{"$in": bson.A{"$status", bson.A{"ready", "rework", "submitted"}}}}} + mine := bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$assignedUserId", q.UserID}}, bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$latestAuthorUserId", q.UserID}}, bson.M{"$eq": bson.A{"$currentStageId", "quality_review"}}, bson.M{"$in": bson.A{"$status", bson.A{"ready", "submitted"}}}}}}} + pipeline := mongo.Pipeline{{{Key: "$match", Value: adminPoolScopeFilter(org, q)}}, {{Key: "$group", Value: bson.M{"_id": bson.M{"batch": "$batchId", "stage": "$currentStageId"}, "all": bson.M{"$sum": 1}, "available": bson.M{"$sum": bson.M{"$cond": bson.A{available, 1, 0}}}, "mine": bson.M{"$sum": bson.M{"$cond": bson.A{mine, 1, 0}}}}}}} + cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second)) + if err != nil { + return AdminAnnotationPoolCounts{}, err + } + defer cur.Close(ctx) + var groups []struct { + ID struct{ Batch, Stage string } `bson:"_id"` + AdminAnnotationPoolCounts `bson:",inline"` + } + if err := cur.All(ctx, &groups); err != nil { + return AdminAnnotationPoolCounts{}, err + } + ids := []string{} + seen := map[string]bool{} + for _, g := range groups { + if g.ID.Batch != "" && !seen[g.ID.Batch] { + ids = append(ids, g.ID.Batch) + seen[g.ID.Batch] = true + } + } + batches, err := s.ListHumanBatchAccessRecords(ctx, org, ids) + if err != nil { + return AdminAnnotationPoolCounts{}, err + } + byID := map[string]domain.HumanBatchRecord{} + for _, b := range batches { + byID[b.ID] = b + } + c := AdminAnnotationPoolCounts{} + for _, g := range groups { + if adminPoolValidBatch(domain.HumanItemRecord{CurrentStageID: g.ID.Stage}, byID[g.ID.Batch], org) { + c.All += g.All + c.Available += g.Available + c.Mine += g.Mine + } + } + return c, nil +} diff --git a/internal/store/admin_annotation_pool_test.go b/internal/store/admin_annotation_pool_test.go new file mode 100644 index 00000000..8dfc6617 --- /dev/null +++ b/internal/store/admin_annotation_pool_test.go @@ -0,0 +1,166 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "net/url" + "os" + "reflect" + "strings" + "testing" + "time" +) + +func TestAdminAnnotationPoolMongoParity(t *testing.T) { + uri := os.Getenv("POOL_LOCAL_MONGO_URI") + if uri == "" { + t.Skip("isolated local Mongo opt-in") + } + u, err := url.Parse(uri) + if err != nil || u.Scheme != "mongodb" || u.Host != "127.0.0.1:27018" { + t.Fatal("local Mongo 27018 only") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri)) + if err != nil { + t.Fatal(err) + } + defer client.Disconnect(context.Background()) + db := client.Database(fmt.Sprintf("admin_pool_test_%d", time.Now().UnixNano())) + defer db.Drop(context.Background()) + s := NewMongoStore(db) + if err := s.EnsureIndexes(ctx); err != nil { + t.Fatal(err) + } + mem := NewMemoryStore() + const org = "pool-org" + q := AdminAnnotationPoolQuery{ProjectID: "project", UserID: "admin", Pool: "available", Limit: 201} + batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: org, ProjectID: q.ProjectID, PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages()} + _ = mem.UpsertHumanBatch(ctx, batch) + _ = s.UpsertHumanBatch(ctx, batch) + items, tasks, batchDocs := []any{}, []any{}, []any{} + for n := 0; n < 1500; n++ { + i := domain.HumanItemRecord{ID: fmt.Sprintf("i-%04d", n), OrganizationID: org, ProjectID: q.ProjectID, PluginID: "annotation-qa", BatchID: batch.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemReady, LegacyTaskID: fmt.Sprintf("t-%04d", n), CreatedAt: fmt.Sprintf("2026-10-21T%02d:%02d:%02dZ", n/3600, n/60%60, n%60), Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("CAPTURE_%04d_camera_left_part0001.mp4", n), Payload: map[string]any{"sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}, "annotationWorkspace": map[string]any{"description": strings.Repeat("x", 8192)}}}} + b := batch + b.ID = fmt.Sprintf("batch-%04d", n) + b.RunID = b.ID + batchDocs = append(batchDocs, b) + _ = mem.UpsertHumanBatch(ctx, b) + i.BatchID = b.ID + if n%7 == 0 { + i.AssignedUserID = "admin" + i.Status = domain.HumanItemClaimed + } + if n%11 == 0 { + i.CurrentStageID = "quality_review" + } + if n == 4 { + i.RunID = "custom-run" + } + if n == 1 { + i.BatchID = "missing" + } + if n == 2 { + i.OrganizationID = "other-org" + } + task := domain.AnnotationTaskRecord{ID: i.LegacyTaskID, OrganizationID: i.OrganizationID, ProjectID: i.ProjectID, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: i.Subject.ItemKey} + items = append(items, i) + tasks = append(tasks, task) + _ = mem.UpsertHumanItem(ctx, i) + _ = mem.UpsertAnnotationTask(ctx, task) + } + if _, err := s.humanBatches.InsertMany(ctx, batchDocs); err != nil { + t.Fatal(err) + } + if _, err := s.humanItems.InsertMany(ctx, items); err != nil { + t.Fatal(err) + } + if _, err := s.annotationTasks.InsertMany(ctx, tasks); err != nil { + t.Fatal(err) + } + custom := domain.CaptureRecord{ID: "custom-capture", OrganizationID: org, ProjectID: q.ProjectID, CaptureID: "客户指定Capture"} + run := domain.RunRecord{ID: "custom-run", OrganizationID: org, ProjectID: q.ProjectID, TaskID: custom.ID} + if e := s.UpsertCaptureRecord(ctx, custom); e != nil { + t.Fatal(e) + } + _ = mem.UpsertCaptureRecord(ctx, custom) + if e := s.UpsertRun(ctx, run); e != nil { + t.Fatal(e) + } + _ = mem.UpsertRun(ctx, run) + + start := time.Now() + counts, err := s.CountAdminAnnotationPools(ctx, org, q) + want, _ := mem.CountAdminAnnotationPools(ctx, org, q) + if err != nil || counts != want { + t.Fatalf("counts=%+v want=%+v err=%v", counts, want, err) + } + t.Logf("1500-item counts: %s %+v", time.Since(start), counts) + for _, scenario := range []struct{ pool, search, category string }{{"available", "", ""}, {"all", "CAPTURE_0003", ""}, {"all", "", "pending_slice"}, {"mine", "", ""}, {"all", "[.*", ""}, {"all", "客户指定Capture", ""}} { + q.Pool, q.Search, q.Category = scenario.pool, scenario.search, scenario.category + start = time.Now() + got, err := s.ListAdminAnnotationPool(ctx, org, q) + expected, _ := mem.ListAdminAnnotationPool(ctx, org, q) + if err != nil { + t.Fatal(err) + } + ids := func(items []domain.HumanItemRecord) []string { + out := []string{} + for _, i := range items { + out = append(out, i.ID) + if _, ok := i.Subject.Payload["annotationWorkspace"]; ok { + t.Fatal("heavy workspace leaked") + } + } + return out + } + if !reflect.DeepEqual(ids(got), ids(expected)) { + t.Fatalf("%+v Mongo=%v memory=%v", scenario, ids(got), ids(expected)) + } + t.Logf("%+v page=%d elapsed=%s", scenario, len(got), time.Since(start)) + if len(got) > 201 { + t.Fatal("unbounded page") + } + if scenario.search == "" && len(got) > 0 { + last := got[len(got)-1] + q.Cursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID} + next, err := s.ListAdminAnnotationPool(ctx, org, q) + if err != nil { + t.Fatal(err) + } + for _, i := range next { + for _, old := range got { + if i.ID == old.ID { + t.Fatal("cursor repeated row") + } + } + } + q.Cursor = nil + } + } + scoped := q + scoped.Search = "" + scoped.Category = "" + scoped.Pool = "all" + scoped.Capture = "客户指定Capture" + gotCounts, e := s.CountAdminAnnotationPools(ctx, org, scoped) + wantCounts, _ := mem.CountAdminAnnotationPools(ctx, org, scoped) + if e != nil || gotCounts != wantCounts || gotCounts.All != 1 { + t.Fatalf("capture counts=%+v want=%+v err=%v", gotCounts, wantCounts, e) + } + + // Missing routing IDs must not match an arbitrary batch. + _, err = s.humanItems.InsertOne(ctx, bson.M{"id": "bad", "organizationId": org, "projectId": "project", "pluginId": "annotation-qa", "currentStageId": "annotation_combined", "status": "ready", "createdAt": "z"}) + if err != nil { + t.Fatal(err) + } + again, err := s.CountAdminAnnotationPools(ctx, org, q) + if err != nil || again != counts { + t.Fatalf("missing batch count=%+v err=%v", again, err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index c15e8173..83d29d2a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -309,6 +309,8 @@ type Store interface { GetHumanBatch(context.Context, string) (*domain.HumanBatchRecord, error) UpsertHumanBatch(context.Context, domain.HumanBatchRecord) error ListHumanItems(context.Context, string, domain.HumanItemQuery) ([]domain.HumanItemRecord, error) + CountAdminAnnotationPools(context.Context, string, AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) + ListAdminAnnotationPool(context.Context, string, AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) ListAnnotationClaimCandidates(context.Context, string, string) ([]domain.HumanItemRecord, error) CountAnnotationClaimCandidates(context.Context, string, AnnotationClaimCountQuery) (int, error) GetHumanClaimReceipt(context.Context, string) (*HumanClaimReceipt, error) diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 1abf4870..29b8672b 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -246,6 +246,7 @@ type SliceEditQuickConflict = { dragEdit: { before: SliceEditSnapshot; after?: SliceEditSnapshot } | null; }; +const idleAdminPoolFilters: Parameters[0] = { pool: "available", limit: 200 }; const emptyHumanPools: HumanPools = { available: [], mine: [], all: [], reviewed: [] }; const projectLoadRecoveryAttempts = 3; const projectLoadRecoveryDelayMs = 2_000; @@ -745,6 +746,13 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const [workbenchDrafts, setWorkbenchDrafts] = useState([]); const workbenchHostRef = useRef(null); const [taskSearch, setTaskSearch] = useState(""); + const [adminSearch, setAdminSearch] = useState(""); + const adminPoolAbortRef = useRef(null); + useEffect(() => () => adminPoolAbortRef.current?.abort(), []); + const [adminPoolTotals, setAdminPoolTotals] = useState<{ + scope: string; + counts: { available: number; mine: number; all: number }; + } | null>(null); const [annotatorTaskView, setAnnotatorTaskView] = useState<"active" | "submitted">("active"); const minePoolCursorRef = useRef(""); const [submittedHumanItems, setSubmittedHumanItems] = useState([]); @@ -948,6 +956,47 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: // Acceptance pools are loaded together. Switching their local view must not // reload the same project, evidence prerequisites and package history. const projectPoolView = acceptanceWorkspace ? "all" : taskPoolView; + const managerAnnotationPool = isManager && workspaceMode === "annotate"; + useEffect(() => { + if (!managerAnnotationPool) { + adminPoolAbortRef.current?.abort(); + return; + } + const timer = window.setTimeout(() => setAdminSearch(taskSearch.trim()), 350); + return () => window.clearTimeout(timer); + }, [managerAnnotationPool, taskSearch]); + const managerPoolFilters = useMemo( + () => + managerAnnotationPool + ? { + projectId: projectId || undefined, + workflowId: requestedWorkflowId || undefined, + runId: requestedRunId || undefined, + nodeId: requestedNodeId || undefined, + artifactId: requestedArtifactId || undefined, + capture: requestedCaptureDirectory || undefined, + pool: (taskPoolView === "mine" || taskPoolView === "all" ? taskPoolView : "available") as + | "available" + | "mine" + | "all", + search: adminSearch || undefined, + category: taskStatusFilter, + limit: 200, + } + : idleAdminPoolFilters, + [ + managerAnnotationPool, + projectId, + requestedWorkflowId, + requestedRunId, + requestedNodeId, + requestedArtifactId, + requestedCaptureDirectory, + taskPoolView, + adminSearch, + taskStatusFilter, + ], + ); const projectDataScope = JSON.stringify([ currentUser?.id, role, @@ -958,6 +1007,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: requestedRunId, requestedWorkflowId, annotatorTaskView, + ...(managerAnnotationPool ? [managerPoolFilters] : []), ]); const reviewActionScopeRef = useRef(projectDataScope); if (reviewActionScopeRef.current !== projectDataScope) skipClaimPoolReload.current = ""; @@ -1119,6 +1169,13 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: setLoading(foreground); setRefreshingProject(!foreground); setProjectLoadError(""); + if (managerAnnotationPool) { + adminPoolAbortRef.current?.abort(); + adminPoolAbortRef.current = new AbortController(); + taskPoolCursorRef.current = ""; + setHumanPools(emptyHumanPools); + setTasks([]); + } try { if (blindAnnotator) { // Only load the actor's work, not every available Capture and unrelated @@ -1192,29 +1249,47 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: // database never sorts the whole unassigned backlog (17k docs) before // the role filter drops almost every row. const reviewStageIds = humanItemStageIdsForMode(workspaceMode); - const poolsRequest = acceptanceWorkspace - ? humanLoopApi.listAcceptancePools(humanFilters, currentUser?.id || "", isManager) - : Promise.all([ - humanLoopApi.listItems({ - ...humanFilters, - pool: "available", - ...(reviewStageIds ? { stageIds: reviewStageIds } : {}), - }), - humanLoopApi - .listItemsPage({ + const adminPoolRequest = managerAnnotationPool + ? annotationOperationsApi + .adminPool(managerPoolFilters, { signal: adminPoolAbortRef.current?.signal }) + .then((page) => { + if (requestID === projectLoadRequestRef.current && reviewActionScopeRef.current === projectDataScope) { + taskPoolCursorRef.current = page.nextCursor || ""; + setAdminPoolTotals({ scope: projectDataScope, counts: page.counts }); + } + return page; + }) + : null; + const poolsRequest = adminPoolRequest + ? adminPoolRequest.then((page) => ({ + available: managerPoolFilters.pool === "available" ? page.items : [], + mine: managerPoolFilters.pool === "mine" ? page.items : [], + all: managerPoolFilters.pool === "all" ? page.items : [], + reviewed: [], + })) + : acceptanceWorkspace + ? humanLoopApi.listAcceptancePools(humanFilters, currentUser?.id || "", isManager) + : Promise.all([ + humanLoopApi.listItems({ ...humanFilters, - pool: "mine", - ...(blindAnnotator ? { annotationView: annotatorTaskView } : {}), + pool: "available", ...(reviewStageIds ? { stageIds: reviewStageIds } : {}), - }) - .then((page) => { - if (requestID === projectLoadRequestRef.current && reviewActionScopeRef.current === projectDataScope) - minePoolCursorRef.current = page.nextCursor || ""; - return page.items; }), - isManager ? humanLoopApi.listItems({ ...humanFilters, pool: "all" }) : Promise.resolve([]), - reviewOnly ? humanLoopApi.listItems({ ...humanFilters, pool: "reviewed" }) : Promise.resolve([]), - ]).then(([available, mine, all, reviewed]) => ({ available, mine, all, reviewed })); + humanLoopApi + .listItemsPage({ + ...humanFilters, + pool: "mine", + ...(blindAnnotator ? { annotationView: annotatorTaskView } : {}), + ...(reviewStageIds ? { stageIds: reviewStageIds } : {}), + }) + .then((page) => { + if (requestID === projectLoadRequestRef.current && reviewActionScopeRef.current === projectDataScope) + minePoolCursorRef.current = page.nextCursor || ""; + return page.items; + }), + isManager ? humanLoopApi.listItems({ ...humanFilters, pool: "all" }) : Promise.resolve([]), + reviewOnly ? humanLoopApi.listItems({ ...humanFilters, pool: "reviewed" }) : Promise.resolve([]), + ]).then(([available, mine, all, reviewed]) => ({ available, mine, all, reviewed })); // Task acquisition splits by workspace: every non-manager workspace is // pool-driven and joins to the pool page's legacyTaskIds (the pool // decides visibility; a newest-first org task window can miss pool @@ -1225,6 +1300,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: // role filter empties each fetched window). Only managers keep the // newest-first window with progressive paging. const tasksRequest = (async () => { + if (adminPoolRequest) return (await adminPoolRequest).tasks; if (!isManager) { const pools = await poolsRequest; const ids = [ @@ -1372,6 +1448,8 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: requestedWorkflowId, projectPoolView, annotatorTaskView, + managerAnnotationPool, + managerPoolFilters, ]); useEffect(() => { @@ -1951,6 +2029,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: // List stage — panel search plus status category; scopes only the task list. const matchesPoolListFilters = useCallback( (item: HumanItem) => { + if (managerAnnotationPool) return true; const task = tasks.find((t) => t.id === item.legacyTaskId); const directories = captureDirectoriesFromHumanTask(item, task); if ( @@ -1967,7 +2046,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: : "pending_acceptance"; return taskStatusFilter === "all" || status === taskStatusFilter; }, - [tasks, taskSearch, taskStatusFilter], + [tasks, taskSearch, taskStatusFilter, managerAnnotationPool], ); const matchesPoolFilters = useCallback( (item: HumanItem, pool: TaskPoolView) => matchesPoolScope(item, pool) && matchesPoolListFilters(item), @@ -2510,8 +2589,11 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: [combinedEditorPhase, detail?.humanItem, projects, selectedTaskId, visiblePoolItems, visibleTasks, workflows], ); const filteredVisibleTasks = useMemo( - () => filterAnnotationTasks(visibleTasks, taskContextById, taskSearch, taskStatusFilter), - [taskContextById, taskSearch, taskStatusFilter, visibleTasks], + () => + managerAnnotationPool + ? visibleTasks + : filterAnnotationTasks(visibleTasks, taskContextById, taskSearch, taskStatusFilter), + [taskContextById, taskSearch, taskStatusFilter, visibleTasks, managerAnnotationPool], ); // Progressive rendering cap: a 17k-task pool renders fine in the store but // freezes the tab when every card mounts at once. The pool reveals cards in @@ -2537,6 +2619,31 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: return; setTaskPoolFetchBusy(true); try { + if (managerAnnotationPool) { + const scope = projectDataScope; + const epoch = projectLoadRequestRef.current; + adminPoolAbortRef.current?.abort(); + const controller = new AbortController(); + adminPoolAbortRef.current = controller; + const page = await annotationOperationsApi.adminPool( + { + ...managerPoolFilters, + cursor: taskPoolCursorRef.current, + }, + { signal: controller.signal }, + ); + if (reviewActionScopeRef.current !== scope || projectLoadRequestRef.current !== epoch) return; + taskPoolCursorRef.current = page.nextCursor || ""; + setAdminPoolTotals({ scope, counts: page.counts }); + const pool = managerPoolFilters.pool; + setHumanPools((current) => ({ + ...current, + [pool]: [...current[pool], ...page.items.filter((i) => !current[pool].some((old) => old.id === i.id))], + })); + setTasks((current) => [...current, ...page.tasks.filter((i) => !current.some((old) => old.id === i.id))]); + setTaskRenderLimit((current) => current + 200); + return; + } if (blindAnnotator && taskPoolView === "mine" && minePoolCursorRef.current) { const scope = projectDataScope; const page = await loadAnnotationSidebar(annotationOperationsApi, humanLoopApi, { @@ -2595,8 +2702,16 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const poolCounts = useMemo( // Scope stage only: the search/status panel filters scope the task list // below the tabs, never the pool tab totals (ticket 02). - () => poolTabCounts(humanPools, matchesPoolScope), - [humanPools, matchesPoolScope], + () => + managerAnnotationPool + ? { + ...(adminPoolTotals?.scope === projectDataScope + ? adminPoolTotals.counts + : { available: 0, mine: 0, all: 0 }), + reviewed: 0, + } + : poolTabCounts(humanPools, matchesPoolScope), + [humanPools, matchesPoolScope, managerAnnotationPool, adminPoolTotals, projectDataScope], ); const stageEmptyLabel = workspaceMode === "quality" @@ -6688,11 +6803,13 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: className={`rounded-lg border px-3 py-2 text-xs transition-colors ${taskPoolView === value ? "border-blue-500 bg-blue-50 text-blue-600 dark:bg-blue-500/10 dark:text-blue-200 font-medium" : "border-[color:var(--app-border)] text-slate-600 hover:text-slate-900 dark:text-gray-400 dark:hover:text-white"}`} > {label}{" "} - {blindAnnotator && value === "available" - ? claimableCount.projectId === projectId - ? (claimableCount.count ?? "—") - : "—" - : count} + {managerAnnotationPool && adminPoolTotals?.scope !== projectDataScope + ? "—" + : blindAnnotator && value === "available" + ? claimableCount.projectId === projectId + ? (claimableCount.count ?? "—") + : "—" + : count} {/* Counts only loaded pages; a further page means more exist. */} {blindAnnotator && value === "mine" && minePoolCursorRef.current ? "+" : ""} diff --git a/src/services/api/annotation-operations.ts b/src/services/api/annotation-operations.ts index 2c870d6e..38150c6c 100644 --- a/src/services/api/annotation-operations.ts +++ b/src/services/api/annotation-operations.ts @@ -61,6 +61,29 @@ export const annotationOperationsApi = { apiRequest<{ items: HumanItem[]; tasks: AnnotationTaskRecord[]; nextCursor?: string; hasMore?: boolean }>( `/api/plugins/annotation-qa/sidebar${query(filters)}`, ), + adminPool: ( + filters: { + projectId?: string; + workflowId?: string; + runId?: string; + nodeId?: string; + artifactId?: string; + capture?: string; + search?: string; + category?: string; + pool: "available" | "mine" | "all"; + cursor?: string; + limit?: number; + }, + options: { signal?: AbortSignal } = {}, + ) => + apiRequest<{ + items: HumanItem[]; + tasks: AnnotationTaskRecord[]; + counts: { available: number; mine: number; all: number }; + nextCursor?: string; + hasMore?: boolean; + }>(`/api/plugins/annotation-qa/admin-pool${query(filters)}`, options), claimCount: (projectId: string) => apiRequest<{ projectId: string; count: number }>( `/api/plugins/annotation-qa/claim-count?projectId=${encodeURIComponent(projectId)}`, diff --git a/tests/browser/admin-annotation-pool-replica.mjs b/tests/browser/admin-annotation-pool-replica.mjs new file mode 100644 index 00000000..fc95da50 --- /dev/null +++ b/tests/browser/admin-annotation-pool-replica.mjs @@ -0,0 +1,114 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { chromium } from "playwright"; +import { createServer } from "vite"; +const api = process.env.ADMIN_POOL_API; +assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); +const out = path.resolve("local-data/test-results/admin-annotation-pool"); +fs.mkdirSync(out, { recursive: true }); +const report = { pageErrors: [], requests: [], writes: [], cases: [] }; +const server = await createServer({ + server: { host: "127.0.0.1", port: 5205, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, +}); +let browser, page; +let releaseSlow; +let delaySearch = ""; +try { + await server.listen(); + browser = await chromium.launch({ + executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + headless: true, + args: ["--no-proxy-server"], + }); + page = await browser.newPage({ viewport: { width: 1600, height: 1100 } }); + page.setDefaultTimeout(15000); + page.on("pageerror", (e) => report.pageErrors.push(e.message)); + await page.route("**/*", async (route) => { + const r = route.request(), + u = new URL(r.url()); + if (u.hostname !== "127.0.0.1") return route.abort(); + if (u.pathname.includes("/worker-api/")) return route.fulfill({ json: { status: "ok" } }); + if (!["GET", "HEAD"].includes(r.method())) report.writes.push(u.pathname); + if (u.pathname.includes("/admin-pool")) { + report.requests.push(Object.fromEntries(u.searchParams)); + if (delaySearch && u.searchParams.get("search") === delaySearch) { + delaySearch = ""; + const response = await route.fetch(); + await new Promise((r) => (releaseSlow = r)); + return route.fulfill({ response }); + } + } + return route.continue(); + }); + const base = "http://127.0.0.1:5205/plugins/annotation-qa/workflows/all/annotate?projectId=draft-project"; + await page.goto(base); + const available = page.getByTestId("human-task-pool-available"), + all = page.getByTestId("human-task-pool-all"), + cards = page.getByTestId("annotation-task-card"); + await available.getByText("可领取 606", { exact: true }).waitFor(); + await all.getByText("全部任务 611", { exact: true }).waitFor(); + assert.equal(await cards.count(), 200); + await page.getByTestId("task-pool-show-more").click(); + await page.waitForTimeout(500); + assert.equal(await cards.count(), 400); + assert.match(await available.innerText(), /606/); + report.cases.push("total independent of human cursor pages"); + const search = page.getByRole("textbox", { name: "搜索标注任务", exact: true }); + await search.fill("CAPTURE_0001"); + await cards.filter({ hasText: "CAPTURE_0001" }).waitFor(); + assert.equal(await cards.count(), 1); + assert.match(await available.innerText(), /606/); + report.cases.push("old item search beyond first500"); + await page.getByRole("combobox", { name: "按任务状态筛选", exact: true }).selectOption("pending_acceptance"); + await page.waitForTimeout(700); + assert.equal(await cards.count(), 0); + assert.match(await available.innerText(), /606/); + report.cases.push("category does not change total"); + await page.getByRole("combobox", { name: "按任务状态筛选", exact: true }).selectOption("all"); + await search.fill(""); + await available.getByText("可领取 606", { exact: true }).waitFor(); + await page.waitForTimeout(500); + delaySearch = "CAPTURE_0002"; + await search.fill(delaySearch); + for (let n = 0; n < 40 && !releaseSlow; n++) await page.waitForTimeout(50); + assert.ok(releaseSlow, "held older search response"); + await search.fill("CAPTURE_0003"); + await cards.filter({ hasText: "CAPTURE_0003" }).waitFor(); + releaseSlow(); + releaseSlow = null; + await page.waitForTimeout(300); + assert.equal(await cards.count(), 1); + assert.match(await cards.first().innerText(), /CAPTURE_0003/); + report.cases.push("stale search response ignored"); + await search.fill(""); + await all.click(); + await page.waitForTimeout(700); + await page.getByRole("combobox", { name: "按任务状态筛选", exact: true }).selectOption("pending_quality"); + for (let n = 0; n < 80 && (await cards.count()) !== 4; n++) await page.waitForTimeout(50); + assert.equal(await cards.count(), 4); + assert.match(await all.innerText(), /611/); + assert.match(await available.innerText(), /606/); + report.cases.push("all pool and server status filter include older review-stage rows"); + const claimed = await fetch(api + "/__fixture/claim", { method: "POST" }); + assert.equal(claimed.status, 200); + await page.reload(); + await available.getByText("可领取 605", { exact: true }).waitFor(); + assert.match(await all.innerText(), /611/); + report.cases.push("real claim invalidates cached total"); + assert.deepEqual(report.pageErrors, []); + assert.deepEqual(report.writes, []); + console.log(JSON.stringify(report)); +} catch (e) { + report.failure = e.message; + if (page) { + fs.writeFileSync(path.join(out, "failure.txt"), await page.locator("body").innerText()); + await page.screenshot({ path: path.join(out, "failure.png") }); + } + throw e; +} finally { + releaseSlow?.(); + fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2)); + await browser?.close(); + await server.close(); +} From 3221ac65e4669c2872065b7ac0965694f56a937e Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 21 Oct 2026 16:30:38 +0800 Subject: [PATCH 12/54] =?UTF-8?q?=F0=9F=90=9B=20Preserve=20annotation=20dr?= =?UTF-8?q?afts=20and=20recover=20source=20video=20playback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adr/0010-source-backed-slice-revisions.md | 8 + docs/slice-name-dictionary.md | 21 ++- .../annotation-failures-20261021.md | 112 ++++++++++++ .../api/annotation_confirmation_readiness.go | 51 ++++++ internal/api/annotation_editor_state_test.go | 52 ++++++ .../api/annotation_feedback_names_test.go | 55 ++++++ internal/api/annotation_operations.go | 39 +++- internal/api/annotation_source_certificate.go | 7 +- internal/api/annotation_source_movie_clock.go | 85 +++++++++ .../api/annotation_source_movie_clock_test.go | 78 ++++++++ internal/api/human_media_head_test.go | 48 +++++ internal/api/rework_boundary_browser_test.go | 39 +++- internal/api/routes_human_loop.go | 3 + internal/api/slice_dictionary_names.go | 2 +- internal/api/slice_retry.go | 30 ++++ internal/api/slice_retry_test.go | 148 +++++++++++++++ internal/api/testdata/routes.golden | 6 +- public/vendor/movi-player-0.4.1/README.md | 19 +- src/components/CaptureAnnotationWorkbench.tsx | 14 +- src/components/MoviVideo.tsx | 21 +++ src/hooks/useCaptureVideoTransport.ts | 4 +- src/lib/annotation-workspace.js | 19 ++ src/pages/AnnotationOperations.tsx | 39 ++-- src/services/api/annotation-operations.ts | 6 + src/types.ts | 5 + .../annotation-media-recovery-replica.mjs | 169 ++++++++++++++++++ tests/browser/rework-boundary-replica.mjs | 32 ++++ .../frontend/annotation-editor-state.test.mjs | 45 +++++ 28 files changed, 1120 insertions(+), 37 deletions(-) create mode 100644 docs/verification/annotation-failures-20261021.md create mode 100644 internal/api/annotation_editor_state_test.go create mode 100644 internal/api/annotation_feedback_names_test.go create mode 100644 internal/api/annotation_source_movie_clock.go create mode 100644 internal/api/annotation_source_movie_clock_test.go create mode 100644 internal/api/human_media_head_test.go create mode 100644 tests/browser/annotation-media-recovery-replica.mjs create mode 100644 tests/frontend/annotation-editor-state.test.mjs diff --git a/docs/adr/0010-source-backed-slice-revisions.md b/docs/adr/0010-source-backed-slice-revisions.md index ac2914f0..52f661bc 100644 --- a/docs/adr/0010-source-backed-slice-revisions.md +++ b/docs/adr/0010-source-backed-slice-revisions.md @@ -30,6 +30,14 @@ 新的 JSON-only 源索引交付是 **v2 协议**,含固定完整源的逻辑键、哈希、双目帧区间。它只能交给有权读取完整源、承诺按区间解析的下游。完整源含舍弃区间,因此不能把它视为“物理交付视频不含舍弃区间”的等价替代。该变化尚未得到交付协议批准,必须同时显式设置 `PART_INDEX_DELIVERY=true` 和 `SOURCE_INDEX_DELIVERY=true`;仅有旧开关时拒绝新源索引 JSON 交付。目录交付无需放宽旧契约。上线启用前必须确认实际工作流使用哪种交付方式。 +## 2026-10-21:MP4 呈现终点量化兼容与旧任务恢复 + +逐帧时间戳和解码帧数检查保持原要求。总时长检查还需考虑 MP4 编辑列表使用的 movie timescale,而非只看视频 track time base:新的终点容差为原容差与 `min(movie tick + track tick + 浮点余量, 半个帧周期)` 的较大值。无法读取有效 movie clock 时维持原容差,不扩大放行范围。 + +B06/part0002、B14/part0001、B18/part0003 的本地样本均为连续 30fps,packet 时长全部是 512 track tick;编辑列表导致呈现总时长少 5 个 track tick,约 0.326ms。不得将此误认为丢帧,也不得直接删除时长检查或统一放宽一帧。兼容处理不改源文件、硬件 PTS、统一边界或已有证书,不以忽略编辑列表重新定义播放起点。整帧截短、帧数差异和逐帧断档仍拒绝。 + +没有 `sliceWorkflow` 的旧任务仍按已有正式生成证据识别。只有草稿而没有有效生成证据的任务不能凭前端推断进入标注;允许操作者显式按服务器保存的完整快照恢复生成,复用持久作业、租约和版本竞争机制,并保留动作、描述及分类。已确认工作区之外还存在较新的本人自动保存草稿时,恢复采用该草稿,与页面恢复内容一致;归属、版本、时间或数据格式无法核对时停止,不能猜选或清空。恢复模式需跨失败重试保留,避免再次退回旧工作区。恢复不是清空后重新切片。 + ## 成本、兼容性与回退 - 固定源每 Run/源版本复制一次,不是跨 Run 去重;首用仍消耗解码 CPU、读取带宽与一份源大小的存储,之后修订为 JSON。 diff --git a/docs/slice-name-dictionary.md b/docs/slice-name-dictionary.md index 48ca26df..0953602d 100644 --- a/docs/slice-name-dictionary.md +++ b/docs/slice-name-dictionary.md @@ -7,9 +7,9 @@ for this project's selection, so other projects sharing an entry are unaffected. In the annotator's slice phase, **从切片字典选择** searches active names enabled for the current project and fills the selected slice's existing `name` field. -The name remains editable. Selection does not seek video, change boundaries or +The name is selected from the project dictionary. Selection does not seek video, change boundaries or submit work; **确认全部切片** retains the existing save/materialization flow. -No dictionary means manual naming still works. Editing/removing dictionary +An absent or unavailable dictionary blocks a new slice confirmation. Editing/removing dictionary entries never rewrites already saved slice names or historical delivery data. ## Compatibility and API @@ -26,7 +26,22 @@ entries never rewrites already saved slice names or historical delivery data. - Project selections store `sliceLabelIds` independently from existing `labelIds`. Memory/Mongo updates change only the relevant dictionary selection. - Task/project switches abort stale reads; reopening the picker refreshes names. - Load failures do not disable manual name entry. + Load failures retain saved names and expose retry rather than manual name entry. + +## Post-confirmation naming and submission + +The annotation view has a visible **选择切片名称 / 修改切片名称** button. It opens +the existing atomic slice-edit panel; choosing a name preserves action content, +boundaries and history. E remains a keyboard entry. Renaming never requires +the destructive initial **重新切片** operation. + +Unnamed kept slices may be saved as drafts while editing. Formal submission +requires a nonempty classification name, including after a split/merge was +saved: an unchanged empty workspace name is not a historical-name exemption. +Unchanged nonempty historical names retain their existing compatibility rule; +new/changed names still require the enabled project dictionary. This rule does +not prohibit unnamed or pending actions, which remain distinct from invalid or +discarded data. Existing deliveries and customer records are not rewritten. Tests cover legacy action compatibility, purpose and project isolation, permissions, independent updates, removal without history mutation, and the existing name callback. diff --git a/docs/verification/annotation-failures-20261021.md b/docs/verification/annotation-failures-20261021.md new file mode 100644 index 00000000..b987c7f8 --- /dev/null +++ b/docs/verification/annotation-failures-20261021.md @@ -0,0 +1,112 @@ +# Annotation generation, naming and media recovery + +Scope: October 21 platform-tuning reports B06, B14, B18, B11 and B20. Production +inspection is read-only. Tests use local synthetic media and isolated stores. + +## Evidence + +- B06/B14/B18 downloaded left videos have 6343/7957/11239 continuous 30fps + frames. Their stream duration is approximately 0.326ms short. All packet + durations are 512 ticks; the difference comes from MP4 edit-list presentation + duration. Unedited duration and decoded cadence agree. This does not establish + the state of every right-camera source in production. +- The user confirmed the 15:26 report is B11/part0001. Its task has no + `sliceWorkflow` or completed materialization evidence, no annotation revision, + and only an old slice revision 0. Its saved workspace contains seven slices + and six actions; its draft contains seven slices and ten actions. Frontend + draft inference can show annotation while the write gate rejects it. Merely + seeing numbered slices does not prove successful generation. +- B20/part0001 recording shows repeated HEAD 405 responses on the authenticated + `subject-content` URL during recovery. Both referenced BOS camera objects + exist; metadata and 1KiB range reads succeed (226335649/217067600 bytes). + This verifies the recovery blocker, without claiming the recording proves + what first interrupted native playback. Its later generation job completed. +- A local HTTP flow reproduced named parent → two unnamed children → saved + workspace → formal submission 201. Unchanged blank names bypassed the old + historical-name exception. + +Redacted source metadata and recording are under the main repository's +untracked `local-data/diagnostics/annotation-failures-20261021/`. + +## Behavior and compatibility + +- Keep strict decoded frame/PTS checks; bound presentation endpoint rounding + by the actual MP4 movie clock and preserve rejection of whole-frame loss. +- Detail GET ships `editorState.phase`, `canConfirmDescriptions` and + `canRecoverGeneration`. The existing description gate remains authoritative. + Status-only polling does not perform the additional legacy evidence lookup. +- Explicit `recoverExisting` on slice confirmation is restricted to legacy + full-source tasks missing generation evidence. The server owns its retained + snapshot; client-supplied replacement snapshots are ignored. If a newer, + owned autosaved draft exists alongside the confirmed workspace, recovery uses + that draft, matching the editor's restored view. Ambiguous ownership, version, + timestamps or malformed content stop recovery without mutation. Queueing, + interruption/reclaim, failure/retry and completion preserve the selected work. + Ordinary failed-generation retries retain their existing confirmation policy. +- A visible post-confirmation name button uses the atomic edit application. + Blank kept names block formal submission; nonempty historical compatibility, + pending actions and discard evidence remain intact. +- Subject and scoped evidence media routes accept HEAD with the same access + checks as GET. The existing storage handler returns metadata without reading + the video body. Hidden cameras remain unloaded. +- The movi wrapper clears its exposed error for each new load instead of + retaining the vendor's previous failure. Transport sets the custom element's + source once, avoiding a second concurrent decoder load. Vendor assets and + media sources are unchanged. +- The recovery mode remains marked in `sliceWorkflow` until completion, so a + failure followed by another retry cannot silently select the older workspace. + API and annotation processing components need compatible versions for this + recovery path. Source-validation processes also need the clock fix before + retrying the affected source checks. Do not roll active recovery jobs back to + an API that lacks these retention rules. + +## Verification + +- Focused HTTP/API tests pass for server-owned editor readiness, blank-name + submission rejection, name correction, authenticated HEAD and denied access. +- Real MP4 regressions pass for 6343/7957/11239 frames with movie-clock endpoint + rounding. Mismatched frame counts and a whole-frame endpoint truncation remain + rejected. Invalid movie metadata retains strict tolerance. +- `TestLegacyGenerationRecoveryPreservesAnnotationsAcrossRestart` passes: + a newer autosaved draft survives queue interruption/reclaim, a real processor + rejection, retry and successful generation. Duplicate recovery and replacement + snapshot attempts cannot overwrite it. Ambiguous drafts reject without writes. +- The full local page → HTTP handler → isolated store browser flow passes: + the recovery button preserves the newer draft and waits for generation before + entering annotation; post-confirmation naming, first-pass/rework drag edits, + archived discard-side actions, failed saves, lost responses, reload and + persisted undo/redo also pass. +- Real `` software fallback passes injected media failure → reload + for annotator, quality and acceptance views. HEAD succeeds, the hidden camera + stays unloaded, editing position and unsaved description survive, and playback + advances after recovery. Review views do not expose the new naming control. +- The frontend and Go repository gates report no new failures against + `tests/known-failures.json` (13 frontend failures in the 14-case baseline, + 70 Go failures in the 70-case baseline). The baseline is unchanged. +- A subsequent Go run also exposed + `TestDispatchTwoWorkersAndRestartDoNotDuplicateClaims`: it intermittently + records six distinct claims where the test expects three. The exact failure + reproduces 15 times in 3000 repetitions on pristine commit `2876a14d`, before + this change. Worker, store and domain files are unchanged. This independent + concurrency issue remains unresolved and is not added to the known-failure + baseline; its failed run is retained alongside the successful gate evidence. +- The pristine full gate also exposes the preceding commit's stale route + snapshot (the admin-pool route was already present). This change regenerates + the snapshot from the actual router, including that route and the new HEAD + routes; it does not change admin-pool access behavior. +- Type checking, formatting, Go build, web build and the production-dist browser + boot check pass. The boot check uses loopback API stubs, with no page errors or + unexpected writes. + +Local browser regression commands: + +```sh +REWORK_BOUNDARY_BROWSER_TEST=1 go test ./internal/api -run '^TestReworkBoundaryBrowserReplica$' -count=1 -v +REWORK_BOUNDARY_BROWSER_TEST=1 ANNOTATION_FEEDBACK_BROWSER_SCRIPT=tests/browser/annotation-media-recovery-replica.mjs go test ./internal/api -run '^TestReworkBoundaryBrowserReplica$' -count=1 -v +``` + +The browser runs use local Edge/Chromium and synthetic media; they do not prove +every customer's decoder or network. The original cause of B20's initial native +playback interruption remains unverified. Production B11 generation recovery +has not been executed. No rollout, customer-record repair, index migration or +object-storage write is part of this change. diff --git a/internal/api/annotation_confirmation_readiness.go b/internal/api/annotation_confirmation_readiness.go index 2315d674..53d595f3 100644 --- a/internal/api/annotation_confirmation_readiness.go +++ b/internal/api/annotation_confirmation_readiness.go @@ -7,6 +7,57 @@ import ( "droboticflow/internal/domain" ) +type annotationEditorState struct { + Phase string `json:"phase"` + CanConfirmDescriptions bool `json:"canConfirmDescriptions"` + CanRecoverGeneration bool `json:"canRecoverGeneration"` +} + +// Use the same persisted evidence as the write gate, including legacy tasks +// without sliceWorkflow. Restored draft actions do not prove generation ready. +func (server *server) annotationEditorStateForTask(ctx context.Context, task domain.AnnotationTaskRecord, item *domain.HumanItemRecord) (annotationEditorState, error) { + ready, err := server.legacyAnnotationConfirmationReady(ctx, task, item) + state := annotationEditorState{Phase: "slice", CanConfirmDescriptions: ready} + if err != nil { + return state, err + } + if ready { + state.Phase = "annotate" + return state, nil + } + if item != nil { + state.CanRecoverGeneration = legacyGenerationRecoveryCandidate(task, item) + workflow := mapValue(item.Subject.Payload["sliceWorkflow"]) + status := stringValue(workflow["status"]) + jobStatus := "" + if item.ProcessingJob != nil { + jobStatus = item.ProcessingJob.Status + } + if status == "failed" || jobStatus == "failed" { + return state, nil + } + if status == "processing" || status == "queued" || status == "running" || status == "materializing" || jobStatus == "queued" || jobStatus == "running" || jobStatus == "processing" || item.PendingCombinedSubmissionID != "" { + state.Phase = "materializing" + } + } + return state, nil +} + +func legacyGenerationRecoveryCandidate(task domain.AnnotationTaskRecord, item *domain.HumanItemRecord) bool { + if item == nil || !task.SourceIsFullVideo || item.PendingCombinedSubmissionID != "" || item.Subject.Payload["sliceWorkflow"] != nil { + return false + } + if item.ProcessingJob != nil && item.ProcessingJob.Status != "completed" { + return false + } + for _, workspace := range []map[string]any{annotationWorkspacePayload(item), mapValue(item.Subject.Payload["annotationDraft"])} { + if len(decodeMaterializationValue[domain.SliceSegment](workspace["sliceSegments"])) > 0 && len(decodeMaterializationValue[domain.AnnotationSegment](workspace["segments"])) > 0 { + return true + } + } + return false +} + // legacyAnnotationConfirmationReady recognizes persisted pre-workflow evidence // for tasks whose old records do not have sliceWorkflow. It never mutates the // item; the caller remains responsible for checking the submitted version. diff --git a/internal/api/annotation_editor_state_test.go b/internal/api/annotation_editor_state_test.go new file mode 100644 index 00000000..9750c846 --- /dev/null +++ b/internal/api/annotation_editor_state_test.go @@ -0,0 +1,52 @@ +package api + +import ( + "encoding/json" + "testing" + "time" + + "droboticflow/internal/domain" +) + +func TestAnnotationDetailSharesDescriptionConfirmationReadiness(t *testing.T) { + for _, tc := range []struct { + name, status, phase, expected string + ready bool + }{ + {"completed", "completed", "annotate", "annotate", true}, + {"failed-with-old-phase", "failed", "annotate", "slice", false}, + {"failed-with-draft", "failed", "", "slice", false}, + {"inconsistent-completed", "completed", "slice", "slice", false}, + {"processing", "processing", "slice", "materializing", false}, + {"legacy-confirmed", "", "", "annotate", true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := continuousFixture(t) + item := f.item + item.Subject.Payload["annotationWorkspace"] = map[string]any{"sliceSegments": []domain.SliceSegment{{ID: "s", Name: "测试切片", Start: 0, End: 10}}, "segments": []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Status: "pending"}}} + if tc.status != "" { + item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": tc.status, "phase": tc.phase, "leaseExpiresAt": time.Now().Add(time.Hour).UTC().Format(time.RFC3339Nano)} + } + if err := f.store.UpsertHumanItem(f.ctx, item); err != nil { + t.Fatal(err) + } + r := f.request("GET", "/api/plugins/annotation-qa/tasks/"+f.task.ID, f.annotator, nil) + if r.Code != 200 { + t.Fatalf("detail: %d %s", r.Code, r.Body.String()) + } + var body struct { + EditorState annotationEditorState `json:"editorState"` + } + if err := json.Unmarshal(r.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.EditorState.Phase != tc.expected || body.EditorState.CanConfirmDescriptions != tc.ready { + t.Fatalf("state=%+v", body.EditorState) + } + ready, err := (&server{store: f.store}).legacyAnnotationConfirmationReady(f.ctx, f.task, &item) + if err != nil || ready != body.EditorState.CanConfirmDescriptions { + t.Fatalf("gate differs: %v %v", ready, err) + } + }) + } +} diff --git a/internal/api/annotation_feedback_names_test.go b/internal/api/annotation_feedback_names_test.go new file mode 100644 index 00000000..c6935234 --- /dev/null +++ b/internal/api/annotation_feedback_names_test.go @@ -0,0 +1,55 @@ +package api + +import ( + "encoding/json" + "strings" + "testing" + + "droboticflow/internal/domain" +) + +func TestNewUnnamedSplitCannotBecomeHistoricalNameAtSubmission(t *testing.T) { + f := continuousFixture(t) + seedCompletedMaterialization(t, f, 1) + item, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + item.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "completed", "phase": "annotate"} + item.Subject.Payload["annotationWorkspace"] = map[string]any{ + "sliceSegments": []domain.SliceSegment{{ID: "parent", Name: "测试切片", Start: 0, End: 10}}, + "segments": []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Status: "pending"}}, + } + if err := f.store.UpsertHumanItem(f.ctx, *item); err != nil { + t.Fatal(err) + } + path := "/api/plugins/annotation-qa/tasks/" + f.task.ID + split := []domain.SliceSegment{{ID: "child-a", SourceIDs: []string{"parent"}, Start: 0, End: 5}, {ID: "child-b", SourceIDs: []string{"parent"}, Start: 5, End: 10}} + actions := []domain.AnnotationSegment{{ID: "a1", Start: 0, End: 5, Status: "pending"}, {ID: "a2", Start: 5, End: 10, Status: "pending"}} + r := f.request("POST", path+"/slice-edit-applications", f.annotator, sliceEditApplicationInput{ApplicationID: "unnamed-split", combinedAnnotationDraftInput: combinedAnnotationDraftInput{Version: item.Version, SliceSegments: split, Segments: actions}}) + if r.Code != 200 { + t.Fatalf("save split: %d %s", r.Code, r.Body.String()) + } + var got struct { + Workspace combinedAnnotationDraftInput `json:"workspace"` + } + if err := json.Unmarshal(r.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + item, _ = f.store.GetHumanItem(f.ctx, item.ID) + got.Workspace.Version = item.Version + r = f.request("POST", path+"/combined-submissions", f.annotator, got.Workspace) + if r.Code != 400 || !strings.Contains(r.Body.String(), "slice_dictionary_name_required") { + t.Fatalf("unnamed new slices submitted: %d %s", r.Code, r.Body.String()) + } + unchanged, _ := f.store.GetHumanItem(f.ctx, item.ID) + if unchanged.Version != item.Version || unchanged.CurrentStageID != item.CurrentStageID { + t.Fatal("rejection changed the task") + } + // Naming the slices must preserve the anonymous pending actions, which are + // valid classification results and must never be silently marked invalid. + for i := range got.Workspace.SliceSegments { + got.Workspace.SliceSegments[i].Name = "测试切片" + } + r = f.request("POST", path+"/combined-submissions", f.annotator, got.Workspace) + if r.Code != 201 { + t.Fatalf("named split: %d %s", r.Code, r.Body.String()) + } +} diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index b4146954..7d8c0f0e 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -1090,7 +1090,12 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques } linkedHumanItem.CanClaim = &claimable } - renderJSON(writer, http.StatusOK, map[string]any{"task": item, "sliceRevisions": sliceRevisions, "revisions": revisions, "reviews": reviews, "humanItem": linkedHumanItem}) + editorState, readinessErr := server.annotationEditorStateForTask(request.Context(), *item, linkedHumanItem) + if readinessErr != nil { + renderError(writer, http.StatusInternalServerError, readinessErr) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"task": item, "sliceRevisions": sliceRevisions, "revisions": revisions, "reviews": reviews, "humanItem": linkedHumanItem, "editorState": editorState}) } func annotationFrameTimelinePayload(camera string, rows []hardwarePTSRow) (map[string]any, error) { @@ -1415,6 +1420,7 @@ type combinedAnnotationDraftInput struct { } type sliceConfirmationInput struct { + RecoverExisting bool `json:"recoverExisting,omitempty"` RetryFailed bool `json:"retryFailed,omitempty"` RetryWorkspace map[string]any `json:"retryWorkspace,omitempty"` Version int `json:"version"` @@ -2135,7 +2141,25 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter, } payload.RetryWorkspace = nil // Private durable snapshot is always server-owned. var retained combinedAnnotationDraftInput - if payload.RetryFailed { + if payload.RecoverExisting { + ready, err := server.legacyAnnotationConfirmationReady(request.Context(), *task, item) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if ready || !legacyGenerationRecoveryCandidate(*task, item) { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "当前任务不需要恢复历史切片生成,已有内容仍保留", "code": "slice_recovery_unavailable"}) + return + } + var recoveryErr error + payload, retained, recoveryErr = retainedSliceRecoveryInput(item, payload.Version) + if recoveryErr != nil { + renderJSON(writer, http.StatusUnprocessableEntity, map[string]string{"error": recoveryErr.Error(), "code": "slice_retry_workspace_invalid"}) + return + } + payload.RecoverExisting = true + payload.ChangeReason = "恢复历史切片生成并保留已有标注" + } else if payload.RetryFailed { workflow := mapValue(item.Subject.Payload["sliceWorkflow"]) job := item.ProcessingJob if stringValue(workflow["status"]) != "failed" || (job != nil && (job.Status == "queued" || job.Status == "running" || job.Kind != "slice")) { @@ -2143,7 +2167,13 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter, return } var retryErr error - payload, retained, retryErr = retainedSliceRetryInput(item, payload.Version) + recoverExisting := boolValue(workflow["recoverExisting"]) + if recoverExisting { + payload, retained, retryErr = retainedSliceRecoveryInput(item, payload.Version) + payload.RecoverExisting = true + } else { + payload, retained, retryErr = retainedSliceRetryInput(item, payload.Version) + } if retryErr != nil { renderJSON(writer, http.StatusUnprocessableEntity, map[string]string{"error": retryErr.Error(), "code": "slice_retry_workspace_invalid"}) return @@ -2232,6 +2262,9 @@ func (server *server) handleSliceConfirmationCreate(writer http.ResponseWriter, // mutations remain blocked by the existing processing guard. workflow := mapValue(item.Subject.Payload["sliceWorkflow"]) workflow["retryOf"] = retryOf + if payload.RecoverExisting { + workflow["recoverExisting"] = true + } item.Subject.Payload["sliceWorkflow"] = workflow } else { delete(item.Subject.Payload, "annotationDraft") diff --git a/internal/api/annotation_source_certificate.go b/internal/api/annotation_source_certificate.go index 948828f3..f5246eb7 100644 --- a/internal/api/annotation_source_certificate.go +++ b/internal/api/annotation_source_certificate.go @@ -242,9 +242,10 @@ func validateSourceMediaClock(ctx context.Context, path string, count int, durat period := 1 / rate // r_frame_rate is only a candidate cadence: verify every decoded frame // against it below, rather than trusting the metadata or deriving a cadence - // from rounded total duration. The final frame timestamp and packet duration - // can each round, so allow one time-base tick at the end of the stream. - if math.IsInf(period, 0) || math.Abs(duration-float64(count)*period) > math.Max(0.0001, tick+1e-9) { + // from rounded total duration. MP4 edit lists additionally round the + // presentation endpoint in the movie clock (often 1ms, even for a video + // track at 1/15360). This never relaxes the decoded frame checks below. + if math.IsInf(period, 0) || math.Abs(duration-float64(count)*period) > sourceClockEndTolerance(path, tick, period) { return fmt.Errorf("source video duration does not match its fixed frame cadence") } // Integer timestamps preserve half-tick boundaries; ffprobe's *_time diff --git a/internal/api/annotation_source_movie_clock.go b/internal/api/annotation_source_movie_clock.go new file mode 100644 index 00000000..64ab6ecc --- /dev/null +++ b/internal/api/annotation_source_movie_clock.go @@ -0,0 +1,85 @@ +package api + +import ( + "encoding/binary" + "fmt" + "io" + "math" + "os" +) + +// MP4 edit-list durations use the movie clock, not the video's track clock. +// Read only atom headers and mvhd: a large mdat is skipped without loading it. +func sourceMovieClockTick(path string) (float64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return 0, err + } + var scan func(int64, int64, bool) (float64, error) + scan = func(start, end int64, inMovie bool) (float64, error) { + for offset, boxes := start, 0; offset+8 <= end && boxes < 4096; boxes++ { + var header [16]byte + if _, err := f.ReadAt(header[:8], offset); err != nil { + return 0, err + } + size, headerSize := uint64(binary.BigEndian.Uint32(header[:4])), int64(8) + if size == 1 { + if _, err := f.ReadAt(header[8:], offset+8); err != nil { + return 0, err + } + size, headerSize = binary.BigEndian.Uint64(header[8:]), 16 + } else if size == 0 { + size = uint64(end - offset) + } + if size < uint64(headerSize) || size > uint64(end-offset) { + return 0, fmt.Errorf("invalid MP4 atom size") + } + kind := string(header[4:8]) + if kind == "moov" && !inMovie { + return scan(offset+headerSize, offset+int64(size), true) + } + if kind == "mvhd" && inMovie { + var version [1]byte + if _, err := f.ReadAt(version[:], offset+headerSize); err != nil { + return 0, err + } + clockOffset := int64(12) + if version[0] == 1 { + clockOffset = 20 + } else if version[0] != 0 { + return 0, fmt.Errorf("unsupported MP4 movie clock") + } + if headerSize+clockOffset+4 > int64(size) { + return 0, io.ErrUnexpectedEOF + } + var clock [4]byte + if _, err := f.ReadAt(clock[:], offset+headerSize+clockOffset); err != nil { + return 0, err + } + timescale := binary.BigEndian.Uint32(clock[:]) + if timescale == 0 { + return 0, fmt.Errorf("invalid MP4 movie timescale") + } + return 1 / float64(timescale), nil + } + offset += int64(size) + } + return 0, fmt.Errorf("MP4 movie clock unavailable") + } + return scan(0, info.Size(), false) +} + +func sourceClockEndTolerance(path string, trackTick, period float64) float64 { + tolerance := math.Max(0.0001, trackTick+1e-9) + if movieTick, err := sourceMovieClockTick(path); err == nil { + // Both conversions can round. Never turn the container exception into + // permission to lose a whole final frame; decoded cadence stays strict. + tolerance = math.Max(tolerance, math.Min(movieTick+trackTick+1e-9, period/2)) + } + return tolerance +} diff --git a/internal/api/annotation_source_movie_clock_test.go b/internal/api/annotation_source_movie_clock_test.go new file mode 100644 index 00000000..722adafe --- /dev/null +++ b/internal/api/annotation_source_movie_clock_test.go @@ -0,0 +1,78 @@ +package api + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestSourceClockAcceptsEditListTailRoundingWithoutRelaxingFrames(t *testing.T) { + if _, err := exec.LookPath("ffmpeg"); err != nil { + t.Skip("ffmpeg unavailable") + } + for _, count := range []int{6343, 7957, 11239} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "edit-list.mp4") + args := []string{"-v", "error", "-y", "-f", "lavfi", "-i", "color=c=navy:s=32x32:r=30", "-frames:v", fmt.Sprint(count), "-c:v", "libx264", "-preset", "ultrafast", "-bf", "3", "-video_track_timescale", "15360", "-movie_timescale", "1000", "-use_editlist", "1", path} + if out, err := exec.CommandContext(t.Context(), "ffmpeg", args...).CombinedOutput(); err != nil { + t.Fatalf("encode: %v %s", err, out) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + at := bytes.Index(body, []byte("elst")) + if at < 4 || at+24 > len(body) || body[at+4] != 0 || binary.BigEndian.Uint32(body[at+8:]) != 1 { + t.Fatal("fixture needs one version-0 edit") + } + writeEndpoint := func(milliseconds uint32) { + binary.BigEndian.PutUint32(body[at+12:], milliseconds) + if err := os.WriteFile(path, body, 0600); err != nil { + t.Fatal(err) + } + } + writeEndpoint(uint32(count * 1000 / 30)) + duration, err := ffprobeDurationSeconds(t.Context(), path) + if err != nil { + t.Fatal(err) + } + if delta := float64(count)/30 - duration; delta < .0003 || delta > .0004 { + t.Fatalf("fixture does not reproduce endpoint rounding: %.9f", delta) + } + if err := validateSourceMediaClock(t.Context(), path, count, duration); err != nil { + t.Fatalf("complete CFR source rejected: %v", err) + } + if err := validateSourceMediaClock(t.Context(), path, count-1, duration); err == nil { + t.Fatal("frame/PTS mismatch admitted") + } + writeEndpoint(uint32((count - 1) * 1000 / 30)) + duration, err = ffprobeDurationSeconds(t.Context(), path) + if err != nil { + t.Fatal(err) + } + if err := validateSourceMediaClock(t.Context(), path, count, duration); err == nil { + t.Fatal("whole-frame endpoint truncation admitted") + } + }) + } +} + +func TestSourceMovieClockMalformedInputKeepsStrictTolerance(t *testing.T) { + for _, data := range [][]byte{nil, []byte("not MP4"), {0xff, 0xff, 0xff, 0xff, 'm', 'o', 'o', 'v'}, {0, 0, 0, 1, 'm', 'o', 'o', 'v'}} { + path := filepath.Join(t.TempDir(), "bad.mp4") + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if _, err := sourceMovieClockTick(path); err == nil { + t.Fatal("malformed movie clock accepted") + } + if got := sourceClockEndTolerance(path, 1.0/15360, 1.0/30); math.Abs(got-.0001) > 1e-12 { + t.Fatal(got) + } + } +} diff --git a/internal/api/human_media_head_test.go b/internal/api/human_media_head_test.go new file mode 100644 index 00000000..1ffc8941 --- /dev/null +++ b/internal/api/human_media_head_test.go @@ -0,0 +1,48 @@ +package api + +import ( + "net/http" + "os" + "path/filepath" + "testing" + + "droboticflow/internal/domain" +) + +func TestHumanSubjectMediaHeadSupportsBothCamerasAndRetry(t *testing.T) { + f := continuousFixture(t) + root := t.TempDir() + artifacts := []domain.ExecutionArtifact{} + for _, side := range []string{"left", "right"} { + path := filepath.Join(root, side+".mp4") + if err := os.WriteFile(path, []byte("synthetic-media"), 0600); err != nil { + t.Fatal(err) + } + artifacts = append(artifacts, domain.ExecutionArtifact{ID: side, Name: side + ".mp4", Path: path, Type: domain.PortTypeVideo}) + } + if err := f.store.UpsertRun(f.ctx, domain.RunRecord{ID: f.task.RunID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, Artifacts: artifacts}); err != nil { + t.Fatal(err) + } + item := f.item + item.Subject.ArtifactID = "left" + item.Subject.Payload["rightArtifactId"] = "right" + if err := f.store.UpsertHumanItem(f.ctx, item); err != nil { + t.Fatal(err) + } + for _, side := range []string{"left", "right"} { + url := "/api/human-items/" + item.ID + "/subject-content?side=" + side + "&_mediaRetry=synthetic" + get := f.request(http.MethodGet, url, f.annotator, nil) + if get.Code != 200 { + t.Fatalf("GET %s: %d %s", side, get.Code, get.Body.String()) + } + head := f.request(http.MethodHead, url, f.annotator, nil) + if head.Code != 200 || head.Body.Len() != 0 || head.Header().Get("Content-Length") != get.Header().Get("Content-Length") { + t.Fatalf("HEAD %s: status=%d body=%q headers=%v", side, head.Code, head.Body.String(), head.Header()) + } + foreign := f.annotator + foreign.ID = "unassigned-media-reader" + if r := f.request(http.MethodHead, url, foreign, nil); r.Code == 200 { + t.Fatal("HEAD bypassed subject access control") + } + } +} diff --git a/internal/api/rework_boundary_browser_test.go b/internal/api/rework_boundary_browser_test.go index bb699937..8b5602a9 100644 --- a/internal/api/rework_boundary_browser_test.go +++ b/internal/api/rework_boundary_browser_test.go @@ -24,6 +24,18 @@ func TestReworkBoundaryBrowserReplica(t *testing.T) { } f := continuousFixture(t) reason := dispositionFixturePayload(t, f) + selection, err := f.store.GetProjectAnnotationLabels(f.ctx, f.task.ProjectID) + if err != nil { + t.Fatal(err) + } + label := domain.AnnotationLabelRecord{ID: "browser-extra-slice-name", OrganizationID: f.task.OrganizationID, Kind: "slice", Code: "browser-extra", Name: "补充切片名称", Status: domain.AnnotationLabelActive} + if err := f.store.UpsertAnnotationLabel(f.ctx, label); err != nil { + t.Fatal(err) + } + selection.SliceLabelIDs = append(selection.SliceLabelIDs, label.ID) + if err := f.store.UpsertProjectSliceLabels(f.ctx, *selection); err != nil { + t.Fatal(err) + } if err := f.store.UpsertOrganizationPlugin(f.ctx, domain.OrganizationPluginRecord{OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", Enabled: true, Config: defaultAnnotationPluginConfig()}); err != nil { t.Fatal(err) } @@ -31,6 +43,9 @@ func TestReworkBoundaryBrowserReplica(t *testing.T) { var before []domain.SliceSegment d := domain.SliceSegment{ID: "d", Start: 0, End: 5, DeliveryDisposition: "discard", DiscardReason: "相机被遮挡", DiscardReasonLabelID: reason.ReasonLabelID, DiscardReasonVersion: reason.ReasonVersion} k := domain.SliceSegment{ID: "k", Name: "测试切片", Start: 5, End: 10} + if side == "unnamed" { + k.Name = "" + } if side == "right" { k.Start, k.End = 0, 5 d.Start, d.End = 5, 10 @@ -47,9 +62,19 @@ func TestReworkBoundaryBrowserReplica(t *testing.T) { a.Description = "合成有效动作" } item := f.item + item.Subject.ArtifactID = "left-video" item.ReviewRound = round actions := []domain.AnnotationSegment{a} - item.Subject.Payload = map[string]any{"sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}, "annotationWorkspace": map[string]any{"userId": f.annotator.ID, "savedAt": nowISO(), "sliceSegments": before, "segments": actions, "statuses": map[string]string{"k": "pending", "d": "discarded"}}} + item.Subject.Payload = map[string]any{"rightArtifactId": "right-video", "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}, "annotationWorkspace": map[string]any{"userId": f.annotator.ID, "savedAt": nowISO(), "sliceSegments": before, "segments": actions, "statuses": map[string]string{"k": "pending", "d": "discarded"}}} + if side == "legacy" { + delete(item.Subject.Payload, "sliceWorkflow") + draft := mapValue(item.Subject.Payload["annotationWorkspace"]) + draft["segments"] = []domain.AnnotationSegment{{ID: "newer-described", Start: 5, End: 7, Status: "described", Description: "最新草稿描述"}, {ID: "newer-pending", Start: 7, End: 10, Status: "pending"}} + draft["statuses"] = map[string]string{"k": "pending", "d": "pending"} + draft["savedAt"] = nowISO() + draft["savedVersion"] = item.Version + item.Subject.Payload["annotationDraft"] = draft + } return f.store.UpsertHumanItem(f.ctx, item) } root := t.TempDir() @@ -66,12 +91,14 @@ func TestReworkBoundaryBrowserReplica(t *testing.T) { pts += fmt.Sprintf("%d\n", 1000000+i*100000) } bindings := []domain.NodeInputBinding{} + mediaArtifacts := []domain.ExecutionArtifact{} for _, side := range []string{"left", "right"} { video := filepath.Join(root, side+".mp4") if out, err := exec.Command("ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "color=c=navy:s=160x90:r=10:d=10", "-c:v", "libx264", "-pix_fmt", "yuv420p", video).CombinedOutput(); err != nil { t.Fatalf("video: %v: %s", err, out) } timestamps := filepath.Join(root, side+"_pts.csv") + mediaArtifacts = append(mediaArtifacts, domain.ExecutionArtifact{ID: side + "-video", Name: side + ".mp4", Path: video, Type: domain.PortTypeVideo}) if err := os.WriteFile(timestamps, []byte(pts), 0600); err != nil { t.Fatal(err) } @@ -91,7 +118,7 @@ func TestReworkBoundaryBrowserReplica(t *testing.T) { t.Fatal(err) } bindings = append(bindings, domain.NodeInputBinding{TargetPortID: "imu", Artifacts: []domain.ExecutionArtifact{{ID: "imu", Name: "imu.csv", Path: imuPath, Data: map[string]any{"collectionId": "capture"}}}}) - if err := f.store.UpsertRun(f.ctx, domain.RunRecord{ID: f.task.RunID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, WorkflowID: f.task.WorkflowID, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: f.task.NodeID, InputBindings: bindings}}}); err != nil { + if err := f.store.UpsertRun(f.ctx, domain.RunRecord{ID: f.task.RunID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, WorkflowID: f.task.WorkflowID, Artifacts: mediaArtifacts, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: f.task.NodeID, InputBindings: bindings}}}); err != nil { t.Fatal(err) } if err := f.store.UpsertAnnotationTask(f.ctx, f.task); err != nil { @@ -123,7 +150,7 @@ func TestReworkBoundaryBrowserReplica(t *testing.T) { renderJSON(w, 200, item) return } - if r.Method != "GET" && r.Method != "HEAD" && !strings.HasSuffix(r.URL.Path, "/slice-edit-applications") && !strings.HasSuffix(r.URL.Path, "/slice-edit-restorations") && !strings.HasSuffix(r.URL.Path, "/drafts") { + if r.Method != "GET" && r.Method != "HEAD" && !strings.HasSuffix(r.URL.Path, "/slice-edit-applications") && !strings.HasSuffix(r.URL.Path, "/slice-edit-restorations") && !strings.HasSuffix(r.URL.Path, "/slice-confirmations") && !strings.HasSuffix(r.URL.Path, "/drafts") { renderJSON(w, 403, map[string]string{"error": "fixture write not permitted"}) return } @@ -135,7 +162,11 @@ func TestReworkBoundaryBrowserReplica(t *testing.T) { defer server.Close() ctx, cancel := context.WithTimeout(t.Context(), 4*time.Minute) defer cancel() - cmd := exec.CommandContext(ctx, "node", "tests/browser/rework-boundary-replica.mjs") + script := os.Getenv("ANNOTATION_FEEDBACK_BROWSER_SCRIPT") + if script == "" { + script = "tests/browser/rework-boundary-replica.mjs" + } + cmd := exec.CommandContext(ctx, "node", script) cmd.Dir = "../.." cmd.Env = append(os.Environ(), "REWORK_BOUNDARY_API="+server.URL) out, err := cmd.CombinedOutput() diff --git a/internal/api/routes_human_loop.go b/internal/api/routes_human_loop.go index 41258726..79605d6c 100644 --- a/internal/api/routes_human_loop.go +++ b/internal/api/routes_human_loop.go @@ -15,9 +15,12 @@ func (server *server) mountHumanLoopRoutes(router chi.Router) { router.Get("/api/human-items/{id}/review-draft", server.handleReviewDraft) router.Get("/api/human-items/{id}/review-operations/{operationId}", server.handleReviewOperation) router.Get("/api/human-items/{id}/subject-content", server.handleHumanItemSubjectContent) + router.Head("/api/human-items/{id}/subject-content", server.handleHumanItemSubjectContent) router.Get("/api/human-items/{id}/evidence/{artifactId}/content", server.handleHumanItemEvidenceContent) + router.Head("/api/human-items/{id}/evidence/{artifactId}/content", server.handleHumanItemEvidenceContent) router.Get("/api/human-items/{id}/evidence/{artifactId}/members", server.handleHumanItemEvidenceMembers) router.Get("/api/human-items/{id}/evidence/{artifactId}/members/{memberId}/content", server.handleHumanItemEvidenceMemberContent) + router.Head("/api/human-items/{id}/evidence/{artifactId}/members/{memberId}/content", server.handleHumanItemEvidenceMemberContent) router.Get("/api/human-items/{id}/events", server.handleHumanItemEvents) router.Get("/api/human-rejects", server.handleHumanRejectsList) router.Get("/api/human-rejects/{id}", server.handleHumanRejectGet) diff --git a/internal/api/slice_dictionary_names.go b/internal/api/slice_dictionary_names.go index 5f8f3592..66260255 100644 --- a/internal/api/slice_dictionary_names.go +++ b/internal/api/slice_dictionary_names.go @@ -39,7 +39,7 @@ func (s *server) validateProjectSliceNames(ctx context.Context, item *domain.Hum if c.IsDiscarded() { continue } - if mode != "confirm" { + if mode != "confirm" && strings.TrimSpace(c.Name) != "" { if name, ok := old[c.ID]; ok && name == c.Name { continue } diff --git a/internal/api/slice_retry.go b/internal/api/slice_retry.go index b1ad2b43..daa61f8e 100644 --- a/internal/api/slice_retry.go +++ b/internal/api/slice_retry.go @@ -3,6 +3,7 @@ package api import ( "encoding/json" "fmt" + "time" "droboticflow/internal/domain" ) @@ -10,7 +11,36 @@ import ( // A retry reads the retained server workspace. Client-supplied slices must never // replace it, and malformed historic containers must not become empty content. func retainedSliceRetryInput(item *domain.HumanItemRecord, version int) (sliceConfirmationInput, combinedAnnotationDraftInput, error) { + return retainedSliceInput(annotationWorkspacePayload(item), version) +} + +// Legacy recovery must retain the same newer, owned autosave restored by the +// editor. Failed-job retries keep their separately frozen confirmation snapshot. +func retainedSliceRecoveryInput(item *domain.HumanItemRecord, version int) (sliceConfirmationInput, combinedAnnotationDraftInput, error) { workspace := annotationWorkspacePayload(item) + baseline := mapValue(item.Subject.Payload["annotationWorkspace"]) + if raw := item.Subject.Payload["annotationDraft"]; raw != nil && len(baseline) > 0 { + draft := mapValue(raw) + if len(draft) == 0 || item.AssignedUserID == "" || stringValue(draft["userId"]) != item.AssignedUserID || intValue(draft["savedVersion"]) > item.Version { + return sliceConfirmationInput{}, combinedAnnotationDraftInput{}, fmt.Errorf("自动保存草稿的归属或版本无法核对,请先保存当前进度;原内容仍保留") + } + baselineTime := stringValue(baseline["confirmedAt"]) + if baselineTime == "" { + baselineTime = stringValue(baseline["savedAt"]) + } + confirmedAt, confirmedErr := time.Parse(time.RFC3339Nano, baselineTime) + savedAt, savedErr := time.Parse(time.RFC3339Nano, stringValue(draft["savedAt"])) + if confirmedErr != nil || savedErr != nil { + return sliceConfirmationInput{}, combinedAnnotationDraftInput{}, fmt.Errorf("无法核对自动保存草稿与已确认内容的先后,请先核对保存记录;原内容仍保留") + } + if !savedAt.Before(confirmedAt) { + workspace = draft + } + } + return retainedSliceInput(workspace, version) +} + +func retainedSliceInput(workspace map[string]any, version int) (sliceConfirmationInput, combinedAnnotationDraftInput, error) { var saved combinedAnnotationDraftInput encoded, err := json.Marshal(workspace) if err == nil { diff --git a/internal/api/slice_retry_test.go b/internal/api/slice_retry_test.go index 21c70851..f3070a2e 100644 --- a/internal/api/slice_retry_test.go +++ b/internal/api/slice_retry_test.go @@ -83,6 +83,154 @@ func TestFailedSliceRetryUsesRetainedWorkspaceAndDurableRecovery(t *testing.T) { } } +func TestLegacyGenerationRecoveryPreservesAnnotationsAcrossRestart(t *testing.T) { + t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") + f, root := annotationProcessorFixture(t) + h := seedFailedSliceRetry(t, f) + delete(h.Subject.Payload, "sliceWorkflow") + delete(h.Subject.Payload, "materialization") + workspace := annotationWorkspacePayload(&h) + h.Subject.Payload["annotationWorkspace"] = workspace + draft := map[string]any{} + for key, value := range workspace { + draft[key] = value + } + draft["segments"] = []domain.AnnotationSegment{{ID: "retained-action", Start: 0, End: 2, Status: "pending"}, {ID: "human-description", Start: 2, End: 4, Status: "described", Description: "人工确认的分类内容"}} + draft["savedAt"] = nowISO() + draft["savedVersion"] = h.Version + h.Subject.Payload["annotationDraft"] = draft + if err := f.store.UpsertHumanItem(f.ctx, h); err != nil { + t.Fatal(err) + } + saved, err := f.store.GetHumanItem(f.ctx, h.ID) + if err != nil { + t.Fatal(err) + } + h = *saved + workspace = annotationWorkspacePayload(&h) + draft = mapValue(h.Subject.Payload["annotationDraft"]) + s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true} + state, err := s.annotationEditorStateForTask(f.ctx, f.task, &h) + if err != nil || state.Phase != "slice" || !state.CanRecoverGeneration || state.CanConfirmDescriptions { + t.Fatalf("state=%+v err=%v", state, err) + } + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true, "retryWorkspace": map[string]any{"segments": []any{}}}) + if r.Code != http.StatusAccepted { + t.Fatalf("recover: %d %s", r.Code, r.Body.String()) + } + queued, _ := f.store.GetHumanItem(f.ctx, h.ID) + if !reflect.DeepEqual(annotationWorkspacePayload(queued), workspace) { + t.Fatal("queueing recovery erased annotations") + } + if !reflect.DeepEqual(mapValue(queued.Subject.Payload["annotationDraft"]), draft) { + t.Fatal("queueing recovery erased the newer autosaved draft") + } + if duplicate := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": queued.Version, "recoverExisting": true}); duplicate.Code != 409 { + t.Fatal("duplicate recovery accepted") + } + now := time.Now().UTC() + old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "recovery-old", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano)) + claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "recovery-new", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano)) + if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 { + t.Fatal("recovery did not survive worker restart") + } + // Exercise a real processor rejection before retrying: both snapshots must + // survive failure, and retry must not switch back to the older baseline. + claimed.ProcessingJob.SourceFingerprint = "synthetic-source-changed" + s.executeAnnotationJob(context.Background(), *claimed) + failed, _ := f.store.GetHumanItem(f.ctx, h.ID) + if failed.ProcessingJob.Status != "failed" || !reflect.DeepEqual(annotationWorkspacePayload(failed), workspace) || !reflect.DeepEqual(mapValue(failed.Subject.Payload["annotationDraft"]), draft) { + t.Fatal("failed recovery changed retained snapshots") + } + r = f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": failed.Version, "retryFailed": true}) + if r.Code != http.StatusAccepted { + t.Fatalf("retry recovery: %d %s", r.Code, r.Body.String()) + } + claimed, _ = f.store.ClaimAnnotationProcessing(f.ctx, "recovery-retry", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano)) + if claimed == nil { + t.Fatal("recovery retry was not queued") + } + s.executeAnnotationJob(context.Background(), *claimed) + completed, _ := f.store.GetHumanItem(f.ctx, h.ID) + if completed.ProcessingJob.Status != "completed" { + t.Fatalf("recovery failed: %+v", completed.ProcessingJob) + } + if !reflect.DeepEqual(decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(completed)["segments"]), decodeMaterializationValue[domain.AnnotationSegment](draft["segments"])) { + t.Fatal("generation lost human descriptions or pending actions") + } + state, err = s.annotationEditorStateForTask(f.ctx, f.task, completed) + if err != nil || state.Phase != "annotate" || !state.CanConfirmDescriptions || state.CanRecoverGeneration { + t.Fatalf("completed state=%+v err=%v", state, err) + } +} + +func TestLegacyGenerationRecoveryRecognizesDraftBeyondEmptyBaseline(t *testing.T) { + t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") + f, _ := annotationProcessorFixture(t) + h := seedFailedSliceRetry(t, f) + delete(h.Subject.Payload, "sliceWorkflow") + workspace := annotationWorkspacePayload(&h) + draft := mapValue(workspace) + draft["savedAt"] = nowISO() + workspace["segments"] = []domain.AnnotationSegment{} + h.Subject.Payload["annotationWorkspace"] = workspace + h.Subject.Payload["annotationDraft"] = draft + if err := f.store.UpsertHumanItem(f.ctx, h); err != nil { + t.Fatal(err) + } + state, err := (&server{store: f.store}).annotationEditorStateForTask(f.ctx, f.task, &h) + if err != nil || !state.CanRecoverGeneration || state.CanConfirmDescriptions { + t.Fatalf("draft-only work was not recoverable: %+v %v", state, err) + } + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true}) + if r.Code != http.StatusAccepted { + t.Fatalf("recover draft-only work: %d %s", r.Code, r.Body.String()) + } + queued, _ := f.store.GetHumanItem(f.ctx, h.ID) + if len(decodeMaterializationValue[domain.AnnotationSegment](mapValue(queued.ProcessingJob.Input["retryWorkspace"])["segments"])) != 1 { + t.Fatal("empty baseline replaced the newer draft in the durable job") + } +} + +func TestLegacyGenerationRecoveryRejectsAmbiguousDraftWithoutMutation(t *testing.T) { + t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") + for _, name := range []string{"wrong-owner", "future-version", "missing-time", "malformed-actions"} { + t.Run(name, func(t *testing.T) { + f, _ := annotationProcessorFixture(t) + h := seedFailedSliceRetry(t, f) + delete(h.Subject.Payload, "sliceWorkflow") + draft := map[string]any{} + for key, value := range annotationWorkspacePayload(&h) { + draft[key] = value + } + draft["savedAt"] = nowISO() + switch name { + case "wrong-owner": + draft["userId"] = f.reviewer.ID + case "future-version": + draft["savedVersion"] = h.Version + 1 + case "missing-time": + delete(draft, "savedAt") + case "malformed-actions": + draft["segments"] = "not-a-list" + } + h.Subject.Payload["annotationDraft"] = draft + if err := f.store.UpsertHumanItem(f.ctx, h); err != nil { + t.Fatal(err) + } + before, _ := f.store.GetHumanItem(f.ctx, h.ID) + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true}) + if r.Code != http.StatusUnprocessableEntity { + t.Fatalf("ambiguous draft accepted: %d %s", r.Code, r.Body.String()) + } + after, _ := f.store.GetHumanItem(f.ctx, h.ID) + if !reflect.DeepEqual(before, after) { + t.Fatal("rejected recovery changed saved snapshots") + } + }) + } +} + func TestFailedSliceRetryRejectsUnsafeSnapshotsWithoutMutation(t *testing.T) { t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") f, _ := annotationProcessorFixture(t) diff --git a/internal/api/testdata/routes.golden b/internal/api/testdata/routes.golden index 04628c0a..f7e8f13c 100644 --- a/internal/api/testdata/routes.golden +++ b/internal/api/testdata/routes.golden @@ -1,7 +1,7 @@ # Generated by TestRouteSnapshot; see routes_snapshot_test.go. # access: reach = endpoint reached; otherwise the status the middleware chain returned. global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache] -routes=369 +routes=373 DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403 @@ -111,6 +111,7 @@ GET /api/packages | mw=[] | (*server).handlePackagesList-fm | anon=401 token=401 GET /api/packages/{runId}/{packageName} | mw=[] | (*server).handlePackageDetail-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/packages/{runId}/{packageName}/content | mw=[] | (*server).handlePackageContent-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins | mw=[] | (*server).handlePluginsList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +GET /api/plugins/annotation-qa/admin-pool | mw=[] | (*server).handleAdminAnnotationPool-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/claim-count | mw=[] | (*server).handleAnnotationClaimCount-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/claim-projects | mw=[] | (*server).handleAnnotationClaimProjects-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/sidebar | mw=[] | (*server).handleAnnotationSidebar-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach @@ -186,6 +187,9 @@ GET /worker-api/metrics | mw=[(*server).requireWorkerAPIRole.1] | (*server).hand GET /worker-api/ready | mw=[] | (*server).handleWorkerAPIReady-fm | anon=reach token=reach super=reach admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /worker-api/stats | mw=[(*server).requireWorkerAPIRole.1] | (*server).handleWorkerAPIStats-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403 HEAD /api/delivery-packages/{id}/artifacts/{artifactId}/content | mw=[] | (*server).handleDeliveryPackageArtifactContent-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +HEAD /api/human-items/{id}/evidence/{artifactId}/content | mw=[] | (*server).handleHumanItemEvidenceContent-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +HEAD /api/human-items/{id}/evidence/{artifactId}/members/{memberId}/content | mw=[] | (*server).handleHumanItemEvidenceMemberContent-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +HEAD /api/human-items/{id}/subject-content | mw=[] | (*server).handleHumanItemSubjectContent-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach HEAD /api/object-storage/content | mw=[(*server).requireBusinessRole.1] | (*server).handleObjectStorageContent-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403 HEAD /api/object-sync/client-releases/{id}/artifacts/{platform} | mw=[(*server).requireObjectSyncFeature-fm,(*server).requireBusinessRole.1] | (*server).handleObjectSyncClientReleaseDownload-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=reach quality=403 acceptance=403 HEAD /api/runs/{id}/artifacts/{artifactId}/content | mw=[(*server).requireAdminMonitoringAccess-fm] | (*server).handleRunsArtifactContent-fm | anon=401 token=reach super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403 diff --git a/public/vendor/movi-player-0.4.1/README.md b/public/vendor/movi-player-0.4.1/README.md index 7dd323ed..984cacc4 100644 --- a/public/vendor/movi-player-0.4.1/README.md +++ b/public/vendor/movi-player-0.4.1/README.md @@ -40,9 +40,8 @@ SHA-256 (this directory, element.slim.js includes the patch above): ## Host-element quirks (found in production, 2026-10-12) -Three hotfix rounds traced back to how this custom element behaves from the -outside. None of these are visible in the upstream README; all three re-break -silently on an upgrade unless the bridges below are re-checked: +Several production fixes traced back to how this custom element behaves from +the outside. These need explicit upgrade checks: 1. **Host events are incomplete.** The element forwards `play`/`pause`/`ended` to the host, but NOT `timeupdate`, `seeking`, `seeked`, or `waiting`. The @@ -57,10 +56,22 @@ silently on an upgrade unless the bridges below are re-checked: has a real host size. Bridge: review caption overlay measurement. 3. **The playback-rate cap is hard-coded** (`getMaxAllowedRate`), patched locally to 4 — see the patch section above. +4. **Host `error` retains an earlier failure after a successful reload.** + `MoviVideo.tsx` exposes a per-load error through the host instance, captures + error events, and clears it on a new load or media readiness. It does not + modify vendor private fields. Without this adapter, transport recovery can + reject a successfully loaded replacement source. +5. **Setting host `src` already loads the player.** The transport must not also + call `load()` for this element: two asynchronous decoder loads race. Native + `