From a341def261ef18e7c116b81ecd044ac66e9537e9 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 5 Nov 2026 03:06:18 -0800 Subject: [PATCH 1/8] =?UTF-8?q?=F0=9F=90=9B=20Preserve=20acceptance=20play?= =?UTF-8?q?back=20rate=20across=20slices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../acceptance-playback-rate/verification.md | 27 +++ .scratch/acceptance-playback-rate/work.md | 8 + scripts/verification-scopes.json | 26 +++ src/components/CaptureAnnotationWorkbench.tsx | 10 +- src/components/FinalAcceptanceWorkspace.tsx | 7 + tests/browser/acceptance-playback-rate.mjs | 198 ++++++++++++++++++ .../fixtures/acceptance-playback-rate.tsx | 53 +++++ 7 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 .scratch/acceptance-playback-rate/verification.md create mode 100644 .scratch/acceptance-playback-rate/work.md create mode 100644 tests/browser/acceptance-playback-rate.mjs create mode 100644 tests/browser/fixtures/acceptance-playback-rate.tsx diff --git a/.scratch/acceptance-playback-rate/verification.md b/.scratch/acceptance-playback-rate/verification.md new file mode 100644 index 00000000..cdb36014 --- /dev/null +++ b/.scratch/acceptance-playback-rate/verification.md @@ -0,0 +1,27 @@ +# Acceptance playback-rate verification + +Base: `867d406c`. Synthetic local data only; no production writes. + +## Reproduction and fix + +The new browser regression failed against the unchanged base: after selecting 2x and advancing to slice two, the speed selector returned `1` instead of `2`. Physical acceptance slices intentionally change the media key and remount CaptureAnnotationWorkbench, whose rate was component-local state. + +FinalAcceptanceWorkspace now owns the rate and passes it to its media and pose workbenches. Callers that do not supply a rate retain the existing local-state behavior. No persistence across reloads is introduced. + +## Evidence + +- `npm run verify:task -- --base 867d406c --budget-seconds 600` ran type/architecture checks, changed-component formatting, verification tooling tests, affected frontend tests, the new browser regression, rework-boundary browser, review-draft browser and review-decision browser checks successfully. Its final permissions command could not launch because the default Playwright Chromium was absent. Original report: `local-data/test-results/verify-1793876380248594000/report.json` (retains that failure; not relabeled). +- Reran the permissions command with `BROWSER_EXECUTABLE=/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge`; passed with no errors, failures or unexpected requests. +- Strengthened and reran `tests/browser/acceptance-playback-rate.mjs`: physical clips must mount VIDEO and source-index clips must mount MOVI-PLAYER. Both paths passed three forward/backward transitions retaining 2x/1.5x, actual left/right element rates and play/pause. Report/screenshots: `local-data/test-results/acceptance-playback-rate/`. +- The source components did not change during these follow-ups. Unchanged editing, draft, decision and permission evidence is reused; only the strengthened synthetic fixture/browser test was rerun. +- Visually checked the acceptance page, both cameras, current-slice caption and retained 1.5x control. No customer-facing copy or layout changed. + +## Running the dedicated browser regression + +Requires ffmpeg, installed browser and local Vite on 127.0.0.1:5197 (`npm run dev -- --host 127.0.0.1 --port 5197 --strictPort`). The browser test intercepts only /api requests, permits GET/HEAD only, and supplies synthetic metadata and MP4s. It never claims, saves or submits a task. + +Run `BROWSER_PATH='/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge' node tests/browser/acceptance-playback-rate.mjs`. + +## Limits + +No production verification or deployment. No promise of retaining speed across page reloads. Broad release/capacity checks are not claimed by this local UI fix. The original task-gate report remains failed for its environment issue; the targeted follow-up above supplies the successful missing check. diff --git a/.scratch/acceptance-playback-rate/work.md b/.scratch/acceptance-playback-rate/work.md new file mode 100644 index 00000000..c4642e00 --- /dev/null +++ b/.scratch/acceptance-playback-rate/work.md @@ -0,0 +1,8 @@ +# Preserve acceptance playback speed across slice changes + +Goal: Retain the selected playback rate when acceptance switches slices or media views, including both cameras. +Agent: Codex +Base: origin/dev @ 867d406c (contains live 6eee426e) +Scope: FinalAcceptanceWorkspace playback state, optional workbench rate ownership, isolated browser regression. +Out of scope: Persistence across browser reloads, global player defaults, task/permission changes, V4 migration, production writes or deployment. +Risk: Shared workbench must preserve its existing uncontrolled behavior for annotation and quality review; remounted acceptance media must receive the retained rate before playback. diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 47d11f09..c3721d12 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1,6 +1,32 @@ { "version": 1, "scopes": [ + { + "name": "acceptance-playback-rate", + "paths": [ + "^src/components/FinalAcceptanceWorkspace\\.tsx$", + "^tests/browser/acceptance-playback-rate\\.mjs$", + "^tests/browser/fixtures/acceptance-playback-rate\\.tsx$" + ], + "frontend": [], + "go": {}, + "task": [ + { + "argv": [ + "node", + "tests/browser/acceptance-playback-rate.mjs" + ], + "env": { + "REVIEW_TEST_URL": "http://127.0.0.1:5197", + "BROWSER_PATH": "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge" + }, + "tests": true + } + ], + "warnings": [ + "倍速浏览器回归使用本机5197测试服务与合成媒体;不连接生产。" + ] + }, { "name": "review-navigation", "paths": [ diff --git a/src/components/CaptureAnnotationWorkbench.tsx b/src/components/CaptureAnnotationWorkbench.tsx index a84181e6..a6b67ca0 100644 --- a/src/components/CaptureAnnotationWorkbench.tsx +++ b/src/components/CaptureAnnotationWorkbench.tsx @@ -320,6 +320,8 @@ type Props = ReviewViewingProps & { browserReadyMedia?: boolean; seekRequest?: { time: number; revision: number } | null; cameraPreferenceKey?: string; + playbackRate?: number; + onPlaybackRateChange?: (rate: number) => void; continuousActions?: boolean; progressSaved?: boolean; onSaveProgress?: (segments: AnnotationSegment[]) => Promise; @@ -471,6 +473,8 @@ export function CaptureAnnotationWorkbench({ browserReadyMedia = false, seekRequest, cameraPreferenceKey = "", + playbackRate: preferredPlaybackRate, + onPlaybackRateChange, continuousActions = false, progressSaved = true, onSaveProgress, @@ -587,7 +591,8 @@ export function CaptureAnnotationWorkbench({ const [currentTime, setCurrentTime] = useState(clipStart); const [captionsVisible, setCaptionsVisible] = useState(true); const [sliceZoom, setSliceZoom] = useState(false); - const [playbackRate, setPlaybackRate] = useState(1); + const [localPlaybackRate, setLocalPlaybackRate] = useState(1); + const playbackRate = preferredPlaybackRate ?? localPlaybackRate; const [showLeft, setShowLeft] = useState(initialCameraVisibility?.left ?? true); // Ordinary layouts load the left stream only until the right checkbox is // explicitly selected. Focused camera preferences are applied separately. @@ -2958,7 +2963,8 @@ export function CaptureAnnotationWorkbench({ const sliceBoundaryIssue = timelineReady ? sliceTimelineIssue(slices, clipStart, clipEnd, frameBoundaries) : ""; const changePlaybackRate = (rate: number) => { if (saving || phase === "materializing" || !CAPTURE_PLAYBACK_RATES.includes(rate)) return; - setPlaybackRate(rate); + setLocalPlaybackRate(rate); + onPlaybackRateChange?.(rate); if (leftVideoRef.current) leftVideoRef.current.playbackRate = rate; if (rightVideoRef.current) rightVideoRef.current.playbackRate = rate; }; diff --git a/src/components/FinalAcceptanceWorkspace.tsx b/src/components/FinalAcceptanceWorkspace.tsx index 6562c2c8..83974f0e 100644 --- a/src/components/FinalAcceptanceWorkspace.tsx +++ b/src/components/FinalAcceptanceWorkspace.tsx @@ -156,6 +156,9 @@ export function FinalAcceptanceWorkspace({ setCameras(readReviewCameras()); }, [item.id, cameraPreferenceKey, focusedLayout]); const [seekRequest, setSeekRequest] = useState<{ time: number; revision: number } | null>(null); + // Media keys intentionally remount players between slices; the user's rate + // belongs to the acceptance workspace so each new player inherits it. + const [playbackRate, setPlaybackRate] = useState(1); const seekRevision = useRef(0); const [technicalOpen, setTechnicalOpen] = useState(false); const [reviewsOpen, setReviewsOpen] = useState(false), @@ -743,6 +746,8 @@ export function FinalAcceptanceWorkspace({ (!active.sourceIndex || (currentMetadata && !currentMetadata.loading && !currentMetadata.error)) ? ( errors.push(e.message)); + await page.addInitScript((engine) => { + if (engine === "native") localStorage.setItem("droboticflow.movi.force", "native"); + else localStorage.removeItem("droboticflow.movi.force"); + }, engine); + await page.route("**/api/**", (route) => { + const url = new URL(route.request().url()); + if (!url.pathname.startsWith("/api/")) return route.continue(); + assert.match(route.request().method(), /^(GET|HEAD)$/, "fixture must not claim, save or submit"); + if (url.pathname.endsWith("/members")) { + const items = Array.from({ length: 3 }, (_, i) => { + const slice = `slice-${String(i + 1).padStart(6, "0")}`; + return [ + "left.mp4", + "right.mp4", + "annotation.json", + "synchronized-slice.json", + "left_pts.csv", + "right_pts.csv", + ...(sourceIndex ? ["source-index.json"] : []), + ].map((name) => ({ + id: `${slice}-${name}`, + name, + group: `slices/${slice}`, + sizeBytes: 100, + contentAvailable: true, + })); + }).flat(); + return route.fulfill({ + json: { items, total: items.length, limit: 200, offset: 0, recognized: true, truncated: false }, + }); + } + if (url.pathname.endsWith("/content")) { + if (url.pathname.includes(".mp4/")) { + const bytes = sourceIndex ? sourceMedia : media; + const range = /bytes=(\d+)-(\d*)/.exec(route.request().headers().range || ""); + const start = range ? Number(range[1]) : 0; + const end = range?.[2] ? Math.min(Number(range[2]), bytes.length - 1) : bytes.length - 1; + return route.fulfill({ + status: range ? 206 : 200, + contentType: "video/mp4", + body: route.request().method() === "HEAD" ? undefined : bytes.subarray(start, end + 1), + headers: { + "Accept-Ranges": "bytes", + "Content-Length": String(end - start + 1), + ...(range ? { "Content-Range": `bytes ${start}-${end}/${bytes.length}` } : {}), + }, + }); + } + if (url.pathname.includes("annotation.json")) return route.fulfill({ json: { actions: [] } }); + if (url.pathname.includes("synchronized-slice.json")) { + const index = Number(/slice-(\d+)/.exec(url.pathname)?.[1] || 1) - 1; + return route.fulfill({ + json: { + window: { startTimestampUs: index * 8000000, endTimestampUs: (index + 1) * 8000000 }, + boundaryContract: "hardware-frame-v1", + boundarySeconds: Array.from({ length: 241 }, (_, i) => i / 30), + left: { pts: { rowCount: 240 } }, + ...(sourceIndex + ? { + sourceIndexSchema: "droboticflow.source-slice-index/v1", + sourceMediaOffsets: { left: index * 8, right: index * 8 }, + rightBoundarySeconds: Array.from({ length: 241 }, (_, i) => i / 30), + } + : {}), + }, + }); + } + return route.fulfill({ + contentType: "text/csv", + body: "clip_time_ms\n" + Array.from({ length: 240 }, (_, i) => (i * 1000) / 30).join("\n"), + }); + } + return route.fulfill({ json: { items: [] } }); + }); + await page.route("**/__acceptance_playback_rate", (route) => + route.fulfill({ + contentType: "text/html", + body: `
`, + }), + ); + await page.goto(`${base}/__acceptance_playback_rate`); + await page.getByLabel("播放速度").waitFor(); + await page.getByRole("button", { name: "恢复双画面", exact: true }).click(); + async function assertRate(rate) { + assert.equal(await page.getByLabel("播放速度").inputValue(), String(rate)); + await page.waitForFunction( + ({ rate, tag }) => { + const players = [...document.querySelectorAll("[data-camera]")].filter((p) => + ["VIDEO", "MOVI-PLAYER"].includes(p.tagName), + ); + return ( + players.length === 2 && + players.every((p) => p.tagName === tag && p.readyState >= 2 && Math.abs(p.playbackRate - rate) < 0.001) + ); + }, + { rate, tag: sourceIndex ? "MOVI-PLAYER" : "VIDEO" }, + { timeout: 20000 }, + ); + } + await page.getByLabel("播放速度").selectOption("2"); + await assertRate(2); + await page.getByRole("button", { name: "下一片", exact: true }).click(); + await page.waitForFunction( + () => document.querySelector('[data-testid="review-current-slice"]')?.textContent === "第二片", + ); + await assertRate(2); + await page.getByRole("button", { name: "播放", exact: true }).click(); + await page.waitForFunction(() => + [...document.querySelectorAll("[data-camera]")].some( + (p) => ["VIDEO", "MOVI-PLAYER"].includes(p.tagName) && !p.paused && p.currentTime > 0.2, + ), + ); + await page.getByRole("button", { name: "暂停", exact: true }).click(); + await assertRate(2); + await page.getByLabel("播放速度").selectOption("1.5"); + await page.getByRole("button", { name: "下一片", exact: true }).click(); + await page.waitForFunction( + () => document.querySelector('[data-testid="review-current-slice"]')?.textContent === "第三片", + ); + await assertRate(1.5); + await page.getByRole("button", { name: "上一片", exact: true }).click(); + await page.waitForFunction( + () => document.querySelector('[data-testid="review-current-slice"]')?.textContent === "第二片", + ); + await assertRate(1.5); + assert.deepEqual(errors, []); + await page.screenshot({ path: `${out}/${engine}.png`, fullPage: true }); + cases.push({ engine, sliceChanges: 3, retainedRates: [2, 1.5, 1.5], errors }); + await context.close(); + } + await fs.writeFile(`${out}/report.json`, JSON.stringify({ passed: true, cases }, null, 2)); + console.log(JSON.stringify({ passed: true, cases })); +} finally { + await browser.close(); +} diff --git a/tests/browser/fixtures/acceptance-playback-rate.tsx b/tests/browser/fixtures/acceptance-playback-rate.tsx new file mode 100644 index 00000000..37a806bf --- /dev/null +++ b/tests/browser/fixtures/acceptance-playback-rate.tsx @@ -0,0 +1,53 @@ +import React from "react"; +import { createRoot } from "react-dom/client"; +import { FinalAcceptanceWorkspace } from "/src/components/FinalAcceptanceWorkspace"; +import "/src/index.css"; +import "/src/styles/capture-workspace.css"; +import "/src/styles/review-workspace.css"; + +const slices = ["第一片", "第二片", "第三片"].map((name, i) => ({ + id: `slice-${String(i + 1).padStart(6, "0")}`, + name, + start: i * 8, + end: (i + 1) * 8, +})); +const item = { + id: "acceptance-rate-synthetic", + version: 1, + status: "claimed", + assignedUserId: "synthetic-reviewer", + processingProfile: "raw_human", + subject: { + payload: { + items: [ + { + taskId: "synthetic-task", + sourceHumanItemId: "synthetic-source", + materialization: { + artifactId: "synthetic-materialization", + revision: 1, + manifestSha256: "synthetic", + sliceCount: slices.length, + coverage: { contiguous: true, gapMicroseconds: 0, overlapMicroseconds: 0 }, + }, + finalSlices: slices, + }, + ], + }, + }, +}; + +createRoot(document.getElementById("root")!).render( + {}} + onCommentChange={() => {}} + onApprove={() => {}} + onReject={() => {}} + />, +); From e4969acced2b2a0217cf4d0db4d9b9ea79cf364b Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 5 Nov 2026 07:46:08 -0800 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=90=9B=20Use=20native=20media=20state?= =?UTF-8?q?=20during=20rapid=20slice=20navigation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../acceptance-playback-rate/verification.md | 13 +++++ .scratch/acceptance-playback-rate/work.md | 8 +++ src/components/MoviVideo.tsx | 25 ++++++++++ tests/browser/acceptance-playback-rate.mjs | 50 +++++++++++++++---- 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/.scratch/acceptance-playback-rate/verification.md b/.scratch/acceptance-playback-rate/verification.md index cdb36014..fffe6727 100644 --- a/.scratch/acceptance-playback-rate/verification.md +++ b/.scratch/acceptance-playback-rate/verification.md @@ -25,3 +25,16 @@ Run `BROWSER_PATH='/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edg ## Limits No production verification or deployment. No promise of retaining speed across page reloads. Broad release/capacity checks are not claimed by this local UI fix. The original task-gate report remains failed for its environment issue; the targeted follow-up above supplies the successful missing check. + +## Follow-up: shortcut navigation playback (base a341def2) + +- Built a real-browser keyboard loop (Ctrl+Arrow then immediate Space) using synthetic source-index media. Reproduced repeated failures with all bytes buffered: UI either remained in positioning or reverted to paused. Temporary probes and event traces are under `local-data/test-results/acceptance-shortcut-playback/`. +- Compared the pinned vendor implementation and native video properties: Movi derives seeking/readiness from its asynchronous state machine and paused from play intent. These diverge from the underlying media during rapid navigation; stale pause notifications could cancel a new play. The adapter now uses the current native engine's real paused/seeking/readyState; the software path retains vendor getters. No change to logical annotation times or seek accuracy. +- Extended the permanent browser regression to native clips, source-index clips with integer offsets, and source-index clips with fractional offsets. Each performs 15 transitions, checks both real media clocks advance after immediate keyboard play, and retains the playback rate. Passed twice with the fix (45 transitions per run). Screenshots/report: `local-data/test-results/shortcut-playback-fixed/`; task-run copy: `local-data/test-results/acceptance-playback-rate/`. +- Ran that same strengthened regression against the pre-fix MoviVideo from a341def2: failed waiting for playback after the shortcut (`local-data/test-results/shortcut-playback-baseline/browser-red.log`). Restored the fix automatically afterward. +- Task verification `verify-1793893323697578000/report.json`: type/architecture, format, and browser checks passed. Full frontend gate failed with one additional-to-known-baseline ID: `fanyu-comment-followup.test.mjs :: dashboard routes all workforce roles to the clean production quality panel`. Running the required gate with the pre-fix MoviVideo produces the same failure (`shortcut-playback-baseline/frontend-gate.log`); the tested dashboard sources are unchanged by this task. Known-failures was not expanded and the failed report remains failed. +- Actual acceptance screenshot checked with both cameras and 1.5x rate; no copy/layout changes. Production/client-specific verification remains pending; this is a local reproduction of the reported symptom, not proof of every customer's failure condition. + +- Media recovery across annotation, quality review and acceptance passed with forced software-engine reload, simulated failed loading, bounded retry, retained position, and annotation draft preservation: `REWORK_BOUNDARY_BROWSER_TEST=1 ANNOTATION_FEEDBACK_BROWSER_SCRIPT=local-data/test-results/acceptance-shortcut-playback/media-recovery.mjs go test -count=1 ./internal/api -run '^TestReworkBoundaryBrowserReplica$' -v` (35.16s). The original script failed before mounting due to a hardcoded missing `.vite/deps/react-dom_client.js`; the task-local copy changes only its Vite cache directory and React module import through a Vite-transformed module. No product/fixture assertions were removed. The original harness failure remains recorded, and no shared node_modules cache was repaired or overwritten. + +Local only. No push, PR, deployment, production data change or source-media change. Remaining release limitation: the pre-existing dashboard frontend assertion is still outside the known-failure baseline and blocks claiming the full task gate green. diff --git a/.scratch/acceptance-playback-rate/work.md b/.scratch/acceptance-playback-rate/work.md index c4642e00..2c6fa2de 100644 --- a/.scratch/acceptance-playback-rate/work.md +++ b/.scratch/acceptance-playback-rate/work.md @@ -6,3 +6,11 @@ Base: origin/dev @ 867d406c (contains live 6eee426e) Scope: FinalAcceptanceWorkspace playback state, optional workbench rate ownership, isolated browser regression. Out of scope: Persistence across browser reloads, global player defaults, task/permission changes, V4 migration, production writes or deployment. Risk: Shared workbench must preserve its existing uncontrolled behavior for annotation and quality review; remounted acceptance media must receive the retained rate before playback. + +## Follow-up: intermittent shortcut navigation playback + +Follow-up base: `a341def2`. The user reports occasional inability to play after keyboard slice navigation. Reproduced locally with synthetic full-source clips and Ctrl+Arrow navigation followed immediately by Space; no production mutation. + +Scope: normalize Movi's HTMLMediaElement-shaped paused/seeking/readyState properties from its current native engine when present; preserve the existing software-engine getters, source/seek coordinates, media bytes, annotations and permissions. Restore original property descriptors on cleanup. Extend the browser regression to repeated keyboard navigation with integer and fractional source offsets. + +Compatibility: pinned vendored Movi 0.4.1 exposes the active native video via getHLSVideoElement(), including its direct native wrapper; the software engine returns null. Re-check that seam if the vendor is upgraded. No vendor bundle or network settings changed. diff --git a/src/components/MoviVideo.tsx b/src/components/MoviVideo.tsx index f8e9a435..de226ff7 100644 --- a/src/components/MoviVideo.tsx +++ b/src/components/MoviVideo.tsx @@ -191,9 +191,30 @@ export const MoviVideo = React.forwardRef(func // without modifying the vendor's private state or remounting the editor. const originalOwnError = Object.getOwnPropertyDescriptor(element, "error"); const mediaPrototype = Object.getPrototypeOf(element); + const nativeProperties = ["paused", "seeking", "readyState"] as const; + const originalNativeProperties = nativeProperties.map( + (name) => [name, Object.getOwnPropertyDescriptor(element, name)] as const, + ); let currentMediaError: unknown = null; if (mode === "movi") { Object.defineProperty(element, "error", { configurable: true, get: () => currentMediaError }); + // Movi's state/intent flags can lag the native video during rapid seeks. + // In particular, an old pause notification can cancel a newer play, and + // "seeking" reports HAVE_CURRENT_DATA before the native target is ready. + // Use the active native engine's media state; WASM keeps its own getters. + for (const name of nativeProperties) { + Object.defineProperty(element, name, { + configurable: true, + get() { + const native = ( + element as HTMLElement & { + player?: { getHLSVideoElement?: () => HTMLVideoElement | null }; + } + ).player?.getHLSVideoElement?.(); + return native ? native[name] : Reflect.get(mediaPrototype, name, element); + }, + }); + } } const add = (name: string, listener: EventListener) => { element.addEventListener(name, listener); @@ -328,6 +349,10 @@ export const MoviVideo = React.forwardRef(func if (mode === "movi") { if (originalOwnError) Object.defineProperty(element, "error", originalOwnError); else Reflect.deleteProperty(element, "error"); + for (const [name, descriptor] of originalNativeProperties) { + if (descriptor) Object.defineProperty(element, name, descriptor); + else Reflect.deleteProperty(element, name); + } } }; }, [mode, plainSrc, endAt]); diff --git a/tests/browser/acceptance-playback-rate.mjs b/tests/browser/acceptance-playback-rate.mjs index a5eac599..65b3a66f 100644 --- a/tests/browser/acceptance-playback-rate.mjs +++ b/tests/browser/acceptance-playback-rate.mjs @@ -36,7 +36,7 @@ execFileSync("ffmpeg", [ "-f", "lavfi", "-i", - "testsrc2=s=320x240:r=30:d=24", + "testsrc2=s=320x240:r=30:d=25", "-c:v", "libx264", "-pix_fmt", @@ -53,8 +53,9 @@ const browser = await chromium.launch({ }); const cases = []; try { - for (const engine of ["native", "movi"]) { - const sourceIndex = engine === "movi"; + for (const engine of ["native", "movi", "movi-fractional"]) { + const sourceIndex = engine !== "native"; + const fractionalOffset = engine === "movi-fractional" ? 0.023333333 : 0; const context = await browser.newContext({ viewport: { width: 1500, height: 1000 } }); const page = await context.newPage(); const errors = []; @@ -119,7 +120,7 @@ try { ...(sourceIndex ? { sourceIndexSchema: "droboticflow.source-slice-index/v1", - sourceMediaOffsets: { left: index * 8, right: index * 8 }, + sourceMediaOffsets: { left: index * 8 + fractionalOffset, right: index * 8 + fractionalOffset }, rightBoundarySeconds: Array.from({ length: 241 }, (_, i) => i / 30), } : {}), @@ -168,10 +169,12 @@ try { ); await assertRate(2); await page.getByRole("button", { name: "播放", exact: true }).click(); - await page.waitForFunction(() => - [...document.querySelectorAll("[data-camera]")].some( - (p) => ["VIDEO", "MOVI-PLAYER"].includes(p.tagName) && !p.paused && p.currentTime > 0.2, - ), + await page.waitForFunction( + (offset) => + [...document.querySelectorAll("[data-camera]")].some( + (p) => ["VIDEO", "MOVI-PLAYER"].includes(p.tagName) && !p.paused && p.currentTime > offset + 0.3, + ), + sourceIndex ? 8 + fractionalOffset : 0, ); await page.getByRole("button", { name: "暂停", exact: true }).click(); await assertRate(2); @@ -186,9 +189,38 @@ try { () => document.querySelector('[data-testid="review-current-slice"]')?.textContent === "第二片", ); await assertRate(1.5); + // Actual keyboard events, immediately followed by play: do not wait for + // canplay/seeked first. Older Movi state flags could strand this sequence + // in positioning, or let an old pause event cancel the new play request. + const shortcutChanges = 12; + for (let iteration = 0; iteration < shortcutChanges; iteration++) { + const forward = iteration % 2 === 0; + await page.getByTestId("final-acceptance-workspace").focus(); + await page.keyboard.press(forward ? "Control+ArrowRight" : "Control+ArrowLeft"); + await page.waitForFunction( + (name) => document.querySelector('[data-testid="review-current-slice"]')?.textContent === name, + forward ? "第三片" : "第二片", + ); + await page.keyboard.press("Space"); + const offset = sourceIndex ? (forward ? 16 : 8) + fractionalOffset : 0; + await page.waitForFunction( + (offset) => { + const players = [...document.querySelectorAll("[data-camera]")].filter((p) => + ["VIDEO", "MOVI-PLAYER"].includes(p.tagName), + ); + return ( + players.length === 2 && + players.every((p) => !p.paused && !p.seeking && p.readyState >= 2 && p.currentTime > offset + 0.35) + ); + }, + offset, + { timeout: 8000 }, + ); + await page.keyboard.press("Space"); + } assert.deepEqual(errors, []); await page.screenshot({ path: `${out}/${engine}.png`, fullPage: true }); - cases.push({ engine, sliceChanges: 3, retainedRates: [2, 1.5, 1.5], errors }); + cases.push({ engine, sliceChanges: 3 + shortcutChanges, retainedRates: [2, 1.5, 1.5], errors }); await context.close(); } await fs.writeFile(`${out}/report.json`, JSON.stringify({ passed: true, cases }, null, 2)); From adc63af81c38f0315e9ee0d1f4869dc2c06cdfd9 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 5 Nov 2026 19:21:52 -0800 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=90=9B=20Isolate=20reviewer=20pool=20?= =?UTF-8?q?searches=20and=20batch=20metadata=20joins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/review-pool-search/work.md | 35 +++++ internal/store/human_pool_search.go | 53 ++++---- internal/store/human_pool_search_filter.go | 5 +- internal/store/human_pool_search_join.go | 121 ++++++++++++++++++ internal/store/human_pool_search_payload.go | 84 ++++++++++++ internal/store/human_pool_search_test.go | 92 ++++++++++++- internal/store/human_pool_search_text.go | 37 ++++++ src/lib/annotation-project-loading.js | 2 +- src/pages/AnnotationOperations.tsx | 29 +++-- tests/browser/review-pool-search-replica.mjs | 6 +- .../frontend/annotation-pool-loading.test.mjs | 46 +++++++ .../annotation-pool-pagination.test.mjs | 28 ++++ tests/frontend/review-pool-search.test.mjs | 9 +- 13 files changed, 497 insertions(+), 50 deletions(-) create mode 100644 .scratch/review-pool-search/work.md create mode 100644 internal/store/human_pool_search_join.go create mode 100644 internal/store/human_pool_search_payload.go create mode 100644 internal/store/human_pool_search_text.go diff --git a/.scratch/review-pool-search/work.md b/.scratch/review-pool-search/work.md new file mode 100644 index 00000000..f0437b9e --- /dev/null +++ b/.scratch/review-pool-search/work.md @@ -0,0 +1,35 @@ +# Reviewer pool search isolation and metadata reads + +Goal: Restore the reported reviewer history search without making it wait for unrelated pool queries. +Agent: Codex +Base: live e4969acc (codex/acceptance-playback-rate, PR #72), preserving its playback fixes. +Status: Local candidate; not pushed, submitted as PR, or deployed. + +## Changes + +- Review/acceptance pages explicitly read the selected pool. Switching tabs cancels the old read; an available-pool timeout no longer discards a successful reviewed-pool search. Unread counts show an unknown value instead of zero. +- Keep action ownership scope unchanged when switching pool tabs; stale initial/page results are guarded separately by the active view. +- Separate page-read errors from action errors. A successful retry/refresh clears the read error without clearing an unresolved write error. +- Search sorts compact metadata, batches foreign lookups in groups of 64, uses one conservative text prefilter, and hydrates matching records in original order. Legacy recursive payload names are read in batches only where needed. +- Preserve dynamic legacy run-input reference names, tenant isolation, stable cursors, server eligibility, old response shapes and the 8-second deadline. No production data/index writes, migrations, or resource changes. + +## Verification + +Evidence directory: local-data/pool-search-verification/ (ignored). + +- final-freshness.log: contains current origin/dev and the live version. +- frontend-red.log: the new independent reviewed-search assertion fails against the exact baseline JavaScript (requests available instead of reviewed). +- baseline-regression.log: raw historical input-reference fixture fails against baseline Go source; current Mongo/memory parity passes. +- final-mongo-all.log: Mongo parity, 10,000 modern records, and a >100MB synthetic payload fixture pass, including cancellation, same-time cursor ties, cross-tenant links, category equivalence, payload-only fallback, candidate and identity responses. Find replies are included in traffic accounting. +- final-search-browser.log: selected-pool search, old history, slow/failed unrelated pool, cancellation, pagination retry and cleared read errors pass; screenshot inspected at local-data/test-results/review-pool-search/review-search.png. +- final-permissions-browser.log: review and acceptance authoring permissions pass using installed Edge. Initial default-browser attempt lacked a downloaded Playwright browser; rerun with Edge passed. +- final-lint.log: lint and lint-related frontend checks pass. +- remaining-checks-results.json and task-extra-*.log: source admission Mongo, source validation browser, review trace, rework boundaries, review drafts and decisions pass. The task plan was invoked but stopped at existing opt-in skips (RandomClaimLocalPreview and SourceValidationBrowserReplica); remaining checks were run separately with their prescribed environments. Do not describe the aggregate task gate as green. +- final-frontend-gate.log: the only remaining global frontend failure is fanyu-comment-followup.test.mjs :: dashboard routes all workforce roles to the clean production quality panel. baseline-frontend-gate.log reproduces the identical failure at e4969acc via the official gate. No failure baseline was expanded. + +## Production read-only evidence and limits + +- The reported part exists and is accepted; its current state was not changed. +- reviewer-scoped-readonly.log: the exact source-generated Mongo pipeline, restricted to each of the two actual reviewers' history IDs (79 and 98), finds one matching row in 583ms and 734ms. These are backend pipeline timings, not customer-browser end-to-end acceptance. No impersonation or customer writes. +- Wide administrator/all-history and broad available-pool search can still exceed 8 seconds. The selected-pool isolation fixes the reported history page being blocked by that failure; it does not prove all broad-pool latency or capacity has been resolved. +- No production write acceptance, 81-user load test, or release gates were performed for this task. Push, PR and deployment require new task-specific authorization. diff --git a/internal/store/human_pool_search.go b/internal/store/human_pool_search.go index 704f6d59..9b203412 100644 --- a/internal/store/human_pool_search.go +++ b/internal/store/human_pool_search.go @@ -4,7 +4,6 @@ import ( "context" "droboticflow/internal/domain" "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "regexp" "strings" @@ -94,19 +93,7 @@ func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q doma } filter = bson.M{"$and": bson.A{filter, visibility}} } - pipeline := mongo.Pipeline{ - {{Key: "$match", Value: filter}}, - {{Key: "$sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}}, - {{Key: "$project", Value: humanItemListProjection(domain.HumanItemQuery{ListProjection: true})}}, - adminPoolLookup("annotationTasks", "legacyTaskId", "poolTasks", org, bson.M{"_id": 0, "id": 1, "status": 1, "artifactId": 1, "sourceVideo": 1}), - } - if q.Search != "" || q.Capture != "" { - pipeline = append(pipeline, - adminPoolLookup("runs", "runId", "poolRuns", org, bson.M{"_id": 0, "id": 1, "taskId": 1, "projectName": 1, "captureDirectories": 1, "inputs.ref": 1, "nodeExecutions.artifacts.id": 1, "nodeExecutions.artifacts.name": 1, "nodeExecutions.artifacts.path": 1, "artifacts.id": 1, "artifacts.name": 1, "artifacts.path": 1}), - adminPoolLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org, bson.M{"_id": 0, "captureId": 1, "prefix": 1}), - adminPoolLookup("projects", "projectId", "poolProjects", org, bson.M{"_id": 0, "name": 1}), - ) - } + pipeline := humanPoolSearchPipeline(org, q, filter) for _, text := range []string{q.Search, q.Capture} { if text != "" { if searchFilter := humanPoolSearchPrefilter(text); searchFilter != nil { @@ -122,26 +109,36 @@ func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q doma return nil, err } defer cur.Close(ctx) - out := []domain.HumanItemRecord{} - for cur.Next(ctx) { - var row adminPoolRow - if err := cur.Decode(&row); err != nil { + ids := []string{} + for { + rows := []adminPoolRow{} + for len(rows) < 200 && cur.Next(ctx) { + var row adminPoolRow + if err := cur.Decode(&row); err != nil { + return nil, err + } + rows = append(rows, row) + } + if err := cur.Err(); err != nil { return nil, err } - if !humanPoolSearchRowMatches(row, q) { - continue + if len(rows) == 0 { + break } - item := row.HumanItemRecord - if q.PoolCandidateProjection { - item = item.PoolCandidate() + if err := s.humanPoolSearchLegacyPayloads(ctx, org, q, rows); err != nil { + return nil, err } - if q.IdentityProjection { - item = domain.HumanItemRecord{ID: item.ID} + for _, row := range rows { + if humanPoolSearchRowMatches(row, q) { + ids = append(ids, row.ID) + } + if q.Limit > 0 && len(ids) >= q.Limit { + break + } } - out = append(out, item) - if q.Limit > 0 && len(out) >= q.Limit { + if q.Limit > 0 && len(ids) >= q.Limit { break } } - return out, cur.Err() + return s.hydrateHumanPoolSearch(ctx, org, q, ids) } diff --git a/internal/store/human_pool_search_filter.go b/internal/store/human_pool_search_filter.go index 04abcb6f..3c4c5a57 100644 --- a/internal/store/human_pool_search_filter.go +++ b/internal/store/human_pool_search_filter.go @@ -16,10 +16,7 @@ func humanPoolSearchPrefilter(search string) bson.M { pattern := regexp.QuoteMeta(strings.ToLower(search)) pattern = strings.ReplaceAll(pattern, "/part", `(?:/|_|_camera_(?:left|right)_)part`) match := bson.A{} - for _, field := range []string{"id", "legacyTaskId", "subject.itemKey", "subject.artifactId", "poolTasks.artifactId", "poolTasks.sourceVideo", "poolCaptures.captureId", "poolCaptures.prefix", "poolProjects.name", "poolRuns.projectName", "poolRuns.captureDirectories", "poolRuns.inputs.ref.captureUnit", "poolRuns.inputs.ref.prefix", "poolRuns.inputs.ref.key", "poolRuns.inputs.ref.manifestKey", "poolRuns.artifacts.id", "poolRuns.artifacts.name", "poolRuns.artifacts.path", "poolRuns.nodeExecutions.artifacts.id", "poolRuns.nodeExecutions.artifacts.name", "poolRuns.nodeExecutions.artifacts.path"} { - match = append(match, bson.M{field: bson.M{"$regex": pattern, "$options": "i"}}) - match = append(match, bson.M{field: bson.M{"$regex": "%[0-9a-f]{2}", "$options": "i"}}) - } + match = append(match, bson.M{"$expr": bson.M{"$regexMatch": bson.M{"input": humanPoolSearchText(), "regex": pattern + "|%[0-9a-f]{2}", "options": "i"}}}) if split := strings.LastIndex(strings.ToLower(search), "/part"); split > 0 { captures := bson.A{} for _, field := range []string{"poolCaptures.captureId", "poolCaptures.prefix"} { diff --git a/internal/store/human_pool_search_join.go b/internal/store/human_pool_search_join.go new file mode 100644 index 00000000..07785b81 --- /dev/null +++ b/internal/store/human_pool_search_join.go @@ -0,0 +1,121 @@ +package store + +import ( + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +// Join fixed-size batches instead of sending four foreign queries per item. +// Batching does not limit candidates: every batch is searched, then the original +// item order is restored. Tenant filtering precedes the run -> capture link. +func humanPoolSearchPipeline(org string, q domain.HumanItemQuery, filter bson.M) mongo.Pipeline { + // Only search/category metadata participates in global sorts and joins. + // Hydrate matched IDs afterwards; the public record shape stays unchanged. + projection := bson.M{"_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1} + pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: projection}}} + pipeline = append(pipeline, + // Number candidate batches; restore the stable createdAt/id order below. + bson.D{{Key: "$setWindowFields", Value: bson.M{"sortBy": bson.D{{Key: "createdAt", Value: -1}}, "output": bson.M{"_poolRow": bson.M{"$documentNumber": bson.M{}}}}}}, + bson.D{{Key: "$group", Value: bson.M{"_id": bson.M{"$floor": bson.M{"$divide": bson.A{bson.M{"$subtract": bson.A{"$_poolRow", 1}}, 64}}}, "poolItems": bson.M{"$push": "$$ROOT"}}}}, + humanPoolBatchLookup("annotationTasks", "poolItems.legacyTaskId", "poolTasks", org), + ) + fields := bson.M{"poolTasks": humanPoolTenantProjection("poolTasks", org, bson.M{ + "id": "$$row.id", "status": "$$row.status", "artifactId": "$$row.artifactId", "sourceVideo": "$$row.sourceVideo", + })} + if q.Search != "" || q.Capture != "" { + pipeline = append(pipeline, + humanPoolBatchLookup("runs", "poolItems.runId", "poolRuns", org), + humanPoolBatchLookup("projects", "poolItems.projectId", "poolProjects", org), + bson.D{{Key: "$set", Value: bson.M{"poolRuns": humanPoolTenantProjection("poolRuns", org, bson.M{ + "id": "$$row.id", "taskId": "$$row.taskId", "projectName": "$$row.projectName", "captureDirectories": "$$row.captureDirectories", + "inputs": humanPoolRunInputRefs(), "artifacts": humanPoolArtifactNames("$$row.artifacts"), + "nodeExecutions": bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{"$$row.nodeExecutions", bson.A{}}}, "as": "execution", "in": bson.M{"artifacts": humanPoolArtifactNames("$$execution.artifacts")}}}, + })}}}, + humanPoolBatchLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org), + ) + fields["poolCaptures"] = humanPoolTenantProjection("poolCaptures", org, bson.M{"id": "$$row.id", "captureId": "$$row.captureId", "prefix": "$$row.prefix"}) + fields["poolProjects"] = humanPoolTenantProjection("poolProjects", org, bson.M{"id": "$$row.id", "name": "$$row.name"}) + } + indexes := bson.M{} + for field := range fields { + indexes[field] = humanPoolJoinIndex(field) + } + // Runs were already projected before following their capture references. + if q.Search != "" || q.Capture != "" { + indexes["poolRuns"] = humanPoolJoinIndex("poolRuns") + } + pipeline = append(pipeline, bson.D{{Key: "$set", Value: fields}}, bson.D{{Key: "$set", Value: indexes}}) + pipeline = append(pipeline, bson.D{{Key: "$unwind", Value: "$poolItems"}}) + joined := bson.M{"poolTasks": humanPoolItemJoin("poolTasks", "$poolItems.legacyTaskId")} + if q.Search != "" || q.Capture != "" { + joined["poolRuns"] = humanPoolItemJoin("poolRuns", "$poolItems.runId") + joined["poolProjects"] = humanPoolItemJoin("poolProjects", "$poolItems.projectId") + joined["poolCaptures"] = bson.M{"$let": bson.M{"vars": bson.M{"run": humanPoolIndexedRecord("poolRuns", "$poolItems.runId")}, "in": humanPoolItemJoin("poolCaptures", "$$run.taskId")}} + + } + return append(pipeline, + bson.D{{Key: "$replaceWith", Value: bson.M{"$mergeObjects": bson.A{"$poolItems", joined}}}}, + bson.D{{Key: "$unset", Value: "_poolRow"}}, + bson.D{{Key: "$sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}}, + ) +} + +func humanPoolBatchLookup(from, local, as, org string) bson.D { + projection := bson.M{"_id": 0, "id": 1, "organizationId": 1} + fields := map[string][]string{ + "annotationTasks": {"status", "artifactId", "sourceVideo"}, + "runs": {"taskId", "projectName", "captureDirectories"}, + "captureRecords": {"captureId", "prefix"}, + "projects": {"name"}, + } + for _, field := range fields[from] { + projection[field] = 1 + } + if from == "runs" { + // CaptureDirectoriesFromRun gives materialized directories precedence. + // Only historical records need artifact and input-reference fallbacks. + hasDirectories := bson.M{"$gt": bson.A{bson.M{"$size": bson.M{"$ifNull": bson.A{"$captureDirectories", bson.A{}}}}, 0}} + projection["inputs"] = bson.M{"$cond": bson.A{hasDirectories, bson.M{}, bson.M{"$let": bson.M{"vars": bson.M{"row": "$$ROOT"}, "in": humanPoolRunInputRefs()}}}} + projection["artifacts"] = bson.M{"$cond": bson.A{hasDirectories, bson.A{}, humanPoolArtifactNames("$artifacts")}} + projection["nodeExecutions"] = bson.M{"$cond": bson.A{hasDirectories, bson.A{}, bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{"$nodeExecutions", bson.A{}}}, "as": "execution", "in": bson.M{"artifacts": humanPoolArtifactNames("$$execution.artifacts")}}}}} + } + + return bson.D{{Key: "$lookup", Value: bson.M{"from": from, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$match": bson.M{"organizationId": normalizedOrganizationID(org)}}, bson.M{"$project": projection}}, "as": as}}} +} + +func humanPoolTenantProjection(field, org string, projection bson.M) bson.M { + return bson.M{"$map": bson.M{ + "input": bson.M{"$filter": bson.M{"input": "$" + field, "as": "candidate", "cond": bson.M{"$eq": bson.A{"$$candidate.organizationId", normalizedOrganizationID(org)}}}}, + "as": "row", "in": projection, + }} +} + +func humanPoolArtifactNames(input string) bson.M { + return bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{input, bson.A{}}}, "as": "artifact", "in": bson.M{"id": "$$artifact.id", "name": "$$artifact.name", "path": "$$artifact.path"}}} +} + +func humanPoolRunInputRefs() bson.M { + ref := bson.M{} + for _, field := range []string{"captureUnit", "prefix", "key", "manifestKey"} { + ref[field] = "$$input.v.ref." + field + } + return bson.M{"$arrayToObject": bson.M{"$map": bson.M{ + "input": bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$$row.inputs", bson.M{}}}}, + "as": "input", "in": bson.M{"k": "$$input.k", "v": bson.M{"ref": ref}}, + }}} +} + +func humanPoolIndexedRecord(field, id string) bson.M { + // indexOfArray keeps compatibility with Mongo 7.0; dynamic getField needs 7.2. + return bson.M{"$let": bson.M{"vars": bson.M{"position": bson.M{"$indexOfArray": bson.A{"$" + field + ".ids", bson.M{"$ifNull": bson.A{id, nil}}}}}, "in": bson.M{"$cond": bson.A{bson.M{"$gte": bson.A{"$$position", 0}}, bson.M{"$arrayElemAt": bson.A{"$" + field + ".rows", "$$position"}}, bson.M{}}}}} +} + +func humanPoolItemJoin(field, id string) bson.M { + return bson.M{"$let": bson.M{"vars": bson.M{"record": humanPoolIndexedRecord(field, id)}, "in": bson.M{"$cond": bson.A{bson.M{"$ne": bson.A{"$$record", bson.M{}}}, bson.A{"$$record"}, bson.A{}}}}} +} + +func humanPoolJoinIndex(field string) bson.M { + return bson.M{"$let": bson.M{"vars": bson.M{"rows": "$" + field}, "in": bson.M{"ids": "$$rows.id", "rows": "$$rows"}}} +} diff --git a/internal/store/human_pool_search_payload.go b/internal/store/human_pool_search_payload.go new file mode 100644 index 00000000..004bd8a5 --- /dev/null +++ b/internal/store/human_pool_search_payload.go @@ -0,0 +1,84 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Recursive names in old payloads remain searchable. Read those payloads in +// batches, after the globally sorted search metadata has left Mongo's buffer. +func (s *MongoStore) humanPoolSearchLegacyPayloads(ctx context.Context, org string, q domain.HumanItemQuery, rows []adminPoolRow) error { + if q.Search == "" && q.Capture == "" { + return nil + } + ids := []string{} + for _, row := range rows { + name := "" + if len(row.Captures) > 0 { + name = row.Captures[0].CaptureID + if name == "" { + name = domain.CaptureDirectoryFromDirectoryValue(row.Captures[0].Prefix) + } + } + if name == "" { + ids = append(ids, row.ID) + } + } + if len(ids) == 0 { + return nil + } + items, err := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{}, ids) + if err != nil { + return err + } + payloads := map[string]map[string]any{} + for _, item := range items { + payloads[item.ID] = item.Subject.Payload + } + for i := range rows { + if payload, ok := payloads[rows[i].ID]; ok { + rows[i].Subject.Payload = payload + } + } + return nil +} + +func (s *MongoStore) hydrateHumanPoolSearch(ctx context.Context, org string, q domain.HumanItemQuery, ids []string) ([]domain.HumanItemRecord, error) { + out := []domain.HumanItemRecord{} + if len(ids) == 0 { + return out, nil + } + projectionQuery := domain.HumanItemQuery{ListProjection: true} + if q.PoolCandidateProjection { + projectionQuery = domain.HumanItemQuery{PoolCandidateProjection: true} + } + if q.IdentityProjection { + projectionQuery = domain.HumanItemQuery{IdentityProjection: true} + } + cur, err := s.humanItems.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "id": bson.M{"$in": ids}}, options.Find().SetProjection(humanItemListProjection(projectionQuery)).SetMaxTime(8*time.Second).SetBatchSize(200)) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + byID := map[string]domain.HumanItemRecord{} + for cur.Next(ctx) { + var item domain.HumanItemRecord + if err := cur.Decode(&item); err != nil { + return nil, err + } + byID[item.ID] = item + } + if err := cur.Err(); err != nil { + return nil, err + } + for _, id := range ids { + if item, ok := byID[id]; ok { + out = append(out, item) + } + } + return out, nil +} diff --git a/internal/store/human_pool_search_test.go b/internal/store/human_pool_search_test.go index 906ab139..59d9a546 100644 --- a/internal/store/human_pool_search_test.go +++ b/internal/store/human_pool_search_test.go @@ -3,6 +3,7 @@ package store import ( "context" "droboticflow/internal/domain" + "errors" "fmt" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/event" @@ -11,15 +12,20 @@ import ( "net/url" "os" "reflect" + "strings" "sync/atomic" "testing" "time" ) -func TestHumanPoolSearchMongoParity(t *testing.T) { testHumanPoolSearchMongo(t, 963, false) } -func TestHumanPoolSearchMongoLargeModern(t *testing.T) { testHumanPoolSearchMongo(t, 10000, true) } +func TestHumanPoolSearchMongoParity(t *testing.T) { testHumanPoolSearchMongo(t, 963, false, false) } +func TestHumanPoolSearchMongoLargeModern(t *testing.T) { + testHumanPoolSearchMongo(t, 10000, true, false) +} -func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { +func TestHumanPoolSearchMongoLargePayload(t *testing.T) { testHumanPoolSearchMongo(t, 420, true, true) } + +func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) { uri := os.Getenv("POOL_LOCAL_MONGO_URI") if uri == "" { t.Skip("isolated local Mongo opt-in") @@ -45,7 +51,7 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { }, Failed: func(_ context.Context, e *event.CommandFailedEvent) { t.Logf("mongo failed command=%s duration=%s maxTimeMS=%d failure=%s", e.CommandName, e.Duration, lastMaxTime.Load(), e.Failure) }, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { - if measuring.Load() && (e.CommandName == "aggregate" || e.CommandName == "getMore") { + if measuring.Load() && (e.CommandName == "aggregate" || e.CommandName == "getMore" || e.CommandName == "find") { replyBytes.Add(int64(len(e.Reply))) } }} @@ -71,6 +77,9 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { runs, captures := []any{}, []any{} for n := 0; n < count; n++ { i := domain.HumanItemRecord{ID: fmt.Sprintf("i-%04d", n), OrganizationID: org, PluginID: "annotation-qa", ProjectID: "p", RunID: "run", CurrentStageID: "quality_review", Status: domain.HumanItemReady, CreatedAt: time.Date(2026, 11, 1, 0, 0, n, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("FYLX_ITEM_%04d/part0002/camera_left.mp4", n), Payload: map[string]any{"annotationWorkspace": map[string]any{"secret": "not for lists"}}}} + if heavy && n >= 20 { + i.Subject.Payload["unusedCatalog"] = strings.Repeat("x", 256*1024) + } if modern { i.RunID = fmt.Sprintf("modern-run-%d", n) capture := domain.CaptureRecord{ID: fmt.Sprintf("modern-capture-%d", n), OrganizationID: org, CaptureID: fmt.Sprintf("FYLX_ITEM_%04d", n)} @@ -81,6 +90,9 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { _ = mem.UpsertCaptureRecord(ctx, capture) } + if n >= 63 && n <= 65 { + i.CreatedAt = "2026-11-01T00:01:03Z" + } if n >= 10 && n <= 18 { i.CurrentStageID = "annotation_combined" switch n { @@ -114,6 +126,23 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { if n == 3 { i.Subject.ItemKey = "special-[.*-capture" } + if n == 5 { + i.Subject.ItemKey = "historical-input-reference" + i.RunID = "legacy-input-run" + } + if n == 6 { + i.Subject.ItemKey = "cross-tenant-links" + i.RunID, i.ProjectID, i.LegacyTaskID = "foreign-run", "foreign-project", "foreign-task" + } + if n == 7 { + i.Subject.ItemKey = "cross-tenant-capture" + i.RunID = "foreign-capture-run" + } + if n == 8 { + i.Subject.ItemKey = "opaque-legacy-key" + i.RunID = "missing-historical-run" + i.Subject.Payload["importManifest"] = map[string]any{"sourceDirectory": "archive/payload-only-capture"} + } _ = mem.UpsertHumanItem(ctx, i) docs = append(docs, i) } @@ -131,6 +160,43 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { legacy := domain.RunRecord{ID: "legacy-run", OrganizationID: org, CaptureDirectories: []string{"历史目录"}} _ = mem.UpsertRun(ctx, legacy) _ = s.UpsertRun(ctx, legacy) + legacyInput := domain.RunRecord{ID: "legacy-input-run", OrganizationID: org, TaskID: "legacy-input-capture", Inputs: map[string]domain.ArtifactPayload{ + "sourceDirectory": {Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{CaptureUnit: "FYLX_B07_AZER764008K_20260927_074440/part0002"}}, + }} + _ = mem.UpsertRun(ctx, legacyInput) + // Raw insert models records written before captureDirectories was materialized. + if _, err := s.runs.InsertOne(ctx, legacyInput); err != nil { + t.Fatal(err) + } + for _, capture := range []domain.CaptureRecord{ + {ID: "legacy-input-capture", OrganizationID: org, CaptureID: "other-display-name"}, + {ID: "same-org-poison", OrganizationID: org, CaptureID: "TENANT_LEAK"}, + {ID: "foreign-capture", OrganizationID: "other-org", CaptureID: "TENANT_LEAK"}, + } { + _ = mem.UpsertCaptureRecord(ctx, capture) + if _, err := s.captureRecords.InsertOne(ctx, capture); err != nil { + t.Fatal(err) + } + } + for _, run := range []domain.RunRecord{ + {ID: "foreign-run", OrganizationID: "other-org", TaskID: "same-org-poison", ProjectName: "TENANT_LEAK"}, + {ID: "foreign-capture-run", OrganizationID: org, TaskID: "foreign-capture"}, + } { + _ = mem.UpsertRun(ctx, run) + if _, err := s.runs.InsertOne(ctx, run); err != nil { + t.Fatal(err) + } + } + foreignProject := domain.ProjectRecord{ID: "foreign-project", OrganizationID: "other-org", Name: "TENANT_LEAK"} + _ = mem.UpsertProject(ctx, foreignProject) + if _, err := s.projects.InsertOne(ctx, foreignProject); err != nil { + t.Fatal(err) + } + foreignTask := domain.AnnotationTaskRecord{ID: "foreign-task", OrganizationID: "other-org", ArtifactID: "TENANT_LEAK"} + _ = mem.UpsertAnnotationTask(ctx, foreignTask) + if _, err := s.annotationTasks.InsertOne(ctx, foreignTask); err != nil { + t.Fatal(err) + } ids := func(t *testing.T, rows []domain.HumanItemRecord) []string { out := []string{} for _, i := range rows { @@ -141,9 +207,24 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { } return out } + t.Run("canceled-query", func(t *testing.T) { + canceled, stop := context.WithCancel(ctx) + stop() + if _, err := s.ListHumanItems(canceled, org, domain.HumanItemQuery{Search: "FYLX_ITEM"}); !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + }) for _, q := range []domain.HumanItemQuery{ + {Search: "FYLX_B07_AZER764008K_20260927_074440/part0002", Limit: 200}, + {Search: "FYLX_B07_AZER764008K_20260927_074440/part0002", Limit: 200, PoolCandidateProjection: true}, + {Search: "FYLX_ITEM", Limit: 200, IdentityProjection: true}, + {Search: "NO_SUCH_CAPTURE", Limit: 200, PoolCandidateProjection: true}, {Search: "FYLX_ITEM_0001/part0002", Limit: 200}, + {Search: "FYLX_ITEM", Limit: 10, Cursor: &domain.HumanItemCursor{CreatedAt: "2026-11-01T00:01:03Z", ID: "i-0064"}}, {Search: "NO_SUCH_CAPTURE", Limit: 200}, + {Search: "TENANT_LEAK", Limit: 200}, + {Search: "payload-only-capture", Limit: 200, PoolCandidateProjection: true}, + {Capture: "payload-only-capture", Limit: 200}, {Capture: "FYLX_ITEM_0001", Limit: 200}, {Capture: "历史目录", Limit: 200}, {Search: "FYLX_ITEM_0004/PART0002", Limit: 200}, @@ -156,6 +237,9 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { {Category: "accepted", ReviewedPool: true, Limit: 200}, {Category: "rework_review", Limit: 200}, {Category: "pending_acceptance", Limit: 200}, {Category: "rework", Limit: 200}, {Category: "slicing", Limit: 200}, {Category: "pending_annotation", Limit: 200}, {Category: "pending_slice", Limit: 200}, {Category: "available", Limit: 200}, {Search: "FYLX_ITEM", IDs: []string{}, Limit: 200}, } { + if heavy && !q.PoolCandidateProjection && !q.IdentityProjection { + continue + } t.Run(fmt.Sprintf("search=%s_capture=%s_category=%s_reviewed=%t", q.Search, q.Capture, q.Category, q.ReviewedPool), func(t *testing.T) { q.ListProjection = true var clockBefore time.Time diff --git a/internal/store/human_pool_search_text.go b/internal/store/human_pool_search_text.go new file mode 100644 index 00000000..fd8d19a2 --- /dev/null +++ b/internal/store/human_pool_search_text.go @@ -0,0 +1,37 @@ +package store + +import "go.mongodb.org/mongo-driver/bson" + +// This is a conservative prefilter only. The existing Go matcher still makes +// the final decision. Joining metadata into one string avoids compiling and +// executing dozens of identical regex predicates for every candidate. +func humanPoolSearchText() bson.M { + artifacts := func(input string) bson.M { + return humanPoolTextArray(input, "artifact", humanPoolTextConcat("$$value", "$$artifact.id", "$$artifact.name", "$$artifact.path")) + } + refs := humanPoolTextArray(bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$$run.inputs", bson.M{}}}}, "input", humanPoolTextConcat("$$value", "$$input.v.ref.captureUnit", "$$input.v.ref.prefix", "$$input.v.ref.key", "$$input.v.ref.manifestKey")) + runs := humanPoolTextArray("$poolRuns", "run", humanPoolTextConcat("$$value", "$$run.projectName", + humanPoolTextArray("$$run.captureDirectories", "directory", humanPoolTextConcat("$$value", "$$directory")), + refs, artifacts("$$run.artifacts"), + humanPoolTextArray("$$run.nodeExecutions", "execution", humanPoolTextConcat("$$value", artifacts("$$execution.artifacts"))), + )) + return humanPoolTextConcat("$id", "$legacyTaskId", "$subject.itemKey", "$subject.artifactId", runs, + humanPoolTextArray("$poolTasks", "task", humanPoolTextConcat("$$value", "$$task.artifactId", "$$task.sourceVideo")), + humanPoolTextArray("$poolCaptures", "capture", humanPoolTextConcat("$$value", "$$capture.captureId", "$$capture.prefix")), + humanPoolTextArray("$poolProjects", "project", humanPoolTextConcat("$$value", "$$project.name")), + ) +} + +func humanPoolTextConcat(values ...any) bson.M { + parts := bson.A{} + for _, value := range values { + parts = append(parts, " ", bson.M{"$ifNull": bson.A{value, ""}}) + } + return bson.M{"$concat": parts} +} + +func humanPoolTextArray(input any, name string, expression any) bson.M { + // reduce uses this/value; let gives the current element a distinct name + // so nested artifact/input reductions cannot shadow their parent record. + return bson.M{"$reduce": bson.M{"input": bson.M{"$ifNull": bson.A{input, bson.A{}}}, "initialValue": "", "in": bson.M{"$let": bson.M{"vars": bson.M{name: "$$this"}, "in": expression}}}} +} diff --git a/src/lib/annotation-project-loading.js b/src/lib/annotation-project-loading.js index c3f632ab..21ae1197 100644 --- a/src/lib/annotation-project-loading.js +++ b/src/lib/annotation-project-loading.js @@ -124,7 +124,7 @@ export async function loadOwnedAnnotationProject({ export async function loadAnnotationPoolPage({ annotation, human, scope, view, cursor = "", signal = undefined }) { signal?.throwIfAborted(); if (view.kind === "review") { - const names = cursor + const names = view.pool ? [view.pool] : view.isManager ? ["available", "mine", "all"] diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index ff4a1489..0ba04487 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -954,6 +954,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const [saving, setSaving] = useState(false); const [error, setError] = useState(""); const [projectLoadError, setProjectLoadError] = useState(""); + const [taskPoolReadError, setTaskPoolReadError] = useState(""); const [projectLoadRecoveryNotice, setProjectLoadRecoveryNotice] = useState(""); const [combinedSubmissionNotice, setCombinedSubmissionNotice] = useState(""); // Acceptance-fixes defect 4/5: a dirty review draft used to make 开始/继续审查, @@ -1063,6 +1064,8 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: ...(managerAnnotationPool ? [managerPoolFilters] : []), ...(pagedReviewPool ? [adminSearch, taskStatusFilter, requestedCaptureDirectory] : []), ]); + const reviewPoolViewRef = useRef(taskPoolView); + reviewPoolViewRef.current = taskPoolView; const reviewActionScopeRef = useRef(projectDataScope); if (reviewActionScopeRef.current !== projectDataScope) skipClaimPoolReload.current = ""; reviewActionScopeRef.current = projectDataScope; @@ -1230,13 +1233,15 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const current = () => !controller.signal.aborted && requestID === projectLoadRequestRef.current && - reviewActionScopeRef.current === projectDataScope; + reviewActionScopeRef.current === projectDataScope && + (!pagedReviewPool || reviewPoolViewRef.current === taskPoolView); // A refresh supersedes pagination even when the project ID stays the same. taskPoolFetchOwnerRef.current = null; setTaskPoolFetchBusy(false); taskPoolCursorRef.current = ""; minePoolCursorRef.current = ""; reviewPoolCursorsRef.current = {}; + setTaskPoolReadError(""); projectReadAbortRef.current?.abort(); poolPageAbortRef.current?.abort(); projectReadAbortRef.current = controller; @@ -1310,6 +1315,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: ? { kind: "admin", filters: managerPoolFilters } : { kind: pagedReviewPool ? "review" : "workspace", + pool: pagedReviewPool ? taskPoolView : undefined, search: adminSearch, capture: requestedCaptureDirectory, category: taskStatusFilter, @@ -1442,6 +1448,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: managerAnnotationPool, managerPoolFilters, pagedReviewPool, + taskPoolView, adminSearch, requestedCaptureDirectory, taskStatusFilter, @@ -2734,7 +2741,8 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: !controller.signal.aborted && taskPoolFetchOwnerRef.current === owner && reviewActionScopeRef.current === scope && - projectLoadRequestRef.current === epoch; + projectLoadRequestRef.current === epoch && + (!pagedReviewPool || reviewPoolViewRef.current === taskPoolView); setTaskPoolFetchBusy(true); try { poolPageAbortRef.current?.abort(); @@ -2771,6 +2779,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: signal: controller.signal, }); if (!current()) return; + setTaskPoolReadError(""); taskPoolCursorRef.current = page.taskCursor; minePoolCursorRef.current = page.mineCursor; if (page.poolCursors) reviewPoolCursorsRef.current = { ...reviewPoolCursorsRef.current, ...page.poolCursors }; @@ -2792,7 +2801,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: setTasks((current) => [...current, ...page.tasks.filter((item) => !current.some((old) => old.id === item.id))]); setTaskRenderLimit((current) => current + taskPageLimit); } catch (cause) { - if (current()) setError(getActionErrorMessage(cause, "加载更多任务失败,请重试")); + if (current()) setTaskPoolReadError(getActionErrorMessage(cause, "加载更多任务失败,请重试")); } finally { // An obsolete response cannot release a newer page's busy indicator. if (taskPoolFetchOwnerRef.current === owner) { @@ -6770,10 +6779,13 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: combinedSubmissionNotice || error || projectLoadError || + taskPoolReadError || combinedSubmissionConflicted || projectLoadRecoveryNotice ? (
- 工作状态{error || projectLoadError || stageResultUnknown ? " · 需处理" : ""} + + 工作状态{error || projectLoadError || taskPoolReadError || stageResultUnknown ? " · 需处理" : ""} +
{workPage && randomClaimNotice ? (

@@ -6831,14 +6843,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: )} {combinedSubmissionNotice}

- ) : error || projectLoadError ? ( + ) : error || projectLoadError || taskPoolReadError ? (
- {(error || projectLoadError)?.includes("project annotation membership is required") + {(error || projectLoadError || taskPoolReadError)?.includes("project annotation membership is required") ? "您当前没有该任务所属项目的标注成员权限(或该任务所属项目已归档/移除)" - : error || projectLoadError} + : error || projectLoadError || taskPoolReadError}
) : combinedSubmissionConflicted ? (
{label} {pagedReviewPool ? "已加载 " : ""} - {managerAnnotationPool && adminPoolTotals?.scope !== projectDataScope + {(managerAnnotationPool && adminPoolTotals?.scope !== projectDataScope) || + (pagedReviewPool && !(value in reviewPoolCursorsRef.current)) ? "—" : blindAnnotator && value === "available" ? claimableCount.projectId === projectId diff --git a/tests/browser/review-pool-search-replica.mjs b/tests/browser/review-pool-search-replica.mjs index 8135e57b..4c3199e5 100644 --- a/tests/browser/review-pool-search-replica.mjs +++ b/tests/browser/review-pool-search-replica.mjs @@ -34,6 +34,8 @@ try { if (!["GET", "HEAD"].includes(r.method())) report.writes.push(u.pathname); if (u.pathname === "/api/human-items") { report.requests.push(Object.fromEntries(u.searchParams)); + if (u.searchParams.get("pool") === "available" && u.searchParams.get("search") === "FYLX_SEARCH_0612") + return route.fulfill({ status: 504, json: { error: "合成可领取池超时" } }); if (u.searchParams.get("cursor") && failNext) { failNext = false; return route.fulfill({ status: 500, json: { error: "合成分页失败" } }); @@ -66,6 +68,7 @@ try { await more.click(); await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 600); report.cases.push("failed page retry retains cursor and rows"); + assert.equal(await page.getByText("合成分页失败", { exact: true }).count(), 0); const search = page.getByRole("textbox", { name: "搜索标注任务", exact: true }); await search.fill("FYLX_SEARCH_0001/part0002"); await cards.filter({ hasText: "FYLX_SEARCH_0001" }).waitFor(); @@ -93,7 +96,8 @@ try { await page.getByTestId("human-task-pool-reviewed").click(); await cards.filter({ hasText: "FYLX_SEARCH_0612" }).waitFor(); assert.equal(await cards.count(), 1); - report.cases.push("reviewed search retains old returned-stage history"); + report.cases.push("reviewed search retains old returned-stage history despite unrelated available timeout"); + assert.ok(!report.requests.some((r) => r.pool === "available" && r.search === "FYLX_SEARCH_0612")); assert.ok(report.requests.every((r) => Number(r.limit) <= 200)); assert.deepEqual(report.pageErrors, []); assert.deepEqual(report.writes, []); diff --git a/tests/frontend/annotation-pool-loading.test.mjs b/tests/frontend/annotation-pool-loading.test.mjs index dc0b2323..a0f04489 100644 --- a/tests/frontend/annotation-pool-loading.test.mjs +++ b/tests/frontend/annotation-pool-loading.test.mjs @@ -85,6 +85,52 @@ test("an obsolete owned-project count cannot publish after cancellation", async }); import { loadAnnotationPoolPage } from "../../src/lib/annotation-project-loading.js"; +test("reviewed search never waits for or requests unrelated slow pools", async () => { + const calls = []; + const result = await loadAnnotationPoolPage({ + scope: {}, + view: { + kind: "review", + pool: "reviewed", + search: "FYLX_B07/part0002", + category: "all", + stageIds: ["quality_review"], + }, + human: { + listItemsPage: async (filters) => { + calls.push(filters); + assert.equal(filters.pool, "reviewed", "unrelated pools may time out"); + return { + items: [{ id: "reviewed", legacyTaskId: "task", canClaim: false }], + filtersApplied: true, + hasMore: false, + }; + }, + }, + annotation: { listTasksPage: async () => ({ items: [{ id: "task" }] }) }, + }); + assert.equal(calls.length, 1); + assert.equal(calls[0].stageIds, undefined, "returned-stage review history remains visible"); + assert.equal(result.pools.reviewed[0].id, "reviewed"); + assert.deepEqual(result.poolCursors, { reviewed: "" }); +}); + +test("selected review pool failure propagates instead of presenting an empty search", async () => { + await assert.rejects( + loadAnnotationPoolPage({ + scope: {}, + view: { kind: "review", pool: "mine", search: "capture", category: "all" }, + human: { + listItemsPage: async () => { + throw new Error("selected pool failed"); + }, + }, + annotation: {}, + }), + /selected pool failed/, + ); +}); + const scope = { projectId: "p", workflowId: "w", runId: "r", nodeId: "n", artifactId: "a" }; function fixture(view = {}) { const calls = []; diff --git a/tests/frontend/annotation-pool-pagination.test.mjs b/tests/frontend/annotation-pool-pagination.test.mjs index 9a19006d..c7955e83 100644 --- a/tests/frontend/annotation-pool-pagination.test.mjs +++ b/tests/frontend/annotation-pool-pagination.test.mjs @@ -54,6 +54,7 @@ function fixture(kind) { minePoolCursorRef: { current: kind === "sidebar" ? "old-cursor" : "" }, projectLoadRequestRef: { current: 1 }, reviewActionScopeRef: { current: "scope-a" }, + reviewPoolViewRef: { current: "mine" }, projectReadAbortRef: { current: null }, poolPageAbortRef: { current: null }, annotationOperationsApi: { adminPool: read, getSidebar: read, listTasksPage: read }, @@ -65,6 +66,9 @@ function fixture(kind) { setSubmittedHumanItems: () => updates.push(["submitted"]), setTaskRenderLimit: () => updates.push(["render"]), setError: () => updates.push(["error"]), + setTaskPoolReadError: (value) => { + if (value) updates.push(["error"]); + }, getActionErrorMessage: (e) => e.message, }; const run = new Function("env", `with(env){${js}; return taskPoolLoadMore;}`)( @@ -323,3 +327,27 @@ test("review continuation is single-flight and only advances its pool cursor", a await pending; assert.equal(f.env.reviewPoolCursorsRef.current.mine, "new-cursor"); }); + +test("switching review tabs invalidates the old page before the next refresh starts", async () => { + const f = fixture("review"); + const pending = f.run(); + f.env.reviewPoolViewRef.current = "reviewed"; + f.resolve(); + await pending; + assert.ok(!f.updates.some(([kind]) => ["pools", "tasks", "render", "error"].includes(kind))); + assert.equal(f.env.reviewPoolCursorsRef.current.mine, "review-old"); +}); + +test("successful pool retry clears its own error without clearing an action error", async () => { + const f = fixture("admin"); + const errors = []; + f.env.setTaskPoolReadError = (value) => errors.push(value); + const first = f.run(); + f.reject(new Error("pool failed")); + await first; + assert.deepEqual(errors, ["pool failed"]); + f.env.annotationOperationsApi.adminPool = async () => ({ items: [], tasks: [], counts: {}, nextCursor: "" }); + await f.run(); + assert.deepEqual(errors, ["pool failed", ""]); + assert.ok(!f.updates.some(([kind]) => kind === "error")); +}); diff --git a/tests/frontend/review-pool-search.test.mjs b/tests/frontend/review-pool-search.test.mjs index 134eb6c0..09ebd9a1 100644 --- a/tests/frontend/review-pool-search.test.mjs +++ b/tests/frontend/review-pool-search.test.mjs @@ -31,19 +31,19 @@ function fixture(pool = "available") { }, }; } -test("review initial reads retain all pool cursors and push search before pagination", async () => { +test("review initial reads load only the selected pool and push search before pagination", async () => { const f = fixture(); const signal = new AbortController().signal; const page = await loadAnnotationPoolPage({ ...f.options, signal }); - assert.equal(f.calls.length, 3); - assert.deepEqual(page.poolCursors, { available: "available-next", mine: "mine-next", reviewed: "reviewed-next" }); + assert.equal(f.calls.length, 1); + assert.deepEqual(page.poolCursors, { available: "available-next" }); assert.equal(page.pools.available[0].canClaim, true); for (const { filters, options } of f.calls) { assert.equal(filters.search, "old/part0002"); assert.equal(filters.limit, 200); assert.equal(options.signal, signal); } - assert.equal(f.calls.find(({ filters }) => filters.pool === "reviewed").filters.stageIds, undefined); + assert.deepEqual(f.calls[0].filters.stageIds, ["quality_review", "split_quality_review"]); }); for (const pool of ["available", "mine", "reviewed"]) test(`${pool} consumes only its own cursor and joins just this page`, async () => { @@ -52,6 +52,7 @@ for (const pool of ["available", "mine", "reviewed"]) assert.equal(f.calls.length, 1); assert.equal(f.calls[0].filters.pool, pool); assert.equal(f.calls[0].filters.cursor, "previous"); + if (pool === "reviewed") assert.equal(f.calls[0].filters.stageIds, undefined); assert.equal(page.poolCursors[pool], pool + "-next"); assert.deepEqual(page.tasks, [{ id: pool + "-task" }]); }); From c942a8ced0141974b63e14c068988e9755e44c2e Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 6 Nov 2026 00:25:45 -0800 Subject: [PATCH 4/8] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20Repair=20worktree?= =?UTF-8?q?=20verification=20and=20browser=20guardrails?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/web-ci.yml | 29 ++++++ .scratch/review-pool-search/work.md | 8 +- Dockerfile.api | 1 + docs/agents/verification.md | 10 ++ package.json | 8 +- scripts/check-git-hooks.mjs | 25 +---- scripts/lib/browser-executable.mjs | 31 ++++++ scripts/lib/git-hooks.mjs | 33 +++++++ scripts/test-dashboard-presentation.mjs | 15 ++- scripts/verification-scopes.json | 99 +++++++++++++++++-- scripts/verify-change.py | 62 ++++++++---- tests/browser/acceptance-playback-rate.mjs | 38 +++++-- .../browser/admin-annotation-pool-replica.mjs | 5 +- .../annotation-source-validation-replica.mjs | 42 ++++++-- .../dashboard-presentation-replica.mjs | 4 +- tests/browser/production-quality-replica.mjs | 12 ++- .../browser/review-authoring-permissions.mjs | 5 +- tests/browser/review-decision-failure.mjs | 41 ++++++-- tests/browser/review-decision-lifecycle.mjs | 10 +- tests/browser/review-decision-start-race.mjs | 4 +- tests/browser/review-draft-persistence.mjs | 4 +- tests/browser/review-drafts.mjs | 4 +- tests/browser/review-pool-search-replica.mjs | 5 +- tests/browser/review-trace.mjs | 37 +++++-- tests/browser/rework-boundary-replica.mjs | 5 +- tests/browser/run-editing-regression.mjs | 49 ++++----- tests/browser/run-production-boot.mjs | 4 +- tests/frontend/browser-executable.test.mjs | 49 +++++++++ tests/frontend/dev-guardrails.test.mjs | 20 +++- .../frontend/fanyu-comment-followup.test.mjs | 13 ++- tests/frontend/git-hooks.test.mjs | 40 ++++++++ tests/frontend/guardrails-ci.test.mjs | 20 ++++ tests/tooling/test_verification.py | 34 ++++++- 33 files changed, 638 insertions(+), 128 deletions(-) create mode 100644 scripts/lib/browser-executable.mjs create mode 100644 scripts/lib/git-hooks.mjs create mode 100644 tests/frontend/browser-executable.test.mjs create mode 100644 tests/frontend/git-hooks.test.mjs create mode 100644 tests/frontend/guardrails-ci.test.mjs diff --git a/.gitea/workflows/web-ci.yml b/.gitea/workflows/web-ci.yml index a2d4a8d4..819e627a 100644 --- a/.gitea/workflows/web-ci.yml +++ b/.gitea/workflows/web-ci.yml @@ -24,6 +24,7 @@ jobs: node-version: 22 cache: npm - run: npm ci + - run: npm run test:verification - run: npm run lint - run: npm run format:check - run: npm run test:go:gate @@ -33,6 +34,14 @@ jobs: if: ${{ !cancelled() }} - run: npm run test:browser:editing if: ${{ !cancelled() }} + - run: npm run test:browser:review-permissions + if: ${{ !cancelled() }} + - run: npm run test:browser:dashboard + if: ${{ !cancelled() }} + - run: go test ./internal/api -run '^TestReviewPoolSearchBrowserReplica$' -count=1 -timeout=3m -v + if: ${{ !cancelled() }} + env: + REVIEW_POOL_BROWSER_TEST: '1' - uses: actions/upload-artifact@v4 if: always() with: @@ -40,3 +49,23 @@ jobs: path: local-data/test-results/ if-no-files-found: warn retention-days: 14 + + pool-search-mongo: + runs-on: ubuntu-22.04 + services: + mongo: + image: mongo:8 + ports: + - 27018:27017 + options: >- + --health-cmd "mongosh --quiet --eval 'db.adminCommand({ping:1})'" + --health-interval 5s --health-timeout 5s --health-retries 12 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - run: go test ./internal/store -run '^TestHumanPoolSearchMongo' -count=1 -timeout=3m -v + env: + POOL_LOCAL_MONGO_URI: mongodb://127.0.0.1:27018 diff --git a/.scratch/review-pool-search/work.md b/.scratch/review-pool-search/work.md index f0437b9e..a6b9949b 100644 --- a/.scratch/review-pool-search/work.md +++ b/.scratch/review-pool-search/work.md @@ -3,7 +3,6 @@ Goal: Restore the reported reviewer history search without making it wait for unrelated pool queries. Agent: Codex Base: live e4969acc (codex/acceptance-playback-rate, PR #72), preserving its playback fixes. -Status: Local candidate; not pushed, submitted as PR, or deployed. ## Changes @@ -33,3 +32,10 @@ Evidence directory: local-data/pool-search-verification/ (ignored). - reviewer-scoped-readonly.log: the exact source-generated Mongo pipeline, restricted to each of the two actual reviewers' history IDs (79 and 98), finds one matching row in 583ms and 734ms. These are backend pipeline timings, not customer-browser end-to-end acceptance. No impersonation or customer writes. - Wide administrator/all-history and broad available-pool search can still exceed 8 seconds. The selected-pool isolation fixes the reported history page being blocked by that failure; it does not prove all broad-pool latency or capacity has been resolved. - No production write acceptance, 81-user load test, or release gates were performed for this task. Push, PR and deployment require new task-specific authorization. + +## Retrospective follow-up (base adc63af8) + +Scope: Repair worktree hook bootstrap, verification selection/dependencies/results, browser executable resolution and stale dashboard assertions. Wire the existing synthetic pool Mongo suite into CI. Reuse existing installed dependencies and isolated local services. +Risk: Local/CI tooling only; no business query, delivery contract, customer record, migration, resource or deployment changes in this follow-up. Keep the wide-pool performance limitation above open. + +Verification follow-up evidence: local-data/retro-guardrails/. The unchanged browser/product inputs passed all 23 task checks in verify-1793952872699139000/report.json (338.854s, no source drift). Earlier failures and the 600-second budget exhaustion remain recorded. Subsequent Docker COPY ordering and its regression affect only the dependency-install bootstrap: Docker-prepare regression and final quick checks are run separately; application/browser/Mongo inputs are unchanged. No known-failure baseline was expanded. CI wiring is validated locally; the remote CI has not run because this branch remains local. diff --git a/Dockerfile.api b/Dockerfile.api index 7d79037e..ddebc818 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -6,6 +6,7 @@ WORKDIR /app COPY package*.json ./ COPY scripts/check-git-hooks.mjs ./scripts/check-git-hooks.mjs +COPY scripts/lib/git-hooks.mjs ./scripts/lib/git-hooks.mjs RUN npm ci COPY index.html metadata.json package.json tsconfig.json vite.config.ts ./ diff --git a/docs/agents/verification.md b/docs/agents/verification.md index a41c0533..8986c80e 100644 --- a/docs/agents/verification.md +++ b/docs/agents/verification.md @@ -34,6 +34,16 @@ - 退出码 0 仅表示所选检查通过;1 表示检查失败;2 表示超时、跳过、未映射、执行中源码变化或入口异常。报告中的 deferred/notCovered 始终保留,不把快速通过解释成完整通过。 - 遇到已有测试债务,使用既有 `test:frontend:gate` / `test:go:gate` 比较具体用例。不因失败而扩大 known-failures 基线,不手工忽略断言。 +## 检查选择与环境初始化 + +标准开发入口与本地验证先检查并补齐当前 worktree 的 Husky wrappers,复用依赖目录也执行该步骤。自定义 hooksPath 保持原样并报告冲突;CI、导出源码和显式 HUSKY=0 保留原有行为。 + +快速 Go 检查按包内实际用例展开范围规则,交互预览、Mongo、浏览器及容量用例由专项入口处理。任务映射显式声明所需 opt-in 环境;执行器清理继承的测试开关,避免误连外部服务。领取池 Mongo 等价和大载荷回归在任务级执行,也由 CI 的隔离 Mongo 服务执行。 + +独立检查在其他项失败或跳过后继续执行,报告保留每一项结果;依赖检查仅在前置检查通过后执行。例如构建失败时,生产产物启动检查标为未运行,避免读取旧 dist。预算耗尽仍停止后续执行,失败和未完成仍决定最终退出码。 + +浏览器门禁共用解析器:优先 BROWSER_EXECUTABLE,兼容 BROWSER_PATH;Mac 优先使用已安装的 Edge/Chrome,其余环境优先使用已安装的 Playwright Chromium,再查找本机 Chromium 浏览器。显式路径无效会报错。倍速回归默认创建并清理临时本地 Vite 服务,也支持显式指定本地测试 URL。 + ## 减少重复与环境准备 - 同一批同一输入只做一次成功验证;优先复用已有 npm 测试组、Go 编译/测试缓存和本地隔离 Mongo 服务,不为每次小改动重装依赖、重建镜像或重启整套副本。 diff --git a/package.json b/package.json index 593e3a80..4aabe311 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,13 @@ "verify:quick": "python3 scripts/verify-change.py --mode quick", "verify:task": "python3 scripts/verify-change.py --mode task", "verify:integration": "python3 scripts/verify-change.py --mode integration", - "test:verification": "python3 -m unittest discover -s tests/tooling -p test_verification.py" + "test:verification": "python3 -m unittest discover -s tests/tooling -p test_verification.py", + "predev": "npm run hooks:install", + "predev:full": "npm run hooks:install", + "predev:api": "npm run hooks:install", + "predev:web": "npm run hooks:install", + "predev:worker": "npm run hooks:install", + "predev:api:replica": "npm run hooks:install" }, "dependencies": { "@tailwindcss/vite": "^4.1.14", diff --git a/scripts/check-git-hooks.mjs b/scripts/check-git-hooks.mjs index d2c0afa2..1fc1854c 100644 --- a/scripts/check-git-hooks.mjs +++ b/scripts/check-git-hooks.mjs @@ -1,30 +1,11 @@ #!/usr/bin/env node -import { execFileSync } from "node:child_process"; -import { accessSync, constants, existsSync } from "node:fs"; +import { ensureGitHooks } from "./lib/git-hooks.mjs"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); try { - if (process.env.CI || process.env.HUSKY === "0" || !existsSync(resolve(root, ".git"))) { - console.log("Git hooks: skipped (CI, HUSKY=0, or exported source)"); - } else { - let configured = ""; - try { - configured = execFileSync("git", ["config", "--get", "core.hooksPath"], { cwd: root, encoding: "utf8" }).trim(); - } catch {} - if (process.argv.includes("--install")) { - if (configured && resolve(root, configured) !== resolve(root, ".husky/_")) - throw Error("Custom hooksPath detected; left unchanged"); - execFileSync(process.execPath, [resolve(root, "node_modules/husky/bin.js")], { cwd: root, stdio: "inherit" }); - configured = execFileSync("git", ["config", "--get", "core.hooksPath"], { cwd: root, encoding: "utf8" }).trim(); - } - if (resolve(root, configured) !== resolve(root, ".husky/_")) - throw Error("core.hooksPath must resolve to this checkout's .husky/_"); - for (const name of [".husky/_/pre-commit"]) accessSync(resolve(root, name), constants.X_OK); - accessSync(resolve(root, ".husky/_/h"), constants.R_OK); - accessSync(resolve(root, ".husky/pre-commit"), constants.R_OK); - console.log(`Git hooks: active in ${root}`); - } + const state = ensureGitHooks(root, { install: process.argv.includes("--install") }); + console.log(`Git hooks: ${state}${state === "active" ? ` in ${root}` : " (CI, HUSKY=0, or exported source)"}`); } catch (error) { console.error(`Git hooks unavailable: ${error.message}. Run npm run hooks:install in this checkout.`); process.exitCode = 1; diff --git a/scripts/lib/browser-executable.mjs b/scripts/lib/browser-executable.mjs new file mode 100644 index 00000000..00506c87 --- /dev/null +++ b/scripts/lib/browser-executable.mjs @@ -0,0 +1,31 @@ +import { existsSync } from "node:fs"; +import { chromium } from "playwright"; + +export function browserExecutableOptions({ + env = process.env, + platform = process.platform, + exists = existsSync, + bundled = chromium.executablePath(), +} = {}) { + const explicit = env.BROWSER_EXECUTABLE || env.BROWSER_PATH; + if (explicit) { + if (!exists(explicit)) throw Error(`Configured browser does not exist: ${explicit}`); + return { executablePath: explicit }; + } + const candidates = + platform === "darwin" + ? [ + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + ] + : ["/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome"]; + const executablePath = + platform === "darwin" + ? candidates.find(exists) || (exists(bundled) ? bundled : undefined) + : exists(bundled) + ? bundled + : candidates.find(exists); + if (!executablePath) + throw Error("No Chromium browser found. Set BROWSER_EXECUTABLE or run npx playwright install chromium."); + return { executablePath }; +} diff --git a/scripts/lib/git-hooks.mjs b/scripts/lib/git-hooks.mjs new file mode 100644 index 00000000..81ecd397 --- /dev/null +++ b/scripts/lib/git-hooks.mjs @@ -0,0 +1,33 @@ +import { execFileSync } from "node:child_process"; +import { accessSync, constants, existsSync } from "node:fs"; +import { resolve } from "node:path"; + +export function ensureGitHooks(root, { install = false, env = process.env } = {}) { + if (env.CI || env.HUSKY === "0" || !existsSync(resolve(root, ".git"))) return "skipped"; + const git = (...args) => + execFileSync("git", args, { cwd: root, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + let configured = ""; + try { + configured = git("config", "--get", "core.hooksPath"); + } catch {} + const expected = resolve(root, ".husky/_"); + if (configured && resolve(root, configured) !== expected) throw Error("Custom hooksPath detected; left unchanged"); + if (install && configured && resolve(root, configured) === expected) { + try { + accessSync(resolve(root, ".husky/_/pre-commit"), constants.X_OK); + accessSync(resolve(root, ".husky/_/h"), constants.R_OK); + accessSync(resolve(root, ".husky/pre-commit"), constants.R_OK); + return "active"; + } catch {} + } + if (install) { + execFileSync(process.execPath, [resolve(root, "node_modules/husky/bin.js")], { cwd: root, env, stdio: "pipe" }); + configured = git("config", "--get", "core.hooksPath"); + } + if (!configured || resolve(root, configured) !== expected) + throw Error("core.hooksPath must resolve to this checkout's .husky/_"); + accessSync(resolve(root, ".husky/_/pre-commit"), constants.X_OK); + accessSync(resolve(root, ".husky/_/h"), constants.R_OK); + accessSync(resolve(root, ".husky/pre-commit"), constants.R_OK); + return "active"; +} diff --git a/scripts/test-dashboard-presentation.mjs b/scripts/test-dashboard-presentation.mjs index 878f0ac1..6064c66d 100644 --- a/scripts/test-dashboard-presentation.mjs +++ b/scripts/test-dashboard-presentation.mjs @@ -3,7 +3,7 @@ import { createServer } from "node:net"; import { spawn } from "node:child_process"; import { once } from "node:events"; -import { mkdirSync, openSync, closeSync, existsSync } from "node:fs"; +import { mkdirSync, openSync, closeSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -25,10 +25,8 @@ const freePort = async () => { const apiPort = await freePort(); let webPort = await freePort(); while (webPort === apiPort) webPort = await freePort(); -const edge = "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"; const env = { ...process.env, - ...(process.env.BROWSER_EXECUTABLE ? {} : existsSync(edge) ? { BROWSER_EXECUTABLE: edge } : {}), VITE_API_BASE_URL: "", VITE_DEV_API_BASE_URL: `http://127.0.0.1:${apiPort}`, VITE_DEV_WORKER_BASE_URL: "http://127.0.0.1:1", @@ -86,7 +84,16 @@ try { ); const web = start( process.execPath, - ["node_modules/vite/bin/vite.js", "--host", "127.0.0.1", "--port", String(webPort), "--strictPort"], + [ + "node_modules/vite/bin/vite.js", + "--config", + "tests/browser/vite.platform-feedback.config.ts", + "--host", + "127.0.0.1", + "--port", + String(webPort), + "--strictPort", + ], "web.log", ); await Promise.all([ready(`http://127.0.0.1:${apiPort}/api/health`, api), ready(`http://127.0.0.1:${webPort}/`, web)]); diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index c3721d12..c804413f 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1,6 +1,79 @@ { "version": 1, "scopes": [ + { + "name": "verification-tooling", + "paths": [ + "^scripts/(check-git-hooks\\.mjs|lib/(git-hooks|browser-executable)\\.mjs|verify-change\\.py|verification-scopes\\.json)$", + "^tests/tooling/", + "^tests/frontend/(git-hooks|browser-executable|guardrails-ci)\\.test\\.mjs$", + "^\\.gitea/workflows/web-ci\\.yml$", + "^Dockerfile\\.api$" + ], + "frontend": [ + "tests/frontend/git-hooks.test.mjs", + "tests/frontend/browser-executable.test.mjs", + "tests/frontend/guardrails-ci.test.mjs", + "tests/frontend/dev-guardrails.test.mjs" + ], + "go": {}, + "task": [] + }, + { + "name": "verification-browser-harnesses", + "paths": [ + "^tests/browser/(review-drafts|dashboard-presentation-replica|run-editing-regression)\\.mjs$", + "^scripts/test-dashboard-presentation\\.mjs$" + ], + "frontend": [ + "tests/frontend/browser-executable.test.mjs" + ], + "go": {}, + "task": [ + { + "argv": [ + "npm", + "run", + "test:browser:editing" + ], + "tests": true + } + ] + }, + { + "name": "production-boot-harness", + "paths": [ + "^tests/browser/run-production-boot\\.mjs$" + ], + "frontend": [ + "tests/frontend/browser-executable.test.mjs" + ], + "go": {}, + "task": [ + { + "argv": [ + "npm", + "run", + "build:web" + ] + }, + { + "argv": [ + "npm", + "run", + "test:production:boot" + ], + "requires": [ + [ + "npm", + "run", + "build:web" + ] + ], + "tests": true + } + ] + }, { "name": "acceptance-playback-rate", "paths": [ @@ -16,15 +89,12 @@ "node", "tests/browser/acceptance-playback-rate.mjs" ], - "env": { - "REVIEW_TEST_URL": "http://127.0.0.1:5197", - "BROWSER_PATH": "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge" - }, + "env": {}, "tests": true } ], "warnings": [ - "倍速浏览器回归使用本机5197测试服务与合成媒体;不连接生产。" + "倍速浏览器回归默认自建临时本地测试服务与合成媒体;不连接生产。" ] }, { @@ -132,7 +202,8 @@ ], "frontend": [ "tests/frontend/annotation-pool-*.test.mjs", - "tests/frontend/review-pool-preview.test.mjs" + "tests/frontend/review-pool-preview.test.mjs", + "tests/frontend/review-pool-search.test.mjs" ], "go": { "./internal/api": [ @@ -156,6 +227,22 @@ "REVIEW_POOL_BROWSER_TEST": "1" }, "tests": true + }, + { + "argv": [ + "go", + "test", + "./internal/store", + "-run", + "^TestHumanPoolSearchMongo", + "-count=1", + "-timeout=3m", + "-v" + ], + "env": { + "POOL_LOCAL_MONGO_URI": "mongodb://127.0.0.1:27018" + }, + "tests": true } ] }, diff --git a/scripts/verify-change.py b/scripts/verify-change.py index 7ff8e4e8..177f4bae 100644 --- a/scripts/verify-change.py +++ b/scripts/verify-change.py @@ -28,7 +28,10 @@ def changed_files(root, base): def command(argv, env=None, tests=False): - return {'argv': argv, 'env': env or {}, 'tests': tests} + result = {'argv': argv, 'env': env or {}, 'tests': tests} + if argv == ['npm', 'run', 'test:production:boot']: + result['requires'] = [['npm', 'run', 'build:web']] + return result def tooling_package_change(root, base): @@ -39,12 +42,12 @@ def tooling_package_change(root, base): return False old_scripts, new_scripts = old.pop('scripts', {}), new.pop('scripts', {}) names = {name for name in set(old_scripts) | set(new_scripts) if old_scripts.get(name) != new_scripts.get(name)} - allowed = {'verify:plan', 'verify:quick', 'verify:task', 'verify:integration', 'test:verification'} + allowed = {'verify:plan', 'verify:quick', 'verify:task', 'verify:integration', 'test:verification', 'predev', 'predev:full', 'predev:api', 'predev:web', 'predev:worker', 'predev:api:replica'} return old == new and bool(names) and names <= allowed def plan_checks(root, files, policy, mode, base, tooling_package=False): - quick = [command(['git', 'diff', '--check', base, '--'])] + quick = [command(['git', 'diff', '--check', base, '--']), command(['node', 'scripts/check-git-hooks.mjs', '--install'])] task = [] warnings = [] unresolved = [] @@ -89,7 +92,7 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): fast = [] if test_file.exists(): names = re.findall(r'^func (Test\w+)\(', test_file.read_text(), re.M) - fast = [name for name in names if not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark', name)] + fast = [name for name in names if not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark|Preview', name)] if fast: go_patterns.setdefault(pkg, set()).update('^' + name + '$' for name in fast) if len(fast) != len(names) and f not in matched: @@ -100,6 +103,9 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): # Package-wide fallback is explicit at task level, not every edit. task.append(command(['go', 'test', pkg, '-timeout=5m', '-v'], tests=True)) continue + if f in ('scripts/verify-change.py', 'scripts/verification-scopes.json') or f.startswith('tests/tooling/'): + quick.append(command([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests/tooling', '-p', 'test_verification.py'], tests=True)) + continue if f in matched or f.endswith('.md') or f.startswith('docs/') and f.endswith(('.png', '.jpg', '.svg')): continue if f.startswith('src/') and f.endswith('.css'): @@ -120,9 +126,6 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): command(['npm', 'run', 'build:web']), command(['npm', 'run', 'test:production:boot'])]) warnings.append('依赖/公共配置变化:任务级扩大到集成检查,核对安装与锁文件。') continue - if f in ('scripts/verify-change.py', 'scripts/verification-scopes.json') or f.startswith('tests/tooling/'): - quick.append(command([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests/tooling', '-p', 'test_verification.py'], tests=True)) - continue if path.exists() and f.endswith('.py'): quick.append(command([sys.executable, 'scripts/verify-change.py', '--python-syntax', f])) elif path.exists() and f.endswith('.mjs'): @@ -131,9 +134,20 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): quick.append(command(['bash', '-n', f])) unresolved.append(f + ': 尚无行为验证映射;补充映射/专项证据,语法通过不代表功能通过。') if frontend: - quick.append(command(['node', '--test', *sorted(frontend)], tests=True)) + known = json.loads((root/'tests/known-failures.json').read_text()).get('frontend', []) + known_files = {item.split(' :: ', 1)[0] for item in known} + if any(Path(name).name in known_files for name in frontend): + quick.append(command(['npm', 'run', 'test:frontend:gate'])) + else: + quick.append(command(['node', '--test', *sorted(frontend)], tests=True)) for pkg, patterns in sorted(go_patterns.items()): - quick.append(command(['go', 'test', pkg, '-run', '|'.join(sorted(patterns)), '-timeout=2m', '-v'], tests=True)) + names = set() + for test_file in (root/pkg).glob('*_test.go'): + names.update(re.findall(r'^func (Test\w+)\(', test_file.read_text(), re.M)) + selected_names = sorted(name for name in names if any(re.search(p, name) for p in patterns) + and not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark|Preview', name)) + if selected_names: + quick.append(command(['go', 'test', pkg, '-run', '|'.join('^' + name + '$' for name in selected_names), '-timeout=2m', '-v'], tests=True)) integration = [command(['npm', 'run', x]) for x in ['lint:web', 'format:check', 'test:frontend:gate', 'test:go:gate', 'build:web', 'test:production:boot']] def unique(commands): @@ -216,6 +230,24 @@ def run_check(check, root, logfile, timeout): return {**check, 'status': status, 'exitCode': code, 'seconds': round(time.monotonic()-start, 3), 'log': str(logfile)} +def run_checks(checks, root, folder, budget): + results = [] + start = time.monotonic() + for index, check in enumerate(checks): + remaining = budget - (time.monotonic() - start) + if remaining <= 0: + results.append({**check, 'status': 'not-run', 'reason': 'budget exhausted'}) + continue + if any(not any(r['argv'] == required and r['status'] == 'passed' for r in results) + for required in check.get('requires', [])): + results.append({**check, 'status': 'not-run', 'reason': 'dependency did not pass'}) + continue + result = run_check(check, root, folder/f'{index:02}.log', remaining) + results.append(result) + print(f"{result['status']}: {' '.join(check['argv'])} ({result['seconds']}s)", flush=True) + return results + + def runtime_identity(root): versions = {'python': sys.version.split()[0]} for name, argv in [('node', ['node', '--version']), ('go', ['go', 'version'])]: @@ -259,17 +291,7 @@ def main(argv=None): 'environmentDigest': hashlib.sha256(json.dumps(dict(os.environ), sort_keys=True).encode()).hexdigest(), 'startedAt': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'results': []} start = time.monotonic() - for index, check in enumerate(plan['commands']): - remaining = budget - (time.monotonic()-start) - if remaining <= 0: - report['results'].append({**check, 'status': 'not-run', 'reason': 'budget exhausted'}) - continue - result = run_check(check, ROOT, folder/f'{index:02}.log', remaining) - report['results'].append(result) - print(f"{result['status']}: {' '.join(check['argv'])} ({result['seconds']}s)", flush=True) - if result['status'] != 'passed': - report['results'].extend({**c, 'status': 'not-run', 'reason': 'previous check did not pass'} for c in plan['commands'][index+1:]) - break + report['results'] = run_checks(plan['commands'], ROOT, folder, budget) report['inputsChangedDuringRun'] = fingerprint(ROOT) != before statuses = {r['status'] for r in report['results']} code = 1 if 'failed' in statuses else 2 if (statuses - {'passed'} or plan['unmapped'] or report['inputsChangedDuringRun']) else 0 diff --git a/tests/browser/acceptance-playback-rate.mjs b/tests/browser/acceptance-playback-rate.mjs index 65b3a66f..3c7cd612 100644 --- a/tests/browser/acceptance-playback-rate.mjs +++ b/tests/browser/acceptance-playback-rate.mjs @@ -1,11 +1,18 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; // Actual acceptance component and media reloads; API responses/media are synthetic. +import { createServer } from "vite"; import { chromium } from "playwright"; import assert from "node:assert/strict"; import fs from "node:fs/promises"; import { execFileSync } from "node:child_process"; -const base = process.env.REVIEW_TEST_URL || "http://127.0.0.1:5197"; -assert(["localhost", "127.0.0.1"].includes(new URL(base).hostname)); +let base = process.env.REVIEW_TEST_URL; +if (base) { + const url = new URL(base); + assert( + url.protocol === "http:" && ["localhost", "127.0.0.1"].includes(url.hostname) && !url.username && !url.password, + ); +} const out = process.env.REVIEW_TEST_OUTPUT || "local-data/test-results/acceptance-playback-rate"; await fs.mkdir(out, { recursive: true }); execFileSync("ffmpeg", [ @@ -46,13 +53,27 @@ execFileSync("ffmpeg", [ `${out}/source.mp4`, ]); const sourceMedia = await fs.readFile(`${out}/source.mp4`); -const browser = await chromium.launch({ - ...(process.env.BROWSER_PATH ? { executablePath: process.env.BROWSER_PATH } : { channel: "chrome" }), - headless: true, - args: ["--no-proxy-server"], -}); +let browser, server; +const launchBrowser = () => + chromium.launch({ + ...browserExecutableOptions(), + + headless: true, + args: ["--no-proxy-server"], + }); const cases = []; try { + if (!base) { + server = await createServer({ + configFile: "tests/browser/vite.platform-feedback.config.ts", + cacheDir: `${out}/vite-cache`, + optimizeDeps: { entries: ["tests/browser/fixtures/acceptance-playback-rate.tsx"] }, + server: { host: "127.0.0.1", port: 0 }, + }); + await server.listen(); + base = server.resolvedUrls.local[0].replace(/\/$/, ""); + } + browser = await launchBrowser(); for (const engine of ["native", "movi", "movi-fractional"]) { const sourceIndex = engine !== "native"; const fractionalOffset = engine === "movi-fractional" ? 0.023333333 : 0; @@ -226,5 +247,6 @@ try { await fs.writeFile(`${out}/report.json`, JSON.stringify({ passed: true, cases }, null, 2)); console.log(JSON.stringify({ passed: true, cases })); } finally { - await browser.close(); + await browser?.close(); + await server?.close(); } diff --git a/tests/browser/admin-annotation-pool-replica.mjs b/tests/browser/admin-annotation-pool-replica.mjs index d79b52a4..a33adf53 100644 --- a/tests/browser/admin-annotation-pool-replica.mjs +++ b/tests/browser/admin-annotation-pool-replica.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; @@ -9,6 +10,7 @@ const out = path.resolve("local-data/test-results/admin-annotation-pool"); fs.mkdirSync(out, { recursive: true }); const report = { pageErrors: [], requests: [], writes: [], cases: [] }; const server = await createServer({ + cacheDir: "local-data/test-results/vite-admin-annotation-pool-replica", server: { host: "127.0.0.1", port: 5205, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, }); let browser, page; @@ -18,7 +20,8 @@ let pageFault = ""; try { await server.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/annotation-source-validation-replica.mjs b/tests/browser/annotation-source-validation-replica.mjs index 80604ca9..408362a9 100644 --- a/tests/browser/annotation-source-validation-replica.mjs +++ b/tests/browser/annotation-source-validation-replica.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import { createServer } from "vite"; @@ -7,6 +8,7 @@ assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); const output = "local-data/test-results/source-validation-browser"; fs.mkdirSync(output, { recursive: true }); const vite = await createServer({ + cacheDir: "local-data/test-results/vite-annotation-source-validation-replica", server: { host: "127.0.0.1", port: 5208, @@ -21,7 +23,8 @@ const report = { pageErrors: [], writes: [], cases: [] }; try { await vite.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); @@ -48,11 +51,19 @@ try { await notice.getByRole("button", { name: "刷新状态", exact: true }).click(); await notice.getByText(/418451.*gyr/).waitFor(); assert.equal(report.writes.filter((path) => path.endsWith("source-validation-retries")).length, 0); - report.cases.push("annotator sees exact source fault and repair responsibility; refresh does not enqueue revalidation"); + report.cases.push( + "annotator sees exact source fault and repair responsibility; refresh does not enqueue revalidation", + ); await page.screenshot({ path: output + "/annotator-source-failed.png", fullPage: true }); await page.goto(base + "/plugins/annotation-qa/workflows/all/work?projectId=draft-project&taskId=draft-task"); - await page.getByRole("combobox", { name: "个人任务切换", exact: true }).getByRole("option", { name: /源数据异常/ }).waitFor({ state: "attached" }); - await page.getByRole("region", { name: "恢复工作会话" }).getByRole("button", { name: /capture-custom/ }).click(); + await page + .getByRole("combobox", { name: "个人任务切换", exact: true }) + .getByRole("option", { name: /源数据异常/ }) + .waitFor({ state: "attached" }); + await page + .getByRole("region", { name: "恢复工作会话" }) + .getByRole("button", { name: /capture-custom/ }) + .click(); await notice.getByText(/418451.*gyr/).waitFor(); await page.screenshot({ path: output + "/owned-source-failed.png", fullPage: true }); report.cases.push("owned task remains selectable for inspection and its buffer option shows source failure"); @@ -80,7 +91,9 @@ try { await new Promise((resolve) => setTimeout(resolve, 100)); } assert.equal(validation?.status, "valid", "actual source validation completed"); - const readyFrame = await context.request.get(base + "/api/plugins/annotation-qa/tasks/draft-task/frame-timeline?camera=left&encoding=compact"); + const readyFrame = await context.request.get( + base + "/api/plugins/annotation-qa/tasks/draft-task/frame-timeline?camera=left&encoding=compact", + ); assert.equal(readyFrame.status(), 200, "validated source exposes its frame timeline"); await notice.getByRole("button", { name: "刷新状态", exact: true }).click(); await notice.waitFor({ state: "detached" }); @@ -111,13 +124,19 @@ try { await page.route("**/api/plugins/annotation-qa/tasks/draft-task", async (route) => { const response = await route.fetch(); const data = await response.json(); - data.humanItem.subject.payload.sliceWorkflow = { status: generationRecovered ? "completed" : "failed", phase: "annotate", error: "合成生成失败" }; + data.humanItem.subject.payload.sliceWorkflow = { + status: generationRecovered ? "completed" : "failed", + phase: "annotate", + error: "合成生成失败", + }; data.editorState = { ...data.editorState, phase: "annotate", canConfirmDescriptions: generationRecovered }; await route.fulfill({ response, json: data }); }); await page.reload(); await page.getByTestId("human-task-pool-all").click(); - const generationFault = page.getByTestId("capture-annotation-workbench").getByText("切片生成失败:合成生成失败", { exact: true }); + const generationFault = page + .getByTestId("capture-annotation-workbench") + .getByText("切片生成失败:合成生成失败", { exact: true }); await generationFault.waitFor(); await page.screenshot({ path: output + "/generation-failed.png", fullPage: true }); generationRecovered = true; @@ -125,10 +144,15 @@ try { assert.ok(generationReads > 0); assert.equal(await page.getByText("切片生成失败:合成生成失败", { exact: true }).count(), 0); assert.equal(report.writes.filter((path) => path.endsWith("source-validation-retries")).length, 1); - assert.equal(report.writes.some((path) => /drafts|slice-confirmations|combined-submissions/.test(path)), false); + assert.equal( + report.writes.some((path) => /drafts|slice-confirmations|combined-submissions/.test(path)), + false, + ); assert.deepEqual(report.pageErrors, []); await page.screenshot({ path: output + "/generation-recovered.png", fullPage: true }); - report.cases.push("synthetic failed generation automatically observes external completion without regeneration or annotation writes"); + report.cases.push( + "synthetic failed generation automatically observes external completion without regeneration or annotation writes", + ); console.log(JSON.stringify(report)); } catch (error) { if (page) { diff --git a/tests/browser/dashboard-presentation-replica.mjs b/tests/browser/dashboard-presentation-replica.mjs index 7a49f172..e399cf1e 100644 --- a/tests/browser/dashboard-presentation-replica.mjs +++ b/tests/browser/dashboard-presentation-replica.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import { readFile, writeFile } from "node:fs/promises"; import { chromium } from "playwright"; @@ -13,7 +14,8 @@ assert.match(accounts.origin, /^http:\/\/127\.0\.0\.1:\d+$/); assert.match(accounts.database, /^annotation_output_browser_\d+$/); const scope = (id) => JSON.stringify(["output-browser-org", id, id === "admin" ? "admin" : "annotator"]); const browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || undefined, + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/production-quality-replica.mjs b/tests/browser/production-quality-replica.mjs index de1185ed..70e34ecf 100644 --- a/tests/browser/production-quality-replica.mjs +++ b/tests/browser/production-quality-replica.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; @@ -19,6 +20,7 @@ assert.equal(accounts.origin, origin); const server = process.env.ANNOTATION_OUTPUT_ACCOUNTS ? null : await createServer({ + cacheDir: "local-data/test-results/vite-production-quality-replica", server: { host: "127.0.0.1", port: 5197, @@ -33,7 +35,8 @@ const report = { roles: [], errors: [], unexpectedWrites: [] }; try { await server?.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); @@ -61,6 +64,13 @@ try { const data = await (await response).json(); const panel = page.getByRole("region", { name: "生产与质量看板", exact: true }); await panel.getByRole("button", { name: "导出", exact: true }).waitFor(); + await panel + .getByRole("heading", { + name: ["u1", "qc", "acceptor"].includes(id) ? "我的批次质量" : "生产与批次质量总览", + exact: true, + }) + .waitFor(); + assert.ok(!(await panel.innerText()).includes("durationSeconds")); await page.waitForFunction(() => !document.querySelector('.pq-dashboard [role="status"]')); assert.ok(data.snapshot.id); assert.equal(await panel.locator(".pq-cards>div").count(), ["qc", "acceptor"].includes(id) ? 3 : 5); diff --git a/tests/browser/review-authoring-permissions.mjs b/tests/browser/review-authoring-permissions.mjs index 484d7d54..1714051b 100644 --- a/tests/browser/review-authoring-permissions.mjs +++ b/tests/browser/review-authoring-permissions.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; @@ -29,6 +30,7 @@ assert.equal( ); process.env.CAPTURE_WORKBENCH_MOCK_VIDEO = media; const server = await createServer({ + cacheDir: "local-data/test-results/vite-review-authoring-permissions", configFile: "tests/browser/vite.platform-feedback.config.ts", server: { host: "127.0.0.1", port: 5194, strictPort: true }, }); @@ -41,7 +43,8 @@ try { await server.listen(); const origin = "http://127.0.0.1:5194"; browser = await chromium.launch({ - ...(process.env.BROWSER_EXECUTABLE ? { executablePath: process.env.BROWSER_EXECUTABLE } : {}), + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/review-decision-failure.mjs b/tests/browser/review-decision-failure.mjs index f3746f52..aee1fae0 100644 --- a/tests/browser/review-decision-failure.mjs +++ b/tests/browser/review-decision-failure.mjs @@ -1,9 +1,11 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; // Local-only hook regression: synthetic responses, no customer API calls. import assert from "node:assert/strict"; import { createServer } from "vite"; import { chromium } from "playwright"; const server = await createServer({ + cacheDir: "local-data/test-results/vite-review-decision-failure", configFile: false, server: { allowedHosts: ["review-errors.test"], host: "127.0.0.1", port: 5197 }, plugins: [ @@ -38,7 +40,8 @@ let browser; try { await server.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_PATH || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server", "--host-resolver-rules=MAP review-errors.test 127.0.0.1"], }); @@ -49,16 +52,34 @@ try { for (const stage of ["quality", "acceptance"]) { await page.goto(base + "review-error-test?stage=" + stage); await page.waitForFunction(() => window.decision); - for (const [httpStatus,receipt] of [[500,'failed_before_effects'],[409,'failed_before_effects'],[500,'completed'],[500,'unknown']]) { - await page.evaluate(({httpStatus,receipt})=>{window.httpStatus=httpStatus;window.receipt=receipt;void window.decision.request({itemId:'test-item',path:'/test',payload:{},title:'test',decision:'rejected',scopeText:'test'},()=>({}));},{httpStatus,receipt}); - await page.waitForFunction(()=>window.decision.confirmation); - await page.evaluate(()=>window.decision.confirm()); - await page.waitForFunction(()=>!window.decision.busy&&!window.decision.confirmation); - assert.equal(await page.evaluate(()=>Boolean(window.decision.pending)),receipt==='unknown'); - assert.equal(await page.evaluate(()=>Object.keys(localStorage).some(k=>k.startsWith('review-decision:'))),receipt==='unknown'); + for (const [httpStatus, receipt] of [ + [500, "failed_before_effects"], + [409, "failed_before_effects"], + [500, "completed"], + [500, "unknown"], + ]) { + await page.evaluate( + ({ httpStatus, receipt }) => { + window.httpStatus = httpStatus; + window.receipt = receipt; + void window.decision.request( + { itemId: "test-item", path: "/test", payload: {}, title: "test", decision: "rejected", scopeText: "test" }, + () => ({}), + ); + }, + { httpStatus, receipt }, + ); + await page.waitForFunction(() => window.decision.confirmation); + await page.evaluate(() => window.decision.confirm()); + await page.waitForFunction(() => !window.decision.busy && !window.decision.confirmation); + assert.equal(await page.evaluate(() => Boolean(window.decision.pending)), receipt === "unknown"); + assert.equal( + await page.evaluate(() => Object.keys(localStorage).some((k) => k.startsWith("review-decision:"))), + receipt === "unknown", + ); } - await page.evaluate(()=>localStorage.clear()); - console.log('PASS',stage,'500/409 terminal failure unlock, completed receipt, unknown remains protected'); + await page.evaluate(() => localStorage.clear()); + console.log("PASS", stage, "500/409 terminal failure unlock, completed receipt, unknown remains protected"); } } finally { await browser?.close(); diff --git a/tests/browser/review-decision-lifecycle.mjs b/tests/browser/review-decision-lifecycle.mjs index e9d240f8..54153964 100644 --- a/tests/browser/review-decision-lifecycle.mjs +++ b/tests/browser/review-decision-lifecycle.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; @@ -13,7 +14,11 @@ const server = await createServer({ configFile: false, cacheDir: "local-data/vite-review-decision-controller", server: { host: "127.0.0.1", port: 0, proxy: { "/api": api, "/__fixture": api } }, - optimizeDeps: { noDiscovery: true, entries: [], include: ["react", "react-dom/client", "react/jsx-runtime", "react/jsx-dev-runtime"] }, + optimizeDeps: { + noDiscovery: true, + entries: [], + include: ["react", "react-dom/client", "react/jsx-runtime", "react/jsx-dev-runtime"], + }, plugins: [ { name: "review-decision-controller-fixture", @@ -51,7 +56,8 @@ let browser, page, releaseReceipt; try { await server.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/review-decision-start-race.mjs b/tests/browser/review-decision-start-race.mjs index 76522170..60e51efd 100644 --- a/tests/browser/review-decision-start-race.mjs +++ b/tests/browser/review-decision-start-race.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; // Local browser regression of the real continuation + durable decision hooks. import assert from "node:assert/strict"; import { createServer } from "vite"; @@ -53,7 +54,8 @@ let browser; try { await server.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_PATH || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/review-draft-persistence.mjs b/tests/browser/review-draft-persistence.mjs index de107eeb..0b21c50d 100644 --- a/tests/browser/review-draft-persistence.mjs +++ b/tests/browser/review-draft-persistence.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; @@ -56,7 +57,8 @@ let fault = "", try { await server.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/review-drafts.mjs b/tests/browser/review-drafts.mjs index 33079ee0..fdde3243 100644 --- a/tests/browser/review-drafts.mjs +++ b/tests/browser/review-drafts.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; // Actual HTTP application/API with transport failure injection only. import { chromium } from "playwright"; import fs from "node:fs/promises"; @@ -13,7 +14,8 @@ const state = JSON.parse( ); state.cookies = state.cookies.map((cookie) => ({ ...cookie, domain: hostname })); const browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: [`--host-resolver-rules=MAP ${hostname} 127.0.0.1`, "--no-proxy-server"], }); diff --git a/tests/browser/review-pool-search-replica.mjs b/tests/browser/review-pool-search-replica.mjs index 4c3199e5..a6a90896 100644 --- a/tests/browser/review-pool-search-replica.mjs +++ b/tests/browser/review-pool-search-replica.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; @@ -9,6 +10,7 @@ const out = path.resolve("local-data/test-results/review-pool-search"); fs.mkdirSync(out, { recursive: true }); const report = { cases: [], requests: [], pageErrors: [], writes: [] }; const server = await createServer({ + cacheDir: "local-data/test-results/vite-review-pool-search-replica", server: { host: "127.0.0.1", port: 5207, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, }); let browser, @@ -19,7 +21,8 @@ let browser, try { await server.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/review-trace.mjs b/tests/browser/review-trace.mjs index b231fd74..257abec0 100644 --- a/tests/browser/review-trace.mjs +++ b/tests/browser/review-trace.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import { chromium } from "playwright"; @@ -7,6 +8,7 @@ assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); const out = "local-data/test-results/review-trace"; fs.mkdirSync(out, { recursive: true }); const vite = await createServer({ + cacheDir: "local-data/test-results/vite-review-trace", server: { host: "127.0.0.1", port: 5206, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, }); let browser; @@ -21,7 +23,8 @@ const selection = try { await vite.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); @@ -94,16 +97,34 @@ try { await page.waitForTimeout(500); assert.deepEqual(unexpectedPoolReads, [], "layout entry must not reload the task pool"); await panel.waitFor(); - if (mode === "quality") assert.equal(await editor.inputValue(), "Synthetic unsaved opinion retained across focus entry", "unsaved opinion survives layout entry"); - assert.ok(!sessionWrites.slice(before).some((a) => ["release", "acquire", "takeover"].includes(a)), "layout entry must retain the review lease"); + if (mode === "quality") + assert.equal( + await editor.inputValue(), + "Synthetic unsaved opinion retained across focus entry", + "unsaved opinion survives layout entry", + ); + assert.ok( + !sessionWrites.slice(before).some((a) => ["release", "acquire", "takeover"].includes(a)), + "layout entry must retain the review lease", + ); if (mode === "quality") { - const renewal = await page.waitForResponse((response) => { - const request = response.request(); - return new URL(response.url()).pathname.endsWith("/review-continuation") && request.method() === "POST" && request.postDataJSON()?.action === "renew"; - }, { timeout: 15000 }); + const renewal = await page.waitForResponse( + (response) => { + const request = response.request(); + return ( + new URL(response.url()).pathname.endsWith("/review-continuation") && + request.method() === "POST" && + request.postDataJSON()?.action === "renew" + ); + }, + { timeout: 15000 }, + ); assert.equal(renewal.status(), 200, "continuation renew succeeds after focus entry"); await page.getByText("连续审查中", { exact: true }).first().waitFor(); - assert.ok(!continuationWrites.slice(continuationBefore).includes("pause"), "focus entry must not pause a running queue"); + assert.ok( + !continuationWrites.slice(continuationBefore).includes("pause"), + "focus entry must not pause a running queue", + ); assert.equal(await editor.inputValue(), "Synthetic unsaved opinion retained across focus entry"); await page.getByRole("button", { name: "取消修改", exact: true }).click(); } diff --git a/tests/browser/rework-boundary-replica.mjs b/tests/browser/rework-boundary-replica.mjs index 43a47a11..2657e460 100644 --- a/tests/browser/rework-boundary-replica.mjs +++ b/tests/browser/rework-boundary-replica.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; @@ -8,6 +9,7 @@ assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); const out = path.resolve("local-data/test-results/rework-boundary"); fs.mkdirSync(out, { recursive: true }); const server = await createServer({ + cacheDir: "local-data/test-results/vite-rework-boundary-replica", server: { host: "127.0.0.1", port: 5202, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, }); let browser, @@ -28,7 +30,8 @@ const waitFor = async (check) => { try { await server.listen(); browser = await chromium.launch({ - executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/browser/run-editing-regression.mjs b/tests/browser/run-editing-regression.mjs index 3aaf52de..fcf83802 100644 --- a/tests/browser/run-editing-regression.mjs +++ b/tests/browser/run-editing-regression.mjs @@ -1,11 +1,12 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; // CI-only browser launcher. Local agent interactions use the exported CUA driver. import { chromium } from "playwright"; -import { spawn } from "node:child_process"; -import { mkdirSync, openSync, closeSync } from "node:fs"; +import { createServer as createViteServer } from "vite"; +import { mkdirSync, writeFileSync } from "node:fs"; import { runDiscardFocusTrackRegression, runEditingHistoryRegression } from "./discard-focus-track.mjs"; const output = "local-data/test-results/browser-editing"; mkdirSync(output, { recursive: true }); -const log = openSync(output + "/vite.log", "w"); + // Refuse an occupied port before starting, so a stale fixture cannot be reused. const { createServer } = await import("node:net"); await new Promise((resolve, reject) => { @@ -13,29 +14,34 @@ await new Promise((resolve, reject) => { probe.once("error", reject); probe.listen(5188, "127.0.0.1", () => probe.close(resolve)); }); -const server = spawn( - process.execPath, - ["node_modules/vite/bin/vite.js", "--host", "127.0.0.1", "--port", "5188", "--strictPort"], - { stdio: ["ignore", log, log] }, -); +const server = await createViteServer({ + configFile: "tests/browser/vite.platform-feedback.config.ts", + cacheDir: output + "/vite-cache", + optimizeDeps: { entries: ["tests/browser/fixtures/discard-focus-track.tsx"] }, + server: { host: "127.0.0.1", port: 5188, strictPort: true }, +}); const origin = "http://127.0.0.1:5188"; let browser, page; try { - const deadline = Date.now() + 20000; - while (true) { - try { - if ((await fetch(origin, { signal: AbortSignal.timeout(1000) })).ok) break; - } catch {} - if (server.exitCode !== null || Date.now() > deadline) - throw Error("Synthetic Vite fixture server unavailable; inspect vite.log"); - await new Promise((done) => setTimeout(done, 200)); - } - browser = await chromium.launch({ headless: true }); + await server.listen(); + browser = await chromium.launch({ ...browserExecutableOptions(), headless: true, args: ["--no-proxy-server"] }); page = await browser.newPage(); - page.setDefaultTimeout(8000); + page.setDefaultTimeout(20000); + const failures = []; + page.on("pageerror", (error) => failures.push(error.message)); + page.on("requestfailed", (request) => + failures.push(`${request.method()} ${new URL(request.url()).pathname}: ${request.failure()?.errorText}`), + ); + page.on("console", (message) => { + if (message.type() === "error") failures.push(message.text()); + }); + page.on("close", () => writeFileSync(output + "/browser-errors.json", JSON.stringify(failures, null, 2))); + await page.route("**/*", (route) => + ["127.0.0.1", "localhost"].includes(new URL(route.request().url()).hostname) ? route.continue() : route.abort(), + ); const keyMap = { Return: "Enter", "ctrl+z": "Control+z", "ctrl+shift+z": "Control+Shift+z" }; const tab = { - goto: (url) => page.goto(url), + goto: (url) => page.goto(url, { waitUntil: "domcontentloaded" }), playwright: page, pressKey: (_element, key) => page.keyboard.press(keyMap[key] || key), }; @@ -47,6 +53,5 @@ try { process.exitCode = 1; } finally { await browser?.close(); - server.kill("SIGTERM"); - closeSync(log); + await server.close(); } diff --git a/tests/browser/run-production-boot.mjs b/tests/browser/run-production-boot.mjs index 4b0d49e1..de0fac45 100644 --- a/tests/browser/run-production-boot.mjs +++ b/tests/browser/run-production-boot.mjs @@ -1,3 +1,4 @@ +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; // Boot the built production assets, not a Vite/component fixture. This catches // dependency resolution failures that tsc/build and aliased fixtures can miss. import assert from "node:assert/strict"; @@ -55,7 +56,8 @@ try { }); const origin = "http://127.0.0.1:" + server.address().port; browser = await chromium.launch({ - ...(process.env.BROWSER_EXECUTABLE ? { executablePath: process.env.BROWSER_EXECUTABLE } : {}), + ...browserExecutableOptions(), + headless: true, args: ["--no-proxy-server"], }); diff --git a/tests/frontend/browser-executable.test.mjs b/tests/frontend/browser-executable.test.mjs new file mode 100644 index 00000000..e8c60609 --- /dev/null +++ b/tests/frontend/browser-executable.test.mjs @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; + +test("browser selection respects both explicit variables and reports invalid configuration", () => { + const exists = (p) => p === "/explicit" || p === "/legacy"; + assert.deepEqual( + browserExecutableOptions({ env: { BROWSER_EXECUTABLE: "/explicit", BROWSER_PATH: "/legacy" }, exists }), + { executablePath: "/explicit" }, + ); + assert.deepEqual(browserExecutableOptions({ env: { BROWSER_PATH: "/legacy" }, exists }), { + executablePath: "/legacy", + }); + assert.throws( + () => browserExecutableOptions({ env: { BROWSER_EXECUTABLE: "/missing" }, exists }), + /Configured browser/, + ); +}); + +test("installed browsers work without downloads and installed Mac Edge takes precedence over a cached bundle", () => { + assert.equal( + browserExecutableOptions({ env: {}, bundled: "/bundled", exists: (p) => p === "/bundled" }).executablePath, + "/bundled", + ); + const edge = "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"; + assert.equal( + browserExecutableOptions({ env: {}, platform: "darwin", exists: (p) => p === edge }).executablePath, + edge, + ); + assert.equal( + browserExecutableOptions({ + env: {}, + platform: "darwin", + bundled: "/bundled", + exists: (p) => p === edge || p === "/bundled", + }).executablePath, + edge, + ); + assert.throws(() => browserExecutableOptions({ env: {}, exists: () => false }), /playwright install chromium/); +}); + +test("gated browser fixtures use the shared resolver and worktree-local Vite cache", () => { + for (const name of ["review-pool-search-replica", "review-authoring-permissions", "review-decision-failure"]) { + const source = readFileSync(new URL(`../browser/${name}.mjs`, import.meta.url), "utf8"); + assert.match(source, /browserExecutableOptions\(\)/); + assert.match(source, /cacheDir: "local-data\/test-results\/vite-/); + } +}); diff --git a/tests/frontend/dev-guardrails.test.mjs b/tests/frontend/dev-guardrails.test.mjs index 1630a398..8c19bc11 100644 --- a/tests/frontend/dev-guardrails.test.mjs +++ b/tests/frontend/dev-guardrails.test.mjs @@ -17,10 +17,11 @@ const run = (file, args, cwd, env = {}) => test("missing Husky wrappers are detected and installation produces a working pre-commit hook", () => { const dir = mkdtempSync(join(tmpdir(), "hook-guard-")); try { - mkdirSync(join(dir, "scripts")); + mkdirSync(join(dir, "scripts/lib"), { recursive: true }); mkdirSync(join(dir, ".husky")); mkdirSync(join(dir, "node_modules")); copyFileSync(join(root, "scripts/check-git-hooks.mjs"), join(dir, "scripts/check-git-hooks.mjs")); + copyFileSync(join(root, "scripts/lib/git-hooks.mjs"), join(dir, "scripts/lib/git-hooks.mjs")); cpSync(join(root, "node_modules/husky"), join(dir, "node_modules/husky"), { recursive: true }); spawnSync("git", ["init", "-q"], { cwd: dir }); writeFileSync(join(dir, ".husky/pre-commit"), "echo hook-was-executed\nexit 1\n"); @@ -124,3 +125,20 @@ test("local stack check rejects a missing or stale API version and accepts match await new Promise((done) => server.close(done)); } }); + +test("Docker npm prepare has all hook bootstrap modules before dependency installation", () => { + const dir = mkdtempSync(join(tmpdir(), "docker-hook-prepare-")); + try { + const beforeInstall = readFileSync(join(root, "Dockerfile.api"), "utf8").split("RUN npm ci")[0]; + for (const [, source, target] of beforeInstall.matchAll(/^COPY (scripts\/\S+) (\S+)$/gm)) { + const destination = join(dir, target); + mkdirSync(join(destination, ".."), { recursive: true }); + cpSync(join(root, source), destination, { recursive: true }); + } + const result = run(join(dir, "scripts/check-git-hooks.mjs"), [], dir); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Git hooks: skipped/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/frontend/fanyu-comment-followup.test.mjs b/tests/frontend/fanyu-comment-followup.test.mjs index 999e201c..c9455ea8 100644 --- a/tests/frontend/fanyu-comment-followup.test.mjs +++ b/tests/frontend/fanyu-comment-followup.test.mjs @@ -24,10 +24,17 @@ test("dashboard routes all workforce roles to the clean production quality panel const panel = await readFile(new URL("../../src/components/ProductionQualityDashboard.tsx", import.meta.url), "utf8"); assert.match(source, /ProductionQualityDashboard/); assert.match(source, /isOrganizationAdminRole\(currentUser\?\.role\)/); - assert.match(panel, /生产与质量总览/); - assert.match(panel, /我的工作记录/); + assert.match(panel, /我的批次质量/); + assert.match(panel, /生产与批次质量总览/); + assert.match(panel, /每日作业记录/); + const workforce = await readFile( + new URL("../../src/components/WorkforceStatisticsPanel.tsx", import.meta.url), + "utf8", + ); + assert.match(panel, /import\("\.\/WorkforceStatisticsPanel"\)/); + assert.match(workforce, /我的工作记录/); assert.match(panel, /供应商总体产出/); - assert.doesNotMatch(panel, /评分、排名和结算未启用|durationSeconds/); + assert.doesNotMatch(panel, /评分、排名和结算未启用|>\s*durationSeconds\s* { diff --git a/tests/frontend/git-hooks.test.mjs b/tests/frontend/git-hooks.test.mjs new file mode 100644 index 00000000..37f0f1b4 --- /dev/null +++ b/tests/frontend/git-hooks.test.mjs @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, mkdirSync, writeFileSync, symlinkSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { execFileSync } from "node:child_process"; +import { ensureGitHooks } from "../../scripts/lib/git-hooks.mjs"; + +test("verification bootstrap repairs hooks in a new worktree sharing dependencies", (t) => { + const dir = mkdtempSync(join(tmpdir(), "hooks-worktree-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const main = join(dir, "main"), + worktree = join(dir, "task"); + mkdirSync(main); + const git = (cwd, ...args) => execFileSync("git", args, { cwd, stdio: "pipe" }); + git(main, "init"); + mkdirSync(join(main, ".husky")); + writeFileSync(join(main, ".husky/pre-commit"), "printf ran > hook-ran\n"); + git(main, "add", ".husky/pre-commit"); + git(main, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-m", "fixture"); + git(main, "worktree", "add", "-b", "task", worktree); + symlinkSync(resolve("node_modules"), join(worktree, "node_modules")); + const env = { ...process.env, CI: "", HUSKY: "1" }; + assert.throws(() => ensureGitHooks(worktree, { env })); + assert.equal(ensureGitHooks(worktree, { install: true, env }), "active"); + assert.ok(existsSync(join(worktree, ".husky/_/pre-commit"))); + assert.equal(ensureGitHooks(worktree, { env }), "active"); + writeFileSync(join(worktree, "verified"), "fixture"); + git(worktree, "add", "verified"); + git(worktree, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-m", "verify hook"); + assert.ok(existsSync(join(worktree, "hook-ran")), "commit must actually invoke the hook"); + git(worktree, "config", "core.hooksPath", "/custom/user-hooks"); + assert.throws(() => ensureGitHooks(worktree, { install: true, env }), /Custom hooksPath/); + assert.equal(git(worktree, "config", "--get", "core.hooksPath").toString().trim(), "/custom/user-hooks"); +}); + +test("explicit hook opt-out and exported sources remain supported", () => { + assert.equal(ensureGitHooks("/missing/export", { env: {} }), "skipped"); + assert.equal(ensureGitHooks(process.cwd(), { install: true, env: { HUSKY: "0" } }), "skipped"); +}); diff --git a/tests/frontend/guardrails-ci.test.mjs b/tests/frontend/guardrails-ci.test.mjs new file mode 100644 index 00000000..eb95f9bf --- /dev/null +++ b/tests/frontend/guardrails-ci.test.mjs @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; + +test("CI wires verification tooling and required pool Mongo regressions", () => { + const source = readFileSync(new URL("../../.gitea/workflows/web-ci.yml", import.meta.url), "utf8"); + assert.match(source, /npm run test:verification/); + assert.match(source, /POOL_LOCAL_MONGO_URI/); + assert.match(source, /TestHumanPoolSearchMongo/); + assert.match(source, /mongo:8/); + assert.match(source, /test:browser:review-permissions/); + assert.match(source, /test:browser:dashboard/); + assert.match(source, /REVIEW_POOL_BROWSER_TEST/); +}); + +test("all standard development entry points bootstrap worktree hooks", () => { + const { scripts } = JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf8")); + for (const entry of ["dev", "dev:full", "dev:api", "dev:web", "dev:worker", "dev:api:replica"]) + assert.equal(scripts["pre" + entry], "npm run hooks:install"); +}); diff --git a/tests/tooling/test_verification.py b/tests/tooling/test_verification.py index 4e661219..8fdf5ac9 100644 --- a/tests/tooling/test_verification.py +++ b/tests/tooling/test_verification.py @@ -20,7 +20,7 @@ class VerificationTest(unittest.TestCase): def test_documentation_is_small(self): plan = self.plan(['docs/example.md', 'AGENTS.md']) - self.assertEqual(len(plan['commands']), 1) + self.assertEqual(len(plan['commands']), 2) self.assertEqual(plan['deferred'], []) self.assertEqual(plan['unmapped'], []) @@ -33,6 +33,28 @@ class VerificationTest(unittest.TestCase): self.assertEqual(len(node_tests), 1) self.assertEqual(len(node_tests[0]['argv'][2:]), len(set(node_tests[0]['argv'][2:]))) + def test_quick_scope_excludes_opt_in_and_interactive_previews(self): + plan = self.plan(['src/pages/AnnotationOperations.tsx']) + patterns = [c['argv'][c['argv'].index('-run') + 1] for c in plan['commands'] if c['argv'][:2] == ['go', 'test']] + import re + for name in ['TestSourceValidationBrowserReplica', 'TestRandomClaimLocalPreview']: + self.assertFalse(any(re.search(p, name) for p in patterns), name) + self.assertTrue(any(re.search(p, 'TestHumanPoolSearchBeyondFirst500') for p in patterns)) + + def test_pool_task_explicitly_runs_mongo_parity_and_large_payload(self): + plan = self.plan(['internal/store/human_pool_search.go'], 'task') + self.assertTrue(any(c['env'].get('POOL_LOCAL_MONGO_URI', '').startswith('mongodb://127.0.0.1:') for c in plan['commands'])) + + def test_failed_check_does_not_hide_independent_checks_or_allow_stale_boot(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + commands = [v.command([sys.executable, '-c', 'raise SystemExit(1)']), + v.command([sys.executable, '-c', 'print("independent")']), + {**v.command([sys.executable, '-c', 'raise RuntimeError("must not run")']), 'requires': [[sys.executable, '-c', 'raise SystemExit(1)']]}] + results = v.run_checks(commands, root, root, 5) + self.assertEqual([x['status'] for x in results], ['failed', 'passed', 'not-run']) + self.assertIn('dependency', results[-1]['reason']) + def test_source_task_includes_real_mongo_and_browser(self): plan = self.plan(['internal/store/annotation_source_admission.go'], 'task') self.assertTrue(any(c['env'].get('SOURCE_ADMISSION_LOCAL_MONGO') for c in plan['commands'])) @@ -40,6 +62,11 @@ class VerificationTest(unittest.TestCase): self.assertIn('source-and-buffer', plan['scopes']) self.assertEqual(plan['unmapped'], []) + def test_dependency_task_boot_requires_the_current_build(self): + plan = self.plan(['package-lock.json'], 'task') + boot = next(c for c in plan['commands'] if c['argv'] == ['npm', 'run', 'test:production:boot']) + self.assertEqual(boot['requires'], [['npm', 'run', 'build:web']]) + def test_script_alias_only_does_not_trigger_dependency_suite(self): plan = v.plan_checks(ROOT, ['package.json'], POLICY, 'quick', 'HEAD', tooling_package=True) self.assertEqual(plan['deferred'], []) @@ -69,6 +96,11 @@ class VerificationTest(unittest.TestCase): self.assertTrue(self.plan(['scripts/new-data-migration.py'])['unmapped']) self.assertTrue(self.plan(['tests/browser/new-flow.mjs'])['unmapped']) + def test_known_failure_file_uses_official_gate_without_expanding_baseline(self): + plan = self.plan(['tests/frontend/fanyu-comment-followup.test.mjs']) + self.assertTrue(any(c['argv'] == ['npm', 'run', 'test:frontend:gate'] for c in plan['commands'])) + self.assertFalse(any(c['argv'][:2] == ['node', '--test'] for c in plan['commands'])) + def test_changed_frontend_test_is_always_included(self): f = 'tests/frontend/annotation-buffer-summary.test.mjs' plan = self.plan([f]) From 6d8b011c67ecadfb9354cd1871d20c21b58d1f87 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 8 Nov 2026 09:02:49 -0800 Subject: [PATCH 5/8] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20Add=20resumable=20V?= =?UTF-8?q?4=20previews=20and=20record=20pool=20query=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/review-pool-search/index-plan.md | 39 +++++ .scratch/review-pool-search/work.md | 10 ++ scripts/delivery_v4_preview.py | 201 ++++++++++++++++++++++ scripts/verification-scopes.json | 27 +++ tests/backend/test_delivery_v4_preview.py | 179 +++++++++++++++++++ 5 files changed, 456 insertions(+) create mode 100644 .scratch/review-pool-search/index-plan.md create mode 100644 scripts/delivery_v4_preview.py create mode 100644 tests/backend/test_delivery_v4_preview.py diff --git a/.scratch/review-pool-search/index-plan.md b/.scratch/review-pool-search/index-plan.md new file mode 100644 index 00000000..40f95f7b --- /dev/null +++ b/.scratch/review-pool-search/index-plan.md @@ -0,0 +1,39 @@ +# Broad pool search: rejected metadata index experiment + +Base: c942a8ce on codex/review-pool-search; live source e4969acc. This is an opt-in follow-up, not a startup migration. + +## Evidence + +Production read-only phase counts on 2026-11-08 found about 11,900 quality/semantic candidates. The current source-generated wide search still exceeds the 8-second deadline. Root explain examined 11,903 full documents in 1,018 ms in one warm sample; the window/group prefix and foreign joins add more work. These samples are diagnostic, not an end-to-end p95 or a fixed dataset benchmark. Random claim uses a separate path. + +Hash batching and removing foreign lookup pipelines did not resolve the timeout and were not adopted. An initial synthetic index with an arbitrary prefix still fetched all documents. Ordering equality scope first is material: organizationId/pluginId/currentStageId, then createdAt/id, followed by projected metadata. + +`TestHumanPoolSearchMongoCoveringIndex` uses the actual aggregation with 1,000 synthetic 64 KiB payloads. With the candidate index, its root cursor examines zero documents, and matching results remain identical before creation, after creation, and after dropping the index. There is no hint in the tested aggregation. A separate 10,000-row projection experiment also examined zero documents with the correctly ordered index. It does not establish full-query latency or concurrency capacity. + +## Exact proposed production change + +- Target: Fanyu cluster, namespace droboticflow-fanyu; Mongo database drobodata-fanyu, collection humanItems. +- Create one non-unique, non-sparse index `pool_search_metadata_v1` using the ordered keys in `local-data/pool-v4-followup-20261108/rejected-index/index-spec.json`. +- No customer record updates, backfill, deletion, existing index removal, Pod/bandwidth changes, or deployment. +- The candidate API code's compact projection is still local; the live legacy projection may not gain the same coverage. After index creation, run the **local candidate's source-generated pipeline read-only** against the same filter/cursor and compare all matching IDs and count. Also check live API latency/errors because planner selection can change. +- Do not automatically add this index in MongoStore startup. Creation requires separate production-change authorization under AGENTS.md. + +## Cost, stop and rollback + +A 10,000-row synthetic 16 KiB-payload experiment measured about 1.8 MiB for the 16-field version and a 51 ms local build. The final spec adds supplier/author scope fields. These figures do not predict production build time, disk size, replication lag, CPU or write amplification; inspect actual capacity and active builds before creation. All indexed fields are scalar by contract; abort if historical array values make this a multikey/parallel-array index. + +Build adds CPU, storage I/O, disk usage and ongoing index maintenance. Monitor the existing service and cancel a build on sustained pressure rather than changing resources simultaneously. If the index worsens plans or gives no useful gain, drop **only this newly created index** with the same authorization. No customer-data rewrite is needed for rollback. + +Root coverage alone is not acceptance. If foreign lookups still dominate or total p95 does not meet the agreed threshold, retain pending status and profile the next phase. Do not widen permissions, omit legacy rows, change pagination, or increase the 8-second timeout. + +## Executed result and rollback + +The user explicitly authorized creation of this index and dropping only this index if ineffective. Preflight: humanItems had 25,802 records, about 250 MB uncompressed data, 62.5 MB storage, 19.3 MB existing indexes; Mongo cache used 3.54 / 7.86 GB. A paged read-only field check examined all 25,802 records and found no arrays in the indexed fields. The original one-shot type aggregate exceeded its 8-second diagnostic bound; the 52 bounded batches completed in 11.3 seconds. + +Creation took 4,597 ms; the new index occupied 15,835,136 bytes. The source-generated production root explain really used this index and examined zero documents, but the full **actual Go Store** candidate queries still took 8,001 ms and timed out for broad/all and available scopes. The acceptance example returned one match in 5,915 ms. These are scoped backend reads, not a customer browser/capacity pass. + +A direct old/new root projection comparison measured old index: 2,065 ms, 11,903 documents, 11,904 keys, 1 seek; candidate: 3,377 ms, zero documents, 11,904 keys, 9 seeks. Coverage did not translate to reduced overall cost. Hash batching and native equality batching also still timed out. Production activity and warm caches vary, so these do not establish a fixed-dataset percentage regression; they are sufficient to reject retaining an unproven extra index. + +The authorized rollback dropped only pool_search_metadata_v1 in 140 ms and verified it absent. Health/readiness returned HTTP 200 in 38/32 ms afterwards. No customer record, old index, resource or application image was changed. The experiment and local synthetic test are retained only under ignored local-data/, not installed by startup or made a permanent CI requirement. + +Next work remains: avoid performing full candidate-set sorting/grouping and foreign metadata joins before selective search. Candidate discovery must preserve direct IDs, capture/part combinations, historical payload/input names, categories, ordering and tenant/claim rules. A new implementation needs differential tests and realistic latency evidence before another production change. The broad-pool issue is **open**, not fixed by this experiment. diff --git a/.scratch/review-pool-search/work.md b/.scratch/review-pool-search/work.md index a6b9949b..faf31d26 100644 --- a/.scratch/review-pool-search/work.md +++ b/.scratch/review-pool-search/work.md @@ -39,3 +39,13 @@ Scope: Repair worktree hook bootstrap, verification selection/dependencies/resul Risk: Local/CI tooling only; no business query, delivery contract, customer record, migration, resource or deployment changes in this follow-up. Keep the wide-pool performance limitation above open. Verification follow-up evidence: local-data/retro-guardrails/. The unchanged browser/product inputs passed all 23 task checks in verify-1793952872699139000/report.json (338.854s, no source drift). Earlier failures and the 600-second budget exhaustion remain recorded. Subsequent Docker COPY ordering and its regression affect only the dependency-install bootstrap: Docker-prepare regression and final quick checks are run separately; application/browser/Mongo inputs are unchanged. No known-failure baseline was expanded. CI wiring is validated locally; the remote CI has not run because this branch remains local. + +## 2026-11-08: wide-pool and all-delivery continuation (base c942a8ce) + +- Production freshness was confirmed against origin/dev and live e4969acc before investigation. No application deployment or customer-record mutation was performed. +- Full production delivery inventory: 35 records (27 accepted, 8 legacy no_deliverables). Added a local-only resumable V4 preview converter that reuses the native builder, preserves fixed revision checksums, refuses conflicting output, records every blocked item and never claims a complete audit/publication. All-discard conversion requires the fixed child-run confirmation to name the exact revision/artifact/hash. Historical source recovery requires the published Capture snapshot and an exact persisted Run captureUnit/objectKeys selection; it never uses the mutable current Capture association or current dictionaries. +- Converted and independently verified 18 previews: 12 published packages and 6 all-discard records. All 18 pass original left/right PTS coverage and calibration byte checks. The 6 all-discard cases have no historical calibration SHA: their checksums come from current bytes of the frozen BOS object selection, explicitly recorded. Historical object-byte version proof, complete internal audit, video decoding/downstream cuts and formal publication remain unverified. No online package/pointer was replaced. +- 17 records remain blocked: 10 missing frozen slice classification dictionaries; 6 unavailable/incomplete fixed revision JSONs (5 published + 1 all-discard); 1 older all-discard record lacks original-source and dictionary snapshots. Historical original selection was recovered for 9 of the 10 dictionary-blocked packages by cross-checking the published Capture snapshot with the original Run's explicit selection and the existing Go validator; this does not supply missing historical classification tags. A per-record Chinese follow-up list and historical-name confirmation list are included with the local evidence. +- Preview archive: local-data/delivery-v4-all-preview-20261108/V4交付包_18个已校验预览.zip. Full reports/evidence: local-data/pool-v4-followup-20261108/ and local-data/delivery-v4-all-preview-20261108/. Customer ZIP contains only 18 part.json files, README and public validation summary; no video/IMU/audit files. +- Broad pool remains open. The separately authorized metadata-index experiment really achieved zero root document fetches, but actual Go Store broad/all and available queries still timed out at 8,001 ms, acceptance example 5,915 ms. The index was explicitly rolled back and verified absent; health/readiness 200 afterwards. No unproven index or runtime query rewrite was retained. See index-plan.md for exact scope, cost, evidence and rollback. +- Earlier full task evidence (including synthetic covering-index, pool Mongo parity, permission and browser checks): verify-1794155814156557000/report.json passed. The rejected index experiment was then moved to ignored local-data rather than installed/added as a permanent CI requirement. Final converter/compatibility/tooling checks, including exact historical source recovery and all-discard acceptance guards: verify-1794157259610133000/report.json passed (8 converter cases plus native V4/real-materialization/legacy compatibility suites). These are scoped verification, not release or 81-user capacity acceptance. diff --git a/scripts/delivery_v4_preview.py b/scripts/delivery_v4_preview.py new file mode 100644 index 00000000..444ba9c6 --- /dev/null +++ b/scripts/delivery_v4_preview.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Resume a local-only V4 conversion of inventoried, frozen delivery revisions. + +Input: inventory.json and packages//{old/,pinned-revision/,pinned-provenance.json}. +This produces customer JSON previews, never a complete audit or publishable run. +No network, current dictionaries, video downloads, or production writes are used. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from pathlib import Path +import tempfile + +from part_index_migration import load_operator_module, resolve_pose_policy, split_slices + + +def load(path): + return json.loads(path.read_text(encoding="utf8")) + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def convert(package, directory, output): + if package["status"] not in {"accepted", "no_deliverables"}: + raise ValueError("no published accepted manifest; frozen revision recovery required") + revision = directory / "pinned-revision" + manifest_bytes = (revision / "revision-manifest.json").read_bytes() + manifest = json.loads(manifest_bytes) + provenance = load(directory / "pinned-provenance.json") + delivery = load(directory / "old/packages-manifest.json") + lineage = delivery["lineage"] + if (provenance["sha256"] != sha(manifest_bytes) + or provenance["artifactId"] != lineage["acceptedSliceDirectoryArtifactId"] + or provenance["revision"] != manifest["revision"]): + raise ValueError("frozen revision provenance mismatch") + # Verify every input JSON against its accepted manifest before the builder reads it. + before = {} + for row in manifest["items"]: + relative = Path(row["directory"]) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError("unsafe accepted slice directory") + verified = set() + files = row.get("files", []) + for entry in files.values() if isinstance(files, dict) else files: + if entry["name"] not in {"annotation.json", "synchronized-slice.json"}: + continue + path = revision / relative / entry["name"] + if not path.resolve().is_relative_to(revision.resolve()): + raise ValueError("accepted file escapes revision directory") + raw = path.read_bytes() + if not entry.get("sha256") or sha(raw) != entry["sha256"]: + raise ValueError("frozen annotation checksum mismatch") + before[path] = sha(raw) + verified.add(entry["name"]) + required = {"synchronized-slice.json"} + if row["deliveryDisposition"] == "keep": + required.add("annotation.json") + if not required <= verified: + raise ValueError("missing frozen JSON checksums") + source_basis = "accepted_revision_manifest" + recovery_path = directory / "source-selection-recovery.json" + if not manifest.get("originalSource") and recovery_path.exists(): + recovery = load(recovery_path) + raw_input = (directory / "source-selection-input.json").read_bytes() + evidence = json.loads(raw_input) + run, capture = evidence["run"], evidence["capture"] + frozen = delivery["capture"] + refs = [v["ref"] for v in run["inputs"].values()] + source = recovery["originalSource"] + if (recovery.get("basis") != "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys" + or recovery.get("inputSha256") != sha(raw_input) + or recovery.get("manifestSha256") != sha(manifest_bytes) + or run["id"] != lineage["sourceRunId"] or run["id"] != manifest["runId"] + or run["organizationId"] != package["organizationId"] + or capture["runId"] != run["id"] or capture["organizationId"] != run["organizationId"] + or capture["id"] != frozen["captureRecordId"] + or capture["captureId"] != frozen["captureId"] or capture["prefix"] != frozen["prefix"] + or len(refs) != 1): + raise ValueError("historical source recovery provenance mismatch") + ref = refs[0] + if (ref["provider"] != "BOS" or ref["kind"] != "directory" + or ref["captureUnit"] != frozen["captureId"] + or not frozen["prefix"].startswith(ref["prefix"] + "/") + or source["bucket"] != ref["bucket"] or source["prefix"] != ref["prefix"] + or source["partPrefix"] != frozen["prefix"] or source["captureUnit"] != frozen["captureId"] + or source["objectKeys"] != sorted(ref["objectKeys"]) + or source.get("snapshotFingerprint", "") != ref.get("versionId", "")): + raise ValueError("historical source selection mismatch") + manifest = copy.deepcopy(manifest) + manifest["originalSource"] = source + source_basis = "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys" + acceptance = load(directory / "old/acceptance-record.json") + kept, discarded = split_slices(manifest) + if package["status"] == "no_deliverables": + proof = load(directory / "empty-acceptance-proof.json") + accepted = proof.get("acceptedMaterialization", {}) + if (kept or not discarded or acceptance["finalAcceptance"].get("confirmed") is not True + or proof.get("packageId") != package["id"] or proof.get("confirmed") is not True + or proof.get("summaryArtifactId") not in package.get("artifactIds", []) + or accepted.get("artifactId") != lineage["acceptedSliceDirectoryArtifactId"] + or accepted.get("revision") != manifest["revision"] + or accepted.get("manifestSha256") != sha(manifest_bytes)): + raise ValueError("unconfirmed or mismatched all-discard acceptance evidence") + operator = load_operator_module() + final = acceptance["finalAcceptance"] + # Deliberately incomplete audit input: never export it or claim formal readiness. + preview_context = {"trace": [], "finalAcceptance": final} + bodies = [] + with tempfile.TemporaryDirectory(prefix="v4-preview-") as temp: + for attempt in range(2): + staging = Path(temp) / str(attempt) + staging.mkdir() + payload, _ = operator._build_part_index( + manifest, manifest["coverage"], kept, revision, + copy.deepcopy(delivery["calibration"]), resolve_pose_policy(acceptance), + {}, final, copy.deepcopy(delivery["capture"]), lineage, discarded, + classification_v4=True, + ) + def internal(*args, **kwargs): + return {"portId": "deliveryAudit", "previewOnly": True} + def virtual(run, node, name, path, kind, size, data=None, output=None): + return {"path": str(path), "portId": output["id"]} + artifacts = operator._emit_part_index_directory( + "local-preview", "package", staging, + {"outputs": [{"id": "deliveryAudit"}, {"id": "deliveryDirectory"}]}, + {"write_json_artifact": internal, "build_virtual_artifact": virtual}, + payload, preview_context, manifest, revision, + ) + folder = Path(artifacts[0]["path"]) + if {p.name for p in folder.iterdir()} != {"part.json"}: + raise ValueError("customer directory must contain only part.json") + bodies.append((folder / "part.json").read_bytes()) + if bodies[0] != bodies[1]: + raise ValueError("non-deterministic conversion") + for path, digest in before.items(): + if sha(path.read_bytes()) != digest: + raise ValueError("source changed during conversion") + if (revision / "revision-manifest.json").read_bytes() != manifest_bytes: + raise ValueError("source manifest changed during conversion") + calibration_proof = directory / "calibration-provenance.json" + calibration_basis = "published_manifest_checksums" + if calibration_proof.exists(): + evidence = load(calibration_proof) + if not all(entry.get("historicalStoredSHAAvailable") is True for entry in evidence): + calibration_basis = "original_BOS_selection_read_at_conversion; historical_checksum_unavailable" + document = json.loads(bodies[0]) + target = output / package["id"] / "part.json" + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != bodies[0]: + raise ValueError("refusing to replace an existing different preview") + target.write_bytes(bodies[0]) + return {"packageId": package["id"], "status": "converted-preview", "file": str(target), + "captureId": document["capture"]["captureId"], "partId": document["partId"], + "revision": document["revision"], "sha256": sha(bodies[0]), "bytes": len(bodies[0]), + "keptSlices": len(kept), "discardedSlices": len(discarded), + "manifestSha256": sha(manifest_bytes), "deterministic": True, + "calibrationEvidence": calibration_basis, + "originalSourceEvidence": source_basis, + "formalPublicationReady": False, "completeInternalAuditValidated": False, + "onlinePublished": False} + + +def run(root, output): + root, output = root.resolve(), output.resolve() + if output == root or root in output.parents: + raise ValueError("output must be outside the evidence directory") + inventory = load(root / "inventory.json") + ids = [p["id"] for p in inventory] + if len(ids) != len(set(ids)): + raise ValueError("duplicate inventory package IDs") + results = [] + output.mkdir(parents=True, exist_ok=True) + for package in inventory: + # Do not use an untrusted ID as a filesystem path. + if Path(package["id"]).name != package["id"] or package["id"] in {".", ".."}: + raise ValueError("unsafe package ID") + try: + result = convert(package, root / "packages" / package["id"], output) + except (ValueError, RuntimeError, KeyError, OSError, SystemExit) as error: + result = {"packageId": package["id"], "status": "blocked", "reason": str(error)} + results.append(result) + report = {"total": len(inventory), "processed": len(results), + "converted": sum(r["status"] == "converted-preview" for r in results), + "productionWrites": False, "formalPublicationReady": False, "packages": results} + (output / "conversion-report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + return report + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = run(args.evidence, args.out) + print(json.dumps({k: v for k, v in report.items() if k != "packages"}, ensure_ascii=False)) + raise SystemExit(0 if report["converted"] == report["total"] else 2) diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index c804413f..b0abb6e6 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1,6 +1,33 @@ { "version": 1, "scopes": [ + { + "name": "delivery-v4-preview", + "paths": [ + "^scripts/delivery_v4_preview\\.py$", + "^tests/backend/test_delivery_v4_preview\\.py$" + ], + "frontend": [], + "go": {}, + "task": [ + { + "argv": [ + "python3", + "-m", + "pytest", + "tests/backend/test_delivery_v4_preview.py", + "tests/backend/test_delivery_v4_package.py", + "tests/backend/test_delivery_v4_real_materialization.py", + "tests/backend/test_legacy_package_compatibility.py", + "-q" + ], + "tests": true + } + ], + "warnings": [ + "仅验证本地 V4 转换预览;完整内部审计、真实视频切片与正式发布另行验收。" + ] + }, { "name": "verification-tooling", "paths": [ diff --git a/tests/backend/test_delivery_v4_preview.py b/tests/backend/test_delivery_v4_preview.py new file mode 100644 index 00000000..eca13fa8 --- /dev/null +++ b/tests/backend/test_delivery_v4_preview.py @@ -0,0 +1,179 @@ +"""Batch conversion must preserve revisions, resume safely and report every failure.""" +import hashlib +import importlib.util +import json +from pathlib import Path + +import pytest + +from tests.backend.test_delivery_v4_package import prepare + + +def converter(monkeypatch): + root = Path(__file__).resolve().parents[2] + monkeypatch.syspath_prepend(str(root / "scripts")) + spec = importlib.util.spec_from_file_location("preview", root / "scripts/delivery_v4_preview.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def evidence(tmp_path): + root = tmp_path / "evidence" + directory = root / "packages/package-accepted" + directory.mkdir(parents=True) + inputs = prepare(directory, [{"description": "拿取"}, {"description": "", "isInvalid": True}]) + revision = Path(inputs[0]["path"]) + revision.rename(directory / "pinned-revision") + revision = directory / "pinned-revision" + manifest = json.loads((revision / "revision-manifest.json").read_text()) + for row in manifest["items"]: + row["files"] = [{"name": name, "sha256": hashlib.sha256((revision / row["directory"] / name).read_bytes()).hexdigest()} + for name in ["annotation.json", "synchronized-slice.json"]] + raw = json.dumps(manifest).encode() + (revision / "revision-manifest.json").write_bytes(raw) + (directory / "pinned-provenance.json").write_text(json.dumps({ + "sha256": hashlib.sha256(raw).hexdigest(), "revision": manifest["revision"], "artifactId": "frozen-artifact"})) + old = directory / "old" + old.mkdir() + (old / "packages-manifest.json").write_text(json.dumps({ + "lineage": {"acceptedSliceDirectoryArtifactId": "frozen-artifact"}, + "capture": inputs[1]["data"]["capture"], "calibration": []})) + (old / "acceptance-record.json").write_text(json.dumps({"finalAcceptance": inputs[1]["data"]["finalAcceptance"]})) + inventory = [{"id": "package-accepted", "status": "accepted"}, {"id": "package-empty", "status": "no_deliverables"}] + (root / "inventory.json").write_text(json.dumps(inventory)) + return root, directory + + +def test_batch_resumes_exactly_and_keeps_all_blocked_rows(tmp_path, monkeypatch): + module = converter(monkeypatch) + root, directory = evidence(tmp_path) + output = tmp_path / "output" + before = {p: p.read_bytes() for p in root.rglob("*.json")} + report = module.run(root, output) + assert report["processed"] == report["total"] == 2 + assert report["converted"] == 1 + assert report["packages"][1]["status"] == "blocked" + assert report["productionWrites"] is report["formalPublicationReady"] is False + customer = output / "package-accepted/part.json" + doc = json.loads(customer.read_text()) + assert doc["schemaVersion"].endswith("/v4") + assert not {"lineage", "acceptance", "reviewHistory"} & doc.keys() + assert not list(output.rglob("review-history.json")) + assert "PRIVATE_REWORK" not in customer.read_text() + assert "PRIVATE_BASELINE" not in customer.read_text() + assert module.run(root, output) == report + assert all(p.read_bytes() == body for p, body in before.items()) + customer.write_text("other customer bytes") + blocked = module.run(root, output) + assert blocked["converted"] == 0 + assert "refusing to replace" in blocked["packages"][0]["reason"] + assert customer.read_text() == "other customer bytes" + + +@pytest.mark.parametrize("mutation", ["annotation", "provenance", "source", "classification", "checksums"]) +def test_corrupt_or_missing_frozen_evidence_blocks_without_guessing(tmp_path, monkeypatch, mutation): + module = converter(monkeypatch) + root, directory = evidence(tmp_path) + revision = directory / "pinned-revision" + path = revision / "revision-manifest.json" + manifest = json.loads(path.read_text()) + if mutation == "annotation": + (revision / manifest["items"][0]["directory"] / "annotation.json").write_text("{}") + elif mutation == "provenance": + (directory / "pinned-provenance.json").write_text('{}') + else: + if mutation == "source": + manifest.pop("originalSource") + elif mutation == "classification": + manifest["items"][0].pop("dictionary", None) + sync = revision / manifest["items"][0]["directory"] / "synchronized-slice.json" + value = json.loads(sync.read_text()); value.pop("dictionary", None); sync.write_text(json.dumps(value)) + manifest["items"][0]["files"][1]["sha256"] = hashlib.sha256(sync.read_bytes()).hexdigest() + else: + manifest["items"][0].pop("files") + path.write_text(json.dumps(manifest)) + provenance = json.loads((directory / "pinned-provenance.json").read_text()) + provenance["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + (directory / "pinned-provenance.json").write_text(json.dumps(provenance)) + report = module.run(root, tmp_path / "output") + assert report["processed"] == 2 + assert report["converted"] == 0 + assert not list((tmp_path / "output").rglob("part.json")) + + +def test_all_discard_preview_requires_matching_fixed_acceptance_proof(tmp_path, monkeypatch): + import copy + module = converter(monkeypatch) + root, directory = evidence(tmp_path) + revision = directory / "pinned-revision" + path = revision / "revision-manifest.json" + manifest = json.loads(path.read_text()) + row = manifest["items"][0] + row["deliveryDisposition"] = "discard" + row["discardEvidence"] = copy.deepcopy(manifest["items"][1]["discardEvidence"]) + sync = revision / row["directory"] / "synchronized-slice.json" + doc = json.loads(sync.read_text()) + doc.update(deliveryDisposition="discard", discardEvidence=row["discardEvidence"]) + sync.write_text(json.dumps(doc)) + row["files"][1]["sha256"] = hashlib.sha256(sync.read_bytes()).hexdigest() + path.write_text(json.dumps(manifest)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + provenance = json.loads((directory / "pinned-provenance.json").read_text()) + provenance["sha256"] = digest + (directory / "pinned-provenance.json").write_text(json.dumps(provenance)) + package = {"id": "package-accepted", "status": "no_deliverables", "artifactIds": ["frozen-summary"]} + (root / "inventory.json").write_text(json.dumps([package])) + assert module.run(root, tmp_path / "out")["converted"] == 0 + proof = {"packageId": package["id"], "confirmed": True, "summaryArtifactId": "frozen-summary", + "acceptedMaterialization": {"artifactId": "frozen-artifact", "revision": manifest["revision"], "manifestSha256": digest}} + (directory / "empty-acceptance-proof.json").write_text(json.dumps(proof)) + report = module.run(root, tmp_path / "out") + assert report["converted"] == 1 + doc = json.loads(Path(report["packages"][0]["file"]).read_text()) + assert doc["slices"] == [] and len(doc["discardedSlices"]) == 2 + assert all(not item["segments"] for item in doc["discardedSlices"]) + proof["acceptedMaterialization"]["manifestSha256"] = "wrong-revision" + (directory / "empty-acceptance-proof.json").write_text(json.dumps(proof)) + assert module.run(root, tmp_path / "other")["converted"] == 0 + + +def test_recovers_source_only_from_matching_published_capture_and_fixed_run(tmp_path, monkeypatch): + module = converter(monkeypatch) + root, directory = evidence(tmp_path) + manifest_path = directory / "pinned-revision/revision-manifest.json" + manifest = json.loads(manifest_path.read_text()) + original = manifest.pop("originalSource") + original["objectKeys"] = sorted(original["objectKeys"]) + manifest_path.write_text(json.dumps(manifest)) + digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + provenance = json.loads((directory / "pinned-provenance.json").read_text()) + provenance["sha256"] = digest + (directory / "pinned-provenance.json").write_text(json.dumps(provenance)) + delivery_path = directory / "old/packages-manifest.json" + delivery = json.loads(delivery_path.read_text()) + delivery["lineage"]["sourceRunId"] = manifest["runId"] + delivery_path.write_text(json.dumps(delivery)) + package = {"id": "package-accepted", "status": "accepted", "organizationId": "org"} + (root / "inventory.json").write_text(json.dumps([package])) + frozen = delivery["capture"] + source = {"run": {"id": manifest["runId"], "organizationId": "org", "inputs": {"source": {"ref": { + "provider": "BOS", "kind": "directory", "bucket": original["bucket"], "prefix": original["prefix"], + "objectKeys": original["objectKeys"], "captureUnit": original["captureUnit"], "versionId": original["snapshotFingerprint"]}}}}, + "capture": {"id": frozen["captureRecordId"], "organizationId": "org", "runId": manifest["runId"], + "captureId": frozen["captureId"], "prefix": frozen["prefix"]}} + recovery = {"basis": "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys", + "manifestSha256": digest, "originalSource": original} + def save(): + raw = json.dumps(source).encode() + (directory / "source-selection-input.json").write_bytes(raw) + recovery["inputSha256"] = hashlib.sha256(raw).hexdigest() + (directory / "source-selection-recovery.json").write_text(json.dumps(recovery)) + save() + report = module.run(root, tmp_path / "out") + assert report["converted"] == 1 + assert report["packages"][0]["originalSourceEvidence"] == recovery["basis"] + assert "originalSource" not in json.loads(manifest_path.read_text()) + source["run"]["organizationId"] = "different-tenant" + save() + assert module.run(root, tmp_path / "foreign")["converted"] == 0 From f4e147e355dc14049c9cb5ca3d50a5629af81c53 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 8 Nov 2026 10:00:31 -0800 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=90=9B=20Enforce=20V4=20delivery=20fo?= =?UTF-8?q?r=20legacy=20release=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/adr/0005-part-index-delivery-shape.md | 6 + docs/delivery-classification-v4.zh-CN.md | 8 +- internal/domain/types.go | 11 + internal/worker/capture_cutover_test.go | 25 +- internal/worker/capture_delivery.go | 21 ++ internal/worker/delivery_audit.go | 6 +- internal/worker/executor.go | 17 +- internal/worker/human_release.go | 5 + internal/worker/legacy_delivery_recovery.go | 144 ++++++++ internal/worker/legacy_delivery_v4.go | 183 ++++++++++ internal/worker/legacy_delivery_v4_test.go | 348 ++++++++++++++++++++ internal/worker/original_source_recheck.go | 81 ++++- internal/worker/runtime.go | 56 +++- 13 files changed, 882 insertions(+), 29 deletions(-) create mode 100644 internal/worker/legacy_delivery_recovery.go create mode 100644 internal/worker/legacy_delivery_v4.go create mode 100644 internal/worker/legacy_delivery_v4_test.go diff --git a/docs/adr/0005-part-index-delivery-shape.md b/docs/adr/0005-part-index-delivery-shape.md index 0d3bbcae..c9eb08f0 100644 --- a/docs/adr/0005-part-index-delivery-shape.md +++ b/docs/adr/0005-part-index-delivery-shape.md @@ -59,3 +59,9 @@ status: accepted 下游需要仅凭总 JSON 和原始数据定位、切分、按类归组。新 v4 使用任务原始输入的 BOS 选择冻结八类源文件及 bucket/prefix,主文件不再依赖平台 workspace。动作同时明确切片内秒数、原始 part 秒数与原始硬件时间戳;用原始左右 PTS 各自解析硬件区间,避免用左视频秒数误切另一相机。输入快照指纹不是单个 BOS 对象内容版本的保证。实际切割和分类打包仍由下游执行。 缺少可靠来源、分类标签或时间证据时,选择拒绝新 v4 交付,而非“尽量输出”后让客户承受丢分类或错切的风险。发布同时验证消费契约、来源归属和文件存在。已交付结果不原地改写;历史只按证据补齐并产生新批次。这个要求适用于尚未发布的 v4,不改变 v1–v3;旧本地 v4 候选须重新生成。实现和本地消费验证见 `docs/delivery-classification-v4.zh-CN.md`。 + +## 2026-11-08 修订:旧任务完成后的 V4 格式约束 + +同一组织、项目、工作流版本家族已启用已发布且可见的 V4 工作流时,尚未交付的旧任务也必须生成 V4。人工任务、来源 Run、已验收修订和原始输入仍属于原工作流,不重绑、不重跑标注。放行子 Run 单独冻结兼容的打包节点、启用参数及摘要镜像;执行、重启恢复与重试使用该快照。冻结的节点仅替换打包运行配置,不修改原工作流。缺少原始输入选择、冻结分类字典或已验收证据时,明确失败并恢复证据,禁止输出旧格式兜底。 + +Worker 在写入交付目标前检查实际格式;V4 客户目录只允许 `part.json`,审核历史仍为独立内部产物。已完整发布的旧包继续支持逐成员归属、SHA-256、大小和完成清单校验后的**只读状态恢复**;不续传未完成旧包、不接管冲突目录、不改写旧完成清单。历史转换仍产生独立候选/新批次,生产转换与正式发布分别验收。 diff --git a/docs/delivery-classification-v4.zh-CN.md b/docs/delivery-classification-v4.zh-CN.md index ee12a88b..99195c59 100644 --- a/docs/delivery-classification-v4.zh-CN.md +++ b/docs/delivery-classification-v4.zh-CN.md @@ -52,10 +52,16 @@ 回退旧 API 不是无损写入保证:新版本保存分类后,旧版本重建草稿数组可能丢弃字典快照。回退前应暂停相关写操作或准备能保留新字段的兼容回退版本;只保留恢复历史不等于能自动恢复所有后续修改。已交付不可变文件不因此原地迁移或覆盖。 -## 2026-10-22 启用决定 +## 2026-10-22 启用决定(在途任务范围已由 2026-11-08 修订替代) 用户随后确认启用 v4,范围仅为切换后新启动的工作流;在途任务、它们后续的审核/打包子运行和已交付文件继续原协议。此决定替代上节“继续保持关闭”的准备范围,不代表生产开关已经启用。实际发布仍需完成镜像、兼容性和目标核对,并审查具体部署及工作流切换计划。 用户明确确认当前无 tags 的启用词条以词条名称作为第一个独立 tags 标签。该初始化是一次有界字典变更:保留词条 ID、名称、用途、category、aliases 和既有历史,递增版本并记录新修订;已有非空 tags 不覆盖。历史任务中的冻结快照不因此重写,亦不反查新词条替代历史分类。后续 tags 仍由管理员维护。 新工作流使用新的定义版本及固定打包镜像。旧定义不原地改参数或镜像;切换必须保留已启动采集的原工作流归属,并阻止新监听器重新接入这些目录。新 v4 接入还需完整的原始文件选择,包含采集 meta;旧协议保留原有接入要求。准备与验收步骤见 [启用核对表](release/delivery-v4-activation.md)。 + +## 2026-11-08 旧任务交付修订 + +用户进一步要求不再新生成旧格式交付包。同一组织、项目和版本家族已启用 V4 时,旧任务后续放行也冻结兼容的 V4 打包配置;原工作流、人工任务归属和已验收内容不变。执行、重试与重启沿用冻结镜像及参数。无法证明原始来源、冻结字典、标签和时间证据时停在明确失败状态,先恢复证据,不回退旧格式。 + +已完整发布的旧文件仅允许校验完成清单、成员哈希和归属后只读恢复状态;缺失或冲突的旧上传不得继续补发。转换生成独立候选或新交付批次,不覆盖旧完成清单。线上替换、历史证据恢复和新批次正式发布需要分别验证;本地转换通过不代表全部历史包可正式交付。 diff --git a/internal/domain/types.go b/internal/domain/types.go index 36ccfe28..7f425499 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -790,6 +790,7 @@ type LightweightOperatorCodeManifest struct { } type RunMetadata struct { + CaptureDeliveryPackage *CaptureDeliveryPackageRuntime `json:"captureDeliveryPackage,omitempty" bson:"captureDeliveryPackage,omitempty"` LightweightOperatorCodeManifests map[string]LightweightOperatorCodeManifest `json:"lightweightOperatorCodeManifests,omitempty" bson:"lightweightOperatorCodeManifests,omitempty"` TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"` AnnotationTaskIDsByNode map[string][]string `json:"annotationTaskIdsByNode,omitempty" bson:"annotationTaskIdsByNode,omitempty"` @@ -797,6 +798,16 @@ type RunMetadata struct { HumanReleaseUniqueKey string `json:"humanReleaseUniqueKey,omitempty" bson:"humanReleaseUniqueKey,omitempty"` } +// A release child keeps its source workflow identity while freezing the newer +// package runtime. It never changes the source task or the stored workflow. +type CaptureDeliveryPackageRuntime struct { + SourceRunID string `json:"sourceRunId" bson:"sourceRunId"` + SourceWorkflowID string `json:"sourceWorkflowId" bson:"sourceWorkflowId"` + PolicyWorkflowID string `json:"policyWorkflowId" bson:"policyWorkflowId"` + PolicyVersion int `json:"policyVersion" bson:"policyVersion"` + Node WorkflowNode `json:"node" bson:"node"` +} + // RunFailure is the machine-readable counterpart to RunRecord.Error. Error is // intentionally retained for backwards compatibility and human-readable logs. type RunFailure struct { diff --git a/internal/worker/capture_cutover_test.go b/internal/worker/capture_cutover_test.go index d981c523..8f05d2d8 100644 --- a/internal/worker/capture_cutover_test.go +++ b/internal/worker/capture_cutover_test.go @@ -138,7 +138,19 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin t.Fatal(err) } } + publisher := &deliveryPublisher{} + f.runtime.artifactPublisher = publisher + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy bytes"), 0600); err != nil { + t.Fatal(err) + } + if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil { + t.Fatalf("old child cannot publish: %v", err) + } f.activate() + if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil { + t.Fatalf("completed legacy recovery failed: %v", err) + } for _, before := range []domain.CaptureRecord{active, done, launching} { after, _ := f.db.GetCaptureRecord(f.ctx, before.ID) if after == nil || after.WorkflowID != f.old.ID || after.RunID != before.RunID { @@ -149,15 +161,6 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin if err != nil || lookup == nil || lookup.ID != active.ID { t.Fatalf("old child lost Capture: %+v %v", lookup, err) } - publisher := &deliveryPublisher{} - f.runtime.artifactPublisher = publisher - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy bytes"), 0600); err != nil { - t.Fatal(err) - } - if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil { - t.Fatalf("old child cannot publish: %v", err) - } child.Status = domain.RunStatusCompleted if err := f.db.UpsertRun(f.ctx, child); err != nil { t.Fatal(err) @@ -180,8 +183,8 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin t.Fatalf("wrong new Run binding: %+v", newRun) } oldMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, child.WorkflowID) - if err != nil || oldMode { - t.Fatalf("old child selected v4: %v %v", oldMode, err) + if err != nil || !oldMode { + t.Fatalf("old child did not select v4 policy: %v %v", oldMode, err) } newMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, newRun.WorkflowID) if err != nil || !newMode { diff --git a/internal/worker/capture_delivery.go b/internal/worker/capture_delivery.go index de152519..605ee0d8 100644 --- a/internal/worker/capture_delivery.go +++ b/internal/worker/capture_delivery.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -88,6 +89,7 @@ type deliveryTarget struct { stageRoot string marker string publish func(key string, body []byte) error + completed *domain.ExecutionArtifact } // prepareDeliveryTarget performs every check that must hold before the first @@ -96,6 +98,9 @@ type deliveryTarget struct { // ownership and the conflict rules of the shared destination prefix. func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (*deliveryTarget, error) { record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID) + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + record, err = runtime.frozenDeliveryCapture(ctx, run) + } if err != nil { return nil, err } @@ -139,6 +144,9 @@ func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.R if run.Trigger == domain.RunTriggerHumanRelease && run.ParentRunID != "" { ownerRunID = run.ParentRunID } + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + ownerRunID = run.Metadata.CaptureDeliveryPackage.SourceRunID + } owner := record.ID + ":" + ownerRunID // PutInternal is conditional-create. A failed invocation may leave staging // objects behind, so each retry needs its own private staging namespace. @@ -166,6 +174,13 @@ func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.R return publisher.Delete(ctx, *connection, target.Bucket, stage) } if err := runtime.recheckV4DeliveryArtifact(ctx, run, artifact); err != nil { + if errors.Is(err, errLegacyDeliveryRequiresV4) { + completed, recoveryErr := completedLegacyDelivery(ctx, prepared, artifact) + if recoveryErr == nil { + prepared.completed = &completed + return prepared, nil + } + } return nil, err } known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker) @@ -196,6 +211,9 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain. if err != nil { return artifact, err } + if prepared.completed != nil { + return *prepared.completed, nil + } target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner publishBytes := prepared.publish marker := prepared.marker @@ -399,6 +417,9 @@ func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *dom if err != nil { return artifact, err } + if prepared.completed != nil { + return *prepared.completed, nil + } target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner publishBytes := prepared.publish fileName := filepath.Base(routing.FileName) diff --git a/internal/worker/delivery_audit.go b/internal/worker/delivery_audit.go index cd234bc9..f5cf1aa1 100644 --- a/internal/worker/delivery_audit.go +++ b/internal/worker/delivery_audit.go @@ -76,7 +76,11 @@ func (runtime *Runtime) classificationDeliveryEnabled(ctx context.Context, workf if workflow == nil { return false, nil } - return workflowUsesV4CaptureDelivery(*workflow), nil + if workflowUsesV4CaptureDelivery(*workflow) { + return true, nil + } + policy, err := runtime.activeV4DeliveryPolicy(ctx, workflow) + return policy != nil, err } // Reuse normal internal-artifact persistence rather than exceeding the 64 KiB diff --git a/internal/worker/executor.go b/internal/worker/executor.go index 808922c6..eb228685 100644 --- a/internal/worker/executor.go +++ b/internal/worker/executor.go @@ -1094,6 +1094,18 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, if workflowErr != nil { workflow = nil } + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + if workflowErr != nil { + return executionResult{}, nil, workflowErr + } + workflow, err = deliveryRuntimeWorkflow(workflow, run) + if err != nil || workflow == nil { + if err == nil { + err = fmt.Errorf("missing delivery workflow") + } + return executionResult{}, nil, err + } + } settings, err := runtime.store.GetSettings(ctx) if err != nil { return executionResult{}, nil, err @@ -1152,7 +1164,10 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, return executionResult{}, nil, fmt.Errorf("project deletion in progress") } } - image, err := sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage) + image, frozen, err := frozenDeliveryPackageImage(run, node) + if err == nil && !frozen { + image, err = sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage) + } if err != nil { return executionResult{}, nil, err } diff --git a/internal/worker/human_release.go b/internal/worker/human_release.go index 5cdb04c9..27539464 100644 --- a/internal/worker/human_release.go +++ b/internal/worker/human_release.go @@ -157,6 +157,10 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch if gateExecution == nil { return fmt.Errorf("human release gate execution %s is missing", batch.GateNodeID) } + frozenPackage, err := runtime.freezeLegacyDeliveryRuntime(ctx, workflow, parent, target) + if err != nil { + return err + } artifacts, err := runtime.releasePartitionArtifacts(ctx, parent, *gateNode, *gateExecution, partition) if err != nil { return err @@ -189,6 +193,7 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch if child.Metadata == nil { child.Metadata = &domain.RunMetadata{} } + child.Metadata.CaptureDeliveryPackage = frozenPackage child.Metadata.HumanReleasePartitionID = partition.ID child.Metadata.HumanReleaseUniqueKey = partition.UniqueKey child.UpdatedAt = nowISO() diff --git a/internal/worker/legacy_delivery_recovery.go b/internal/worker/legacy_delivery_recovery.go new file mode 100644 index 00000000..200faebb --- /dev/null +++ b/internal/worker/legacy_delivery_recovery.go @@ -0,0 +1,144 @@ +package worker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "droboticflow/internal/domain" +) + +// Recover only a byte-identical, fully completed immutable publication. This +// path cannot resume an incomplete legacy upload after a V4 family cutover. +func completedLegacyDelivery(ctx context.Context, target *deliveryTarget, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) { + info, err := os.Stat(artifact.Path) + if err != nil { + return artifact, err + } + paths := []string{artifact.Path} + kind := "part-index" + if info.IsDir() { + kind = "directory" + paths = nil + err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error { + if e != nil { + return e + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("legacy delivery symlink") + } + if !entry.IsDir() { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return artifact, err + } + sort.Slice(paths, func(i, j int) bool { + a, b := filepath.Base(paths[i]) == "packages-manifest.json", filepath.Base(paths[j]) == "packages-manifest.json" + if a != b { + return !a + } + return paths[i] < paths[j] + }) + } + if len(paths) == 0 { + return artifact, fmt.Errorf("empty legacy delivery") + } + entries := make([]directoryManifestEntry, 0, len(paths)) + check := func(key, sha string, size int64) error { + meta, e := target.publisher.Head(ctx, *target.connection, target.target.Bucket, key) + if e != nil { + return e + } + if !deliveryObjectMatches(meta, target.owner, sha, size) { + return fmt.Errorf("legacy delivery incomplete or changed") + } + return nil + } + for _, path := range paths { + relative := filepath.Base(path) + if info.IsDir() { + relative, err = filepath.Rel(artifact.Path, path) + if err != nil { + return artifact, err + } + relative = filepath.ToSlash(relative) + } else { + routing, e := partIndexPayloadPeek(path) + if e != nil { + return artifact, e + } + relative = filepath.Base(routing.FileName) + } + if relative == "manifest.json" || relative == ".delivery-owner.json" || strings.HasPrefix(relative, "../") { + return artifact, fmt.Errorf("reserved legacy member") + } + f, e := os.Open(path) + if e != nil { + return artifact, e + } + hash := sha256.New() + size, e := io.Copy(hash, f) + closeErr := f.Close() + if e != nil { + return artifact, e + } + if closeErr != nil { + return artifact, closeErr + } + entry := directoryManifestEntry{Path: relative, Key: target.target.Prefix + "/" + relative, SizeBytes: size, SHA256: hex.EncodeToString(hash.Sum(nil))} + if e = check(entry.Key, entry.SHA256, size); e != nil { + return artifact, e + } + entries = append(entries, entry) + } + marker, _ := json.Marshal(map[string]string{"owner": target.owner}) + body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: kind, FileCount: len(entries), TotalBytes: sumManifestBytes(entries), Entries: entries}) + expected := map[string]bool{} + for _, entry := range entries { + expected[entry.Key] = true + } + manifestKey := target.target.Prefix + "/manifest.json" + for key, value := range map[string][]byte{target.marker: marker, manifestKey: body} { + hash := sha256.Sum256(value) + if e := check(key, hex.EncodeToString(hash[:]), int64(len(value))); e != nil { + return artifact, e + } + expected[key] = true + } + keys, err := target.lister.ListPrefix(ctx, *target.connection, target.target.Bucket, target.target.Prefix+"/") + if err != nil { + return artifact, err + } + if len(keys) != len(expected) { + return artifact, fmt.Errorf("legacy delivery member count differs") + } + for _, key := range keys { + if !expected[key] { + return artifact, fmt.Errorf("unexpected legacy delivery member") + } + } + artifact.Path = "bos://" + target.target.Bucket + "/" + target.target.Prefix + artifact.SizeBytes = sumManifestBytes(entries) + ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.target.ConnectionID, Bucket: target.target.Bucket, Prefix: target.target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory} + if !info.IsDir() { + entry := entries[0] + artifact.Path = "bos://" + target.target.Bucket + "/" + entry.Key + ref.Prefix = "" + ref.Key = entry.Key + ref.SizeBytes = entry.SizeBytes + ref.SHA256 = entry.SHA256 + ref.Kind = domain.ArtifactRefKindObject + } + artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref} + return artifact, nil +} diff --git a/internal/worker/legacy_delivery_v4.go b/internal/worker/legacy_delivery_v4.go new file mode 100644 index 00000000..9f354fc5 --- /dev/null +++ b/internal/worker/legacy_delivery_v4.go @@ -0,0 +1,183 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "droboticflow/internal/domain" +) + +// The active released version is a delivery policy for its exact tenant, +// project and definition family. It does not rebind any in-flight source Run. +func (r *Runtime) activeV4DeliveryPolicy(ctx context.Context, source *domain.WorkflowRecord) (*domain.WorkflowRecord, error) { + if source == nil || source.ProjectID == "" || source.OrganizationID == "" || source.CaptureAutomation == nil { + return nil, nil + } + normalized := *source + domain.NormalizeWorkflowDefinitionMetadata(&normalized) + versions, err := r.store.ListWorkflowsByProject(ctx, source.ProjectID) + if err != nil { + return nil, err + } + var chosen *domain.WorkflowRecord + for _, candidate := range versions { + domain.NormalizeWorkflowDefinitionMetadata(&candidate) + if candidate.OrganizationID != source.OrganizationID || candidate.ProjectID != source.ProjectID || candidate.DefinitionFamilyID != normalized.DefinitionFamilyID || candidate.DefinitionState != domain.WorkflowDefinitionStateReleased || candidate.Hidden || !workflowUsesV4CaptureDelivery(candidate) { + continue + } + if chosen != nil { + return nil, fmt.Errorf("同一工作流版本家族有多个启用的 V4 交付配置,请先确认") + } + copy := candidate + chosen = © + } + return chosen, nil +} + +func (r *Runtime) freezeLegacyDeliveryRuntime(ctx context.Context, source *domain.WorkflowRecord, parent *domain.RunRecord, targetID string) (*domain.CaptureDeliveryPackageRuntime, error) { + if source == nil || parent == nil || workflowUsesV4CaptureDelivery(*source) || parent.Mode.Normalized() == domain.RunModeDebug { + return nil, nil + } + policy, err := r.activeV4DeliveryPolicy(ctx, source) + if err != nil || policy == nil { + return nil, err + } + old := domain.GetNodeByID(source.Nodes, targetID) + next := domain.GetNodeByID(policy.Nodes, targetID) + if old == nil || next == nil || old.Data.OperatorID != "operator-builtin-package" || next.Data.OperatorID != old.Data.OperatorID { + return nil, fmt.Errorf("旧任务缺少兼容的 V4 打包节点,请先核对交付配置") + } + image, err := sourceIndexPackageImage(*next, nil, r.operatorRunnerImage) + if err != nil { + return nil, err + } + expected := strings.TrimSpace(next.Data.OperatorImageID) + if expected == "" { + expected = strings.TrimSpace(next.Data.Image) + } + if image != expected { + return nil, fmt.Errorf("部署的 V4 打包镜像与已发布工作流不一致,请先核对") + } + var node domain.WorkflowNode + body, err := json.Marshal(old) + if err != nil { + return nil, err + } + if err = json.Unmarshal(body, &node); err != nil { + return nil, err + } + node.Data.Image, node.Data.OperatorImageID = image, image + node.Data.OperatorVersion = next.Data.OperatorVersion + node.Data.PublicationEvidence = next.Data.PublicationEvidence + node.Data.PublicationStatus = next.Data.PublicationStatus + node.Data.GenerationSource = next.Data.GenerationSource + node = applyNodeOverridesToNode(node, domain.NodeOverride{Hyperparams: []domain.HyperParam{ + {Key: "PIN_OPERATOR_IMAGE", Value: "true"}, {Key: "PART_INDEX_DELIVERY", Value: "true"}, {Key: "DELIVERY_JSON_V4", Value: "true"}, {Key: "DELIVERY_JSON_V3", Value: "false"}, {Key: "SOURCE_INDEX_DELIVERY", Value: "true"}, + }}) + audit := false + for _, port := range node.Data.Outputs { + if port.ID == "deliveryAudit" { + audit = true + } + } + if !audit { + node.Data.Outputs = append(node.Data.Outputs, domain.PortDefinition{ID: "deliveryAudit", Name: "deliveryAudit", Type: domain.PortTypeJSON}) + } + return &domain.CaptureDeliveryPackageRuntime{SourceRunID: parent.ID, SourceWorkflowID: source.ID, PolicyWorkflowID: policy.ID, PolicyVersion: policy.DefinitionVersion, Node: node}, nil +} + +func deliveryRuntimeWorkflow(workflow *domain.WorkflowRecord, run *domain.RunRecord) (*domain.WorkflowRecord, error) { + if workflow == nil || run == nil || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil { + return workflow, nil + } + frozen := run.Metadata.CaptureDeliveryPackage + if workflow.ID != run.WorkflowID || workflow.ID != frozen.SourceWorkflowID || workflow.OrganizationID != run.OrganizationID || workflow.ProjectID != run.ProjectID || frozen.SourceRunID == "" || frozen.PolicyWorkflowID == "" || frozen.Node.Data.OperatorID != "operator-builtin-package" || !workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{frozen.Node}}) || !sourcePackageDigestImage.MatchString(frozen.Node.Data.Image) || frozen.Node.Data.Image != frozen.Node.Data.OperatorImageID { + return nil, fmt.Errorf("V4 交付运行快照不完整或归属不符") + } + copy := *workflow + copy.Nodes = append([]domain.WorkflowNode(nil), workflow.Nodes...) + found := false + for i, node := range copy.Nodes { + if node.ID == frozen.Node.ID { + if node.Data.OperatorID != "operator-builtin-package" { + return nil, fmt.Errorf("V4 交付节点类型不符") + } + body, err := json.Marshal(frozen.Node) + if err != nil { + return nil, err + } + var replacement domain.WorkflowNode + if err = json.Unmarshal(body, &replacement); err != nil { + return nil, err + } + copy.Nodes[i] = replacement + found = true + } + } + if !found { + return nil, fmt.Errorf("V4 交付节点不存在") + } + return ©, nil +} + +func frozenDeliveryPackageImage(run *domain.RunRecord, node domain.WorkflowNode) (string, bool, error) { + if run == nil || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil || run.Metadata.CaptureDeliveryPackage.Node.ID != node.ID { + return "", false, nil + } + frozen := run.Metadata.CaptureDeliveryPackage.Node + if frozen.Data.OperatorID != "operator-builtin-package" || !workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{node}}) || !sourcePackageDigestImage.MatchString(frozen.Data.Image) || node.Data.Image != frozen.Data.Image { + return "", true, fmt.Errorf("V4 打包运行快照已改变,请先核对") + } + return frozen.Data.Image, true, nil +} + +// A retry with an explicit immutable delivery snapshot shares the accepted +// release attempt. Ordinary retry runs retain the Store's strict isolation. +func (r *Runtime) frozenDeliveryCapture(ctx context.Context, run *domain.RunRecord) (*domain.CaptureRecord, error) { + _, lineage, err := r.originalDeliveryRun(ctx, run) + if err != nil { + return nil, err + } + capture, err := r.store.GetCaptureRecord(ctx, run.TaskID) + if err != nil { + return nil, err + } + if capture == nil || capture.OrganizationID != run.OrganizationID || capture.ProjectID != run.ProjectID || capture.WorkflowID != run.WorkflowID || !lineage[capture.RunID] { + return nil, fmt.Errorf("frozen delivery capture scope mismatch") + } + return capture, nil +} + +// Persist the newest retry association before it launches. A restart repeats +// this safely; an unrelated or newer branch cannot be adopted or overwritten. +func (r *Runtime) attachFrozenReleaseRetry(ctx context.Context, run *domain.RunRecord) error { + if run.Trigger != domain.RunTriggerRetry || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil { + return nil + } + _, lineage, err := r.originalDeliveryRun(ctx, run) + if err != nil { + return err + } + partition, err := r.store.GetReleasePartitionByUniqueKey(ctx, run.OrganizationID, run.Metadata.HumanReleaseUniqueKey) + if err != nil { + return err + } + if partition == nil || partition.ID != run.Metadata.HumanReleasePartitionID || partition.UniqueKey != run.Metadata.HumanReleaseUniqueKey || !lineage[partition.ChildRunID] || partition.Status != domain.ReleasePartitionDispatched { + return fmt.Errorf("V4 retry release partition scope mismatch") + } + batch, err := r.store.GetHumanBatch(ctx, partition.BatchID) + if err != nil { + return err + } + if batch == nil || batch.OrganizationID != run.OrganizationID || batch.ProjectID != run.ProjectID || batch.WorkflowID != run.WorkflowID || batch.RunID != run.Metadata.CaptureDeliveryPackage.SourceRunID { + return fmt.Errorf("V4 retry release batch scope mismatch") + } + if partition.ChildRunID == run.ID { + return nil + } + partition.ChildRunID = run.ID + partition.UpdatedAt = nowISO() + return r.store.UpsertReleasePartition(ctx, *partition) +} diff --git a/internal/worker/legacy_delivery_v4_test.go b/internal/worker/legacy_delivery_v4_test.go new file mode 100644 index 00000000..35917b73 --- /dev/null +++ b/internal/worker/legacy_delivery_v4_test.go @@ -0,0 +1,348 @@ +package worker + +import ( + "context" + "droboticflow/internal/objectsync" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "droboticflow/internal/domain" +) + +func TestLegacyCaptureReleaseRequiresV4AfterFamilyCutover(t *testing.T) { + f := newCaptureCutoverFixture(t, "legacy-source") + record := f.seed("legacy-source", domain.CaptureRecordWaiting, "legacy-parent") + parent := domain.RunRecord{ID: record.RunID, OrganizationID: record.OrganizationID, ProjectID: record.ProjectID, WorkflowID: f.old.ID, TaskID: record.ID, Status: domain.RunStatusWaitingForHuman} + child := parent + child.ID = "legacy-child" + child.ParentRunID = parent.ID + child.Trigger = domain.RunTriggerHumanRelease + child.Status = domain.RunStatusRunning + for _, run := range []domain.RunRecord{parent, child} { + if err := f.db.UpsertRun(f.ctx, run); err != nil { + t.Fatal(err) + } + } + f.activate() + publisher := &deliveryPublisher{} + f.runtime.artifactPublisher = publisher + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy payload"), 0600); err != nil { + t.Fatal(err) + } + _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}) + if err == nil { + t.Fatal("a newly completed legacy task published an old-format customer package after V4 cutover") + } + if len(publisher.objects) != 0 { + t.Fatal("legacy rejection must precede owner markers and staging writes") + } + current, _ := f.db.GetCaptureRecord(f.ctx, record.ID) + if current.WorkflowID != f.old.ID { + t.Fatal("source ownership must not be rebound") + } +} + +func TestLegacyCaptureReleaseCollectsV4AuditAfterFamilyCutover(t *testing.T) { + f := newCaptureCutoverFixture(t, "legacy-audit") + f.activate() + enabled, err := f.runtime.classificationDeliveryEnabled(f.ctx, f.old.ID) + if err != nil || !enabled { + t.Fatalf("old task must collect the new customer format's internal audit after cutover: enabled=%v err=%v", enabled, err) + } +} + +func TestLegacyCaptureRuntimeIsFrozenAndScoped(t *testing.T) { + f := newCaptureCutoverFixture(t, "frozen") + f.activate() + parent := &domain.RunRecord{ID: "parent", OrganizationID: f.old.OrganizationID, ProjectID: f.old.ProjectID, WorkflowID: f.old.ID, TaskID: "capture"} + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", f.next.Nodes[0].Data.OperatorImageID) + frozen, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package") + if err != nil || frozen == nil { + t.Fatalf("freeze: %+v %v", frozen, err) + } + run := *parent + run.ID = "child" + run.ParentRunID = parent.ID + run.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: frozen} + effective, err := deliveryRuntimeWorkflow(&f.old, &run) + if err != nil { + t.Fatal(err) + } + if !workflowUsesV4CaptureDelivery(*effective) || workflowUsesV4CaptureDelivery(f.old) || run.WorkflowID != f.old.ID { + t.Fatal("execution must upgrade without mutating source ownership/configuration") + } + if len(effective.Nodes[0].Data.Outputs) != 1 || effective.Nodes[0].Data.Outputs[0].ID != "deliveryAudit" { + t.Fatal("missing internal audit output") + } + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "invalid-drift") + image, handled, err := frozenDeliveryPackageImage(&run, effective.Nodes[0]) + if err != nil || !handled || image != frozen.Node.Data.Image { + t.Fatalf("restart changed package image: %s %v", image, err) + } + run.OrganizationID = "foreign" + if _, err = deliveryRuntimeWorkflow(&f.old, &run); err == nil { + t.Fatal("foreign runtime accepted") + } + parent.Mode = domain.RunModeDebug + if frozen, err = f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package"); err != nil || frozen != nil { + t.Fatal("debug selection must be unchanged") + } +} + +func TestLegacyCaptureRuntimeRejectsIncompatiblePublishedImage(t *testing.T) { + f := newCaptureCutoverFixture(t, "image") + f.activate() + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "registry.example/wrong@sha256:"+strings.Repeat("d", 64)) + if _, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, &domain.RunRecord{ID: "parent"}, "package"); err == nil { + t.Fatal("unexpected operator image accepted") + } +} + +func TestLegacyCapturePolicyNeverCrossesFamilyOrTenant(t *testing.T) { + for _, scope := range []string{"family", "organization", "project", "hidden"} { + t.Run(scope, func(t *testing.T) { + f := newCaptureCutoverFixture(t, "scope") + f.activate() + next := f.next + next.Hidden = false + switch scope { + case "family": + next.DefinitionFamilyID = "foreign" + case "organization": + next.OrganizationID = "foreign" + case "project": + next.ProjectID = "foreign" + case "hidden": + next.Hidden = true + } + if err := f.db.UpsertWorkflow(f.ctx, next); err != nil { + t.Fatal(err) + } + policy, err := f.runtime.activeV4DeliveryPolicy(f.ctx, &f.old) + if err != nil || policy != nil { + t.Fatalf("crossed %s: %+v %v", scope, policy, err) + } + }) + } +} + +func TestLegacyCaptureRetryPreservesPackageAndAcceptedBaseline(t *testing.T) { + f := newCaptureCutoverFixture(t, "retry") + f.activate() + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", f.next.Nodes[0].Data.OperatorImageID) + parent := &domain.RunRecord{ID: "source", OrganizationID: f.old.OrganizationID, ProjectID: f.old.ProjectID, WorkflowID: f.old.ID, TaskID: "capture"} + parent.Inputs = map[string]domain.ArtifactPayload{"sourceDirectory": {Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeFile, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: "cutover-storage", Bucket: "synthetic", Prefix: "incoming/retry", Kind: domain.ArtifactRefKindDirectory}}} + if err := f.db.UpsertRun(f.ctx, *parent); err != nil { + t.Fatal(err) + } + frozen, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package") + if err != nil { + t.Fatal(err) + } + failed := *parent + failed.ID = "failed" + failed.ParentRunID = parent.ID + failed.RestartFromNodeID = "package" + failed.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: frozen, HumanReleasePartitionID: "partition", HumanReleaseUniqueKey: "key"} + failed.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "gate", Status: domain.NodeExecutionStatusCompleted, Artifacts: []domain.ExecutionArtifact{{Name: "accepted", Type: domain.PortTypeJSON, Data: map[string]any{"accepted": true}}}}, {NodeID: "package", Status: domain.NodeExecutionStatusFailed}} + if err := f.db.UpsertRun(f.ctx, failed); err != nil { + t.Fatal(err) + } + retry, err := f.runtime.enqueueRetryRun(f.ctx, &f.old, &failed) + if err != nil { + t.Fatal(err) + } + if retry.Metadata == nil || retry.Metadata.CaptureDeliveryPackage == nil || retry.Metadata.CaptureDeliveryPackage.SourceRunID != parent.ID || retry.Metadata.HumanReleasePartitionID != "partition" || retry.RestartFromNodeID != "package" || len(retry.NodeExecutions) != 1 || retry.NodeExecutions[0].NodeID != "gate" { + t.Fatalf("retry lost accepted delivery state: %+v", retry) + } + allowed := rerunAllowedNodeSet(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{ID: "source"}, {ID: "gate"}, {ID: "package"}}, Edges: []domain.WorkflowEdge{{Source: "source", Target: "gate"}, {Source: "gate", Target: "package"}}}, retry) + if _, ok := allowed["package"]; !ok { + t.Fatal("retry lost package subtree") + } + if _, ok := allowed["gate"]; ok { + t.Fatal("retry reruns accepted human gate") + } + retry.Metadata.CaptureDeliveryPackage.Node.Data.Hyperparams[0].Value = "changed" + if frozen.Node.Data.Hyperparams[0].Value == "changed" { + t.Fatal("retry aliased previous immutable selection") + } +} + +// Every write method is counted; equal final objects alone would not prove a +// completed legacy recovery is read-only. +type readonlyLegacyPublisher struct { + deliveryPublisher + writes int +} + +func (p *readonlyLegacyPublisher) PutInternal(c context.Context, x domain.StorageConnectionRecord, b, k string, r io.Reader, n int64, t string, m map[string]string) (objectsync.ObjectMetadata, error) { + p.writes++ + return p.deliveryPublisher.PutInternal(c, x, b, k, r, n, t, m) +} +func (p *readonlyLegacyPublisher) PublishConditional(c context.Context, x domain.StorageConnectionRecord, b, s, k string, n int64, e, a, z string, m map[string]string) (objectsync.ObjectMetadata, error) { + p.writes++ + return p.deliveryPublisher.PublishConditional(c, x, b, s, k, n, e, a, z, m) +} +func (p *readonlyLegacyPublisher) Delete(c context.Context, x domain.StorageConnectionRecord, b, k string) error { + p.writes++ + return p.deliveryPublisher.Delete(c, x, b, k) +} + +func TestLegacyCaptureCompletedRecoveryNeverWrites(t *testing.T) { + for _, damage := range []string{"none", "missing", "modified", "extra", "owner", "file"} { + t.Run(damage, func(t *testing.T) { + f := newCaptureCutoverFixture(t, "completed") + record := f.seed("completed", domain.CaptureRecordWaiting, "parent") + run := domain.RunRecord{ID: "parent", OrganizationID: record.OrganizationID, ProjectID: record.ProjectID, WorkflowID: f.old.ID, TaskID: record.ID} + if err := f.db.UpsertRun(f.ctx, run); err != nil { + t.Fatal(err) + } + publisher := &readonlyLegacyPublisher{} + f.runtime.artifactPublisher = publisher + root := t.TempDir() + path := filepath.Join(root, "legacy.json") + if err := os.WriteFile(path, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root} + publish := f.runtime.publishCaptureDelivery + if damage == "file" { + path = filepath.Join(root, "part.json") + if err := os.WriteFile(path, []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"part.json"}}`), 0600); err != nil { + t.Fatal(err) + } + artifact.Name = partIndexArtifactName + artifact.Path = path + publish = f.runtime.publishCaptureDeliveryFile + } + if _, err := publish(f.ctx, &run, "package", artifact); err != nil { + t.Fatal(err) + } + f.activate() + publisher.writes = 0 + key := record.Delivery.Bucket + "/" + record.Delivery.Prefix + "/legacy.json" + switch damage { + case "missing": + delete(publisher.objects, key) + case "modified": + publisher.metadata[key]["sha256"] = "changed" + case "extra": + publisher.objects[record.Delivery.Bucket+"/"+record.Delivery.Prefix+"/unexpected"] = []byte("x") + case "owner": + publisher.metadata[key]["delivery-owner"] = "foreign" + } + _, err := publish(f.ctx, &run, "package", artifact) + if damage == "none" || damage == "file" { + if err != nil { + t.Fatal(err) + } + } else if err == nil { + t.Fatal("damaged legacy publication was adopted") + } + if publisher.writes != 0 { + t.Fatalf("legacy recovery performed %d writes", publisher.writes) + } + }) + } +} + +func TestV4CustomerDirectoryRejectsInternalOrMediaMembers(t *testing.T) { + for _, name := range []string{"review-history.json", "left.mp4", "other.json"} { + t.Run(name, func(t *testing.T) { + r, pub, run, _, artifact := originalWorkerFixture(t) + if err := os.WriteFile(filepath.Join(artifact.Path, name), []byte("internal"), 0600); err != nil { + t.Fatal(err) + } + if _, err := r.publishCaptureDelivery(t.Context(), &run, "package", artifact); err == nil { + t.Fatal("V4 leaked extra member") + } + if len(pub.puts) > 0 || len(pub.published) > 0 { + t.Fatal("format guard ran after writes") + } + }) + } +} + +func TestFrozenV4RetryResolvesOnlyOriginalReleaseAncestry(t *testing.T) { + for _, scenario := range []string{"valid", "foreign", "cycle", "missing"} { + t.Run(scenario, func(t *testing.T) { + r, _, child, _, _ := originalWorkerFixture(t) + retry := child + retry.ID = "retry" + retry.ParentRunID = child.ID + retry.Trigger = domain.RunTriggerRetry + retry.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: &domain.CaptureDeliveryPackageRuntime{SourceRunID: child.ParentRunID}} + switch scenario { + case "foreign": + child.OrganizationID = "other" + case "cycle": + retry.Metadata.CaptureDeliveryPackage.SourceRunID = "unknown" + child.ParentRunID = retry.ID + case "missing": + child.ParentRunID = "unknown" + } + if err := r.store.UpsertRun(t.Context(), child); err != nil { + t.Fatal(err) + } + if err := r.store.UpsertRun(t.Context(), retry); err != nil { + t.Fatal(err) + } + _, err := r.originalDeliverySelection(t.Context(), &retry) + if (err == nil) != (scenario == "valid") { + t.Fatalf("scenario=%s err=%v", scenario, err) + } + }) + } +} + +func TestFrozenReleaseRetryRestoresPartitionAndCaptureAssociation(t *testing.T) { + r, _, child, _, _ := originalWorkerFixture(t) + retry := child + retry.ID = "retry" + retry.ParentRunID = child.ID + retry.Trigger = domain.RunTriggerRetry + retry.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: &domain.CaptureDeliveryPackageRuntime{SourceRunID: child.ParentRunID}, HumanReleasePartitionID: "partition", HumanReleaseUniqueKey: "key"} + if err := r.store.UpsertRun(t.Context(), retry); err != nil { + t.Fatal(err) + } + batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: child.OrganizationID, ProjectID: child.ProjectID, WorkflowID: child.WorkflowID, RunID: child.ParentRunID} + if err := r.store.UpsertHumanBatch(t.Context(), batch); err != nil { + t.Fatal(err) + } + partition := domain.ReleasePartitionRecord{ID: "partition", OrganizationID: child.OrganizationID, BatchID: batch.ID, ChildRunID: child.ID, UniqueKey: "key", Status: domain.ReleasePartitionDispatched} + if err := r.store.UpsertReleasePartition(t.Context(), partition); err != nil { + t.Fatal(err) + } + if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err != nil { + t.Fatal(err) + } + saved, err := r.store.GetReleasePartitionByUniqueKey(t.Context(), child.OrganizationID, "key") + if err != nil || saved.ChildRunID != retry.ID { + t.Fatalf("retry association: %+v %v", saved, err) + } + if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err != nil { + t.Fatal("restart did not preserve association", err) + } + capture, err := r.frozenDeliveryCapture(t.Context(), &retry) + if err != nil { + t.Fatal(err) + } + capture.RunID = retry.ID + if err := r.store.UpsertCaptureRecord(t.Context(), *capture); err != nil { + t.Fatal(err) + } + if _, err := r.originalDeliverySelection(t.Context(), &retry); err != nil { + t.Fatal("reconciled retry lost original source", err) + } + saved.ChildRunID = "unrelated" + if err := r.store.UpsertReleasePartition(t.Context(), *saved); err != nil { + t.Fatal(err) + } + if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err == nil { + t.Fatal("unrelated branch was overwritten") + } +} diff --git a/internal/worker/original_source_recheck.go b/internal/worker/original_source_recheck.go index 95cf5923..d8a58ca7 100644 --- a/internal/worker/original_source_recheck.go +++ b/internal/worker/original_source_recheck.go @@ -3,6 +3,7 @@ package worker import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -14,28 +15,57 @@ import ( "droboticflow/internal/workspacescope" ) -// Release children carry accepted artifacts but no original Inputs. Resolve -// only the persisted human-release edge; retries are independent selections. -func (r *Runtime) originalDeliverySelection(ctx context.Context, run *domain.RunRecord) (*sourceidentity.Selection, error) { +// Follow only validated persisted release ancestry, including retries carrying +// the same frozen package selection. Never infer a source from current intake. +func (r *Runtime) originalDeliveryRun(ctx context.Context, run *domain.RunRecord) (*domain.RunRecord, map[string]bool, error) { if run == nil { - return nil, fmt.Errorf("missing original delivery run") + return nil, nil, fmt.Errorf("missing original delivery run") } original := run - if run.Trigger == domain.RunTriggerHumanRelease { + lineage := map[string]bool{run.ID: true} + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + wanted := run.Metadata.CaptureDeliveryPackage.SourceRunID + for original.ID != wanted { + if len(lineage) >= 32 || original.ParentRunID == "" { + return nil, nil, fmt.Errorf("original delivery lineage incomplete") + } + parent, err := r.store.GetRun(ctx, original.ParentRunID) + if err != nil { + return nil, nil, err + } + if parent == nil || lineage[parent.ID] || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != run.TaskID { + return nil, nil, fmt.Errorf("original delivery parent scope mismatch") + } + lineage[parent.ID] = true + original = parent + } + } else if run.Trigger == domain.RunTriggerHumanRelease { parent, err := r.store.GetRun(ctx, run.ParentRunID) if err != nil { - return nil, err + return nil, nil, err } if parent == nil || parent.ID == run.ID || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != run.TaskID { - return nil, fmt.Errorf("original delivery parent scope mismatch") + return nil, nil, fmt.Errorf("original delivery parent scope mismatch") } original = parent + lineage[parent.ID] = true } - capture, err := r.store.GetCaptureRecordByRunID(ctx, original.ID) + return original, lineage, nil +} + +func (r *Runtime) originalDeliverySelection(ctx context.Context, run *domain.RunRecord) (*sourceidentity.Selection, error) { + original, lineage, err := r.originalDeliveryRun(ctx, run) if err != nil { return nil, err } - if capture == nil || capture.ID != original.TaskID || capture.RunID != original.ID && capture.RunID != run.ID { + capture, err := r.store.GetCaptureRecordByRunID(ctx, original.ID) + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + capture, err = r.frozenDeliveryCapture(ctx, run) + } + if err != nil { + return nil, err + } + if capture == nil || capture.ID != original.TaskID || !lineage[capture.RunID] { return nil, fmt.Errorf("original delivery capture lineage mismatch") } return sourceidentity.FromRun(original, capture) @@ -87,9 +117,25 @@ func (r *Runtime) preflightV4OriginalSource(ctx context.Context, run *domain.Run return r.recheckOriginalDeliverySelection(ctx, run, selection) } -// Detect the actual output schema, independent of mutable workflow switches. +var errLegacyDeliveryRequiresV4 = errors.New("该工作流已启用 V4,禁止新发布旧格式交付包;请使用已验收数据重新生成 V4") + +// Detect the actual output schema as well as the active format contract. // This guard runs before prepareDeliveryTarget writes even its owner marker. func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.RunRecord, artifact domain.ExecutionArtifact) error { + required := run != nil && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil + if !required && run != nil { + var err error + required, err = r.classificationDeliveryEnabled(ctx, run.WorkflowID) + if err != nil { + return err + } + } + legacy := func() error { + if required { + return errLegacyDeliveryRequiresV4 + } + return nil + } name := artifact.Path info, err := os.Stat(name) if err != nil { @@ -98,11 +144,11 @@ func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.Run if info.IsDir() { name = filepath.Join(name, "part.json") } else if filepath.Base(name) != "part.json" { - return nil + return legacy() } f, err := os.Open(name) if os.IsNotExist(err) { - return nil + return legacy() } // Legacy directory has no part.json. if err != nil { return err @@ -133,7 +179,16 @@ func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.Run return fmt.Errorf("invalid delivery part JSON") } if part.Schema != deliveryindex.ClassificationPartSchema { - return nil + return legacy() + } + if info.IsDir() { + members, e := os.ReadDir(artifact.Path) + if e != nil { + return e + } + if len(members) != 1 || members[0].Name() != "part.json" || !members[0].Type().IsRegular() { + return fmt.Errorf("V4 客户交付目录必须只有 part.json") + } } if len(raw) > 8<<20 { return fmt.Errorf("classification part exceeds 8 MiB") diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 8e7c0a6f..5ea35648 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -1400,6 +1400,10 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run Suggestion: "重新从现有发布版本创建工作流后再运行", }) } + workflow, err = deliveryRuntimeWorkflow(workflow, run) + if err != nil { + return runtime.failReconciledRun(ctx, run, err.Error(), domain.RunFailure{Code: "invalid_delivery_runtime", Stage: "orchestration", Retryable: false}) + } node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID) if node == nil { return runtime.failReconciledRun(ctx, run, fmt.Sprintf("node %s was deleted while run %s was active", run.CurrentNodeID, run.ID), domain.RunFailure{ @@ -2288,7 +2292,15 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo if err != nil { return nil, err } - retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.Inputs, failedRun.TaskID) + inputs := failedRun.Inputs + if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil { + source, _, e := runtime.originalDeliveryRun(ctx, failedRun) + if e != nil { + return nil, e + } + inputs = source.Inputs + } + retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, inputs, failedRun.TaskID) if err != nil { return nil, err } @@ -2296,6 +2308,30 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo retryRun.ParentRunID = failedRun.ID retryRun.AvailableAt = nextRunAvailableAt(workflow) lightweight.MergeRunMetadata(retryRun, manifests) + if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil { + // Preserve the immutable package selection and accepted release baseline. + raw, e := json.Marshal(failedRun.Metadata.CaptureDeliveryPackage) + if e != nil { + return nil, e + } + var frozen domain.CaptureDeliveryPackageRuntime + if e = json.Unmarshal(raw, &frozen); e != nil { + return nil, e + } + if retryRun.Metadata == nil { + retryRun.Metadata = &domain.RunMetadata{} + } + retryRun.Metadata.CaptureDeliveryPackage = &frozen + retryRun.Metadata.HumanReleasePartitionID = failedRun.Metadata.HumanReleasePartitionID + retryRun.Metadata.HumanReleaseUniqueKey = failedRun.Metadata.HumanReleaseUniqueKey + retryRun.RestartFromNodeID = failedRun.RestartFromNodeID + for _, execution := range failedRun.NodeExecutions { + if execution.NodeID != frozen.Node.ID && execution.Status == domain.NodeExecutionStatusCompleted { + applyExecution(retryRun, execution) + } + } + } + retryRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil { return nil, err @@ -2557,7 +2593,8 @@ func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) strings.TrimSpace(run.DebugTarget.TargetNodeID): {}, } } - if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease) || strings.TrimSpace(run.RestartFromNodeID) == "" { + frozenDeliveryRetry := run.Trigger == domain.RunTriggerRetry && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil + if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease && !frozenDeliveryRetry) || strings.TrimSpace(run.RestartFromNodeID) == "" { return nil } visited := map[string]struct{}{} @@ -2688,6 +2725,21 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor claimed.UpdatedAt = nowISO() return claimed, runtime.store.UpsertRun(ctx, *claimed) } + workflow, err = deliveryRuntimeWorkflow(workflow, claimed) + if err != nil { + claimed.Status = domain.RunStatusFailed + claimed.EndedAt = nowISO() + claimed.Error = err.Error() + claimed.UpdatedAt = nowISO() + return claimed, runtime.store.UpsertRun(ctx, *claimed) + } + if err = runtime.attachFrozenReleaseRetry(ctx, claimed); err != nil { + claimed.Status = domain.RunStatusFailed + claimed.EndedAt = nowISO() + claimed.Error = err.Error() + claimed.UpdatedAt = nowISO() + return claimed, runtime.store.UpsertRun(ctx, *claimed) + } if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 { currentRun := claimed message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。" From 1ae5d37e60dfbf7728e1d6e861270529839c53f3 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 8 Nov 2026 10:00:49 -0800 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=90=9B=20Freeze=20review=20feedback?= =?UTF-8?q?=20anchors=20and=20retain=20caption=20placement?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/CaptureAnnotationWorkbench.tsx | 24 +++ src/components/FinalAcceptanceWorkspace.tsx | 10 +- src/components/ReviewCaptions.tsx | 5 +- src/components/review-viewing.ts | 1 + src/hooks/useReviewDraft.ts | 11 ++ src/lib/review-caption-preferences.js | 17 ++ src/lib/review-navigation.js | 8 +- src/lib/review-opinions.ts | 5 +- src/styles/review-captions.css | 4 + tests/browser/acceptance-playback-rate.mjs | 148 +++++++++++++++++- .../fixtures/acceptance-playback-rate.tsx | 62 ++++++-- tests/browser/review-draft-persistence.mjs | 4 +- .../review-caption-preferences.test.mjs | 25 +++ .../review-target-interaction.test.mjs | 8 + 14 files changed, 301 insertions(+), 31 deletions(-) create mode 100644 src/lib/review-caption-preferences.js create mode 100644 tests/frontend/review-caption-preferences.test.mjs diff --git a/src/components/CaptureAnnotationWorkbench.tsx b/src/components/CaptureAnnotationWorkbench.tsx index a6b67ca0..396f9ba1 100644 --- a/src/components/CaptureAnnotationWorkbench.tsx +++ b/src/components/CaptureAnnotationWorkbench.tsx @@ -16,6 +16,7 @@ import { userErrorMessage } from "../lib/user-error-message.js"; import { reviewCommandRequiresWrite, reviewShortcut, reviewShortcutHelp } from "../lib/review-shortcuts.js"; import type { ReviewViewingProps, ReviewViewingController, ReviewPosition, ReviewNavigation } from "./review-viewing"; import { ReviewCaptionDetails, ReviewVideoCaption, type ReviewCaption } from "./ReviewCaptions"; +import { readReviewCaptionPlacement, writeReviewCaptionPlacement } from "../lib/review-caption-preferences.js"; import { reviewCaptionAtTime } from "../lib/review-captions.js"; import { sliceEditActionsWithDrafts, workbenchDraftsFingerprint } from "../lib/annotation-autosave.js"; import { DiscardReasonFields, type DiscardReasonChoice } from "./DiscardReasonFields"; @@ -590,6 +591,7 @@ export function CaptureAnnotationWorkbench({ }, [selectedSliceId, onSelectedSliceChange]); const [currentTime, setCurrentTime] = useState(clipStart); const [captionsVisible, setCaptionsVisible] = useState(true); + const [captionPlacement, setCaptionPlacement] = useState(readReviewCaptionPlacement); const [sliceZoom, setSliceZoom] = useState(false); const [localPlaybackRate, setLocalPlaybackRate] = useState(1); const playbackRate = preferredPlaybackRate ?? localPlaybackRate; @@ -2764,6 +2766,7 @@ export function CaptureAnnotationWorkbench({ const reviewSelectedActionId = reviewNavigation?.selectedActionId || editingAnnotation?.id || null; const reviewPosition: ReviewPosition = { time: currentTime, + discarded: selectedSlice.deliveryDisposition === "discard" || providedReviewCaption?.kind === "discarded", timeBasis: mediaOnly ? "slice-seconds" : "capture-seconds", sliceId: selectedSlice.id, navigationSliceId: selectedSlice.id, @@ -2789,6 +2792,7 @@ export function CaptureAnnotationWorkbench({ currentTime, selectedSlice.id, reviewPosition.actionId, + reviewPosition.discarded, reviewSelectedActionId, timelineReady, onReviewPositionChange, @@ -2840,6 +2844,7 @@ export function CaptureAnnotationWorkbench({ } const position: ReviewPosition = { time, + discarded: owner.deliveryDisposition === "discard" || providedReviewCaption?.kind === "discarded", timeBasis: mediaOnly ? "slice-seconds" : "capture-seconds", sliceId: owner.id, navigationSliceId: owner.id, @@ -3994,6 +3999,23 @@ export function CaptureAnnotationWorkbench({ > {captionsVisible ? "隐藏字幕" : "显示字幕"} + {isReviewViewing && captionsVisible && ( + + )} {focusedLayout ? (