fix: close rollout finalization and review publication gaps

Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
This commit is contained in:
ming01.wang 2026-09-29 16:09:01 +08:00
parent e1e73816af
commit 498187569c
23 changed files with 1379 additions and 159 deletions

View File

@ -71,6 +71,7 @@ def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: float = 6
since = st.get("started_at") or st.get("queued_at") or t0
print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}"
f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}"
f"{'(阶段计算已结束,等待数据库登记;无需重跑)' if st.get('registration') == 'PENDING' else ''}"
f" 已 {int(time.time() - since)}s", flush=True)
remaining = deadline - time.monotonic()
if remaining <= 0:

View File

@ -0,0 +1,38 @@
# Rollout 收尾与恢复修复
生产分支保持 `production/s100p-infra-fix-20260928`。Mac、服务器和 Git 远端应对应同一提交;运行参数、密钥、状态和数据继续独立于 Git。分支名不代替提交与进程启动身份。
## 范围与依据
| 已复现问题 | 沿用的入口与修改 |
| --- | --- |
| 编译完成后数据库忙,丢失阶段完成登记 | 缩短 continuation 写事务;复用 stage operation/status/meta 重放登记,不重执行计算 |
| execution 已结束,但 rollout 仍 RUNNING/VERIFYING | reconcile_run 复用 settle_execution / verify_rollout 补真实结局;不得直接设 PASS |
| 数据库忙使 collector 退出,或停止时无限等写库 | 完成 future 保留待登记,暂停新领取;停止时交接已有持久化证据,由下一次启动恢复 |
| 旧任务持续占本机并发槽 | reconcile_leases 检查同 execution 身份、epoch 与全部本机清理证明;未知板端仍隔离 |
| SSH 准备失败扩大为板端工作未知 | 幂等准备步骤有界重试;确未派发工作才记录 not_started |
| 全盘搜索及正常收尾重扫所有历史 | 限定 Agent 搜索范围;运行中恢复只处理刚结束的任务 |
| 损坏阶段 JSON 阻断整批导出 | 原隔离逻辑贯通 source manifest,坏样本不得进入 SFT |
| 静止批次待审或回执待发布无后续触发 | 复用 disposition、审核回执及 export --follow 补审/补发布;缓存不重复付费 |
不新增调度服务、恢复数据库或第二套审核器。保留取消、隔离、产物身份、精度门槛及完整轨迹检查。单个模型质量失败不伪装为基础设施问题,也不强行入 SFT。
## 执行与验收顺序
1. 保存版本、任务身份、数据库与配额账本;通过既有 drain / execution cancel 停止旧任务,核实所属进程退出。
2. 每个故障先复现,再最小修复;测试覆盖 SQLite 持锁、owner 死亡、COMPLETED 判定未落库、SIGTERM 交接、连续槽位补位、SSH 启动边界、审核完成但物化失败。
3. Linux 回归及独立子 Agent 审查通过后提交和部署。原始证据、未知隔离、密钥和有效 SFT 保留。
4. 同一审核组的启动配置与持久化 policy 必须一致。本次审核内部 TPM 调整至 700000、串行 1、RPM 4;这不是服务商额度声明。不得清空账本或把 UNKNOWN 直接改为成功。
5. 使用现成 100 张 ImageNet T1,新 run 先跑 ResNet18、MobileNetV2 两条真实编译、板测、验证、自动审核和 SFT;再验证 8 个 Agent 并行、跨轮补位。板测按可用板槽排队。
6. 重复刷新不得增加审核请求或重复样本。仅对明确选定的 COMPLETE UNKNOWN 使用 `export --review --retry-unknown`,保留旧回执;该入口不能与 `--follow` 同用。
7. 真实验收后,按精确清单复检并删除已终止任务可重建的编译中间文件。保留原对话、故障、回执、最终 HBM、有效 SFT、公共缓存及未知隔离。
故障注入与合成 HTTP 测试不等于实板验收。最终交付应列出实际提交、实板范围、SFT 数量、并发补位和观察时长;不得将短时测试说成所有模型或数日无故障。
## 原有启动入口
在服务器 `/mnt/data/ming01.wang/model-compile-hbdk/bench` 检出上述生产分支,确认工作树干净及同 run 没有存活 collector,并显式设置 `BENCH_ROOT`、`BENCH_STATE_DIR=/home/ming01.wang/bench-state` 和对应 `BENCH_CONFIG`。配置路径见根目录 `AGENTS.md`。
使用 `bin/bench collect --run <run_id> --modes e2e --parallel 8`;需要持续等待新任务时加原有 `--watch`。新批次使用现有 `new-run` / `expand`,不要重置失败行或清除旧 drain 标记来冒充新任务。`--watch` 不会无限重跑已失败的同模型任务。
`bin/bench execution reconcile --run <run_id>` 只恢复已有执行身份与证据;遇到 UNKNOWN 板端证据仍保留隔离。`bin/bench export --run <run_id> --follow --review` 维护同一数据集,复用导出锁与已有回执;不另建训练集或绕过准入检查。

View File

@ -184,7 +184,10 @@ def _managed_settings():
return settings, alias, endpoint["model"]
def review_once(rd: Path, *, timeout=120, max_tokens=32768, managed=False):
def review_once(rd: Path, *, timeout=120, max_tokens=32768, managed=False, retry_unknown=False,
pending_binding=None):
if retry_unknown and not managed:
raise ValueError("REVIEW_RETRY_REQUIRES_MANAGED_BROKER")
with ExitStack() as cleanup:
settings = _managed_settings() if managed else None
if managed:
@ -192,12 +195,18 @@ def review_once(rd: Path, *, timeout=120, max_tokens=32768, managed=False):
lock_dir = paths.STATE / "locks"
lock_dir.mkdir(parents=True, exist_ok=True)
lock = cleanup.enter_context((lock_dir / "content-review.lock").open("a"))
fcntl.flock(lock, fcntl.LOCK_EX)
try:
fcntl.flock(lock, fcntl.LOCK_EX | (fcntl.LOCK_NB if pending_binding else 0))
except BlockingIOError:
return None
if pending_binding and (Path(rd) / "content-reviews" / (pending_binding + ".json")).exists():
return None # The saved disposition can lag a completed or in-flight review.
return _review_once(rd, timeout=timeout, max_tokens=max_tokens,
managed=settings, cleanup=cleanup)
managed=settings, cleanup=cleanup, retry_unknown=retry_unknown,
pending_binding=pending_binding)
def _review_once(rd: Path, *, timeout, max_tokens, managed, cleanup):
def _review_once(rd: Path, *, timeout, max_tokens, managed, cleanup, retry_unknown=False, pending_binding=None):
# Reuse exact export script checks, without making materialization invoke a model.
from export import to_agentrl as ex
from harness.verify import verify_rollout, collect_measurements
@ -222,12 +231,23 @@ def _review_once(rd: Path, *, timeout, max_tokens, managed, cleanup):
if ex.annotation_reasons(rows, verdict["run_id"], verdict["job_id"], verdict["rollout"]):
raise ValueError("SOURCE_EXCLUDED_BY_ANNOTATION")
bound = binding(rd, verdict, frozen, gates)
if pending_binding and digest(bound) != pending_binding:
raise ValueError("CONTENT_REVIEW_BINDING_CHANGED")
path = receipt_path(rd, bound)
path.parent.mkdir(mode=0o700, exist_ok=True)
with (path.parent / ".lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
previous = None
if retry_unknown and not path.exists():
raise ValueError("REVIEW_RETRY_REQUIRES_COMPLETE_UNKNOWN")
if path.exists():
return json.loads(path.read_text())
previous_bytes = path.read_bytes()
previous = json.loads(previous_bytes)
if not retry_unknown:
return previous
if (previous.get("schema") != "bench.content-review/1" or previous.get("binding") != bound
or previous.get("state") != "COMPLETE" or previous.get("decision") != "UNKNOWN"):
raise ValueError("REVIEW_RETRY_REQUIRES_COMPLETE_UNKNOWN")
model = managed[2] if managed else os.environ.get("BENCH_REVIEW_MODEL")
url = os.environ.get("BENCH_REVIEW_URL")
if not model or (not managed and not url):
@ -235,21 +255,35 @@ def _review_once(rd: Path, *, timeout, max_tokens, managed, cleanup):
if managed and json.loads((rd / "rollout.json").read_text()).get("model") not in managed[0]["source_models"]:
raise ValueError("CONTENT_REVIEW_SOURCE_MODEL_EXCLUDED")
context = review_context(job, reconstructed, bound, gates)
max_tokens = max(1, min(int(max_tokens), 32768))
body = json.dumps({"model": model, "messages": [{"role": "system", "content": SYSTEM},
{"role": "user", "content": json.dumps(context, ensure_ascii=False)}],
"max_tokens": max(1, min(int(max_tokens), 32768)), "temperature": 0}).encode()
"max_tokens": max_tokens, "temperature": 0}).encode()
if len(body) > 2_000_000:
raise ValueError("CONTEXT_TOO_LARGE_NO_TRUNCATION")
record = {"schema": "bench.content-review/1", "binding": bound, "state": "STARTED",
"decision": "UNKNOWN", "model": model, "started_at": time.time(), "attempts": 1}
"decision": "UNKNOWN", "model": model, "started_at": time.time(), "attempts": 1,
"request_bytes": len(body), "max_tokens": max_tokens}
if previous is not None:
record["previous_receipt_sha256"] = hashlib.sha256(previous_bytes).hexdigest()
if managed:
from harness.collection_runtime import broker
instance = broker() # Configuration/credentials fail before STARTED; no upstream request.
execution_id = "content-review:" + digest({"source": str(rd), "binding": bound})
if previous is not None:
execution_id += ":" + digest({"previous": record["previous_receipt_sha256"],
"started_at": record["started_at"]})
scope = instance.issue_scope(execution_id, managed[1], model)
cleanup.callback(instance.cancel, execution_id)
generation_id = execution_id + ":0"
record.update(execution_id=execution_id, generation_id=generation_id)
if previous is not None:
archive = path.parent / "history" / (record["previous_receipt_sha256"] + ".json")
archive.parent.mkdir(mode=0o700, exist_ok=True)
# Preserve the original bytes; the active receipt changes only at STARTED.
with archive.open("wb") as stream:
stream.write(previous_bytes)
stream.flush(); os.fsync(stream.fileno())
_atomic(path, record) # crashes stay UNKNOWN; never silently retry
try:
if managed:
@ -281,34 +315,77 @@ def _review_once(rd: Path, *, timeout, max_tokens, managed, cleanup):
return record
def review_pending(run_id: str) -> int:
"""Reuse the exporter's selected candidates; never hold its lock over HTTP."""
def review_pending(run_id: str, *, retry_unknown=False, ready_only=False) -> bool:
"""Review selected candidates; return whether their receipts need publication."""
from harness import paths
settings, _, _ = _managed_settings()
if ready_only and retry_unknown:
raise ValueError("REVIEW_RETRY_MUST_BE_EXPLICIT_ONE_OFF")
settings, alias, _ = _managed_settings()
can_request = True
if ready_only:
from harness.quota import quota_snapshot
config = paths.config()["collection_v1"]
quota = quota_snapshot(Path(config["quota_ledger"]))
group = next((row for row in quota.get("groups", [])
if row["group_id"] == config["credential_groups"][alias]), {})
can_request = not (quota["status"] != "OBSERVED" or group.get("state") not in ("OPEN", "HALF_OPEN", "RECOVERING", "COOLING")
or group.get("cooldown_until", 0) > quota["captured_at"]
or group.get("inflight", 0) >= group.get("target", 0))
dataset = paths.DATASETS / f"{run_id}-training"
with (dataset / ".export.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_SH)
try:
fcntl.flock(lock, fcntl.LOCK_SH | (fcntl.LOCK_NB if ready_only else 0))
except BlockingIOError:
return False
manifest = json.loads((dataset / "manifest.json").read_text())
if manifest.get("filters", {}).get("purpose") not in ("all", "sft"):
return 0
return False
candidates = [json.loads(line) for line in (dataset / "disposition.jsonl").read_text().splitlines() if line]
reviewed = 0
for item in candidates:
if item.get("reasons") not in (["content-review-unreviewed"], ["sft:content-review-unreviewed"]):
reasons = item.get("reasons")
unknown = reasons in (["content-review-unknown"], ["sft:content-review-unknown"])
unreviewed = reasons in (["content-review-unreviewed"], ["sft:content-review-unreviewed"])
retry = retry_unknown and unknown
if not (retry or unreviewed or (ready_only and unknown)):
continue
pending_binding = item.get("review_binding_sha256") if ready_only else None
if ready_only and not re.fullmatch("[0-9a-f]{64}", pending_binding or ""):
continue # Legacy dispositions are refreshed before entering follow mode.
rd = (paths.BENCH_ROOT / item["source"]).resolve()
if item.get("run_id") != run_id or rd.parents[2] != (paths.RUNS / run_id).resolve():
raise ValueError("CONTENT_REVIEW_SOURCE_RUN_MISMATCH")
if ready_only:
try:
saved = (rd / "content-reviews" / (pending_binding + ".json")).read_bytes()
except FileNotFoundError:
saved = None
if saved is not None:
try:
receipt = json.loads(saved)
except ValueError:
continue
if (isinstance(receipt, dict) and receipt.get("schema") == "bench.content-review/1"
and receipt.get("state") == "COMPLETE" and digest(receipt.get("binding")) == pending_binding
and hashlib.sha256(saved).hexdigest() != item.get("review_receipt_sha256")):
return True # A saved result needs export even while the account is paused.
continue # Cached or STARTED receipts never authorize a new request.
if not can_request or not unreviewed:
continue
if json.loads((rd / "rollout.json").read_text()).get("model") not in settings["source_models"]:
continue
result = review_once(rd, managed=True)
result = review_once(rd, managed=True, retry_unknown=retry, pending_binding=pending_binding)
if result is None:
continue
reviewed += 1
print(json.dumps({"event": "content_review", "run_id": run_id, "job_id": item.get("job_id"),
"rollout": item.get("rollout"), "decision": result.get("decision"),
"reason": result.get("reason")}, ensure_ascii=False), flush=True)
if result.get("decision") == "UNKNOWN":
if ready_only:
break # Follow handles one existing candidate without rescanning rollout evidence.
if result.get("decision") == "UNKNOWN" and result.get("reason") != "REQUEST_EXCEEDS_INTERNAL_TPM":
break # Persist and refresh this result; do not consume the remaining candidates.
return reviewed
return bool(reviewed)
if __name__ == "__main__":
@ -316,7 +393,9 @@ if __name__ == "__main__":
parser.add_argument("rollout", type=Path)
parser.add_argument("--timeout", type=float, default=120)
parser.add_argument("--managed", action="store_true", help="use the configured reviewer quota Broker")
parser.add_argument("--retry-unknown", action="store_true", help="explicitly retry a completed UNKNOWN through --managed; preserve its receipt")
args = parser.parse_args()
result = review_once(args.rollout, timeout=args.timeout, managed=args.managed)
result = review_once(args.rollout, timeout=args.timeout, managed=args.managed,
retry_unknown=args.retry_unknown)
print(json.dumps({k: result.get(k) for k in ("state", "decision", "reason", "model", "usage")}, ensure_ascii=False))
raise SystemExit(0 if result.get("decision") == "PASS" else 2)

View File

@ -130,6 +130,23 @@ class MaterializationTests(unittest.TestCase):
stats = ex.materialize(['r'], f'active-{index}')['stats']
self.assertEqual((stats['scanned'], stats['sft_eligible']), (0, 0))
def test_invalid_stage_json_retains_failure_without_blocking_other_rollouts(self):
bad = self.add(kk=0)
self.add(kk=1)
malformed = bad / 'stages/06-hbm/a01/meta.json'
malformed.parent.mkdir()
malformed.write_bytes(b'')
result = ex.materialize(['r'], 'corrupt-stage')
self.assertEqual((result['stats']['scanned'], result['stats']['sft_eligible']), (2, 1))
output = self.root / 'datasets/corrupt-stage'
records = [json.loads(line) for line in (output / 'disposition.jsonl').read_text().splitlines()]
self.assertEqual(records[0]['reasons'], ['reverification_failed:JSONDecodeError'])
self.assertEqual(records[1]['disposition'], 'accepted')
evidence = [json.loads(line) for line in (output / 'source-manifest.jsonl').read_text().splitlines()]
self.assertEqual(evidence[0]['errors'], ['HARD_GATES_UNAVAILABLE:JSONDecodeError'])
self.assertEqual(malformed.read_bytes(), b'')
self.assertEqual(json.loads((output / 'sft.jsonl').read_text())['rollout'], 1)
def test_db_reactivation_during_build_fails_closed(self):
self.add()
reconstruct = ex.preserved.reconstruct

View File

@ -609,7 +609,7 @@ def _write_dataset(a, work: Path, ds_id: str) -> dict:
files[name].write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
if name == "source-manifest":
source_seen.add((rec["run_id"], rec["job_id"], rec["rollout"]))
def exclude(rd, reasons):
def exclude(rd, reasons, review_binding=None, review_receipt=None):
stats["excluded"] += 1
for reason in reasons:
_inc(stats["dropped"], reason)
@ -619,6 +619,9 @@ def _write_dataset(a, work: Path, ds_id: str) -> dict:
disposition = {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name),
"source": str(rd.relative_to(BENCH)), "reasons": sorted(reasons),
"disposition": "no-usable" if "no_usable_messages" in reasons else "retained"}
if review_binding:
disposition["review_binding_sha256"] = review_binding
disposition["review_receipt_sha256"] = review_receipt
write("disposition", disposition)
write("retained", disposition)
if a.purpose == "all":
@ -631,9 +634,15 @@ def _write_dataset(a, work: Path, ds_id: str) -> dict:
ident = (rd.parents[2].name, rd.parents[1].name, int(rd.name))
if ident not in source_seen:
source = preserved.freeze(rd)
try:
gate_sources = _gate_sources(rd)
except (OSError, ValueError, TypeError, KeyError) as error:
# Rejected evidence must not abort the whole run's export.
gate_sources = {}
source.errors.append(f"HARD_GATES_UNAVAILABLE:{type(error).__name__}")
write("source-manifest", {"run_id": ident[0], "job_id": ident[1], "rollout": ident[2],
"source": str(rd.relative_to(BENCH)), "files": source.manifest,
"errors": source.errors, "hard_gates": _gate_sources(rd)})
"errors": source.errors, "hard_gates": gate_sources})
file_info = {}
for name in names:
path = work / f"{name}.jsonl"
@ -809,7 +818,8 @@ def _write_agent(rd, a, stats, annotations, write, exclude):
validation["pass"] = all(check["pass"] for check in validation["checks"].values())
write("validation", {**ident, **validation})
if a.purpose == "sft" and not sft_ok:
exclude(rd, {f"sft:{sft_reason}"}); return
exclude(rd, {f"sft:{sft_reason}"}, content_review.get("binding_sha256"),
content_review.get("receipt_sha256")); return
reward_candidate = verdict["terminal"] in ("PASS", "FAIL") and mode != "repair" and not rw.get("unverifiable")
if a.purpose == "rl" and not reward_candidate:
exclude(rd, {"rl:historical-repair-or-unverified-outcome"}); return
@ -869,6 +879,8 @@ def _write_agent(rd, a, stats, annotations, write, exclude):
write("sft" if sft_ok else "retained", rec)
write("disposition", {"run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"],
"source": str(rd.relative_to(BENCH)), "disposition": "accepted" if sft_ok else "retained",
"review_binding_sha256": content_review.get("binding_sha256"),
"review_receipt_sha256": content_review.get("receipt_sha256"),
"reasons": [] if sft_ok else [sft_reason]})
stats["written"] += 1
stats["agent_written"] += 1

View File

@ -326,6 +326,11 @@ def cmd_export(a) -> int:
import fcntl
import os
from .pipeline import refresh_run_dataset
retry_unknown = getattr(a, "retry_unknown", False)
if retry_unknown and (not getattr(a, "review", False) or a.follow):
raise SystemExit("--retry-unknown requires --review and cannot be used with --follow")
if retry_unknown and not paths.config().get("content_review"):
raise SystemExit("--retry-unknown requires configured managed content_review")
with db.readonly() as con:
rid = _run_id(con, a.run)
lock = None
@ -344,18 +349,22 @@ def cmd_export(a) -> int:
jobs = con.execute("SELECT job_id,state,finished_at,result FROM jobs WHERE run_id=? AND mode='oracle' ORDER BY job_id", (rid,)).fetchall()
rollouts = con.execute("SELECT job_id,kk,state,finished_at,verdict FROM rollouts WHERE run_id=? ORDER BY job_id,kk", (rid,)).fetchall()
signature = hashlib.sha256(json.dumps([list(map(tuple, jobs)), list(map(tuple, rollouts))]).encode()).hexdigest()
if signature != previous:
pending_only = signature == previous
if not pending_only or (paths.config().get("content_review") or {}).get("enabled") or getattr(a, "review", False):
try:
# 导出器只读重核旧结果;历史 verdict 和数据库保持原样。
manifest = refresh_run_dataset(rid, strict=True, review=getattr(a, "review", False))
manifest = refresh_run_dataset(rid, strict=True, review=getattr(a, "review", False),
**({"retry_unknown": True} if retry_unknown else {}),
**({"pending_only": True} if pending_only else {}))
except Exception as exc:
if not a.follow:
raise
print(f"dataset export failed: {type(exc).__name__}: {exc}; {a.interval}s 后重试", file=sys.stderr, flush=True)
time.sleep(a.interval)
continue
print(json.dumps({"run_id": rid, "dataset": str(paths.DATASETS / f"{rid}-training"),
"stats": manifest["stats"]}, ensure_ascii=False), flush=True)
if manifest is not None:
print(json.dumps({"run_id": rid, "dataset": str(paths.DATASETS / f"{rid}-training"),
"stats": manifest["stats"]}, ensure_ascii=False), flush=True)
previous = signature
if not a.follow:
return 0
@ -508,6 +517,7 @@ def main(argv=None) -> int:
x = sp.add_parser("export", help="导出本 run 已完成的流程与 Agent 数据;--follow 跟随当前批次更新同一目录")
x.add_argument("--run"); x.add_argument("--follow", action="store_true")
x.add_argument("--review", action="store_true", help="对脚本合格候选执行一次内容审核,须配置 reviewer 环境")
x.add_argument("--retry-unknown", action="store_true", help="显式补审已完成 UNKNOWN 并保留旧回执;须 --review,不能与 --follow 同用")
x.add_argument("--until-pid", type=int, help="跟随到指定批次进程退出;避免队列暂空时提前结束")
x.add_argument("--interval", type=int, default=30); x.set_defaults(f=cmd_export)
x = sp.add_parser("rollout"); x.add_argument("job_id"); x.add_argument("--kk", type=int, default=0)

View File

@ -88,7 +88,7 @@ class Observer:
return result
for key in ('phases', 'execution_id', 'epoch', 'activity', 'execution_state', 'execution_outcome',
'cleanup_state', 'stop_reason', 'cancel_requested', 'config_version', 'updated_at',
'deadline_monotonic', 'activity_details', 'progress', 'health'):
'deadline_monotonic', 'activity_details', 'progress', 'health', 'cleanup_errors'):
result[key] = record.get(key)
identities = list(record.get('processes', []))
if record.get('owner'):
@ -122,7 +122,7 @@ class Observer:
elif expected_agent and now - started > 30 and not any(p['status'] == 'ALIVE' for p in agents):
result.update(health='STATE_MISMATCH', reason_code='EXPECTED_AGENT_MISSING')
elif record.get('cleanup_state') in ('UNKNOWN', 'QUARANTINED'):
result.update(health='QUARANTINED', reason_code='CLEANUP_UNCONFIRMED')
result.update(health='QUARANTINED', reason_code=record.get('stop_reason') or 'CLEANUP_UNCONFIRMED')
elif active and result['sources']['progress']['status'] == 'STALE':
result.update(health='STALE', reason_code='PROGRESS_SNAPSHOT_STALE')
elif record.get('health') in ('SUSPECT', 'UNKNOWN'):
@ -130,6 +130,10 @@ class Observer:
else:
result.update(health='WAITING' if str(record.get('activity', '')).startswith('WAITING') else 'OBSERVED',
reason_code='IDENTITY_CHECKED')
result['pending_registration'] = [p.parent.name for p in sorted((rd / 'handles').glob('*/status.json'))
if isinstance(status := read_json(p), dict) and status.get('registration') == 'PENDING']
if result['pending_registration'] and result['health'] not in ('QUARANTINED', 'STATE_MISMATCH', 'UNKNOWN'):
result.update(health='WAITING', reason_code='STAGE_REGISTRATION_PENDING')
# Exported and actual execution facts stay separate. No inference from exit 0.
verdict = read_json(rd / 'verdict.json', {})
result['hard_gates'] = {k: {'verdict': v.get('verdict'), 'reason': v.get('reason')}

View File

@ -9,6 +9,7 @@ import hashlib
import json
import os
import signal
import sqlite3
import shutil
import subprocess
import sys
@ -31,24 +32,30 @@ def _stamped(msg: str) -> None:
print(time.strftime("%H:%M:%S"), msg, flush=True)
def refresh_run_dataset(run_id: str, *, strict: bool = False, review: bool = False) -> dict | None:
def refresh_run_dataset(run_id: str, *, strict: bool = False, review: bool = False,
retry_unknown: bool = False, pending_only: bool = False) -> dict | None:
"""同一 run 始终更新同一数据集;失败可补导,不回滚量编上板结果。"""
try:
from export.to_agentrl import materialize
review_config = paths.config().get("content_review") or {}
result = materialize([run_id], f"{run_id}-training", refresh=True,
review=bool(review and not review_config))
if review_config and (review or review_config.get("enabled", False)):
result = None if pending_only else materialize([run_id], f"{run_id}-training", refresh=True,
review=bool(review and not review_config))
if review_config and (review or retry_unknown or review_config.get("enabled", False)):
from export.content_review import review_pending
try:
reviewed = review_pending(run_id)
options = {"retry_unknown": True} if retry_unknown else {}
if pending_only:
options["ready_only"] = True
publish_needed = review_pending(run_id, **options)
except Exception:
# Earlier candidates may already have receipts; expose them before
# propagating the error so follow retries remain visible and safe.
materialize([run_id], f"{run_id}-training", refresh=True, review=False)
raise
if reviewed:
if publish_needed:
result = materialize([run_id], f"{run_id}-training", refresh=True, review=False)
if pending_only and result is None:
return None
# Optional, explicitly enabled inventory. Failure cannot discard training data.
if (paths.STATE / "coverage-master.enabled").exists():
try:
@ -476,6 +483,7 @@ def collect_drain_file(run_id: str) -> Path:
def _collection_snapshot(run_id: str, job_ids: list[str] | None, modes: list[str] | None) -> tuple[list[dict], bool]:
"""每次填槽读当前状态;优先不同模型的第一条轨迹,再补剩余 k。"""
with db.readonly() as con:
con.execute("PRAGMA busy_timeout=0")
rows = [dict(r) for r in con.execute(
"SELECT r.*, j.state AS job_state, j.error_class AS job_ec, j.estimate_s,"
" COALESCE(h.completed,0) AS completed, COALESCE(h.started,0) AS started"
@ -529,6 +537,7 @@ def _resume_snapshot(run_id, job_ids=None, modes=None):
from .execution import Execution
from .collection_runtime import broker, resume_policy
with db.readonly() as con:
con.execute("PRAGMA busy_timeout=0")
rows = [dict(row) for row in con.execute("SELECT * FROM rollouts WHERE run_id=? AND state='PAUSED_INFRA'", (run_id,))]
ready, pending = [], False
for row in rows:
@ -543,7 +552,7 @@ def _resume_snapshot(run_id, job_ids=None, modes=None):
if state['execution_state'] in ('PAUSED_INFRA', 'CANCEL_REQUESTED', 'CLEANING') and reason in ('EXECUTION_DEADLINE_EXCEEDED', 'CANCEL_REQUESTED'):
from .rollout import settle_execution
settle_execution(run_id, row['job_id'], row['kk'], execution, reason,
cancel=bool(state.get('cancel_requested')))
cancel=bool(state.get('cancel_requested')), timeout_s=0)
continue
if (state['execution_state'] != 'PAUSED_INFRA' or state['cleanup_state'] == 'QUARANTINED'
or state.get('cancel_requested') or state.get('auto_resume_count', 0) >= state['config'].get('max_auto_resumes', 3)):
@ -570,8 +579,6 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[
from .runners import get as get_runner
from .continuation import receipt
from .quota import QuotaError
with db.readonly() as con:
db.assert_run_open(con, run_id)
if not dry_run:
from .collection_runtime import config as collection_config
collection_config()
@ -619,107 +626,231 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[
from .rollout import resume_rollout
return resume_rollout(run_id, r['job_id'], r['kk'])
return run_rollout(run_id, r["job_id"], r["kk"], r["mode"], r["policy"], dry_run=dry_run, _claimed=not dry_run, runner=r.get("runner", "pi"))
except (SystemExit, Exception) as e: # 前置/运行错误也必须释放认领,不能遗留 RUNNING
if r.get('_resume'):
return {'terminal': 'PAUSED_INFRA', 'reason': type(e).__name__ + ': ' + str(e)}
result = {"terminal": "UNVERIFIED", "reason": repr(e)}
if not dry_run:
with db.connect() as con:
db.rollout_set(con, run_id, r["job_id"], r["kk"], state="UNVERIFIED", finished_at=time.time())
db.annotate(con, run_id, r["job_id"], f"rollout#{r['kk']:02d} collector error: {type(e).__name__}: {e}")
rj = paths.RUNS / run_id / r["job_id"] / "rollouts" / f"{r['kk']:02d}" / "rollout.json"
if rj.exists():
try:
from .verify import verify_rollout
meta = json.loads(rj.read_text())
meta.update(state="ERROR", finished_at=time.time(), error=str(e))
rj.write_text(json.dumps(meta, indent=2))
result.update(verify_rollout(rj.parent), rollout_dir=str(rj.parent))
except Exception as verify_error:
log(f"[{r['job_id']}#{r['kk']:02d}] 异常后复核未完成:{verify_error}")
except (SystemExit, Exception) as exc:
# Keep the completed future; the collector retries only finalization.
# Calling run_rollout again could replay an already dispatched command.
return {"_collector_error": f"{type(exc).__name__}: {exc}"}
def settle_error(row: dict, result: dict) -> dict:
completed_execution = False
error = result["_collector_error"]
result = {"terminal": "UNVERIFIED", "reason": error}
if dry_run:
return result
rd = paths.RUNS / run_id / row['job_id'] / 'rollouts' / f"{row['kk']:02d}"
with db.readonly() as con:
con.execute('PRAGMA busy_timeout=0')
recorded = con.execute('SELECT state,verdict FROM rollouts WHERE run_id=? AND job_id=? AND kk=?',
(run_id, row['job_id'], row['kk'])).fetchone()
if recorded and recorded['verdict']:
return {**result, 'terminal': recorded['state']}
if (rd / 'execution.json').exists():
from .execution import Execution
from .rollout import settle_execution
execution = Execution(rd).attach()
state = execution.snapshot()
if state['execution_state'] == 'PAUSED_INFRA':
with db.connect(timeout_s=0) as con:
db.rollout_set(con, run_id, row['job_id'], row['kk'], state='PAUSED_INFRA')
return {**result, 'terminal': 'PAUSED_INFRA'}
state = settle_execution(run_id, row['job_id'], row['kk'], execution,
state.get('stop_reason') or 'INFRA_EXECUTION_EXCEPTION', timeout_s=0)
if state.get('execution_outcome') != 'COMPLETED':
return {**result, 'terminal': state['execution_outcome'],
'reason': state.get('stop_reason') or error}
completed_execution = True
with db.connect(timeout_s=0) as con:
# Probe writability before expensive file verification, and never erase
# a verdict already committed before a later finalization exception.
con.execute('BEGIN IMMEDIATE')
current = con.execute('SELECT state,verdict FROM rollouts WHERE run_id=? AND job_id=? AND kk=?',
(run_id, row['job_id'], row['kk'])).fetchone()
if current and (current['verdict'] or current['state'] not in ('RUNNING', 'VERIFYING', 'UNVERIFIED')):
return {**result, 'terminal': current['state']}
db.rollout_set(con, run_id, row['job_id'], row['kk'],
state='VERIFYING' if completed_execution else 'UNVERIFIED', finished_at=time.time())
db.annotate(con, run_id, row['job_id'], f"rollout#{row['kk']:02d} collector error: {error}")
rj = rd / 'rollout.json'
if rj.exists():
try:
from .verify import verify_rollout
from .execution import atomic_json
meta = json.loads(rj.read_text())
if not (rd / 'execution.json').exists():
meta.update(state='ERROR', finished_at=time.time(), error=error)
atomic_json(rj, meta)
result.update(verify_rollout(rd, timeout_s=0), rollout_dir=str(rd))
except sqlite3.OperationalError as exc:
if db.is_busy(exc):
raise
log(f"[{row['job_id']}#{row['kk']:02d}] 异常后复核未完成:{exc}")
except Exception as exc:
log(f"[{row['job_id']}#{row['kk']:02d}] 异常后复核未完成:{exc}")
return result
def preserve_finished() -> bool:
"""A stopped collector may hand off durable, finished work to startup recovery."""
from .execution import Execution, atomic_json
if any(not future.done() for future in futures):
return False
try:
for future, row in futures.items():
rd = paths.RUNS / run_id / row['job_id'] / 'rollouts' / f"{row['kk']:02d}"
if (rd / 'execution.json').exists():
state = Execution(rd).snapshot()
if state['execution_state'] not in ('TERMINAL', 'PAUSED_INFRA') or not scheduler._local_cleanup_verified(state):
return False
else:
result = future.result()
if '_collector_error' not in result:
return False
# Before execution admission, the existing rollout metadata is
# the only durable place to retain the collector's failure.
rj = rd / 'rollout.json'
meta = json.loads(rj.read_text()) if rj.exists() else {'state': 'INFRA_ERROR', 'collection_hold': True}
for key, value in (('run_id', run_id), ('job_id', row['job_id']), ('kk', row['kk'])):
if meta.setdefault(key, value) != value:
return False
meta.update(collector_error=result['_collector_error'])
meta.setdefault('finished_at', time.time())
atomic_json(rj, meta)
except (OSError, ValueError, KeyError, TypeError) as exc:
log(f"collector drain waiting for durable evidence: {type(exc).__name__}: {exc}")
return False
return True
def reconcile(attempts=None) -> bool:
from .recovery import reconcile_run
report = reconcile_run(run_id) if attempts is None else reconcile_run(run_id, attempts=attempts)
released = [r for r in report.get('resources', []) if r.get('state') == 'RELEASED']
if released:
log(f"resource recovery: released={len(released)} " + ', '.join(Path(r['lease']).name for r in released))
if report['ready']:
return True
reasons = [r.get('reason') or (r.get('execution') or {}).get('stop_reason') or 'CLEANUP_UNCONFIRMED'
for r in report.get('executions', []) if not r.get('ready')]
if report.get('reason'):
reasons.append(report['reason'])
reason = ', '.join(sorted(set(reasons))) or 'EXECUTION_RECONCILIATION_REQUIRED'
if reasons and all(r in ('STAGE_REGISTRATION_PENDING', 'ROLLOUT_REGISTRATION_PENDING') for r in reasons):
raise sqlite3.OperationalError('database is locked: ' + reason)
raise ValueError('EXECUTION_RECONCILIATION_REQUIRED: ' + reason)
summary: dict = {}
submitted, seen = 0, set()
initialized, database_wait, deferred = False, False, False
recovery_needed = set()
try:
if not dry_run:
from .recovery import reconcile_run
if not reconcile_run(run_id)['ready']:
raise ValueError('EXECUTION_RECONCILIATION_REQUIRED')
if coverage == "all" and (not modes or "e2e" in modes):
added = enqueue_e2e_coverage(run_id, job_ids)
log(f"coverage=all: 补入 {added} 条 e2e 轨迹,原 k 不变")
log(f"collect {run_id}: parallel={n}, coverage={coverage}, watch={watch}, first_pass={first_pass}")
with ThreadPoolExecutor(max_workers=n) as ex, ThreadPoolExecutor(max_workers=1) as exporter:
futures, export_future, export_dirty = {}, None, False
while True:
db_busy, busy_reason = False, ''
completed = [f for f in futures if f.done()]
for future in completed:
row = futures.pop(future)
res = future.result()
row = futures[future]
try:
res = future.result()
if '_collector_error' in res:
res = settle_error(row, res)
except sqlite3.OperationalError as exc:
if not db.is_busy(exc):
raise
db_busy, busy_reason = True, f"{row['job_id']}#{row['kk']:02d}: {exc}"
continue
del futures[future]
terminal = res.get("terminal", "DRY_RUN" if dry_run else "?")
summary[terminal] = summary.get(terminal, 0) + 1
log(f"[{row['job_id']}#{row['kk']:02d}] → {terminal} "
f"claim_consistent={res.get('claim_consistent')} review={res.get('review')} "
f"{res.get('reason', '')}")
export_dirty = not dry_run
if completed and not dry_run:
scheduler.reconcile_leases()
if not dry_run:
recovery_needed.add((row['job_id'], row['kk']))
ready, pending, resume_pending = [], False, False
if not stopped():
if limit is None or submitted < limit:
ready, pending = _collection_snapshot(run_id, job_ids, modes)
if first_pass:
# Keep the original k and QUEUED rows. A later ordinary collect
# resumes repeats; a failed attempt never resets kk to zero.
ready = [r for r in ready if not r["started"] and not r["completed"]]
if not dry_run:
resumes, resume_pending = _resume_snapshot(run_id, job_ids, modes)
ready = resumes + ready
pending = pending or resume_pending
active_jobs = {r["job_id"] for r in futures.values()}
free = n - len(futures)
if not dry_run:
with db.connect() as con:
active = con.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0]
free = min(free, pool_n - active)
for row in ready:
if free <= 0 or (not row.get('_resume') and limit is not None and submitted >= limit) or stopped():
break
key = (row["job_id"], row["kk"])
if (key in seen and not row.get('_resume')) or row["job_id"] in active_jobs:
continue
# Configuration/version failures do not consume an attempt.
try:
get_runner(row.get("runner", "pi")).check(row["policy"], execute=not dry_run)
except QuotaError:
# Keep QUEUED; the existing watch loop rechecks the shared ledger.
continue
if not dry_run and not row.get('_resume'):
with db.connect() as con:
if not db.claim_rollout(con, run_id, row["job_id"], row["kk"], max_active=pool_n):
continue
futures[ex.submit(one, row)] = row
active_jobs.add(row["job_id"])
seen.add(key)
submitted += int(not row.get('_resume'))
free -= 1
log(f"[{row['job_id']}#{row['kk']:02d}] started {row['mode']}; oracle={row['job_state']}")
try:
if not initialized:
with db.readonly() as con:
con.execute('PRAGMA busy_timeout=0')
db.assert_run_open(con, run_id)
if not dry_run:
reconcile()
if coverage == 'all' and (not modes or 'e2e' in modes):
added = enqueue_e2e_coverage(run_id, job_ids)
log(f"coverage=all: 补入 {added} 条 e2e 轨迹,原 k 不变")
initialized = True
elif recovery_needed and not db_busy:
reconcile(tuple(sorted(recovery_needed)))
recovery_needed.clear()
if not stopped() and not db_busy:
if limit is None or submitted < limit:
ready, pending = _collection_snapshot(run_id, job_ids, modes)
if first_pass:
# Keep the original k and QUEUED rows. A later ordinary collect
# resumes repeats; a failed attempt never resets kk to zero.
ready = [r for r in ready if not r["started"] and not r["completed"]]
if not dry_run:
resumes, resume_pending = _resume_snapshot(run_id, job_ids, modes)
ready = resumes + ready
pending = pending or resume_pending
active_jobs = {r["job_id"] for r in futures.values()}
free = n - len(futures)
if not dry_run:
with db.connect(timeout_s=0) as con:
active = con.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0]
free = min(free, pool_n - active)
for row in ready:
if free <= 0 or (not row.get('_resume') and limit is not None and submitted >= limit) or stopped():
break
key = (row["job_id"], row["kk"])
if (key in seen and not row.get('_resume')) or row["job_id"] in active_jobs:
continue
# Configuration/version failures do not consume an attempt.
try:
get_runner(row.get("runner", "pi")).check(row["policy"], execute=not dry_run)
except QuotaError:
# Keep QUEUED; the existing watch loop rechecks the shared ledger.
continue
if not dry_run and not row.get('_resume'):
with db.connect(timeout_s=0) as con:
if not db.claim_rollout(con, run_id, row["job_id"], row["kk"], max_active=pool_n):
continue
futures[ex.submit(one, row)] = row
active_jobs.add(row["job_id"])
seen.add(key)
submitted += int(not row.get('_resume'))
free -= 1
log(f"[{row['job_id']}#{row['kk']:02d}] started {row['mode']}; oracle={row['job_state']}")
except sqlite3.OperationalError as exc:
if not db.is_busy(exc):
raise
db_busy, busy_reason = True, str(exc)
if db_busy != database_wait:
log(f'database busy: {busy_reason}; 新认领暂停;保留完成结果,5 秒后仅重试登记' if db_busy
else 'database recovered: 已完成任务仅补登记,恢复认领')
database_wait = db_busy
# 先填满运行槽,再用独立线程刷新;慢导出不占 agent worker,也不拖延下一条领取。
if export_future is not None and export_future.done():
finish_export(export_future)
export_future = None
if export_dirty and export_future is None:
if export_dirty and export_future is None and not db_busy:
export_future = exporter.submit(refresh_run_dataset, run_id)
export_dirty = False
if not futures:
if db_busy:
if stopped() and preserve_finished():
deferred = True
log(f"collector drained while database unavailable: {len(futures)} finished result(s) "
"retained on disk for startup recovery; QUEUED unchanged")
break
stop.wait(5.0) if not stop.is_set() else time.sleep(5.0)
elif not futures:
if (stopped() or dry_run or not (ready or pending)
or (not resume_pending and (not watch or limit is not None and submitted >= limit))):
break
stop.wait(5.0)
else:
wait(futures, timeout=5.0, return_when=FIRST_COMPLETED)
if export_dirty:
if export_dirty and not deferred:
if export_future is not None:
finish_export(export_future)
export_future = exporter.submit(refresh_run_dataset, run_id)

View File

@ -146,13 +146,18 @@ def recover(run_id: str, job_id: str, kk: int, *, reason: str,
"runner": report["runner"], "state": "QUEUED"}
def reconcile_run(run_id: str) -> dict:
"""Gate this run's indexed executions; reconcile global resources per slot."""
def reconcile_run(run_id: str, *, attempts=None) -> dict:
"""Gate indexed executions; scope normal harvests and reconcile global slots."""
reports = []
try:
with db.readonly() as con:
con.execute('PRAGMA busy_timeout=0')
rows = list(con.execute("SELECT job_id,kk,state FROM rollouts WHERE run_id=?", (run_id,)))
query, args = "SELECT job_id,kk,state,verdict FROM rollouts WHERE run_id=?", [run_id]
if attempts is not None:
attempts = sorted(set(attempts))
query += " AND (" + (" OR ".join("(job_id=? AND kk=?)" for _ in attempts) or "0") + ")"
args.extend(value for pair in attempts for value in pair)
rows = list(con.execute(query, args))
except sqlite3.OperationalError as exc:
if not db.is_busy(exc):
raise
@ -163,7 +168,8 @@ def reconcile_run(run_id: str) -> dict:
if (rd / "execution.json").exists():
owner = execution.Execution(rd)
record = owner.snapshot()
if (execution.process_status(record.get('owner', {})) == 'DEAD'
if ((execution.process_status(record.get('owner', {})) == 'DEAD'
or record['execution_state'] == 'TERMINAL')
and execution.owned_processes(record) == []):
# Reuse protected Worker replay before fencing the dead epoch.
# Bare historical meta.json is never promoted to a completion.
@ -180,19 +186,67 @@ def reconcile_run(run_id: str) -> dict:
"ready": False, "reason": "STAGE_REGISTRATION_PENDING"})
continue
record = owner.reconcile()
settled_state = row["state"]
if (record["execution_state"] == "TERMINAL" and record.get("execution_outcome") is not None
and (row["state"] in ACTIVE | {"PAUSED_INFRA"} or
record["execution_outcome"] == "COMPLETED" and row["state"] == "UNVERIFIED" and row["verdict"] is None)):
from .rollout import settle_execution
try:
if record["execution_outcome"] == "COMPLETED":
# Check writability before hashing HBM; no artifact IO in
# this transaction, and a later lock is still retryable.
with db.connect(timeout_s=0) as con:
con.execute('BEGIN IMMEDIATE')
from .verify import verify_rollout
verdict = verify_rollout(rd, timeout_s=0)
settled_state = verdict['execution_control']['db_state']
else:
settle_execution(run_id, row["job_id"], row["kk"], owner,
record.get("stop_reason"), timeout_s=0)
settled_state = record["execution_outcome"]
except sqlite3.OperationalError as exc:
if not db.is_busy(exc):
raise
reports.append({"job_id": row["job_id"], "kk": row["kk"], "execution": record,
"ready": False, "reason": "ROLLOUT_REGISTRATION_PENDING"})
continue
ready = record["cleanup_state"] != "QUARANTINED"
# Settled history must not block independent work on healthy slots.
# Preserve execution evidence and every resource isolation marker.
if (not ready and record["execution_state"] == "TERMINAL"
and row["state"] == "INFRA_ERROR"
and execution.process_status(record.get("owner", {})) == "DEAD"
and ((row["state"] == "INFRA_ERROR"
and execution.process_status(record.get("owner", {})) == "DEAD")
or (settled_state == record.get("execution_outcome")
and settled_state in {"INFRA_ERROR", "CANCELLED", "LOST", "AGENT_ERROR",
"DEADLINE_EXCEEDED", "BUDGET_EXHAUSTED"}
and scheduler._local_cleanup_verified(record)))
and execution.owned_processes(record) == []):
ready = True
reports.append({"job_id": row["job_id"], "kk": row["kk"], "execution": record,
"ready": ready})
"ready": ready, **({"reason": record.get("stop_reason") or "RESOURCE_QUARANTINED"}
if not ready else {})})
elif row["state"] in ACTIVE:
reports.append({"job_id": row["job_id"], "kk": row["kk"], "ready": False,
"reason": "LEGACY_EXECUTION_OWNERSHIP_UNKNOWN"})
try:
meta = json.loads((rd / 'rollout.json').read_text())
except (OSError, ValueError):
meta = {}
deferred_error = (isinstance(meta.get('collector_error'), str) and bool(meta['collector_error'])
and isinstance(meta.get('finished_at'), (int, float)) and all(meta.get(key) == value
for key, value in (('run_id', run_id), ('job_id', row['job_id']), ('kk', row['kk']))))
if deferred_error:
try:
with db.connect(timeout_s=0) as con:
con.execute("UPDATE rollouts SET state='INFRA_ERROR',finished_at=? "
"WHERE run_id=? AND job_id=? AND kk=? AND state IN ('RUNNING','VERIFYING')",
(meta['finished_at'], run_id, row['job_id'], row['kk']))
except sqlite3.OperationalError as exc:
if not db.is_busy(exc):
raise
reports.append({"job_id": row["job_id"], "kk": row["kk"], "ready": False,
"reason": "ROLLOUT_REGISTRATION_PENDING"})
continue
reports.append({"job_id": row["job_id"], "kk": row["kk"], "ready": bool(deferred_error),
**({"reason": "LEGACY_EXECUTION_OWNERSHIP_UNKNOWN"} if not deferred_error else {})})
leases = scheduler.reconcile_leases()
# Other runs' isolated slots remain unavailable at scheduler acquisition;
# they must not block this run from using independent, healthy slots.

View File

@ -369,38 +369,41 @@ def run_rollout(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_r
raise
def settle_execution(run_id, job_id, kk, execution, reason, *, cancel=False):
"""Close control-plane stops without claiming a task result or a model answer."""
def settle_execution(run_id, job_id, kk, execution, reason, *, cancel=False, timeout_s=None):
"""Close control-plane stops and retry their registration without rerunning work."""
from .execution import atomic_json
from .collection_runtime import broker
snapshot = execution.snapshot()
if snapshot['execution_state'] == 'TERMINAL':
if snapshot['execution_state'] != 'TERMINAL':
if cancel:
snapshot = execution.cancel(reason)
if snapshot['execution_state'] != 'TERMINAL':
try:
broker().cancel(execution.snapshot()['execution_id'])
except (OSError, ValueError):
# Durable cancellation still fences new work; never refund an unknown request.
pass
execution.cleanup()
state = execution.snapshot()
snapshot = execution.finish(None, protocol_complete=False, stop_reason=reason,
cleanup_state=state['cleanup_state'])
if snapshot['execution_outcome'] == 'COMPLETED':
return {k: v for k, v in snapshot.items() if k != 'token'}
if cancel:
snapshot = execution.cancel(reason)
if snapshot['execution_state'] == 'TERMINAL':
return {k: v for k, v in snapshot.items() if k != 'token'}
try:
broker().cancel(execution.snapshot()['execution_id'])
except (OSError, ValueError):
# Durable execution cancellation still fences every new tool/runner.
# Do not refund an unreachable broker's unknown upstream request.
pass
execution.cleanup()
state = execution.snapshot()
finished = execution.finish(None, protocol_complete=False, stop_reason=reason,
cleanup_state=state['cleanup_state'])
if finished['execution_outcome'] == 'COMPLETED':
return {k: v for k, v in finished.items() if k != 'token'}
rd = rollout_dir(run_id, job_id, kk)
metadata = json.loads((rd / 'rollout.json').read_text())
metadata.update(state=finished['execution_outcome'], execution_outcome=finished['execution_outcome'],
stop_reason=finished['stop_reason'], cleanup_state=finished['cleanup_state'],
finished_at=finished['finished_at'], collection_hold=True)
atomic_json(rd / 'rollout.json', metadata)
with db.connect() as con:
db.rollout_set(con, run_id, job_id, kk, state=finished['execution_outcome'], finished_at=finished['finished_at'])
return {k: v for k, v in finished.items() if k != 'token'}
if rd.resolve() != execution.rd.resolve():
raise ValueError('rollout settlement identity mismatch')
if metadata.get('state') in ('RUNNING', 'VERIFYING', 'PAUSED_INFRA'):
metadata.update(state=snapshot['execution_outcome'], execution_outcome=snapshot['execution_outcome'],
stop_reason=snapshot['stop_reason'], cleanup_state=snapshot['cleanup_state'],
finished_at=snapshot['finished_at'], collection_hold=True)
atomic_json(rd / 'rollout.json', metadata)
with db.connect(**({'timeout_s': timeout_s} if timeout_s is not None else {})) as con:
# A verified task verdict is not replaced by a later control-plane retry.
con.execute("UPDATE rollouts SET state=?,finished_at=? WHERE run_id=? AND job_id=? AND kk=? "
"AND state IN ('RUNNING','VERIFYING','PAUSED_INFRA')",
(snapshot['execution_outcome'], snapshot['finished_at'], run_id, job_id, kk))
return {k: v for k, v in snapshot.items() if k != 'token'}
def resume_rollout(run_id: str, job_id: str, kk: int) -> dict:

View File

@ -276,16 +276,23 @@ def reconcile_leases() -> list[dict]:
quarantine = json.loads(quarantine_path.read_text()) if quarantine_path.exists() else identity
verified = (isinstance(identity['execution_id'], str) and bool(identity['execution_id']) and
type(identity['epoch']) is int and identity['epoch'] > 0 and
all(state[key] == quarantine.get(key) == value for key, value in identity.items()) and
state['execution_id'] == quarantine.get('execution_id') == identity['execution_id'] and
type(state.get('epoch')) is int and identity['epoch'] <= state['epoch'] and
type(quarantine.get('epoch')) is int and
identity['epoch'] <= quarantine['epoch'] <= state['epoch'] and
state['execution_state'] == 'TERMINAL' and
lease_path.name == f"{lease['pool']}-{lease['slot']}.lease.json")
if lease['pool'] in ('agent', 'cpu', 'gpu'):
# Reconcile fences a dead owner by advancing epoch. The
# complete local ledger must prove every retained scope
# empty; an old lease never authorizes board reuse.
verified = verified and _local_cleanup_verified(state)
elif lease['pool'] == 'board':
handle = lease.get('operation_id')
receipt_dir = owner.rd / 'stages' / '_queue' / (handle or '') / 'board-receipts'
receipts = list(receipt_dir.glob('*.json')) if handle else []
verified = (verified and state['cleanup_state'] == 'VERIFIED' and
verified = (verified and identity['epoch'] == quarantine['epoch'] == state['epoch'] and
state['cleanup_state'] == 'VERIFIED' and
execution.process_status(lease['holder']) == 'DEAD' and
execution.owned_processes(state) == [] and bool(receipts) and all(
json.loads(p.read_text()).get('cleanup_state') == 'VERIFIED' for p in receipts))

View File

@ -295,6 +295,7 @@ def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: float = 6
since = st.get("started_at") or st.get("queued_at") or t0
print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}"
f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}"
f"{'(阶段计算已结束,等待数据库登记;无需重跑)' if st.get('registration') == 'PENDING' else ''}"
f" 已 {int(time.time() - since)}s", flush=True)
remaining = deadline - time.monotonic()
if remaining <= 0:

View File

@ -115,6 +115,11 @@ class Worker:
if self.execution:
self.execution.quarantine('WORKER_JOIN_UNCONFIRMED')
raise RuntimeError('worker did not stop within cleanup budget')
if self.execution:
# The scanner may have stopped with complete work waiting on SQLite.
# Spend one shared cleanup budget on registration, never dispatch.
if not self.reconcile_completed(deadline=time.monotonic() + budget):
self.execution.quarantine('INFRA_STAGE_REGISTRATION_PENDING')
def run(self):
while not self.stop_event.is_set():
@ -430,7 +435,7 @@ class Worker:
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', result)
return result
def reconcile_completed(self):
def reconcile_completed(self, *, deadline=None):
"""Register protected completions only; never accept or dispatch a ticket."""
for hid, ledger in self.execution.snapshot().get('operations', {}).items():
queue = self.rd / 'stages' / '_queue' / hid
@ -453,7 +458,7 @@ class Worker:
name = json.loads((queue / 'request.json').read_text())['stage']
if 'meta_sha256' not in result:
self._validate_recovered_receipt(hid, name, previous, result)
result = self._reconcile_stage(hid, name, previous, result)
result = self._reconcile_stage(hid, name, previous, result, deadline=deadline)
if result.get('db_registered') is False:
return False
self._complete_operation(hid, operation, previous)

View File

@ -312,7 +312,7 @@ def _execution_control(owner, metadata, measured_terminal):
'reason': 'EXECUTION_HOLD' if db_state != measured_terminal else 'MEASURED_TERMINAL'}
def verify_rollout(rd: Path, write: bool = True) -> dict:
def verify_rollout(rd: Path, write: bool = True, *, timeout_s=None) -> dict:
rd = rd.resolve()
jd = rd.parent.parent
job = json.loads((jd / "job.json").read_text())
@ -383,7 +383,7 @@ def verify_rollout(rd: Path, write: bool = True) -> dict:
if control:
verdict['execution_control'] = control
(rd / "verdict.json").write_text(json.dumps(verdict, indent=2, ensure_ascii=False, default=str))
with db.connect() as con:
with db.connect(**({'timeout_s': timeout_s} if timeout_s is not None else {})) as con:
db.rollout_upsert(con, job["run_id"], job["job_id"], kk, rmeta.get("mode"), rmeta.get("policy"), runner=rmeta.get("runner", "pi"))
db.rollout_set(con, job["run_id"], job["job_id"], kk, state=control['db_state'] if control else terminal,
claim_terminal=(claim or {}).get("terminal"), verdict_terminal=terminal,

View File

@ -2,7 +2,11 @@
from contextlib import contextmanager, nullcontext
import hashlib
import json
import os
import signal
import subprocess
import sys
import sqlite3
from pathlib import Path
import tempfile
import threading
@ -10,7 +14,9 @@ import time
import unittest
from unittest.mock import patch
from harness import db, pipeline, registry, rollout
from harness import db, pipeline, registry, rollout, recovery
REAL_RECONCILE = recovery.reconcile_run
class Collection(unittest.TestCase):
@ -44,10 +50,10 @@ class Collection(unittest.TestCase):
pass
@contextmanager
def connect(self):
con = db._open(self.db_path)
con.executescript(db.SCHEMA)
def connect(self, *, timeout_s=60):
con = db._open(self.db_path, timeout_s=timeout_s)
try:
con.executescript(db.SCHEMA)
yield con
con.commit()
finally:
@ -98,6 +104,296 @@ class Collection(unittest.TestCase):
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'eligible')[0]['state'], 'QUEUED')
def test_real_database_lock_keeps_failed_future_without_replaying_or_claiming(self):
for jid in ('a', 'b', 'c'):
self.job(jid, k=1)
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
self.addCleanup(locker.close)
entered = threading.Barrier(2)
good_committed, locked, waiting = (threading.Event() for _ in range(3))
launched, logs, outcome, errors = [], [], [], []
def run(*args, **kw):
jid = args[1]
launched.append(jid)
if jid in ('a', 'b'):
entered.wait(timeout=3)
if jid == 'a':
result = self.complete(*args, **kw)
good_committed.set()
self.assertTrue(locked.wait(3))
return result
if jid == 'b':
self.assertTrue(good_committed.wait(3))
locker.execute('BEGIN EXCLUSIVE')
locked.set()
raise RuntimeError('fixture agent ended after accepted work')
return self.complete(*args, **kw)
def log(message):
logs.append(message)
if 'database busy:' in message:
waiting.set()
def collect():
try:
outcome.append(pipeline.collect_rollouts('run', parallel=2, log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout', run):
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(4), logs)
self.assertTrue(thread.is_alive())
self.assertCountEqual(launched, ['a', 'b'])
self.assertTrue(any('→ PASS' in message for message in logs), logs)
finally:
locker.rollback()
thread.join(timeout=8)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcome, [{'PASS': 2, 'UNVERIFIED': 1}])
self.assertCountEqual(launched, ['a', 'b', 'c'])
self.assertTrue(any('database recovered:' in message for message in logs))
with self.connect() as con:
self.assertEqual([(r['job_id'], r['state']) for r in con.execute('SELECT * FROM rollouts ORDER BY job_id')],
[('a', 'PASS'), ('b', 'UNVERIFIED'), ('c', 'PASS')])
def test_real_database_lock_at_start_snapshot_and_claim_waits_without_launch(self):
self.job('a', k=1)
for boundary in ('start', 'snapshot', 'claim'):
with self.subTest(boundary=boundary):
with self.connect() as con:
con.execute("UPDATE rollouts SET state='QUEUED', started_at=NULL, finished_at=NULL")
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
waiting, logs, outcomes, errors, calls = threading.Event(), [], [], [], []
target = pipeline if boundary == 'snapshot' else db
method = '_collection_snapshot' if boundary == 'snapshot' else 'claim_rollout'
original = getattr(target, method)
locked = boundary == 'start'
if locked:
locker.execute('BEGIN EXCLUSIVE')
def inject(*args, **kwargs):
nonlocal locked
if not locked:
locker.execute('BEGIN EXCLUSIVE')
locked = True
return original(*args, **kwargs)
def log(message):
logs.append(message)
if 'database busy:' in message:
waiting.set()
def launch(*args, **kw):
calls.append(args[1])
return self.complete(*args, **kw)
def collect():
try:
outcomes.append(pipeline.collect_rollouts('run', parallel=1, log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout', launch), \
patch.object(target, method, inject) if boundary != 'start' else nullcontext():
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(4), (logs, errors))
self.assertEqual(calls, [])
self.assertTrue(thread.is_alive())
finally:
locker.rollback()
thread.join(timeout=8)
locker.close()
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcomes, [{'PASS': 1}])
self.assertEqual(calls, ['a'])
def test_drain_during_database_wait_preserves_unclaimed_work(self):
self.job('a', k=1)
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
locker.execute('BEGIN EXCLUSIVE')
waiting, outcomes, errors = threading.Event(), [], []
def log(message):
if 'database busy:' in message:
waiting.set()
def collect():
try:
outcomes.append(pipeline.collect_rollouts('run', log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout') as launch:
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(3))
pipeline.collect_drain_file('run').touch()
thread.join(timeout=7)
self.assertFalse(thread.is_alive(), 'empty collector must drain while database remains locked')
finally:
locker.rollback()
locker.close()
thread.join(timeout=8)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcomes, [{}])
launch.assert_not_called()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'QUEUED')
def test_runtime_recovery_scans_only_just_finished_attempts(self):
for jid in ('a', 'b'):
self.job(jid, k=1)
with patch.object(rollout, 'run_rollout', self.complete):
self.assertEqual(self.collect(parallel=1), {'PASS': 2})
calls = self.reconciliation.call_args_list
self.assertEqual(calls[0].args, ('run',))
self.assertEqual(calls[0].kwargs, {})
self.assertEqual([call.kwargs['attempts'] for call in calls[1:]], [(('a', 0),), (('b', 0),)])
def test_drain_persists_pending_prelaunch_failure_and_exits_before_unlock(self):
self.job('a', k=1)
self.job('b', k=1)
locker = sqlite3.connect(self.db_path, timeout=2, check_same_thread=False)
waiting, outcomes, errors, launched = threading.Event(), [], [], []
def launch(*args, **kw):
launched.append(args[1])
locker.execute('BEGIN EXCLUSIVE')
raise RuntimeError('completed worker needs failure registration')
def log(message):
if 'database busy:' in message:
waiting.set()
def collect():
try:
outcomes.append(pipeline.collect_rollouts('run', parallel=1, log=log))
except BaseException as exc:
errors.append(exc)
with patch.object(rollout, 'run_rollout', launch):
thread = threading.Thread(target=collect)
thread.start()
try:
self.assertTrue(waiting.wait(3))
pipeline.collect_drain_file('run').touch()
thread.join(timeout=7)
self.assertFalse(thread.is_alive(), 'drain must persist completed work and exit while DB stays locked')
finally:
locker.rollback()
locker.close()
thread.join(timeout=8)
self.assertFalse(thread.is_alive())
self.assertEqual(errors, [])
self.assertEqual(outcomes, [{}])
self.assertEqual(launched, ['a'])
from harness import recovery
with patch.object(recovery, 'reconcile_run', REAL_RECONCILE), \
patch.object(rollout, 'run_rollout', side_effect=AssertionError('restart dispatched work')):
self.assertEqual(self.collect(parallel=1, limit=0), {})
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'INFRA_ERROR')
self.assertEqual(db.list_rollouts(con, 'run', 'b')[0]['state'], 'QUEUED')
def test_sigterm_hands_off_completed_execution_and_restart_reverifies_once(self):
program = """
import sys
from scripts.test_collection import Collection
case = Collection(); case.setUp()
try: case.sigterm_restart_probe(sys.argv[1])
finally: case.doCleanups()
"""
for boundary in ('first_db', 'during_verify'):
with self.subTest(boundary=boundary):
result = subprocess.run([sys.executable, '-c', program, boundary], cwd=Path(__file__).resolve().parents[1],
capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('SIGTERM_RECOVERY_ONCE', result.stdout)
def sigterm_restart_probe(self, boundary):
from harness import execution, verify
from scripts.test_execution import policy
self.job('a', k=1); self.job('b', k=1)
locker = sqlite3.connect(self.db_path, timeout=0, check_same_thread=False)
self.addCleanup(locker.close)
rd = self.root/'runs/run/a/rollouts/00'
observed = []
def interrupt():
locker.execute('BEGIN EXCLUSIVE')
os.kill(os.getpid(), signal.SIGTERM)
real_verify = verify.verify_rollout
def interrupted_verify(*args, **kwargs):
with self.connect() as con:
observed.append(db.list_rollouts(con, 'run', 'a')[0]['state'])
interrupt()
return real_verify(*args, **kwargs)
def launch(*args, **kw):
owner = execution.Execution(rd); owner.start(policy())
(rd/'effects').write_text('one\n')
owner.finish(0, protocol_complete=True)
execution.atomic_json(rd/'rollout.json', {'state': 'DONE', 'mode': 'e2e', 'policy': 'teacher',
'execution_managed': True, 'execution_id': owner.snapshot()['execution_id']})
spec = registry.JobSpec('a', 'cls', 'fixture', 'nash-m', 'ptq', 'int8', 'A', 'e2e')
execution.atomic_json(rd.parent.parent/'job.json', {'run_id': 'run', 'job_id': 'a',
'gates': registry.gate_snapshot(spec)})
if boundary == 'first_db':
interrupt()
raise RuntimeError('fixture final registration interrupted')
with patch.object(rollout, 'run_rollout', launch), \
patch.object(verify, 'verify_rollout', interrupted_verify) if boundary == 'during_verify' else nullcontext():
self.assertEqual(self.collect(parallel=1), {})
if boundary == 'during_verify':
self.assertEqual(observed, ['VERIFYING'])
self.assertTrue(locker.in_transaction) # Collector exited before DB recovery.
locker.rollback()
with patch.object(recovery, 'reconcile_run', REAL_RECONCILE), \
patch.object(rollout, 'run_rollout', side_effect=AssertionError('restart dispatched work')):
self.assertEqual(self.collect(parallel=1, limit=0), {})
self.assertEqual((rd/'effects').read_text(), 'one\n')
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'UNVERIFIED')
self.assertIsNotNone(db.list_rollouts(con, 'run', 'a')[0]['verdict'])
self.assertEqual(db.list_rollouts(con, 'run', 'b')[0]['state'], 'QUEUED')
print('SIGTERM_RECOVERY_ONCE')
def test_error_after_committed_verdict_preserves_database_terminal(self):
self.job('a', k=1)
def launch(*args, **kw):
with self.connect() as con:
db.rollout_set(con, 'run', 'a', 0, state='FAIL', verdict='{"terminal":"FAIL"}', finished_at=1)
rd = self.root / 'runs/run/a/rollouts/00'
rd.mkdir(parents=True)
# A pre-existing verdict wins; control-plane outcome is not task quality.
(rd / 'execution.json').write_text('{"execution_outcome":"AGENT_ERROR"}')
raise RuntimeError('fixture failure after verdict commit')
with patch.object(rollout, 'run_rollout', launch), patch.object(rollout, 'settle_execution') as settle:
self.assertEqual(self.collect(), {'FAIL': 1})
settle.assert_not_called()
with self.connect() as con:
self.assertEqual(db.list_rollouts(con, 'run', 'a')[0]['state'], 'FAIL')
def test_nonbusy_database_error_is_not_retried(self):
self.job('a', k=1)
with patch.object(pipeline, '_collection_snapshot', side_effect=sqlite3.OperationalError('disk I/O error')) as snapshot, \
patch.object(rollout, 'run_rollout') as launch:
with self.assertRaisesRegex(sqlite3.OperationalError, 'disk I/O error'):
self.collect()
self.assertEqual(snapshot.call_count, 1)
launch.assert_not_called()
def test_recovery_reason_and_released_resources_are_visible(self):
self.job('a', k=1)
logs = []
self.reconciliation.return_value = {'ready': False, 'executions': [
{'ready': False, 'reason': 'LEGACY_EXECUTION_OWNERSHIP_UNKNOWN'}],
'resources': [{'lease': '/fixture/agent-2.lease.json', 'state': 'RELEASED'}]}
with self.assertRaisesRegex(ValueError, 'LEGACY_EXECUTION_OWNERSHIP_UNKNOWN'):
pipeline.collect_rollouts('run', log=logs.append)
self.assertTrue(any('released=1' in message and 'agent-2.lease.json' in message for message in logs))
def test_all_coverage_is_idempotent_preserves_repairs_and_non_priority_filters(self):
self.job("eligible", priority="C")
for jid, kw in [("wrong-task", {"task": "occ"}), ("wrong-march", {"march": "nash-p"}),

View File

@ -52,7 +52,7 @@ class CollectionCrash(unittest.TestCase):
stack.enter_context(patch.object(paths,name,value))
stack.enter_context(patch.object(paths,'config',return_value=cfg))
stack.enter_context(patch.object(paths,'secret',return_value='upstream-fixture'))
stack.enter_context(patch.object(db,'connect',lambda:original_connect(self.state/'jobs.sqlite')))
stack.enter_context(patch.object(db,'connect',lambda **kwargs:original_connect(self.state/'jobs.sqlite',**kwargs)))
stack.enter_context(patch.object(db,'readonly',lambda:original_readonly(self.state/'jobs.sqlite')))
stack.enter_context(patch.object(runtime,'_broker',self.joint.broker))
stack.enter_context(patch.object(runtime,'_configuration',json.dumps(cfg['collection_v1'],sort_keys=True)))

View File

@ -9,8 +9,11 @@ import fcntl
import hashlib
import json
import shutil
import sqlite3
import threading
import time
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from export import content_review, test_materialization as fixtures, to_agentrl
@ -85,9 +88,13 @@ class ContentReviewIntegration(unittest.TestCase):
@staticmethod
def response(finish='stop', decision='PASS'):
issues = [] if decision != 'FAIL' else [{
'quote': 'The measured subset passed within the task tolerance.',
'evidence': 'Synthetic disagreement fixture, not real measurement evidence.',
'reason': 'Synthetic rejected-review scenario.'}]
value = {'model': 'deepseek-flash', 'choices': [{'finish_reason': finish,
'message': {'role': 'assistant', 'content': json.dumps({
'decision': decision, 'reason': 'Synthetic integration response; not a real review.', 'issues': []})}}],
'decision': decision, 'reason': 'Synthetic integration response; not a real review.', 'issues': issues})}}],
'usage': {'prompt_tokens': 101, 'completion_tokens': 17, 'total_tokens': 118}}
return 200, {'Content-Type': 'application/json'}, json.dumps(value).encode()
@ -177,6 +184,259 @@ class ContentReviewIntegration(unittest.TestCase):
self.assertEqual(first['files']['sft.jsonl'], second['files']['sft.jsonl'])
self.assert_manifest(second)
def test_explicit_unknown_retry_archives_bytes_and_uses_new_generation(self):
rd = self.candidate()
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
self.refresh()
path, original = self.receipt(rd)
original_bytes = path.read_bytes()
digest = hashlib.sha256(original_bytes).hexdigest()
result = self.refresh(review=True, retry_unknown=True)
_, reviewed = self.receipt(rd)
self.assertEqual((reviewed['decision'], result['stats']['sft_eligible']), ('PASS', 1))
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
self.assertEqual(reviewed['previous_receipt_sha256'], digest)
self.assertNotEqual(reviewed['execution_id'], original['execution_id'])
self.assertTrue(reviewed['execution_id'].endswith(content_review.digest(
{'previous': digest, 'started_at': reviewed['started_at']})))
self.assertEqual(reviewed['request_bytes'], len(json.dumps(provider.requests[-1]['body']).encode()))
self.assertEqual(reviewed['max_tokens'], 32768)
self.refresh()
self.assertEqual(len(provider.requests), 2)
self.assertEqual(path.read_bytes(), json.dumps(reviewed, ensure_ascii=False, indent=2).encode())
def test_local_tpm_refusal_is_explicitly_retryable_after_policy_migration(self):
rd = self.candidate()
provider = self.provider()
self.config['collection_v1']['quota_groups']['review-only']['tpm'] = 1000
self.refresh()
path, original = self.receipt(rd)
original_bytes = path.read_bytes()
self.assertEqual(original['reason'], 'REQUEST_EXCEEDS_INTERNAL_TPM')
self.assertEqual(len(provider.requests), 0)
self.close_broker()
# This is an isolated, drained ledger migration, not a provider-limit claim.
policy = self.config['collection_v1']['quota_groups']['review-only']
policy['tpm'] = 1000000
from dataclasses import asdict
from harness.quota import QuotaPolicy
with sqlite3.connect(self.config['collection_v1']['quota_ledger']) as con:
con.execute('UPDATE quota_groups SET policy=? WHERE group_id=?',
(json.dumps(asdict(QuotaPolicy(**policy)), sort_keys=True), 'review-only'))
con.close()
result = self.refresh(review=True, retry_unknown=True)
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
digest = hashlib.sha256(original_bytes).hexdigest()
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
def test_retry_interrupted_after_archive_preserves_current_receipt_and_can_retry(self):
rd = self.candidate()
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
self.refresh()
path, _ = self.receipt(rd)
original_bytes = path.read_bytes()
digest = hashlib.sha256(original_bytes).hexdigest()
with patch.object(content_review, '_atomic', side_effect=OSError('injected receipt publication failure')):
with self.assertRaises(OSError):
content_review.review_once(rd, managed=True, retry_unknown=True)
self.assertEqual(path.read_bytes(), original_bytes)
self.assertEqual((path.parent / 'history' / (digest + '.json')).read_bytes(), original_bytes)
self.assertEqual(len(provider.requests), 1)
result = self.refresh(review=True, retry_unknown=True)
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (2, 1))
def test_retry_rejects_started_and_success_receipts_without_request(self):
rd = self.candidate()
provider = self.provider()
self.refresh()
path, receipt = self.receipt(rd)
for state, decision in [('COMPLETE', 'PASS'), ('COMPLETE', 'FAIL'), ('STARTED', 'UNKNOWN')]:
with self.subTest(state=state, decision=decision):
receipt.update(state=state, decision=decision)
path.write_text(json.dumps(receipt))
before = path.read_bytes()
with self.assertRaisesRegex(ValueError, 'REVIEW_RETRY_REQUIRES_COMPLETE_UNKNOWN'):
content_review.review_once(rd, managed=True, retry_unknown=True)
self.assertEqual(path.read_bytes(), before)
self.assertEqual(len(provider.requests), 1)
self.assertFalse((path.parent / 'history').exists())
def test_retry_cli_requires_explicit_one_off_review(self):
from harness.cli import cmd_export
for review, follow in [(False, False), (False, True), (True, True)]:
with self.subTest(review=review, follow=follow):
with self.assertRaisesRegex(SystemExit, 'requires --review'):
cmd_export(SimpleNamespace(retry_unknown=True, review=review, follow=follow))
def test_local_tpm_refusal_does_not_starve_smaller_candidate(self):
large = self.candidate(kk=0)
self.candidate(kk=1)
system = (large / 'agent/system.md').read_text() + (' synthetic context' * 6000)
(large / 'agent/system.md').write_text(system)
events = [json.loads(line) for line in (large / 'agent/events.jsonl').read_text().splitlines()]
events[0]['bench']['system_prompt'] = system
(large / 'agent/events.jsonl').write_text('\n'.join(map(json.dumps, events)))
self.config['collection_v1']['quota_groups']['review-only']['tpm'] = 50000
provider = self.provider()
result = self.refresh()
self.assertEqual(self.receipt(large)[1]['reason'], 'REQUEST_EXCEEDS_INTERNAL_TPM')
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (1, 1))
def test_pending_only_continues_one_after_unknown_without_repeating_unknown(self):
first = self.candidate(kk=0)
self.candidate(kk=1)
self.candidate(kk=2)
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
self.refresh()
old = self.receipt(first)[0].read_bytes()
for expected in (2, 3):
with patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
result = self.refresh(pending_only=True)
self.assertEqual(materialize.call_count, 1) # Only publish after one new review.
self.assertEqual((len(provider.requests), result['stats']['sft_eligible']), (expected, expected - 1))
self.assertEqual(self.receipt(first)[0].read_bytes(), old)
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('idle export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('idle source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 3)
def test_pending_only_paused_account_preserves_unreviewed_candidates(self):
self.candidate(kk=0)
later = self.candidate(kk=1)
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
self.refresh()
with collection_runtime.broker()._db() as con:
con.execute("UPDATE quota_groups SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN'")
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('paused export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('paused source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(list((later / 'content-reviews').glob('*.json')), [])
self.assertEqual(len(provider.requests), 1)
def test_pending_only_keeps_started_receipt_without_review_or_publication(self):
rd = self.candidate()
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
to_agentrl.materialize(['r'], 'r-training')
content_review.review_once(rd, managed=True)
path, receipt = self.receipt(rd)
receipt.update(state='STARTED', decision='UNKNOWN')
path.write_text(json.dumps(receipt))
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('STARTED export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('STARTED source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 1)
def test_saved_reviews_publish_after_materialization_failure_even_when_account_paused(self):
decision = ['PASS']
provider = self.provider(lambda *_: self.response(decision=decision[0]))
broker = collection_runtime.broker()
cases = [(value, state) for value in ('PASS', 'FAIL', 'UNKNOWN') for state in ('PAUSED', 'RECONCILING')]
for kk, (value, state) in enumerate(cases):
with self.subTest(decision=value, account=state):
with broker._db() as con:
con.execute("UPDATE quota_groups SET state='OPEN',reason=NULL")
rd = self.candidate(kk=kk)
decision[0] = value
to_agentrl.materialize(['r'], 'r-training', refresh=True)
before_db = self.fixture.con.execute('SELECT * FROM rollouts').fetchall()
with patch.object(to_agentrl, 'materialize', side_effect=OSError('injected publish failure')):
with self.assertRaisesRegex(OSError, 'injected publish failure'):
self.refresh(pending_only=True)
path, receipt = self.receipt(rd)
self.assertEqual((receipt['state'], receipt['decision']), ('COMPLETE', value))
saved = path.read_bytes()
requests = len(provider.requests)
with broker._db() as con:
con.execute('UPDATE quota_groups SET state=?,reason=?', (state, 'REQUEST_OUTCOME_UNKNOWN'))
with patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
result = self.refresh(pending_only=True)
self.assertIsNotNone(result)
self.assertEqual(materialize.call_count, 1)
disposition = next(row for row in self.rows('disposition') if row['rollout'] == kk)
self.assertEqual(disposition['reasons'], [] if value == 'PASS' else ['content-review-' + value.lower()])
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('published receipt rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('published source read')):
for _ in range(3):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual((len(provider.requests), path.read_bytes()), (requests, saved))
self.assertEqual(self.fixture.con.execute('SELECT * FROM rollouts').fetchall(), before_db)
def test_retried_unknown_receipt_is_republished_once_even_with_same_decision(self):
rd = self.candidate()
provider = self.provider(lambda *_: self.response(decision='UNKNOWN'))
self.refresh()
to_agentrl.materialize(['r'], 'r-training', refresh=True, purpose='sft')
old_hash = self.rows('disposition')[0]['review_receipt_sha256']
original = to_agentrl.materialize
calls = []
def fail_after_review(*args, **kwargs):
calls.append(True)
if len(calls) == 2:
raise OSError('injected publish failure')
return original(*args, **kwargs)
with patch.object(to_agentrl, 'materialize', side_effect=fail_after_review):
with self.assertRaisesRegex(OSError, 'injected publish failure'):
self.refresh(review=True, retry_unknown=True)
path, _ = self.receipt(rd)
new_hash = hashlib.sha256(path.read_bytes()).hexdigest()
self.assertNotEqual(new_hash, old_hash)
with collection_runtime.broker()._db() as con:
con.execute("UPDATE quota_groups SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN'")
self.assertIsNotNone(self.refresh(pending_only=True))
self.assertEqual(self.rows('disposition')[0]['review_receipt_sha256'], new_hash)
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('published receipt rescanned')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 2)
def test_follow_unchanged_database_continues_pending_review(self):
from harness import cli
self.candidate(kk=0)
self.candidate(kk=1)
provider = self.provider(lambda number, _: self.response(decision='UNKNOWN' if number == 1 else 'PASS'))
before = self.fixture.con.execute('SELECT * FROM rollouts').fetchall()
read_only, sleep = cli.db.readonly, time.sleep
intervals = []
def next_interval(seconds):
if seconds != 30:
return sleep(seconds)
intervals.append(seconds)
if len(intervals) == 2:
raise SystemExit('follow test complete')
args = SimpleNamespace(run='r', follow=True, interval=30, review=True, retry_unknown=False, until_pid=None)
(paths.STATE / 'locks').mkdir(exist_ok=True)
with (paths.STATE / 'locks/export-follow-r.lock').open('a') as follow_lock, \
patch('harness.cli.open', return_value=follow_lock, create=True), \
patch.object(cli.db, 'readonly', side_effect=lambda: read_only(self.root / 'state/jobs.sqlite')), \
patch.object(time, 'sleep', side_effect=next_interval), \
patch.object(to_agentrl, 'materialize', wraps=to_agentrl.materialize) as materialize:
with self.assertRaisesRegex(SystemExit, 'follow test complete'):
cli.cmd_export(args)
self.assertEqual(materialize.call_count, 3) # Initial snapshot + UNKNOWN, then one new PASS.
self.assertEqual(self.fixture.con.execute('SELECT * FROM rollouts').fetchall(), before)
self.assertEqual((len(provider.requests), len(self.rows('sft'))), (2, 1))
def test_pending_only_busy_account_does_not_read_sources(self):
first = self.candidate(kk=0)
self.candidate(kk=1)
entered, release = threading.Event(), threading.Event()
def response(*_):
entered.set()
self.assertTrue(release.wait(5))
return self.response()
provider = self.provider(response)
to_agentrl.materialize(['r'], 'r-training')
with ThreadPoolExecutor(max_workers=1) as worker:
future = worker.submit(content_review.review_once, first, managed=True)
self.assertTrue(entered.wait(5))
try:
with patch.object(to_agentrl, 'materialize', side_effect=AssertionError('busy export rescanned')), \
patch.object(content_review.trajectory, 'freeze', side_effect=AssertionError('busy source read')):
self.assertIsNone(self.refresh(pending_only=True))
self.assertEqual(len(provider.requests), 1)
finally:
release.set()
self.assertEqual(future.result(timeout=5)['decision'], 'PASS')
def test_incomplete_response_is_unknown_retained_and_never_resent(self):
rd = self.candidate()
provider = self.provider(lambda *_: self.response(finish='length'))

View File

@ -34,7 +34,7 @@ class ContinuationTests(unittest.TestCase):
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
database = paths.STATE / "jobs.sqlite"
connect, readonly = db.connect, db.readonly
self.stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database)))
self.stack.enter_context(patch.object(db, "connect", side_effect=lambda **kwargs: connect(database, **kwargs)))
self.stack.enter_context(patch.object(db, "readonly", side_effect=lambda: readonly(database)))
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
"e2e": {"enabled": True, "k": 4, "policies": ["teacher"]},

View File

@ -12,10 +12,11 @@ import time
import unittest
from unittest.mock import patch
from harness import db, execution, paths, pipeline, rollout, scheduler, watchdog
from harness import db, execution, paths, pipeline, recovery, rollout, scheduler, watchdog
from scripts import test_collection, test_execution
ACQUIRE = scheduler.acquire
RECONCILE = recovery.reconcile_run
class LeaseRecovery(unittest.TestCase):
@ -107,6 +108,45 @@ class LeaseRecovery(unittest.TestCase):
self.assertEqual(scheduler.reconcile_leases(), [{'lease': str(lease),
'state': 'RELEASED' if mode == 'old complete' else 'QUARANTINED'}])
def test_fenced_epoch_releases_only_proven_local_resources(self):
self.state['epoch'] = 2
self.write_state()
leases = {pool: self.lease(pool) for pool in ('agent', 'cpu', 'gpu', 'board')}
before = {path: path.read_bytes() for path in (leases['board'], self.locks/'board-0.quarantine.json')}
results = scheduler.reconcile_leases()
self.assertEqual({Path(row['lease']).name: row['state'] for row in results},
{f'{pool}-0.lease.json': 'QUARANTINED' if pool == 'board' else 'RELEASED'
for pool in leases})
self.assertEqual(before, {path: path.read_bytes() for path in before})
def test_settled_failures_allow_healthy_work_only_with_complete_local_proof(self):
from harness import recovery
self.rd = self.root/'runs/r/j/rollouts/00'; self.rd.mkdir(parents=True)
database = self.root/'recovery.sqlite'
connect, readonly = db.connect, db.readonly
for outcome in ('CANCELLED', 'LOST', 'AGENT_ERROR', 'DEADLINE_EXCEEDED', 'BUDGET_EXHAUSTED'):
for proof in (False, True):
with self.subTest(outcome=outcome, proof=proof):
state = copy.deepcopy(self.state)
state['execution_outcome'] = outcome
if not proof:
state['process_scopes']['scope']['receipt_path'] = str(self.rd/'missing')
self.write_state(state)
lease = self.lease('board')
preserved = {p: p.read_bytes() for p in (self.rd/'execution.json', lease,
self.locks/'board-0.quarantine.json')}
with connect(database) as con:
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher')
db.rollout_set(con, 'r', 'j', 0, state=outcome)
with patch.object(paths, 'RUNS', self.root/'runs'), \
patch.object(db, 'readonly', lambda: readonly(database)):
report = recovery.reconcile_run('r')
self.assertEqual(report['ready'], proof)
self.assertEqual(preserved, {p: p.read_bytes() for p in preserved})
self.assertEqual(report['resources'], [{'lease': str(lease), 'state': 'QUARANTINED'}])
with connect(database) as con:
self.assertEqual(con.execute('SELECT state FROM rollouts').fetchone()[0], outcome)
def test_incomplete_or_wrong_scope_evidence_remains_isolated(self):
mutations = {
'legacy missing processes': lambda s: s.pop('processes'),
@ -229,6 +269,7 @@ class CollectorRecovery(unittest.TestCase):
lifecycle = test_execution.policy()
lifecycle.update(queue_deadline_s=5, execution_deadline_s=30)
owner.start(lifecycle)
execution.atomic_json(rd/'rollout.json', {'run_id': run_id, 'job_id': job_id, 'kk': kk, 'state': 'RUNNING'})
failed = job_id.startswith('failed')
with owner.scope(), ACQUIRE('agent', timeout_s=5), ACQUIRE('cpu', timeout_s=5):
barrier.wait(timeout=8) # Exercise every slot in each wave before any task completes.
@ -256,6 +297,7 @@ class CollectorRecovery(unittest.TestCase):
config = {'pools': {'agent': {'workers': pool_size}, 'cpu': {'workers': pool_size},
'board': {'workers': 1}}, 'endpoints': {'teacher': {}}}
with patch.object(paths, 'config', return_value=config), \
patch.object(recovery, 'reconcile_run', RECONCILE), \
patch.object(scheduler, 'board_slots', return_value={0: {'name': 'fixture'}}), \
patch.object(rollout, 'run_rollout', run):
self.assertEqual(fixture.collect(parallel=pool_size), {'INFRA_ERROR': model_count // 2, 'PASS': model_count // 2})

View File

@ -216,7 +216,7 @@ def forbidden(*args,**kwargs):
db.claim_rollout=forbidden
try:pipeline.collect_rollouts('r',log=lambda _:None)
except ValueError as error:
assert str(error)=='EXECUTION_RECONCILIATION_REQUIRED',str(error)
assert str(error)=='EXECUTION_RECONCILIATION_REQUIRED: OWNERSHIP_UNKNOWN',str(error)
else:raise AssertionError('unknown old boot admitted')
print(json.dumps({'new_claims':len(claims),'result':'QUARANTINED'}))
'''

View File

@ -90,6 +90,31 @@ class MonitorTests(unittest.TestCase):
self.assertEqual(observed['health'], 'UNKNOWN')
self.assertEqual(observed['reason_code'], 'IO_SOURCE_UNAVAILABLE')
def test_quarantine_preserves_specific_stop_and_cleanup_reason(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
record.update(cleanup_state='QUARANTINED', stop_reason='INFRA_STAGE_REGISTRATION_PENDING',
cleanup_errors=['board receipt remains unknown'])
path.write_text(json.dumps(record))
self.observer.sample()
row = self.observer.detail('r', 'j0', 0)
self.assertEqual(row['reason_code'], 'INFRA_STAGE_REGISTRATION_PENDING')
self.assertEqual(row['cleanup_errors'], ['board receipt remains unknown'])
def test_registration_pending_is_visible_without_forging_done(self):
path = self.root / 'runs/r/j0/rollouts/00/execution.json'
record = json.loads(path.read_text())
record.update(activity='VERIFYING', health='OK', cleanup_state='NOT_NEEDED', progress={'sampled_at': 100})
path.write_text(json.dumps(record))
handle = path.parent / 'handles/h1/status.json'
handle.parent.mkdir(parents=True)
handle.write_text('{"state":"RUNNING","registration":"PENDING"}')
self.observer.sample()
row = self.observer.detail('r', 'j0', 0)
self.assertEqual(row['reason_code'], 'STAGE_REGISTRATION_PENDING')
self.assertEqual(row['pending_registration'], ['h1'])
self.assertEqual(row['state'], 'RUNNING')
def test_legal_phase_without_agent_does_not_signal_mismatch(self):
path=self.root/'runs/r/j1/rollouts/00/execution.json'
record=json.loads(path.read_text());record.update(activity='VERIFYING',processes=[])

View File

@ -59,6 +59,84 @@ class DatabaseSetup(unittest.TestCase):
self.fail('setup succeeded')
con.close.assert_called_once_with()
def test_scoped_recovery_avoids_unrelated_historical_executions(self):
from harness import paths, recovery
with tempfile.TemporaryDirectory() as td:
root = Path(td); runs = root/'runs'; database = root/'jobs.sqlite'
for job, kk in (('j', 0), ('j', 1), ('other', 0)):
rd = runs/'r'/job/'rollouts'/f'{kk:02d}'
owner = execution.Execution(rd); owner.start(policy())
owner.finish(0, protocol_complete=True)
if (job, kk) != ('j', 0):
owner.path.write_text('unreadable historical record')
with CONNECT(database) as con:
db.rollout_upsert(con, 'r', job, kk, 'e2e', 'teacher', state='PASS')
with patch.object(paths, 'RUNS', runs), patch.object(paths, 'STATE', root/'state'), \
patch.object(db, 'readonly', lambda: READONLY(database)):
report = recovery.reconcile_run('r', attempts=[('j', 0), ('j', 0)])
self.assertTrue(report['ready'])
self.assertEqual([(r['job_id'], r['kk']) for r in report['executions']], [('j', 0)])
self.assertEqual(recovery.reconcile_run('r', attempts=[])['executions'], [])
with self.assertRaises(ValueError):
recovery.reconcile_run('r') # Startup still inspects all indexed executions.
def test_prelaunch_handoff_requires_bound_metadata_and_preserves_committed_verdict(self):
from harness import paths, recovery
with tempfile.TemporaryDirectory() as td:
root = Path(td); database = root/'jobs.sqlite'; rd = root/'runs/r/j/rollouts/00'
rd.mkdir(parents=True)
baseline = dict(run_id='r', job_id='j', kk=0, collector_error='fixture prelaunch failure', finished_at=1)
for change in ({'run_id': 'other'}, {'kk': 1}, {'collector_error': None}, {'finished_at': None}, {}):
with self.subTest(change=change):
with CONNECT(database) as con:
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher')
db.rollout_set(con, 'r', 'j', 0, state='RUNNING')
execution.atomic_json(rd/'rollout.json', dict(baseline, **change))
with patch.object(paths, 'RUNS', root/'runs'), patch.object(paths, 'STATE', root/'state'), \
patch.object(db, 'readonly', lambda: READONLY(database)), \
patch.object(db, 'connect', lambda **kw: CONNECT(database, **kw)):
self.assertEqual(recovery.reconcile_run('r')['ready'], not change)
with CONNECT(database) as con:
self.assertEqual(con.execute('SELECT state FROM rollouts').fetchone()[0], 'RUNNING' if change else 'INFRA_ERROR')
for terminal in ('PASS', 'FAIL'):
with CONNECT(database) as con:
db.rollout_set(con, 'r', 'j', 0, state=terminal)
with patch.object(paths, 'RUNS', root/'runs'), patch.object(paths, 'STATE', root/'state'), \
patch.object(db, 'readonly', lambda: READONLY(database)):
self.assertTrue(recovery.reconcile_run('r')['ready'])
with CONNECT(database) as con:
self.assertEqual(con.execute('SELECT state FROM rollouts').fetchone()[0], terminal)
def test_terminal_settlement_keeps_real_outcome_and_committed_verdict(self):
from harness import paths, rollout
for outcome in ('CANCELLED', 'LOST', 'AGENT_ERROR'):
with self.subTest(outcome=outcome), tempfile.TemporaryDirectory() as td:
root = Path(td); rd = root/'runs/r/j/rollouts/00'
owner = execution.Execution(rd); owner.start(policy())
if outcome == 'CANCELLED': owner.cancel('fixture cancel')
owner.finish(1, protocol_complete=False,
stop_reason='OWNER_LOST' if outcome == 'LOST' else None)
database = root/'jobs.sqlite'
with CONNECT(database) as con:
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
(rd/'rollout.json').write_text('{"state":"RUNNING"}')
original = owner.path.read_bytes()
with patch.object(paths, 'RUNS', root/'runs'), \
patch.object(db, 'connect', lambda **kw: CONNECT(database, **kw)):
for _ in range(2):
self.assertEqual(rollout.settle_execution('r', 'j', 0, owner, 'retry')['execution_outcome'], outcome)
self.assertEqual(json.loads((rd/'rollout.json').read_text())['state'], outcome)
with CONNECT(database) as con:
self.assertEqual(con.execute('SELECT state FROM rollouts').fetchone()[0], outcome)
db.rollout_set(con, 'r', 'j', 0, state='FAIL')
(rd/'rollout.json').write_text('{"state":"FAIL","verified":"original"}')
verdict_metadata = (rd/'rollout.json').read_bytes()
rollout.settle_execution('r', 'j', 0, owner, 'late cancel', cancel=True)
self.assertEqual((rd/'rollout.json').read_bytes(), verdict_metadata)
with CONNECT(database) as con:
self.assertEqual(con.execute('SELECT state FROM rollouts').fetchone()[0], 'FAIL')
self.assertEqual(owner.path.read_bytes(), original)
def test_unmanaged_protected_stage_keeps_direct_registration(self):
from harness.stages.base import Stage, StageOutcome
with tempfile.TemporaryDirectory() as td:
@ -85,6 +163,93 @@ class DatabaseSetup(unittest.TestCase):
self.assertIsInstance(json.loads(Path(result['meta']).read_text())['path'], str)
class CompletedRecovery(unittest.TestCase):
def setUp(self):
from harness import paths, recovery
from scripts.test_hbm_provenance import HbmProvenance
fixture = HbmProvenance(); fixture.setUp(); self.addCleanup(fixture.doCleanups)
fixture.stage_records()
self.fixture, self.rd, self.root = fixture, fixture.rd, fixture.root
self.database = self.root/'jobs.sqlite'
# A real owner has exited. Recovery has no in-memory future to harvest.
program = "from pathlib import Path;from harness.execution import Execution;from scripts.test_execution import policy;import sys;rd=Path(sys.argv[1]);owner=Execution(rd);owner.start(policy());(rd/'effects').write_text('one\\n');owner.finish(0,protocol_complete=True)"
subprocess.run([sys.executable, '-c', program, str(self.rd)], cwd=ROOT, check=True,
env=dict(os.environ, BENCH_ROOT=str(ROOT), PYTHONPATH=str(ROOT)))
self.owner = execution.Execution(self.rd).attach()
fixture.save(self.rd/'rollout.json', {'state': 'DONE', 'mode': 'e2e', 'policy': 'teacher',
'execution_id': self.owner.snapshot()['execution_id'], 'execution_managed': True})
self.reset()
for override in (patch.object(paths, 'RUNS', self.root/'runs'),
patch.object(paths, 'STATE', self.root/'state'),
patch.object(db, 'connect', lambda **kw: CONNECT(self.database, **kw)),
patch.object(db, 'readonly', lambda: READONLY(self.database))):
override.start(); self.addCleanup(override.stop)
self.reconcile = lambda: recovery.reconcile_run('r')
def reset(self, state='VERIFYING', verdict=None):
with CONNECT(self.database) as con:
db.rollout_upsert(con, 'r', 'job', 0, 'e2e', 'teacher')
db.rollout_set(con, 'r', 'job', 0, state=state, verdict=verdict)
def row(self):
with READONLY(self.database) as con:
return dict(con.execute('SELECT * FROM rollouts').fetchone())
def test_dead_completed_owner_reverifies_running_and_verifying_exactly_once(self):
from harness import verify
for state in ('RUNNING', 'VERIFYING', 'UNVERIFIED'):
with self.subTest(state=state):
self.reset(state)
self.assertTrue(self.reconcile()['ready'])
self.assertEqual(self.row()['state'], 'PASS')
self.assertEqual(json.loads(self.row()['verdict'])['execution_control']['execution_outcome'], 'COMPLETED')
committed = self.row()
with patch.object(verify, 'verify_rollout', side_effect=AssertionError('reverified committed verdict')):
self.assertTrue(self.reconcile()['ready'])
self.assertEqual(self.row(), committed)
self.assertEqual((self.rd/'effects').read_text(), 'one\n')
for terminal in ('PASS', 'FAIL', 'UNVERIFIED'):
self.reset(terminal, '{"terminal":"' + terminal + '"}')
with patch.object(verify, 'verify_rollout', side_effect=AssertionError('replaced committed verdict')):
self.assertTrue(self.reconcile()['ready'])
self.assertEqual(self.row()['state'], terminal)
def test_busy_preprobe_skips_artifact_io_and_late_lock_reverifies_changed_hbm(self):
from harness import verify
lock = sqlite3.connect(self.database, timeout=0)
self.addCleanup(lock.close)
lock.execute('BEGIN IMMEDIATE')
try:
with patch.object(verify, 'verify_rollout', side_effect=AssertionError('read artifacts while locked')):
report = self.reconcile()
self.assertFalse(report['ready'])
self.assertEqual(self.row()['state'], 'VERIFYING')
finally:
lock.rollback()
validate = verify.validate_hbm_measurements
def relock(*args, **kwargs):
result = validate(*args, **kwargs)
lock.execute('BEGIN IMMEDIATE')
return result
started = time.monotonic()
with patch.object(verify, 'validate_hbm_measurements', side_effect=relock):
report = self.reconcile()
self.assertLess(time.monotonic() - started, 1)
self.assertFalse(report['ready'])
self.assertEqual(json.loads((self.rd/'verdict.json').read_text())['terminal'], 'PASS')
lock.rollback()
self.fixture.hbm.write_bytes(b'changed after pending verdict')
self.assertTrue(self.reconcile()['ready'])
self.assertEqual(self.row()['state'], 'UNVERIFIED')
self.assertEqual((self.rd/'effects').read_text(), 'one\n')
def test_real_failed_measurement_is_not_promoted_by_completed_execution(self):
self.fixture.meas['G3']['quant'] = 0
self.fixture.stage_records()
self.assertTrue(self.reconcile()['ready'])
self.assertEqual(self.row()['state'], 'FAIL')
@unittest.skipUnless(sys.platform == 'linux', 'real kernel process scope receipts')
class StageRegistration(unittest.TestCase):
def setUp(self):
@ -179,18 +344,19 @@ class StageRegistration(unittest.TestCase):
self.assertEqual(self.owner.snapshot()['operations']['h1']['state'], 'DONE')
def test_dead_owner_recovery_preserves_busy_receipt_then_registers_once(self):
from harness import paths, recovery, scheduler
from harness import paths, recovery, rollout, scheduler
rd = self.root / 'restart/r/j/rollouts/00'
(rd / 'handles/h1').mkdir(parents=True); (rd / 'workspace').mkdir()
(rd / 'handles/h1/request.json').write_text('{"stage":"05-compile"}')
(rd.parent.parent / 'job.json').write_bytes((self.rd.parent.parent / 'job.json').read_bytes())
(rd / 'rollout.json').write_text('{"run_id":"r","job_id":"j","kk":0,"state":"RUNNING"}')
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
controller = r"""
import sys,time
from pathlib import Path
from functools import partial
from harness import db,execution,paths,stageworker,watchdog
from harness import db,execution,paths,scheduler,stageworker,watchdog
from scripts.test_stage_registration import CHILD
from scripts.test_execution import policy
rd, database = Path(sys.argv[1]), Path(sys.argv[2])
@ -200,6 +366,10 @@ owner = execution.Execution(rd)
cfg = policy(); cfg['execution_deadline_s'] = 60
for phase in cfg['phases'].values(): phase.update(queue_deadline_s=30, execution_deadline_s=30)
owner.start(cfg)
paths.STATE = rd.parents[4] / 'lease-state'
with owner.scope():
retained_agent_handle, _ = scheduler._try_slots('agent', [0])
assert retained_agent_handle is not None
run = watchdog.run
def dispatch(command, **kw):
queue = rd / 'stages/_queue/h1'
@ -236,7 +406,7 @@ stageworker.Worker(rd, execution=owner).scan()
proof = proof_path.read_bytes()
with patch.object(db, 'readonly', lambda: READONLY(self.database)), \
patch.object(paths, 'RUNS', self.root / 'restart'), \
patch.object(scheduler, 'reconcile_leases', return_value=[]), \
patch.object(paths, 'STATE', self.root / 'lease-state'), \
patch.object(stageworker.watchdog, 'run', side_effect=AssertionError('recovery dispatched work')):
report = recovery.reconcile_run('r')
self.assertFalse(report['ready'])
@ -257,14 +427,70 @@ stageworker.Worker(rd, execution=owner).scan()
recovery.reconcile_run('r')
self.assertEqual(json.loads((proof_path.parent / 'operation.json').read_text())['state'], 'DONE')
self.assertNotEqual(execution.Execution(rd).snapshot()['operations']['h1']['state'], 'DONE')
settle = rollout.settle_execution
def lock_settlement(*args, **kwargs):
self.lock = sqlite3.connect(self.database, timeout=0)
self.lock.execute('BEGIN IMMEDIATE')
return settle(*args, **kwargs)
with patch.object(rollout, 'settle_execution', side_effect=lock_settlement):
report = recovery.reconcile_run('r')
self.assertFalse(report['ready'])
self.assertEqual(report['executions'][0]['reason'], 'ROLLOUT_REGISTRATION_PENDING')
fenced_epoch = execution.Execution(rd).snapshot()['epoch']
self.release()
report = recovery.reconcile_run('r')
self.assertEqual(execution.Execution(rd).snapshot()['epoch'], fenced_epoch)
self.assertTrue(report['ready'])
record = execution.Execution(rd).snapshot()
self.assertEqual((record['execution_outcome'], record['cleanup_state']), ('LOST', 'VERIFIED'))
with self.connect(self.database) as con:
self.assertEqual(con.execute('SELECT state FROM rollouts').fetchone()[0], 'LOST')
self.assertEqual(json.loads((rd / 'rollout.json').read_text())['state'], 'LOST')
self.assertFalse((self.root / 'lease-state/locks/agent-0.lease.json').exists())
self.assertEqual(self.row()['state'], 'OK')
self.assertTrue(json.loads(proof_path.read_text())['db_registered'])
self.assertEqual((rd / 'effects').read_text(), 'one\n')
def test_worker_exit_drains_completed_registration_within_cleanup_budget(self):
self.dispatch()
self.worker.stop_event.set() # Work is already complete; only registration remains.
timer = threading.Timer(.15, self.release)
timer.start(); self.addCleanup(timer.join)
with self.worker:
pass
self.assertEqual(self.row()['state'], 'OK')
self.assertEqual(self.owner.snapshot()['operations']['h1']['state'], 'DONE')
self.assertEqual(self.owner.finish(0, protocol_complete=True)['execution_outcome'], 'COMPLETED')
self.assertEqual((self.rd / 'effects').read_text(), 'one\n')
def test_bounded_worker_exit_can_recover_after_shared_owner_stays_alive(self):
from harness import paths, recovery
(self.rd / 'rollout.json').write_text('{"run_id":"r","job_id":"j","kk":0,"state":"RUNNING"}')
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
self.dispatch()
self.worker.stop_event.set()
started = time.monotonic()
with self.worker:
pass
self.assertLess(time.monotonic() - started, 3)
record = self.owner.finish(0, protocol_complete=True)
self.assertEqual(record['stop_reason'], 'INFRA_STAGE_REGISTRATION_PENDING')
self.assertEqual(record['execution_outcome'], 'INFRA_ERROR')
self.assertEqual(execution.process_status(record['owner']), 'ALIVE')
self.release()
with patch.object(db, 'readonly', lambda: READONLY(self.database)), \
patch.object(paths, 'RUNS', self.root / 'runs'), \
patch.object(paths, 'STATE', self.root / 'lease-state'), \
patch.object(stageworker.watchdog, 'run', side_effect=AssertionError('recovery dispatched work')):
report = recovery.reconcile_run('r')
self.assertTrue(report['ready'])
self.assertEqual(self.row()['state'], 'OK')
self.assertEqual(self.owner.snapshot()['execution_outcome'], 'INFRA_ERROR')
with self.connect(self.database) as con:
self.assertEqual(con.execute('SELECT state FROM rollouts').fetchone()[0], 'INFRA_ERROR')
self.assertEqual((self.rd / 'effects').read_text(), 'one\n')
def test_agent_exit_while_registration_is_busy_keeps_conservative_failure(self):
self.dispatch()
proof = (self.queue / 'status.json').read_bytes()

View File

@ -53,6 +53,15 @@ class StageWaitTests(unittest.TestCase):
self.assertIn('这不是失败', output.getvalue())
self.assertNotIn('result: OK', output.getvalue())
def test_wait_explains_registration_pending_without_claiming_done(self):
self.status.write_text('{"state":"RUNNING","registration":"PENDING"}')
for module in (client, stagerun):
output = io.StringIO()
with redirect_stdout(output):
self.assertEqual(module.wait('h1', max_wait_s=.01), 2)
self.assertIn('等待数据库登记', output.getvalue())
self.assertNotIn('result: OK', output.getvalue())
def test_cli_uses_bounded_default(self):
elapsed = [0]
def sleep(seconds):