data-gen-bpu/scripts/test_recovery.py
2026-10-03 21:17:40 +08:00

245 lines
14 KiB
Python

"""Recovery is an explicit audited new attempt; no live execution in tests."""
import hashlib
import json
import os
import socket
import unittest
from unittest.mock import patch
from harness import db, execution, paths, recovery, scheduler
from scripts import test_plan_publication as publication
class Recovery(unittest.TestCase):
connect = publication.PlanPublication.connect
spec = publication.PlanPublication.spec
snapshot = publication.PlanPublication.snapshot
rows = publication.PlanPublication.rows
def setUp(self):
publication.PlanPublication.setUp(self)
self.stack.enter_context(patch.object(paths, "STATE", self.root / "state"))
self.spec(runner="opencode")
from harness import pipeline
pipeline.expand("r")
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state="RUNNING", started_at=1)
self.rd = self.root / "runs/r/j/rollouts/00"
(self.rd / "agent").mkdir(parents=True)
(self.rd / "agent/events.jsonl").write_text('{"original":"evidence"}\n')
def test_alive_owner_cannot_be_overridden(self):
with self.connect() as con:
con.execute("UPDATE rollout_context SET owner_host=?,owner_pid=? WHERE kk=0", (socket.gethostname(), os.getpid()))
with self.assertRaisesRegex(ValueError, "owner is alive"):
recovery.recover("r", "j", 0, reason="test", confirm_owner_dead=True, confirm_work_stopped=True)
self.assertTrue(recovery.diagnose("r")[0]["blocked"])
def test_live_handle_process_cannot_be_overridden(self):
hd = self.rd / "handles/h1"
hd.mkdir(parents=True)
(hd / "status.json").write_text(json.dumps({"state": "RUNNING", "worker_pid": os.getpid(),
"host": socket.gethostname()}))
with self.assertRaisesRegex(ValueError, "stage/handle process is alive"):
recovery.recover("r", "j", 0, reason="test", confirm_owner_dead=True, confirm_work_stopped=True)
self.assertTrue(recovery.diagnose("r")[0]["blocked"])
def test_unknown_owner_and_work_require_separate_explicit_confirmation(self):
with self.assertRaisesRegex(ValueError, "owner death is unverified"):
recovery.recover("r", "j", 0, reason="test")
with self.assertRaisesRegex(ValueError, "work may remain active"):
recovery.recover("r", "j", 0, reason="test", confirm_owner_dead=True)
self.assertEqual(self.rows()[0]["state"], "RUNNING")
def test_permission_denied_is_unknown_not_dead(self):
with patch.object(recovery.os, "kill", side_effect=PermissionError):
self.assertEqual(recovery._owner_status(socket.gethostname(), 234), "unknown")
self.assertEqual(recovery._owner_status("other-machine", 234), "unknown")
def test_recovery_preserves_all_source_bytes_and_identity(self):
with self.connect() as con:
db.stage_start(con, "r", "j", "05-compile", rollout=0)
hd = self.rd / "handles/h1"
hd.mkdir(parents=True)
(hd / "status.json").write_text('{"state":"RUNNING"}')
before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in self.rd.rglob("*") if p.is_file()}
diag = recovery.diagnose("r")[0]
self.assertEqual(len(diag["unresolved_work"]), 2)
result = recovery.recover("r", "j", 0, reason="operator verified all work stopped",
confirm_owner_dead=True, confirm_work_stopped=True)
self.assertEqual(result["replacement_kk"], 2)
rows = self.rows()
self.assertEqual([(r["kk"], r["runner"], r["state"]) for r in rows],
[(0, "opencode", "ABANDONED"), (1, "opencode", "QUEUED"), (2, "opencode", "QUEUED")])
self.assertEqual(before, {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in before})
with self.connect() as con:
audit = json.loads(con.execute("SELECT note FROM annotations").fetchone()[0].split(" ", 1)[1])
self.assertEqual(audit["source_kk"], 0)
self.assertEqual(con.execute("SELECT state FROM stages").fetchone()[0], "RUNNING")
with self.assertRaisesRegex(ValueError, "requires one explicit"):
recovery.recover("r", "j", 0, reason="repeat", confirm_owner_dead=True, confirm_work_stopped=True)
def test_corrupt_ticket_is_unresolved_and_diagnosis_does_not_reset(self):
hd = self.rd / "handles/h1"
hd.mkdir(parents=True)
(hd / "status.json").write_text('partial json')
self.assertEqual(recovery.diagnose("r")[0]["unresolved_work"][0]["state"], "UNKNOWN")
self.assertEqual(self.rows()[0]["state"], "RUNNING")
def test_other_run_quarantine_does_not_block_new_run_or_release_its_slot(self):
with self.connect() as con:
db.create_run(con, "new", {})
alien = paths.RUNS / "other/j/rollouts/00"
alien.mkdir(parents=True)
(alien / "execution.json").write_text(json.dumps({
"execution_id": "other-execution", "epoch": 1,
"execution_state": "TERMINAL", "cleanup_state": "QUARANTINED"}))
locks = paths.STATE / "locks"
locks.mkdir(parents=True)
lease, quarantine = locks / "cpu-0.lease.json", locks / "cpu-0.quarantine.json"
lease.write_text(json.dumps({"execution_id": "other-execution", "execution_dir": str(alien),
"pool": "cpu", "slot": 0, "holder": {"pid": 99999999}}))
quarantine.write_text('{"reason":"SIDE_EFFECT_UNKNOWN","execution_id":"other-execution"}\n')
before = {path: path.read_bytes() for path in (lease, quarantine, alien / "execution.json")}
report = recovery.reconcile_run("new")
self.assertTrue(report["ready"])
self.assertEqual(report["executions"], [])
self.assertEqual(report["resources"], [{"lease": str(lease), "state": "QUARANTINED"}])
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 1}}}):
with self.assertRaisesRegex(execution.ExecutionStopped, "QUEUE_DEADLINE_EXCEEDED"):
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.02):
self.fail("quarantined slot was acquired")
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 2}}}):
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.1) as slot:
self.assertEqual(slot, 1)
self.assertEqual(before, {path: path.read_bytes() for path in before})
def test_current_run_quarantine_still_blocks_startup(self):
(self.rd / "execution.json").write_text(json.dumps({
"execution_id": "current-execution", "epoch": 1,
"execution_state": "TERMINAL", "cleanup_state": "QUARANTINED"}))
report = recovery.reconcile_run("r")
self.assertFalse(report["ready"])
self.assertEqual(report["resources"], [])
self.assertFalse(report["executions"][0]["ready"])
def test_historical_infra_error_allows_healthy_slot_without_releasing_quarantine(self):
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state="INFRA_ERROR")
(self.rd / "execution.json").write_text(json.dumps({
"execution_id": "historical-execution", "epoch": 1, "owner": {"pid": 99999999},
"execution_state": "TERMINAL", "cleanup_state": "QUARANTINED"}))
locks = paths.STATE / "locks"
locks.mkdir(parents=True)
lease, quarantine = locks / "cpu-0.lease.json", locks / "cpu-0.quarantine.json"
lease.write_text(json.dumps({"execution_id": "historical-execution", "execution_dir": str(self.rd),
"pool": "cpu", "slot": 0, "holder": {"pid": 99999999}}))
quarantine.write_text('{"reason":"SIDE_EFFECT_UNKNOWN","execution_id":"historical-execution"}\n')
preserved = (lease, quarantine, self.rd / "execution.json", self.rd / "agent/events.jsonl")
before, rows = {path: path.read_bytes() for path in preserved}, self.rows()
with patch.object(execution, "process_status", return_value="DEAD"):
report = recovery.reconcile_run("r")
self.assertTrue(report["ready"])
self.assertTrue(report["executions"][0]["ready"])
self.assertEqual(report["resources"], [{"lease": str(lease), "state": "QUARANTINED"}])
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 1}}}):
with self.assertRaisesRegex(execution.ExecutionStopped, "QUEUE_DEADLINE_EXCEEDED"):
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.02):
self.fail("historical quarantined slot was acquired")
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 2}}}):
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.1) as slot:
self.assertEqual(slot, 1)
self.assertEqual(before, {path: path.read_bytes() for path in preserved})
self.assertEqual(rows, self.rows())
def test_historical_quarantine_exception_requires_every_settlement_condition(self):
cases = (
("unfinished", "PAUSED_INFRA", "INFRA_ERROR", "DEAD", []),
("rollout_running", "TERMINAL", "RUNNING", "DEAD", []),
("rollout_pass", "TERMINAL", "PASS", "DEAD", []),
("owner_alive", "TERMINAL", "INFRA_ERROR", "ALIVE", []),
("owner_unknown", "TERMINAL", "INFRA_ERROR", "UNKNOWN", []),
("owned_process", "TERMINAL", "INFRA_ERROR", "DEAD", [{"pid": 123}]),
("owned_unknown", "TERMINAL", "INFRA_ERROR", "DEAD", None),
)
for name, state, rollout_state, owner_status, members in cases:
with self.subTest(name=name):
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state=rollout_state)
record = {"execution_id": "historical-execution", "epoch": 1,
"execution_state": state, "cleanup_state": "QUARANTINED", "owner": {"pid": 123}}
path = self.rd / "execution.json"
path.write_text(json.dumps(record))
before, rows = path.read_bytes(), self.rows()
with patch.object(execution.Execution, "reconcile", return_value=record), \
patch.object(execution, "process_status", return_value=owner_status), \
patch.object(execution, "owned_processes", return_value=members):
report = recovery.reconcile_run("r")
self.assertFalse(report["ready"])
self.assertFalse(report["executions"][0]["ready"])
self.assertEqual(before, path.read_bytes())
self.assertEqual(rows, self.rows())
def test_legacy_active_unmanaged_execution_still_blocks_startup(self):
report = recovery.reconcile_run("r")
self.assertFalse(report["ready"])
self.assertEqual(report["executions"], [{"job_id": "j", "kk": 0, "ready": False,
"reason": "LEGACY_EXECUTION_OWNERSHIP_UNKNOWN"}])
def test_shared_owner_agent_error_uses_recorded_infra_state(self):
from harness.stageworker import Worker
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state="INFRA_ERROR")
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
"execution_state": "TERMINAL", "execution_outcome": "AGENT_ERROR",
"cleanup_state": "QUARANTINED"}
p = self.rd / "execution.json"
p.write_text(json.dumps(record))
before = p.read_bytes()
for verified in (True, False):
with self.subTest(local_cleanup_verified=verified), \
patch.object(execution.Execution, "reconcile", return_value=record), \
patch.object(execution, "process_status", return_value="ALIVE"), \
patch.object(execution, "owned_processes", return_value=[]), \
patch.object(scheduler, "_local_cleanup_verified", return_value=verified), \
patch.object(scheduler, "reconcile_leases", return_value=[]), \
patch.object(Worker, "reconcile_completed", return_value=True):
self.assertEqual(recovery.reconcile_run("r")["ready"], verified)
self.assertEqual(p.read_bytes(), before)
def test_dead_failed_quarantine_does_not_replay_unverifiable_stage(self):
from harness.stageworker import Worker
cases = [("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [], True),
("CANCELLED", "CANCELLED", "TERMINAL", "DEAD", [], True),
("RUNNING", "AGENT_ERROR", "TERMINAL", "DEAD", [], False),
("PASS", "COMPLETED", "TERMINAL", "DEAD", [], False),
("INFRA_ERROR", "AGENT_ERROR", "PAUSED_INFRA", "DEAD", [], False),
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "UNKNOWN", [], False),
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [{"pid": 1}], False)]
for db_state, outcome, status, owner, members, allowed in cases:
with self.subTest(db_state=db_state, status=status, owner=owner, members=members):
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state=db_state)
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
"execution_state": status, "execution_outcome": outcome,
"cleanup_state": "QUARANTINED"}
p = self.rd / "execution.json"
p.write_text(json.dumps(record))
before, rows = p.read_bytes(), self.rows()
with patch.object(execution.Execution, "reconcile", return_value=record), \
patch.object(execution.Execution, "quarantine", return_value=record), \
patch.object(execution, "process_status", return_value=owner), \
patch.object(execution, "owned_processes", return_value=members), \
patch.object(scheduler, "_local_cleanup_verified", return_value=False), \
patch.object(scheduler, "reconcile_leases", return_value=[]), \
patch.object(Worker, "reconcile_completed", side_effect=ValueError("scope cleanup unconfirmed")) as replay:
self.assertEqual(recovery.reconcile_run("r")["ready"], allowed)
if allowed:
replay.assert_not_called()
self.assertEqual(p.read_bytes(), before)
self.assertEqual(self.rows(), rows)
if __name__ == "__main__":
import unittest
unittest.main()