245 lines
14 KiB
Python
245 lines
14 KiB
Python
"""Recovery is an explicit audited new attempt; no live execution in tests."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import socket
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, execution, paths, recovery, scheduler
|
|
from scripts import test_plan_publication as publication
|
|
|
|
|
|
class Recovery(unittest.TestCase):
|
|
connect = publication.PlanPublication.connect
|
|
spec = publication.PlanPublication.spec
|
|
snapshot = publication.PlanPublication.snapshot
|
|
rows = publication.PlanPublication.rows
|
|
def setUp(self):
|
|
publication.PlanPublication.setUp(self)
|
|
self.stack.enter_context(patch.object(paths, "STATE", self.root / "state"))
|
|
self.spec(runner="opencode")
|
|
from harness import pipeline
|
|
pipeline.expand("r")
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "r", "j", 0, state="RUNNING", started_at=1)
|
|
self.rd = self.root / "runs/r/j/rollouts/00"
|
|
(self.rd / "agent").mkdir(parents=True)
|
|
(self.rd / "agent/events.jsonl").write_text('{"original":"evidence"}\n')
|
|
|
|
def test_alive_owner_cannot_be_overridden(self):
|
|
with self.connect() as con:
|
|
con.execute("UPDATE rollout_context SET owner_host=?,owner_pid=? WHERE kk=0", (socket.gethostname(), os.getpid()))
|
|
with self.assertRaisesRegex(ValueError, "owner is alive"):
|
|
recovery.recover("r", "j", 0, reason="test", confirm_owner_dead=True, confirm_work_stopped=True)
|
|
self.assertTrue(recovery.diagnose("r")[0]["blocked"])
|
|
|
|
def test_live_handle_process_cannot_be_overridden(self):
|
|
hd = self.rd / "handles/h1"
|
|
hd.mkdir(parents=True)
|
|
(hd / "status.json").write_text(json.dumps({"state": "RUNNING", "worker_pid": os.getpid(),
|
|
"host": socket.gethostname()}))
|
|
with self.assertRaisesRegex(ValueError, "stage/handle process is alive"):
|
|
recovery.recover("r", "j", 0, reason="test", confirm_owner_dead=True, confirm_work_stopped=True)
|
|
self.assertTrue(recovery.diagnose("r")[0]["blocked"])
|
|
|
|
def test_unknown_owner_and_work_require_separate_explicit_confirmation(self):
|
|
with self.assertRaisesRegex(ValueError, "owner death is unverified"):
|
|
recovery.recover("r", "j", 0, reason="test")
|
|
with self.assertRaisesRegex(ValueError, "work may remain active"):
|
|
recovery.recover("r", "j", 0, reason="test", confirm_owner_dead=True)
|
|
self.assertEqual(self.rows()[0]["state"], "RUNNING")
|
|
|
|
def test_permission_denied_is_unknown_not_dead(self):
|
|
with patch.object(recovery.os, "kill", side_effect=PermissionError):
|
|
self.assertEqual(recovery._owner_status(socket.gethostname(), 234), "unknown")
|
|
self.assertEqual(recovery._owner_status("other-machine", 234), "unknown")
|
|
|
|
def test_recovery_preserves_all_source_bytes_and_identity(self):
|
|
with self.connect() as con:
|
|
db.stage_start(con, "r", "j", "05-compile", rollout=0)
|
|
hd = self.rd / "handles/h1"
|
|
hd.mkdir(parents=True)
|
|
(hd / "status.json").write_text('{"state":"RUNNING"}')
|
|
before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in self.rd.rglob("*") if p.is_file()}
|
|
diag = recovery.diagnose("r")[0]
|
|
self.assertEqual(len(diag["unresolved_work"]), 2)
|
|
result = recovery.recover("r", "j", 0, reason="operator verified all work stopped",
|
|
confirm_owner_dead=True, confirm_work_stopped=True)
|
|
self.assertEqual(result["replacement_kk"], 2)
|
|
rows = self.rows()
|
|
self.assertEqual([(r["kk"], r["runner"], r["state"]) for r in rows],
|
|
[(0, "opencode", "ABANDONED"), (1, "opencode", "QUEUED"), (2, "opencode", "QUEUED")])
|
|
self.assertEqual(before, {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in before})
|
|
with self.connect() as con:
|
|
audit = json.loads(con.execute("SELECT note FROM annotations").fetchone()[0].split(" ", 1)[1])
|
|
self.assertEqual(audit["source_kk"], 0)
|
|
self.assertEqual(con.execute("SELECT state FROM stages").fetchone()[0], "RUNNING")
|
|
with self.assertRaisesRegex(ValueError, "requires one explicit"):
|
|
recovery.recover("r", "j", 0, reason="repeat", confirm_owner_dead=True, confirm_work_stopped=True)
|
|
|
|
def test_corrupt_ticket_is_unresolved_and_diagnosis_does_not_reset(self):
|
|
hd = self.rd / "handles/h1"
|
|
hd.mkdir(parents=True)
|
|
(hd / "status.json").write_text('partial json')
|
|
self.assertEqual(recovery.diagnose("r")[0]["unresolved_work"][0]["state"], "UNKNOWN")
|
|
self.assertEqual(self.rows()[0]["state"], "RUNNING")
|
|
|
|
def test_other_run_quarantine_does_not_block_new_run_or_release_its_slot(self):
|
|
with self.connect() as con:
|
|
db.create_run(con, "new", {})
|
|
alien = paths.RUNS / "other/j/rollouts/00"
|
|
alien.mkdir(parents=True)
|
|
(alien / "execution.json").write_text(json.dumps({
|
|
"execution_id": "other-execution", "epoch": 1,
|
|
"execution_state": "TERMINAL", "cleanup_state": "QUARANTINED"}))
|
|
locks = paths.STATE / "locks"
|
|
locks.mkdir(parents=True)
|
|
lease, quarantine = locks / "cpu-0.lease.json", locks / "cpu-0.quarantine.json"
|
|
lease.write_text(json.dumps({"execution_id": "other-execution", "execution_dir": str(alien),
|
|
"pool": "cpu", "slot": 0, "holder": {"pid": 99999999}}))
|
|
quarantine.write_text('{"reason":"SIDE_EFFECT_UNKNOWN","execution_id":"other-execution"}\n')
|
|
before = {path: path.read_bytes() for path in (lease, quarantine, alien / "execution.json")}
|
|
report = recovery.reconcile_run("new")
|
|
self.assertTrue(report["ready"])
|
|
self.assertEqual(report["executions"], [])
|
|
self.assertEqual(report["resources"], [{"lease": str(lease), "state": "QUARANTINED"}])
|
|
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 1}}}):
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, "QUEUE_DEADLINE_EXCEEDED"):
|
|
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.02):
|
|
self.fail("quarantined slot was acquired")
|
|
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 2}}}):
|
|
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.1) as slot:
|
|
self.assertEqual(slot, 1)
|
|
self.assertEqual(before, {path: path.read_bytes() for path in before})
|
|
|
|
def test_current_run_quarantine_still_blocks_startup(self):
|
|
(self.rd / "execution.json").write_text(json.dumps({
|
|
"execution_id": "current-execution", "epoch": 1,
|
|
"execution_state": "TERMINAL", "cleanup_state": "QUARANTINED"}))
|
|
report = recovery.reconcile_run("r")
|
|
self.assertFalse(report["ready"])
|
|
self.assertEqual(report["resources"], [])
|
|
self.assertFalse(report["executions"][0]["ready"])
|
|
|
|
def test_historical_infra_error_allows_healthy_slot_without_releasing_quarantine(self):
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "r", "j", 0, state="INFRA_ERROR")
|
|
(self.rd / "execution.json").write_text(json.dumps({
|
|
"execution_id": "historical-execution", "epoch": 1, "owner": {"pid": 99999999},
|
|
"execution_state": "TERMINAL", "cleanup_state": "QUARANTINED"}))
|
|
locks = paths.STATE / "locks"
|
|
locks.mkdir(parents=True)
|
|
lease, quarantine = locks / "cpu-0.lease.json", locks / "cpu-0.quarantine.json"
|
|
lease.write_text(json.dumps({"execution_id": "historical-execution", "execution_dir": str(self.rd),
|
|
"pool": "cpu", "slot": 0, "holder": {"pid": 99999999}}))
|
|
quarantine.write_text('{"reason":"SIDE_EFFECT_UNKNOWN","execution_id":"historical-execution"}\n')
|
|
preserved = (lease, quarantine, self.rd / "execution.json", self.rd / "agent/events.jsonl")
|
|
before, rows = {path: path.read_bytes() for path in preserved}, self.rows()
|
|
with patch.object(execution, "process_status", return_value="DEAD"):
|
|
report = recovery.reconcile_run("r")
|
|
self.assertTrue(report["ready"])
|
|
self.assertTrue(report["executions"][0]["ready"])
|
|
self.assertEqual(report["resources"], [{"lease": str(lease), "state": "QUARANTINED"}])
|
|
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 1}}}):
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, "QUEUE_DEADLINE_EXCEEDED"):
|
|
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.02):
|
|
self.fail("historical quarantined slot was acquired")
|
|
with patch.object(paths, "config", return_value={"pools": {"cpu": {"workers": 2}}}):
|
|
with scheduler.acquire("cpu", poll_s=.005, timeout_s=.1) as slot:
|
|
self.assertEqual(slot, 1)
|
|
self.assertEqual(before, {path: path.read_bytes() for path in preserved})
|
|
self.assertEqual(rows, self.rows())
|
|
|
|
def test_historical_quarantine_exception_requires_every_settlement_condition(self):
|
|
cases = (
|
|
("unfinished", "PAUSED_INFRA", "INFRA_ERROR", "DEAD", []),
|
|
("rollout_running", "TERMINAL", "RUNNING", "DEAD", []),
|
|
("rollout_pass", "TERMINAL", "PASS", "DEAD", []),
|
|
("owner_alive", "TERMINAL", "INFRA_ERROR", "ALIVE", []),
|
|
("owner_unknown", "TERMINAL", "INFRA_ERROR", "UNKNOWN", []),
|
|
("owned_process", "TERMINAL", "INFRA_ERROR", "DEAD", [{"pid": 123}]),
|
|
("owned_unknown", "TERMINAL", "INFRA_ERROR", "DEAD", None),
|
|
)
|
|
for name, state, rollout_state, owner_status, members in cases:
|
|
with self.subTest(name=name):
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "r", "j", 0, state=rollout_state)
|
|
record = {"execution_id": "historical-execution", "epoch": 1,
|
|
"execution_state": state, "cleanup_state": "QUARANTINED", "owner": {"pid": 123}}
|
|
path = self.rd / "execution.json"
|
|
path.write_text(json.dumps(record))
|
|
before, rows = path.read_bytes(), self.rows()
|
|
with patch.object(execution.Execution, "reconcile", return_value=record), \
|
|
patch.object(execution, "process_status", return_value=owner_status), \
|
|
patch.object(execution, "owned_processes", return_value=members):
|
|
report = recovery.reconcile_run("r")
|
|
self.assertFalse(report["ready"])
|
|
self.assertFalse(report["executions"][0]["ready"])
|
|
self.assertEqual(before, path.read_bytes())
|
|
self.assertEqual(rows, self.rows())
|
|
|
|
def test_legacy_active_unmanaged_execution_still_blocks_startup(self):
|
|
report = recovery.reconcile_run("r")
|
|
self.assertFalse(report["ready"])
|
|
self.assertEqual(report["executions"], [{"job_id": "j", "kk": 0, "ready": False,
|
|
"reason": "LEGACY_EXECUTION_OWNERSHIP_UNKNOWN"}])
|
|
|
|
def test_shared_owner_agent_error_uses_recorded_infra_state(self):
|
|
from harness.stageworker import Worker
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "r", "j", 0, state="INFRA_ERROR")
|
|
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
|
|
"execution_state": "TERMINAL", "execution_outcome": "AGENT_ERROR",
|
|
"cleanup_state": "QUARANTINED"}
|
|
p = self.rd / "execution.json"
|
|
p.write_text(json.dumps(record))
|
|
before = p.read_bytes()
|
|
for verified in (True, False):
|
|
with self.subTest(local_cleanup_verified=verified), \
|
|
patch.object(execution.Execution, "reconcile", return_value=record), \
|
|
patch.object(execution, "process_status", return_value="ALIVE"), \
|
|
patch.object(execution, "owned_processes", return_value=[]), \
|
|
patch.object(scheduler, "_local_cleanup_verified", return_value=verified), \
|
|
patch.object(scheduler, "reconcile_leases", return_value=[]), \
|
|
patch.object(Worker, "reconcile_completed", return_value=True):
|
|
self.assertEqual(recovery.reconcile_run("r")["ready"], verified)
|
|
self.assertEqual(p.read_bytes(), before)
|
|
|
|
def test_dead_failed_quarantine_does_not_replay_unverifiable_stage(self):
|
|
from harness.stageworker import Worker
|
|
cases = [("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [], True),
|
|
("CANCELLED", "CANCELLED", "TERMINAL", "DEAD", [], True),
|
|
("RUNNING", "AGENT_ERROR", "TERMINAL", "DEAD", [], False),
|
|
("PASS", "COMPLETED", "TERMINAL", "DEAD", [], False),
|
|
("INFRA_ERROR", "AGENT_ERROR", "PAUSED_INFRA", "DEAD", [], False),
|
|
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "UNKNOWN", [], False),
|
|
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [{"pid": 1}], False)]
|
|
for db_state, outcome, status, owner, members, allowed in cases:
|
|
with self.subTest(db_state=db_state, status=status, owner=owner, members=members):
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "r", "j", 0, state=db_state)
|
|
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
|
|
"execution_state": status, "execution_outcome": outcome,
|
|
"cleanup_state": "QUARANTINED"}
|
|
p = self.rd / "execution.json"
|
|
p.write_text(json.dumps(record))
|
|
before, rows = p.read_bytes(), self.rows()
|
|
with patch.object(execution.Execution, "reconcile", return_value=record), \
|
|
patch.object(execution.Execution, "quarantine", return_value=record), \
|
|
patch.object(execution, "process_status", return_value=owner), \
|
|
patch.object(execution, "owned_processes", return_value=members), \
|
|
patch.object(scheduler, "_local_cleanup_verified", return_value=False), \
|
|
patch.object(scheduler, "reconcile_leases", return_value=[]), \
|
|
patch.object(Worker, "reconcile_completed", side_effect=ValueError("scope cleanup unconfirmed")) as replay:
|
|
self.assertEqual(recovery.reconcile_run("r")["ready"], allowed)
|
|
if allowed:
|
|
replay.assert_not_called()
|
|
self.assertEqual(p.read_bytes(), before)
|
|
self.assertEqual(self.rows(), rows)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import unittest
|
|
unittest.main()
|