diff --git a/harness/recovery.py b/harness/recovery.py index dad1988..da0e871 100644 --- a/harness/recovery.py +++ b/harness/recovery.py @@ -168,6 +168,16 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict: if (rd / "execution.json").exists(): owner = execution.Execution(rd) record = owner.snapshot() + # Failed history stays failed and isolated; do not require its unknown + # stage receipt to become trustworthy before scheduling unrelated work. + if (record['execution_state'] == 'TERMINAL' and record['cleanup_state'] == 'QUARANTINED' + and row['state'] in {'INFRA_ERROR', 'CANCELLED'} + and record.get('execution_outcome') in {'AGENT_ERROR', 'INFRA_ERROR', 'CANCELLED'} + and execution.process_status(record.get('owner', {})) == 'DEAD' + and execution.owned_processes(record) == []): + reports.append({'job_id': row['job_id'], 'kk': row['kk'], 'execution': record, + 'ready': True, 'reason': 'FAILED_EXECUTION_ISOLATED'}) + continue if ((execution.process_status(record.get('owner', {})) == 'DEAD' or record['execution_state'] == 'TERMINAL') and execution.owned_processes(record) == []): @@ -216,7 +226,8 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict: if (not ready and record["execution_state"] == "TERMINAL" and ((row["state"] == "INFRA_ERROR" and execution.process_status(record.get("owner", {})) == "DEAD") - or (settled_state == record.get("execution_outcome") + or ((settled_state == record.get("execution_outcome") + or settled_state == 'INFRA_ERROR' and record.get('execution_outcome') == 'AGENT_ERROR') and settled_state in {"INFRA_ERROR", "CANCELLED", "LOST", "AGENT_ERROR", "DEADLINE_EXCEEDED", "BUDGET_EXHAUSTED"} and scheduler._local_cleanup_verified(record))) diff --git a/scripts/test_recovery.py b/scripts/test_recovery.py index 0a34630..e820871 100644 --- a/scripts/test_recovery.py +++ b/scripts/test_recovery.py @@ -185,6 +185,59 @@ class Recovery(unittest.TestCase): self.assertEqual(report["executions"], [{"job_id": "j", "kk": 0, "ready": False, "reason": "LEGACY_EXECUTION_OWNERSHIP_UNKNOWN"}]) + def test_shared_owner_agent_error_uses_recorded_infra_state(self): + from harness.stageworker import Worker + with self.connect() as con: + db.rollout_set(con, "r", "j", 0, state="INFRA_ERROR") + record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123}, + "execution_state": "TERMINAL", "execution_outcome": "AGENT_ERROR", + "cleanup_state": "QUARANTINED"} + p = self.rd / "execution.json" + p.write_text(json.dumps(record)) + before = p.read_bytes() + for verified in (True, False): + with self.subTest(local_cleanup_verified=verified), \ + patch.object(execution.Execution, "reconcile", return_value=record), \ + patch.object(execution, "process_status", return_value="ALIVE"), \ + patch.object(execution, "owned_processes", return_value=[]), \ + patch.object(scheduler, "_local_cleanup_verified", return_value=verified), \ + patch.object(scheduler, "reconcile_leases", return_value=[]), \ + patch.object(Worker, "reconcile_completed", return_value=True): + self.assertEqual(recovery.reconcile_run("r")["ready"], verified) + self.assertEqual(p.read_bytes(), before) + + def test_dead_failed_quarantine_does_not_replay_unverifiable_stage(self): + from harness.stageworker import Worker + cases = [("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [], True), + ("CANCELLED", "CANCELLED", "TERMINAL", "DEAD", [], True), + ("RUNNING", "AGENT_ERROR", "TERMINAL", "DEAD", [], False), + ("PASS", "COMPLETED", "TERMINAL", "DEAD", [], False), + ("INFRA_ERROR", "AGENT_ERROR", "PAUSED_INFRA", "DEAD", [], False), + ("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "UNKNOWN", [], False), + ("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [{"pid": 1}], False)] + for db_state, outcome, status, owner, members, allowed in cases: + with self.subTest(db_state=db_state, status=status, owner=owner, members=members): + with self.connect() as con: + db.rollout_set(con, "r", "j", 0, state=db_state) + record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123}, + "execution_state": status, "execution_outcome": outcome, + "cleanup_state": "QUARANTINED"} + p = self.rd / "execution.json" + p.write_text(json.dumps(record)) + before, rows = p.read_bytes(), self.rows() + with patch.object(execution.Execution, "reconcile", return_value=record), \ + patch.object(execution.Execution, "quarantine", return_value=record), \ + patch.object(execution, "process_status", return_value=owner), \ + patch.object(execution, "owned_processes", return_value=members), \ + patch.object(scheduler, "_local_cleanup_verified", return_value=False), \ + patch.object(scheduler, "reconcile_leases", return_value=[]), \ + patch.object(Worker, "reconcile_completed", side_effect=ValueError("scope cleanup unconfirmed")) as replay: + self.assertEqual(recovery.reconcile_run("r")["ready"], allowed) + if allowed: + replay.assert_not_called() + self.assertEqual(p.read_bytes(), before) + self.assertEqual(self.rows(), rows) + if __name__ == "__main__": import unittest