fix: let dead, quarantined failed executions stop blocking unrelated work
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
10663a800f
commit
2eb29caa83
@ -168,6 +168,16 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict:
|
||||
if (rd / "execution.json").exists():
|
||||
owner = execution.Execution(rd)
|
||||
record = owner.snapshot()
|
||||
# Failed history stays failed and isolated; do not require its unknown
|
||||
# stage receipt to become trustworthy before scheduling unrelated work.
|
||||
if (record['execution_state'] == 'TERMINAL' and record['cleanup_state'] == 'QUARANTINED'
|
||||
and row['state'] in {'INFRA_ERROR', 'CANCELLED'}
|
||||
and record.get('execution_outcome') in {'AGENT_ERROR', 'INFRA_ERROR', 'CANCELLED'}
|
||||
and execution.process_status(record.get('owner', {})) == 'DEAD'
|
||||
and execution.owned_processes(record) == []):
|
||||
reports.append({'job_id': row['job_id'], 'kk': row['kk'], 'execution': record,
|
||||
'ready': True, 'reason': 'FAILED_EXECUTION_ISOLATED'})
|
||||
continue
|
||||
if ((execution.process_status(record.get('owner', {})) == 'DEAD'
|
||||
or record['execution_state'] == 'TERMINAL')
|
||||
and execution.owned_processes(record) == []):
|
||||
@ -216,7 +226,8 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict:
|
||||
if (not ready and record["execution_state"] == "TERMINAL"
|
||||
and ((row["state"] == "INFRA_ERROR"
|
||||
and execution.process_status(record.get("owner", {})) == "DEAD")
|
||||
or (settled_state == record.get("execution_outcome")
|
||||
or ((settled_state == record.get("execution_outcome")
|
||||
or settled_state == 'INFRA_ERROR' and record.get('execution_outcome') == 'AGENT_ERROR')
|
||||
and settled_state in {"INFRA_ERROR", "CANCELLED", "LOST", "AGENT_ERROR",
|
||||
"DEADLINE_EXCEEDED", "BUDGET_EXHAUSTED"}
|
||||
and scheduler._local_cleanup_verified(record)))
|
||||
|
||||
@ -185,6 +185,59 @@ class Recovery(unittest.TestCase):
|
||||
self.assertEqual(report["executions"], [{"job_id": "j", "kk": 0, "ready": False,
|
||||
"reason": "LEGACY_EXECUTION_OWNERSHIP_UNKNOWN"}])
|
||||
|
||||
def test_shared_owner_agent_error_uses_recorded_infra_state(self):
|
||||
from harness.stageworker import Worker
|
||||
with self.connect() as con:
|
||||
db.rollout_set(con, "r", "j", 0, state="INFRA_ERROR")
|
||||
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
|
||||
"execution_state": "TERMINAL", "execution_outcome": "AGENT_ERROR",
|
||||
"cleanup_state": "QUARANTINED"}
|
||||
p = self.rd / "execution.json"
|
||||
p.write_text(json.dumps(record))
|
||||
before = p.read_bytes()
|
||||
for verified in (True, False):
|
||||
with self.subTest(local_cleanup_verified=verified), \
|
||||
patch.object(execution.Execution, "reconcile", return_value=record), \
|
||||
patch.object(execution, "process_status", return_value="ALIVE"), \
|
||||
patch.object(execution, "owned_processes", return_value=[]), \
|
||||
patch.object(scheduler, "_local_cleanup_verified", return_value=verified), \
|
||||
patch.object(scheduler, "reconcile_leases", return_value=[]), \
|
||||
patch.object(Worker, "reconcile_completed", return_value=True):
|
||||
self.assertEqual(recovery.reconcile_run("r")["ready"], verified)
|
||||
self.assertEqual(p.read_bytes(), before)
|
||||
|
||||
def test_dead_failed_quarantine_does_not_replay_unverifiable_stage(self):
|
||||
from harness.stageworker import Worker
|
||||
cases = [("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [], True),
|
||||
("CANCELLED", "CANCELLED", "TERMINAL", "DEAD", [], True),
|
||||
("RUNNING", "AGENT_ERROR", "TERMINAL", "DEAD", [], False),
|
||||
("PASS", "COMPLETED", "TERMINAL", "DEAD", [], False),
|
||||
("INFRA_ERROR", "AGENT_ERROR", "PAUSED_INFRA", "DEAD", [], False),
|
||||
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "UNKNOWN", [], False),
|
||||
("INFRA_ERROR", "AGENT_ERROR", "TERMINAL", "DEAD", [{"pid": 1}], False)]
|
||||
for db_state, outcome, status, owner, members, allowed in cases:
|
||||
with self.subTest(db_state=db_state, status=status, owner=owner, members=members):
|
||||
with self.connect() as con:
|
||||
db.rollout_set(con, "r", "j", 0, state=db_state)
|
||||
record = {"execution_id": "failed", "epoch": 1, "owner": {"pid": 123},
|
||||
"execution_state": status, "execution_outcome": outcome,
|
||||
"cleanup_state": "QUARANTINED"}
|
||||
p = self.rd / "execution.json"
|
||||
p.write_text(json.dumps(record))
|
||||
before, rows = p.read_bytes(), self.rows()
|
||||
with patch.object(execution.Execution, "reconcile", return_value=record), \
|
||||
patch.object(execution.Execution, "quarantine", return_value=record), \
|
||||
patch.object(execution, "process_status", return_value=owner), \
|
||||
patch.object(execution, "owned_processes", return_value=members), \
|
||||
patch.object(scheduler, "_local_cleanup_verified", return_value=False), \
|
||||
patch.object(scheduler, "reconcile_leases", return_value=[]), \
|
||||
patch.object(Worker, "reconcile_completed", side_effect=ValueError("scope cleanup unconfirmed")) as replay:
|
||||
self.assertEqual(recovery.reconcile_run("r")["ready"], allowed)
|
||||
if allowed:
|
||||
replay.assert_not_called()
|
||||
self.assertEqual(p.read_bytes(), before)
|
||||
self.assertEqual(self.rows(), rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import unittest
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user