"""A stage cut off with its collector ends like an executor exit: free compile, told to the agent, board quarantines.""" from contextlib import ExitStack import json from pathlib import Path import tempfile import time import unittest from unittest.mock import patch from harness import db, execution, paths, stagerun from harness.stageworker import Worker from scripts import test_execution class SettleInterrupted(unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup) root = Path(temp.name) self.rd = root / 'runs/r/j/rollouts/00' (self.rd / 'handles').mkdir(parents=True) (self.rd.parent.parent / 'job.json').write_text(json.dumps({'run_id': 'r', 'job_id': 'j'})) stack = ExitStack(); self.addCleanup(stack.close) original = db.connect stack.enter_context(patch.object(db, 'connect', lambda *a, **kw: original(root / 'jobs.sqlite', **kw))) stack.enter_context(patch.object(paths, 'STATE', root)) with db.connect() as con: db.create_run(con, 'r', {}) db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher') self.execution = execution.Execution(self.rd) self.execution.start(test_execution.policy()) dead = dict(host='fixture', boot_id='fixture', pid=999999, proc_start_ticks=1) stack.enter_context(patch.object(execution, 'process_status', return_value='DEAD')) self.dead = dead def operation(self, hid, stage): queue = self.rd / 'stages/_queue' / hid queue.mkdir(parents=True) (self.rd / 'handles' / hid).mkdir() (self.rd / 'handles' / hid / 'request.json').write_text(json.dumps({'stage': stage})) (queue / 'request.json').write_text(json.dumps({'stage': stage, 'config': None, 'recipe': None})) with self.execution.mutation() as state: op = dict(operation_id=hid, payload_sha256='sha', state='DISPATCHED', execution_id=state['execution_id'], epoch=state['epoch'], received_at=time.time(), executor=self.dead) execution.atomic_json(queue / 'operation.json', op) state['operations'][hid] = op self.execution._save(state, 'OPERATION_DISPATCHED', operation_id=hid) execution.atomic_json(queue / 'status.json', dict(state='RUNNING', attempt=0, started_at=time.time(), operation_id=hid, payload_sha256='sha', execution_id=op['execution_id'], epoch=op['epoch'])) with db.connect() as con: db.stage_start(con, 'r', 'j', stage, attempt=0, rollout=0, lane='long') return queue def test_interrupted_compile_is_free_and_the_agent_is_told(self): queue = self.operation('h1', '05-compile') self.assertTrue(Worker(self.rd, execution=self.execution).settle_interrupted()) result = json.loads((queue / 'status.json').read_text()) self.assertEqual((result['state'], result['result'], result['error_class']), ('DONE', 'FAIL', 'STAGE_EXECUTOR_EXIT')) self.assertIn('不计编译次数', result['note']) told = json.loads((self.rd / 'handles/h1/status.json').read_text()) self.assertEqual(told['error_class'], 'STAGE_EXECUTOR_EXIT') state = self.execution.snapshot() self.assertEqual(state['operations']['h1']['state'], 'DONE') self.assertNotEqual(state['cleanup_state'], 'QUARANTINED') with db.connect() as con: row = con.execute("SELECT state,error_class FROM stages WHERE stage='05-compile'").fetchone() self.assertEqual(tuple(row), ('FAIL', 'STAGE_EXECUTOR_EXIT')) self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 0) def test_board_step_without_a_clean_board_receipt_still_quarantines(self): self.operation('h2', '07-board') self.assertTrue(Worker(self.rd, execution=self.execution).settle_interrupted()) state = self.execution.snapshot() self.assertEqual((state['cleanup_state'], state['stop_reason']), ('QUARANTINED', 'BOARD_CLEANUP_UNCONFIRMED')) def test_a_live_executor_is_left_alone(self): self.operation('h3', '05-compile') with patch.object(execution, 'process_status', return_value='ALIVE'): self.assertFalse(Worker(self.rd, execution=self.execution).settle_interrupted()) self.assertEqual(self.execution.snapshot()['operations']['h3']['state'], 'DISPATCHED') if __name__ == '__main__': unittest.main()