diff --git a/harness/quota.py b/harness/quota.py index 2c54fc4..4450095 100644 --- a/harness/quota.py +++ b/harness/quota.py @@ -579,12 +579,27 @@ class Broker: con.execute("UPDATE attempts SET state='UNKNOWN',reason=COALESCE(reason,'REQUEST_OUTCOME_UNKNOWN') WHERE attempt_id=?", (attempt['attempt_id'],)) con.execute("UPDATE generations SET state='PAUSED',reason=COALESCE(reason,'REQUEST_OUTCOME_UNKNOWN'),updated_at=? WHERE generation_id=?", (self._now(), attempt['generation_id'])) stale += [row[0] for row in con.execute("SELECT attempt_id FROM attempts WHERE group_id=? AND state='UNKNOWN' AND ended_at IS NULL AND deadline<=?", (group, self._now() - policy.window_s))] + if self._probe_if_settled(con, group): + # An innocent request was still in flight when the unknown one was resolved. + self._event(con, group, 'UNKNOWN_SETTLED_PROBE_ALLOWED') for attempt_id in stale: # outside the group flock: abandon_unknown takes it itself try: self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed') except QuotaError: pass # another broker settled it first + def _probe_if_settled(self, con, group): + """End an unknown-outcome pause with one probe once nothing in the group is open. + Every resolver and reconcile() call it, so a request that finishes after the + unknown one was resolved cannot leave the group paused forever.""" + remaining = con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'", (group,)).fetchone()[0] + state = con.execute('SELECT * FROM quota_groups WHERE group_id=?', (group,)).fetchone() + if remaining or state['reason'] != 'REQUEST_OUTCOME_UNKNOWN' or state['state'] not in ('RECONCILING', 'PAUSED'): + return False + now = self._now() + con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group)) + return True + def abandon_unknown(self, attempt_id: str, *, operator_reason: str): """Explicit operator release of local capacity, never provider-end proof. @@ -612,11 +627,7 @@ class Broker: con.execute("UPDATE attempts SET state='ABANDONED_UNKNOWN' WHERE attempt_id=?", (attempt_id,)) con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation)) con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,)) - remaining = con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'", (group,)).fetchone()[0] - state = con.execute('SELECT * FROM quota_groups WHERE group_id=?', (group,)).fetchone() - probe_allowed = not remaining and state['reason'] == 'REQUEST_OUTCOME_UNKNOWN' and state['state'] in ('RECONCILING', 'PAUSED') - if probe_allowed: - con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group)) + probe_allowed = self._probe_if_settled(con, group) self._event(con, group, 'OPERATOR_ABANDONED_UNKNOWN', generation, attempt_id=attempt_id, operator_reason=operator_reason, released_at=now, provider_end_confirmed=False, @@ -643,12 +654,9 @@ class Broker: if changed!=1: raise QuotaError('NOT_RECONCILABLE',409) self._event(con, row['group_id'], 'END_CONFIRMED', row['generation_id'], attempt_id=attempt_id, receipt_id=receipt_id) - remaining=con.execute("SELECT COUNT(*) FROM attempts WHERE group_id=? AND state <> 'ABANDONED_UNKNOWN' AND ended_at IS NULL",(row['group_id'],)).fetchone()[0] - group=con.execute('SELECT * FROM quota_groups WHERE group_id=?',(row['group_id'],)).fetchone() - if not remaining and group['reason']=='REQUEST_OUTCOME_UNKNOWN' and group['state'] in ('RECONCILING','PAUSED'): - # The trusted end receipt authorizes one future probe. The old - # generation remains paused and can never replay automatically. - con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?",(self._now(),self._now(),row['group_id'])) + # The trusted end receipt authorizes one future probe. The old + # generation remains paused and can never replay automatically. + if self._probe_if_settled(con, row['group_id']): self._event(con,row['group_id'],'RECONCILED_PROBE_ALLOWED',receipt_id=receipt_id) def _resume_readiness(self, con, execution_id, now): diff --git a/scripts/test_quota_auto_abandon.py b/scripts/test_quota_auto_abandon.py index 86fdced..fbef106 100644 --- a/scripts/test_quota_auto_abandon.py +++ b/scripts/test_quota_auto_abandon.py @@ -42,6 +42,26 @@ class AutoAbandon(unittest.TestCase): self.assertEqual(len([e for e in broker.snapshot()['quota_events'] if e['event'] == 'OPERATOR_ABANDONED_UNKNOWN']), 1) + def test_request_finishing_after_the_abandon_still_releases_the_group(self): + # 2026-10-07: the unknown attempt was abandoned while an innocent one was + # in flight, so no probe was allowed and the group stayed PAUSED for 15h. + _, broker, _, attempt = self.unknown() + self.assertEqual(self.request(broker.issue_scope('victim', 'teacher', 'glm-test'))[0], 503) + with broker._db() as con: + row = dict(con.execute('SELECT * FROM attempts WHERE attempt_id=?', (attempt['attempt_id'],)).fetchone()) + row.update(attempt_id='inflight', state='SENT', deadline=attempt['deadline'] + 3600, ended_at=None) + con.execute(f"INSERT INTO attempts ({','.join(row)}) VALUES ({','.join('?' * len(row))})", tuple(row.values())) + broker._now = lambda: attempt['deadline'] + broker.groups['shared'].window_s + .01 + broker.reconcile() + state = lambda: broker.snapshot()['groups'][0]['state'] + self.assertEqual(state(), 'PAUSED') # something is still open: no probe yet + with broker._db() as con: + con.execute("UPDATE attempts SET state='FINISHED', ended_at=? WHERE attempt_id='inflight'", (broker._now(),)) + broker.reconcile() + self.assertEqual(state(), 'COOLING') + self.assertTrue(broker.resume_readiness('victim')['eligible']) + self.assertFalse(broker.resume_readiness('old')['eligible']) + if __name__ == '__main__': unittest.main()