data-gen-bpu/scripts/test_quota_abandon.py

121 lines
6.0 KiB
Python

"""Explicit local abandonment preserves unknown evidence and fences old work."""
import json
import time
import unittest
from harness.quota import QuotaError, quota_snapshot
from scripts import test_quota as fixtures
class Abandon(unittest.TestCase):
setUp = fixtures.Quota.setUp
provider = fixtures.Quota.provider
policy = fixtures.Quota.policy
broker = fixtures.Quota.broker
request = fixtures.Quota.request
def unknown(self):
def respond(n, body):
if n == 1:
time.sleep(.1)
return fixtures.ok(n, body)
provider = self.provider(respond)
broker = self.broker(provider, self.policy(first_byte_timeout_s=.02))
scope = broker.issue_scope('old', 'teacher', 'glm-test')
self.assertEqual(self.request(scope, 'old-g')[0], 503)
attempt = broker.snapshot()['attempts'][0]
self.assertEqual(attempt['state'], 'UNKNOWN')
return provider, broker, scope, attempt
def expire(self, broker, attempt):
now = attempt['deadline'] + .01
broker._now = lambda: now
def test_release_keeps_unknown_evidence_and_allows_same_key_probe(self):
provider, broker, old, attempt = self.unknown()
self.expire(broker, attempt)
self.assertTrue(broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit operator discard'))
snapshot = quota_snapshot(self.ledger)
self.assertEqual(snapshot['groups'][0]['inflight'], 0)
self.assertEqual(snapshot['groups'][0]['tpm_reserved'], 0)
self.assertEqual(snapshot['groups'][0]['state'], 'COOLING')
after = broker.snapshot()['attempts'][0]
self.assertEqual(after, attempt | {'state': 'ABANDONED_UNKNOWN'})
self.assertIsNone(after['ended_at'])
self.assertEqual(self.request(broker.issue_scope('new', 'teacher', 'glm-test'))[0], 200)
self.assertEqual(len(provider.requests), 2)
self.assertEqual(provider.requests[0]['authorization'], provider.requests[1]['authorization'])
self.assertEqual(broker.snapshot()['attempts'][-1]['state_at_send'], 'HALF_OPEN')
self.assertEqual(self.request(old, 'old-g')[0], 409)
self.assertEqual(self.request(broker.issue_scope('old', 'teacher', 'glm-test'), 'replacement')[0], 409)
self.assertEqual(len(provider.requests), 2)
self.assertFalse(broker.resume_readiness('old')['eligible'])
def test_late_headers_and_completion_cannot_resurrect(self):
_, broker, _, attempt = self.unknown()
self.expire(broker, attempt)
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
before = broker.snapshot()
broker._response_headers('shared', 'old-g', attempt['attempt_id'], broker.groups['shared'], 429, {'Retry-After':'900'})
broker._finish('shared', 'old-g', attempt['attempt_id'], broker.groups['shared'], 'COMPLETE', known_end=True, status=200, usage=3)
self.assertEqual(before, broker.snapshot())
self.assertEqual(broker.execution_status('old')['generations'][0]['state'], 'ABANDONED_UNKNOWN')
def test_idempotence_and_restart_do_not_restore_reservation(self):
provider, broker, _, attempt = self.unknown()
self.expire(broker, attempt)
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
before = broker.snapshot()
self.assertFalse(broker.abandon_unknown(attempt['attempt_id'], operator_reason='again'))
self.assertEqual(before, broker.snapshot())
broker.close()
reopened = self.broker(provider, self.policy(first_byte_timeout_s=.02))
reopened.reconcile()
self.assertEqual(reopened.snapshot()['attempts'][0]['state'], 'ABANDONED_UNKNOWN')
self.assertEqual(quota_snapshot(self.ledger)['groups'][0]['inflight'], 0)
self.assertEqual(self.request(reopened.issue_scope('old', 'teacher', 'glm-test'))[0], 409)
self.assertEqual(len(provider.requests), 1)
def test_requires_explicit_reason_expiry_and_unknown_state(self):
_, broker, _, attempt = self.unknown()
before = broker.snapshot()
with self.assertRaises(ValueError):
broker.abandon_unknown(attempt['attempt_id'], operator_reason=' ')
with self.assertRaises(QuotaError):
broker.abandon_unknown(attempt['attempt_id'], operator_reason='too soon')
with self.assertRaises(QuotaError):
broker.abandon_unknown('absent', operator_reason='explicit')
self.assertEqual(before, broker.snapshot())
self.expire(broker, attempt)
with broker._db() as con:
con.execute("UPDATE attempts SET state='FINISHED', ended_at=1")
before = broker.snapshot()
with self.assertRaises(QuotaError):
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
self.assertEqual(before, broker.snapshot())
def test_unrelated_auth_pause_is_not_cleared(self):
_, broker, _, attempt = self.unknown()
self.expire(broker, attempt)
with broker._db() as con:
con.execute("UPDATE quota_groups SET state='PAUSED', reason='UPSTREAM_AUTH_OR_HARD_QUOTA'")
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
group = broker.snapshot()['groups'][0]
self.assertEqual((group['state'], group['reason']), ('PAUSED', 'UPSTREAM_AUTH_OR_HARD_QUOTA'))
def test_other_unknown_still_holds_group(self):
_, broker, _, attempt = self.unknown()
self.expire(broker, attempt)
with broker._db() as con:
columns = list(attempt)
other = attempt | {'attempt_id':'another-unknown'}
con.execute('INSERT INTO attempts (' + ','.join(columns) + ') VALUES (' + ','.join('?' for _ in columns) + ')', [other[k] for k in columns])
broker.abandon_unknown(attempt['attempt_id'], operator_reason='explicit')
group = quota_snapshot(self.ledger)['groups'][0]
self.assertEqual(group['inflight'], 1)
self.assertEqual(group['reason'], 'REQUEST_OUTCOME_UNKNOWN')
if __name__ == '__main__':
unittest.main()