droboticflow-new/scripts/local/test_poll_pool_readmix.py

92 lines
5.3 KiB
Python

import copy
import importlib.util
from pathlib import Path
import tempfile
import unittest
spec = importlib.util.spec_from_file_location("readmix", Path(__file__).with_name("poll-pool-readmix.py"))
readmix = importlib.util.module_from_spec(spec)
spec.loader.exec_module(readmix)
class ResultValidationTests(unittest.TestCase):
def setUp(self):
self.config = {"duration_seconds": 3, "warmup_seconds": 1,
"authorization_state": "initialized", "run_payload_kib": 256,
"author_history_per_reviewer": 1000}
self.result = {**self.config, "scheduled_seconds": 3, "revision": "fixed", "mode": "summary",
"errors": 0, "mongo_write_commands": 0, "samples": [{"Actor": actor, "Key": key, "Code": 200} for (actor, key), count in readmix.expected_schedule(3).items() for _ in range(count)],
"resource_samples": [{"phase": "finished"}]}
def test_long_schedule_has_all_roles_and_both_scopes(self):
schedule = readmix.expected_schedule(1800)
self.assertEqual(sum(schedule.values()), 48750)
self.assertEqual(schedule[("load-user-00", "annotation.buffer")], 600)
self.assertEqual(schedule[("load-user-60", "quality.all.available")], 75)
self.assertEqual(schedule[("load-user-72", "acceptance.project.prepackage_acceptance.reviewed")], 75)
self.assertEqual(schedule[("load-user-80", "admin.workforce")], 150)
def test_complete_matching_result_is_reusable(self):
readmix.validate_result(self.result, "fixed", "summary", self.config)
def test_121_actor_schedule_includes_exact_counts_and_all_roles(self):
config = {"actor_counts": {"annotation": 90, "quality": 18, "acceptance": 12, "admin": 1}, "claim_counts": True}
schedule = readmix.expected_schedule(1800, config)
self.assertEqual(len({actor for actor, _ in schedule}), 121)
self.assertEqual(sum(schedule.values()), 86550)
self.assertEqual(schedule[("load-user-89", "annotation.buffer")], 600)
self.assertEqual(schedule[("load-user-89", "annotation.claim-count")], 150)
self.assertEqual(schedule[("load-user-90", "quality.all.available")], 75)
self.assertEqual(schedule[("load-user-108", "acceptance.project.prepackage_acceptance.mine")], 75)
self.assertEqual(schedule[("load-user-120", "admin.workforce")], 150)
def test_scaled_results_require_correct_actor_counts_and_count_requests(self):
config = {**self.config, "actor_counts": {"annotation": 90, "quality": 18, "acceptance": 12, "admin": 1}, "claim_counts": True}
result = {**self.result, "actors": 121, "actor_counts": config["actor_counts"], "claim_counts": True,
"samples": [{"Actor": actor, "Key": key, "Code": 200} for (actor, key), n in readmix.expected_schedule(3, config).items() for _ in range(n)]}
readmix.validate_result(result, "fixed", "summary", config)
for change in ({"actors": 81}, {"actor_counts": readmix.DEFAULT_ACTORS}, {"claim_counts": False},
{"samples": [r for r in result["samples"] if r["Key"] != "annotation.claim-count"]}):
with self.subTest(change=next(iter(change))), self.assertRaises(ValueError):
readmix.validate_result({**result, **change}, "fixed", "summary", config)
def test_invalid_concurrency_cannot_label_a_smaller_run_as_capacity(self):
for invalid in ({"annotation": 0}, {"admin": 2}, {"quality": 1.5}, {"annotation": 250}):
with self.subTest(invalid=invalid), self.assertRaises(ValueError):
readmix.expected_schedule(3, {"actor_counts": {**readmix.DEFAULT_ACTORS, **invalid}})
def test_invalid_results_cannot_be_silently_reused(self):
changes = [{"revision": "other"}, {"mode": "legacy"}, {"errors": 1},
{"mongo_write_commands": 1}, {"samples": self.result["samples"][:-1]},
{"resource_samples": []}, {"author_history_per_reviewer": 0},
{"samples": [dict(row, Code=500) for row in self.result["samples"]]},
{"samples": [self.result["samples"][0]] * 145}]
for change in changes:
with self.subTest(change=next(iter(change))), self.assertRaises(ValueError):
result = copy.deepcopy(self.result)
result.update(change)
readmix.validate_result(result, "fixed", "summary", self.config)
def test_concurrent_runner_cannot_overwrite_evidence(self):
with tempfile.TemporaryDirectory() as directory:
output = Path(directory)
first = readmix.acquire_output_lock(output)
try:
with self.assertRaises(RuntimeError):
readmix.acquire_output_lock(output)
finally:
first.close()
resumed = readmix.acquire_output_lock(output)
resumed.close()
def test_checkpoint_replacement_keeps_valid_json(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "status.json"
readmix.write_json(path, {"state": "running"})
readmix.write_json(path, {"state": "complete"})
self.assertEqual(readmix.json.loads(path.read_text()), {"state": "complete"})
self.assertFalse(path.with_suffix(".json.tmp").exists())
if __name__ == "__main__":
unittest.main()