forked from longtao.wu/droboticflow
92 lines
5.3 KiB
Python
92 lines
5.3 KiB
Python
import copy
|
|
import importlib.util
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
spec = importlib.util.spec_from_file_location("readmix", Path(__file__).with_name("poll-pool-readmix.py"))
|
|
readmix = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(readmix)
|
|
|
|
|
|
class ResultValidationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.config = {"duration_seconds": 3, "warmup_seconds": 1,
|
|
"authorization_state": "initialized", "run_payload_kib": 256,
|
|
"author_history_per_reviewer": 1000}
|
|
self.result = {**self.config, "scheduled_seconds": 3, "revision": "fixed", "mode": "summary",
|
|
"errors": 0, "mongo_write_commands": 0, "samples": [{"Actor": actor, "Key": key, "Code": 200} for (actor, key), count in readmix.expected_schedule(3).items() for _ in range(count)],
|
|
"resource_samples": [{"phase": "finished"}]}
|
|
|
|
def test_long_schedule_has_all_roles_and_both_scopes(self):
|
|
schedule = readmix.expected_schedule(1800)
|
|
self.assertEqual(sum(schedule.values()), 48750)
|
|
self.assertEqual(schedule[("load-user-00", "annotation.buffer")], 600)
|
|
self.assertEqual(schedule[("load-user-60", "quality.all.available")], 75)
|
|
self.assertEqual(schedule[("load-user-72", "acceptance.project.prepackage_acceptance.reviewed")], 75)
|
|
self.assertEqual(schedule[("load-user-80", "admin.workforce")], 150)
|
|
|
|
def test_complete_matching_result_is_reusable(self):
|
|
readmix.validate_result(self.result, "fixed", "summary", self.config)
|
|
|
|
def test_121_actor_schedule_includes_exact_counts_and_all_roles(self):
|
|
config = {"actor_counts": {"annotation": 90, "quality": 18, "acceptance": 12, "admin": 1}, "claim_counts": True}
|
|
schedule = readmix.expected_schedule(1800, config)
|
|
self.assertEqual(len({actor for actor, _ in schedule}), 121)
|
|
self.assertEqual(sum(schedule.values()), 86550)
|
|
self.assertEqual(schedule[("load-user-89", "annotation.buffer")], 600)
|
|
self.assertEqual(schedule[("load-user-89", "annotation.claim-count")], 150)
|
|
self.assertEqual(schedule[("load-user-90", "quality.all.available")], 75)
|
|
self.assertEqual(schedule[("load-user-108", "acceptance.project.prepackage_acceptance.mine")], 75)
|
|
self.assertEqual(schedule[("load-user-120", "admin.workforce")], 150)
|
|
|
|
def test_scaled_results_require_correct_actor_counts_and_count_requests(self):
|
|
config = {**self.config, "actor_counts": {"annotation": 90, "quality": 18, "acceptance": 12, "admin": 1}, "claim_counts": True}
|
|
result = {**self.result, "actors": 121, "actor_counts": config["actor_counts"], "claim_counts": True,
|
|
"samples": [{"Actor": actor, "Key": key, "Code": 200} for (actor, key), n in readmix.expected_schedule(3, config).items() for _ in range(n)]}
|
|
readmix.validate_result(result, "fixed", "summary", config)
|
|
for change in ({"actors": 81}, {"actor_counts": readmix.DEFAULT_ACTORS}, {"claim_counts": False},
|
|
{"samples": [r for r in result["samples"] if r["Key"] != "annotation.claim-count"]}):
|
|
with self.subTest(change=next(iter(change))), self.assertRaises(ValueError):
|
|
readmix.validate_result({**result, **change}, "fixed", "summary", config)
|
|
|
|
def test_invalid_concurrency_cannot_label_a_smaller_run_as_capacity(self):
|
|
for invalid in ({"annotation": 0}, {"admin": 2}, {"quality": 1.5}, {"annotation": 250}):
|
|
with self.subTest(invalid=invalid), self.assertRaises(ValueError):
|
|
readmix.expected_schedule(3, {"actor_counts": {**readmix.DEFAULT_ACTORS, **invalid}})
|
|
|
|
def test_invalid_results_cannot_be_silently_reused(self):
|
|
changes = [{"revision": "other"}, {"mode": "legacy"}, {"errors": 1},
|
|
{"mongo_write_commands": 1}, {"samples": self.result["samples"][:-1]},
|
|
{"resource_samples": []}, {"author_history_per_reviewer": 0},
|
|
{"samples": [dict(row, Code=500) for row in self.result["samples"]]},
|
|
{"samples": [self.result["samples"][0]] * 145}]
|
|
for change in changes:
|
|
with self.subTest(change=next(iter(change))), self.assertRaises(ValueError):
|
|
result = copy.deepcopy(self.result)
|
|
result.update(change)
|
|
readmix.validate_result(result, "fixed", "summary", self.config)
|
|
|
|
def test_concurrent_runner_cannot_overwrite_evidence(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = Path(directory)
|
|
first = readmix.acquire_output_lock(output)
|
|
try:
|
|
with self.assertRaises(RuntimeError):
|
|
readmix.acquire_output_lock(output)
|
|
finally:
|
|
first.close()
|
|
resumed = readmix.acquire_output_lock(output)
|
|
resumed.close()
|
|
|
|
def test_checkpoint_replacement_keeps_valid_json(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory) / "status.json"
|
|
readmix.write_json(path, {"state": "running"})
|
|
readmix.write_json(path, {"state": "complete"})
|
|
self.assertEqual(readmix.json.loads(path.read_text()), {"state": "complete"})
|
|
self.assertFalse(path.with_suffix(".json.tmp").exists())
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|