knowledge-platform/tests/unit/test_live_acceptance.py

79 lines
2.4 KiB
Python

"""Offline tests for controlled live-acceptance gates and metrics."""
from pathlib import Path
import pytest
from ekp_api.live_acceptance import (
AcceptanceThresholds,
LiveAcceptanceError,
_build_report,
_load_golden_queries,
_require_safe_live_settings,
)
from ekp_core.config import Settings
def test_live_golden_dataset_contains_only_expected_synthetic_sources() -> None:
records = _load_golden_queries(Path("fixtures/evaluation/golden.jsonl"))
assert len(records) == 32
assert {record.expected_document for record in records} == {
"sample-product-a-policy.md",
"sample-travel-policy.md",
}
assert len({record.query_id for record in records}) == len(records)
def test_live_metrics_apply_binary_source_rank_and_latency_gates() -> None:
report = _build_report(
ranks=[1, 1, 2, None],
latencies_ms=[100.0, 200.0, 300.0, 400.0],
thresholds=AcceptanceThresholds(
min_recall_at_1=0.5,
min_recall_at_5=0.75,
min_mrr=0.6,
min_ndcg_at_5=0.6,
max_p95_ms=400.0,
),
)
assert report.query_count == 4
assert report.recall_at_1 == 0.5
assert report.recall_at_3 == 0.75
assert report.recall_at_5 == 0.75
assert report.mrr == pytest.approx(0.625)
assert report.latency_p50_ms == 200.0
assert report.latency_p95_ms == 400.0
assert report.passed is True
def test_live_acceptance_requires_all_safety_acknowledgements(
monkeypatch: pytest.MonkeyPatch,
isolated_settings_environment: None,
) -> None:
settings = Settings(
app_env="test",
bos_key_prefix="live-acceptance/unit-test",
)
with pytest.raises(LiveAcceptanceError, match="safety flags"):
_require_safe_live_settings(settings)
monkeypatch.setenv("EKP_RUN_LIVE_TESTS", "1")
monkeypatch.setenv("EKP_CREDENTIALS_AUTHORIZED", "1")
monkeypatch.setenv("EKP_LIVE_RESOURCE_ACK", "1")
_require_safe_live_settings(settings)
def test_live_acceptance_rejects_non_dedicated_bos_prefix(
monkeypatch: pytest.MonkeyPatch,
isolated_settings_environment: None,
) -> None:
monkeypatch.setenv("EKP_RUN_LIVE_TESTS", "1")
monkeypatch.setenv("EKP_CREDENTIALS_AUTHORIZED", "1")
monkeypatch.setenv("EKP_LIVE_RESOURCE_ACK", "1")
with pytest.raises(LiveAcceptanceError, match="BOS_KEY_PREFIX"):
_require_safe_live_settings(Settings(app_env="test", bos_key_prefix="shared"))