79 lines
2.4 KiB
Python
79 lines
2.4 KiB
Python
"""Offline tests for controlled live-acceptance gates and metrics."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from ekp_api.live_acceptance import (
|
|
AcceptanceThresholds,
|
|
LiveAcceptanceError,
|
|
_build_report,
|
|
_load_golden_queries,
|
|
_require_safe_live_settings,
|
|
)
|
|
from ekp_core.config import Settings
|
|
|
|
|
|
def test_live_golden_dataset_contains_only_expected_synthetic_sources() -> None:
|
|
records = _load_golden_queries(Path("fixtures/evaluation/golden.jsonl"))
|
|
|
|
assert len(records) == 32
|
|
assert {record.expected_document for record in records} == {
|
|
"sample-product-a-policy.md",
|
|
"sample-travel-policy.md",
|
|
}
|
|
assert len({record.query_id for record in records}) == len(records)
|
|
|
|
|
|
def test_live_metrics_apply_binary_source_rank_and_latency_gates() -> None:
|
|
report = _build_report(
|
|
ranks=[1, 1, 2, None],
|
|
latencies_ms=[100.0, 200.0, 300.0, 400.0],
|
|
thresholds=AcceptanceThresholds(
|
|
min_recall_at_1=0.5,
|
|
min_recall_at_5=0.75,
|
|
min_mrr=0.6,
|
|
min_ndcg_at_5=0.6,
|
|
max_p95_ms=400.0,
|
|
),
|
|
)
|
|
|
|
assert report.query_count == 4
|
|
assert report.recall_at_1 == 0.5
|
|
assert report.recall_at_3 == 0.75
|
|
assert report.recall_at_5 == 0.75
|
|
assert report.mrr == pytest.approx(0.625)
|
|
assert report.latency_p50_ms == 200.0
|
|
assert report.latency_p95_ms == 400.0
|
|
assert report.passed is True
|
|
|
|
|
|
def test_live_acceptance_requires_all_safety_acknowledgements(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
settings = Settings(
|
|
app_env="test",
|
|
bos_key_prefix="live-acceptance/unit-test",
|
|
)
|
|
|
|
with pytest.raises(LiveAcceptanceError, match="safety flags"):
|
|
_require_safe_live_settings(settings)
|
|
|
|
monkeypatch.setenv("EKP_RUN_LIVE_TESTS", "1")
|
|
monkeypatch.setenv("EKP_CREDENTIALS_AUTHORIZED", "1")
|
|
monkeypatch.setenv("EKP_LIVE_RESOURCE_ACK", "1")
|
|
_require_safe_live_settings(settings)
|
|
|
|
|
|
def test_live_acceptance_rejects_non_dedicated_bos_prefix(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
isolated_settings_environment: None,
|
|
) -> None:
|
|
monkeypatch.setenv("EKP_RUN_LIVE_TESTS", "1")
|
|
monkeypatch.setenv("EKP_CREDENTIALS_AUTHORIZED", "1")
|
|
monkeypatch.setenv("EKP_LIVE_RESOURCE_ACK", "1")
|
|
|
|
with pytest.raises(LiveAcceptanceError, match="BOS_KEY_PREFIX"):
|
|
_require_safe_live_settings(Settings(app_env="test", bos_key_prefix="shared"))
|