kb-retrieval-handoff/evowiki/scripts/run_self_evolution.py

114 lines
4.9 KiB
Python

#!/usr/bin/env python3
"""End-to-end self-evolution driver on rdk_qa_dataset.
Story: the forum produces QAs over time. evowiki ingests them as a chronological
STREAM of "generations"; after each generation it consolidates, evaluates recall
on a fixed holdout, and lets the feedback loop re-tune retrieval. Watching
fitness (recall@k) across generations shows the knowledge base *evolving*.
Usage:
python scripts/run_self_evolution.py --generations 4 --holdout 120 \
--distiller heuristic
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from evowiki.config import Config, load_env_files # noqa: E402
from evowiki.distill import build_distiller # noqa: E402
from evowiki.evaluate import fitness_sweep, recall_at_k # noqa: E402
from evowiki.evolve import evolve # noqa: E402
from evowiki.export import export_wiki # noqa: E402
from evowiki.govern import Governor # noqa: E402
from evowiki.store import Store # noqa: E402
def load_records(path: Path) -> list[dict]:
with open(path, encoding="utf-8") as fh:
return [json.loads(l) for l in fh if l.strip()]
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--distiller", choices=["heuristic", "mem0"], default="heuristic")
ap.add_argument("--generations", type=int, default=4)
ap.add_argument("--holdout", type=int, default=120)
ap.add_argument("--db", default=None)
ap.add_argument("--clean", action="store_true")
args = ap.parse_args()
load_env_files()
cfg = Config()
if args.db:
cfg.db_path = Path(args.db)
if args.clean and cfg.db_path.exists():
cfg.db_path.unlink()
cfg.ensure_dirs()
records = load_records(cfg.dataset)
# chronological stream; hold out the most recent N as the eval set
records.sort(key=lambda r: r.get("created", ""))
holdout = records[-args.holdout:]
stream = records[:-args.holdout]
per_gen = max(1, len(stream) // args.generations)
store = Store(cfg.db_path)
governor = Governor(store, cfg)
distiller = build_distiller(args.distiller, cfg)
print(f"[plan] stream={len(stream)} holdout={len(holdout)} "
f"generations={args.generations} distiller={distiller.name}")
print(f"[plan] db={cfg.db_path}\n")
trajectory = []
for gen in range(1, args.generations + 1):
batch = stream[(gen - 1) * per_gen: gen * per_gen] if gen < args.generations \
else stream[(gen - 1) * per_gen:]
stats = evolve(cfg, batch, distiller, store, governor, consolidate_every=10**9)
fb = fitness_sweep(store, holdout, cfg, generation=gen)
cfg.entity_boost = fb["best_entity_boost"] # feedback -> retrieval gene
cfg.retention_floor = fb["best_retention_floor"] # feedback -> governance gene
cfg.graph_weight = fb["best_graph_weight"] # feedback -> structure gene
final = recall_at_k(store, holdout, entity_boost=cfg.entity_boost,
min_effective_conf=cfg.retention_floor,
graph_weight=cfg.graph_weight, graph_max_hops=cfg.graph_max_hops)
trajectory.append({
"gen": gen, "new_facts": stats["ops"].get("ADD", 0),
"updates": stats["ops"].get("UPDATE", 0),
"conflicts": stats["ops"].get("CONFLICT", 0),
"promoted": stats.get("promoted", {}),
"decay": stats.get("decay", {}),
"layers": stats.get("layers", {}),
"recall@1": final["recall"][1], "recall@5": final["recall"][5],
"scorable": final["n_scorable"], "entity_boost": cfg.entity_boost,
"retention_floor": cfg.retention_floor, "graph_weight": cfg.graph_weight,
})
t = trajectory[-1]
prom = "+".join(f"{k[:4]}:{v}" for k, v in sorted(t["promoted"].items())) or "-"
dec = t["decay"]
print(f"[gen {gen}] +facts={t['new_facts']:>3} upd={t['updates']:>2} "
f"conflict={t['conflicts']:>2} | recall@1={t['recall@1']:.3f} "
f"recall@5={t['recall@5']:.3f} (scorable={t['scorable']}) "
f"| boost->{cfg.entity_boost} floor->{cfg.retention_floor} "
f"graph->{cfg.graph_weight} "
f"| promote[{prom}] stale:{dec.get('stale', 0)} arch:{dec.get('archived', 0)}",
flush=True)
print(f" layers={t['layers']}", flush=True)
out = export_wiki(store, cfg.wiki_dir)
print(f"\n[export] {out['pages']} topic pages -> {out['wiki_dir']}")
print("\n=== evolution trajectory ===")
print(json.dumps(trajectory, ensure_ascii=False, indent=2))
print("\n=== final store ===")
print(json.dumps(store.stats(), ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())