114 lines
4.9 KiB
Python
114 lines
4.9 KiB
Python
#!/usr/bin/env python3
|
|
"""End-to-end self-evolution driver on rdk_qa_dataset.
|
|
|
|
Story: the forum produces QAs over time. evowiki ingests them as a chronological
|
|
STREAM of "generations"; after each generation it consolidates, evaluates recall
|
|
on a fixed holdout, and lets the feedback loop re-tune retrieval. Watching
|
|
fitness (recall@k) across generations shows the knowledge base *evolving*.
|
|
|
|
Usage:
|
|
python scripts/run_self_evolution.py --generations 4 --holdout 120 \
|
|
--distiller heuristic
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from evowiki.config import Config, load_env_files # noqa: E402
|
|
from evowiki.distill import build_distiller # noqa: E402
|
|
from evowiki.evaluate import fitness_sweep, recall_at_k # noqa: E402
|
|
from evowiki.evolve import evolve # noqa: E402
|
|
from evowiki.export import export_wiki # noqa: E402
|
|
from evowiki.govern import Governor # noqa: E402
|
|
from evowiki.store import Store # noqa: E402
|
|
|
|
|
|
def load_records(path: Path) -> list[dict]:
|
|
with open(path, encoding="utf-8") as fh:
|
|
return [json.loads(l) for l in fh if l.strip()]
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--distiller", choices=["heuristic", "mem0"], default="heuristic")
|
|
ap.add_argument("--generations", type=int, default=4)
|
|
ap.add_argument("--holdout", type=int, default=120)
|
|
ap.add_argument("--db", default=None)
|
|
ap.add_argument("--clean", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
load_env_files()
|
|
cfg = Config()
|
|
if args.db:
|
|
cfg.db_path = Path(args.db)
|
|
if args.clean and cfg.db_path.exists():
|
|
cfg.db_path.unlink()
|
|
cfg.ensure_dirs()
|
|
|
|
records = load_records(cfg.dataset)
|
|
# chronological stream; hold out the most recent N as the eval set
|
|
records.sort(key=lambda r: r.get("created", ""))
|
|
holdout = records[-args.holdout:]
|
|
stream = records[:-args.holdout]
|
|
per_gen = max(1, len(stream) // args.generations)
|
|
|
|
store = Store(cfg.db_path)
|
|
governor = Governor(store, cfg)
|
|
distiller = build_distiller(args.distiller, cfg)
|
|
|
|
print(f"[plan] stream={len(stream)} holdout={len(holdout)} "
|
|
f"generations={args.generations} distiller={distiller.name}")
|
|
print(f"[plan] db={cfg.db_path}\n")
|
|
|
|
trajectory = []
|
|
for gen in range(1, args.generations + 1):
|
|
batch = stream[(gen - 1) * per_gen: gen * per_gen] if gen < args.generations \
|
|
else stream[(gen - 1) * per_gen:]
|
|
stats = evolve(cfg, batch, distiller, store, governor, consolidate_every=10**9)
|
|
fb = fitness_sweep(store, holdout, cfg, generation=gen)
|
|
cfg.entity_boost = fb["best_entity_boost"] # feedback -> retrieval gene
|
|
cfg.retention_floor = fb["best_retention_floor"] # feedback -> governance gene
|
|
cfg.graph_weight = fb["best_graph_weight"] # feedback -> structure gene
|
|
final = recall_at_k(store, holdout, entity_boost=cfg.entity_boost,
|
|
min_effective_conf=cfg.retention_floor,
|
|
graph_weight=cfg.graph_weight, graph_max_hops=cfg.graph_max_hops)
|
|
trajectory.append({
|
|
"gen": gen, "new_facts": stats["ops"].get("ADD", 0),
|
|
"updates": stats["ops"].get("UPDATE", 0),
|
|
"conflicts": stats["ops"].get("CONFLICT", 0),
|
|
"promoted": stats.get("promoted", {}),
|
|
"decay": stats.get("decay", {}),
|
|
"layers": stats.get("layers", {}),
|
|
"recall@1": final["recall"][1], "recall@5": final["recall"][5],
|
|
"scorable": final["n_scorable"], "entity_boost": cfg.entity_boost,
|
|
"retention_floor": cfg.retention_floor, "graph_weight": cfg.graph_weight,
|
|
})
|
|
t = trajectory[-1]
|
|
prom = "+".join(f"{k[:4]}:{v}" for k, v in sorted(t["promoted"].items())) or "-"
|
|
dec = t["decay"]
|
|
print(f"[gen {gen}] +facts={t['new_facts']:>3} upd={t['updates']:>2} "
|
|
f"conflict={t['conflicts']:>2} | recall@1={t['recall@1']:.3f} "
|
|
f"recall@5={t['recall@5']:.3f} (scorable={t['scorable']}) "
|
|
f"| boost->{cfg.entity_boost} floor->{cfg.retention_floor} "
|
|
f"graph->{cfg.graph_weight} "
|
|
f"| promote[{prom}] stale:{dec.get('stale', 0)} arch:{dec.get('archived', 0)}",
|
|
flush=True)
|
|
print(f" layers={t['layers']}", flush=True)
|
|
|
|
out = export_wiki(store, cfg.wiki_dir)
|
|
print(f"\n[export] {out['pages']} topic pages -> {out['wiki_dir']}")
|
|
print("\n=== evolution trajectory ===")
|
|
print(json.dumps(trajectory, ensure_ascii=False, indent=2))
|
|
print("\n=== final store ===")
|
|
print(json.dumps(store.stats(), ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|