wdb/d_wiki/search.py
2026-05-28 19:02:16 +08:00

470 lines
16 KiB
Python

from __future__ import annotations
import json
import math
import re
from collections import Counter, defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .metadata import (
AliasGroup,
QueryEntry,
frontmatter,
load_alias_groups,
load_query_entries,
normalize,
query_matches_entry,
)
from .dreams import load_dream_query_entries
from .tokenize import tokenize
SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt")
MAX_BLOCK_CHARS = 520
@dataclass(frozen=True)
class Chunk:
title: str
wikilink: str
source_path: Path
page_type: str
heading: str
text: str
tokens: tuple[str, ...]
index: int
def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]:
return {
"status": "ok",
"config": str(config_path),
"knowledge_bases": {
name: {
"root": str(kb.root),
"source_type": kb.source_type,
"exists": kb.root.exists(),
"aliases": str(kb.aliases_path) if kb.aliases_path else None,
"queries": str(kb.queries_dir) if kb.queries_dir else None,
}
for name, kb in bases.items()
},
}
def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]:
chunks = load_chunks(kb)
index_dir = kb.root / ".d_wiki_index"
index_dir.mkdir(exist_ok=True)
payload = {
"kb": kb.name,
"root": str(kb.root),
"source_type": kb.source_type,
"compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None,
"chunks": len(chunks),
"pages": len({chunk.wikilink for chunk in chunks}),
"indexed_at": datetime.now(timezone.utc).isoformat(),
}
(index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return {"status": "indexed", **payload}
def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]:
aliases = load_alias_groups(kb.aliases_path)
query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)]
matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)]
expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries)
chunks = load_chunks(kb)
ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries)
selected = ranked[: max(1, min(max_results, 8))]
context_pack = build_context_pack(query, selected, matched_aliases, matched_entries)
return {
"status": "found" if selected else "no_match",
"query": query,
"knowledge_base": kb.name,
"backend": "lightweight_bm25",
"quality": quality(selected),
"sources": [source_summary(item[1], item[0]) for item in selected],
"context_pack": context_pack,
}
def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]:
chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink]
if not chunks:
return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink}
first = chunks[0]
return {
"status": "found",
"knowledge_base": kb.name,
"id": wikilink,
"title": first.title,
"source_path": str(first.source_path),
"blocks": [
{"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"}
for chunk in chunks
],
}
def expand_query(
query: str,
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> tuple[list[str], list[AliasGroup]]:
terms = tokenize(query)
normalized_query = normalize(query)
matched: list[AliasGroup] = []
for group in aliases:
if any(normalize(term) in normalized_query for term in group.terms):
matched.append(group)
for term in group.terms:
terms.extend(tokenize(term))
for entry in entries:
terms.extend(tokenize(entry.title))
for alias in entry.aliases:
terms.extend(tokenize(alias))
return terms, matched
def rank_chunks(
chunks: list[Chunk],
terms: list[str],
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> list[tuple[float, Chunk]]:
if not chunks or not terms:
return []
term_counts = [Counter(chunk.tokens) for chunk in chunks]
doc_freq: Counter[str] = Counter()
for counts in term_counts:
for term in counts:
doc_freq[term] += 1
avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks))
query_counts = Counter(terms)
ranked: list[tuple[float, Chunk]] = []
related = {item for group in aliases for item in group.related}
primary = {item for entry in entries for item in entry.primary_sources}
supporting = {item for entry in entries for item in entry.supporting_sources}
for chunk, counts in zip(chunks, term_counts):
score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts)
if any(wikilink_matches(chunk.wikilink, item) for item in related):
score += 8.0
if any(wikilink_matches(chunk.wikilink, item) for item in primary):
score += 16.0
if any(wikilink_matches(chunk.wikilink, item) for item in supporting):
score += 5.0
score += heading_score_adjustment(chunk.heading)
if score > 0:
ranked.append((round(score, 3), chunk))
ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index))
return ranked
def heading_score_adjustment(heading: str) -> float:
if heading == "Retrieval Hints":
return -80.0
if heading == "Intent Model":
return -40.0
if heading == "Answer Blueprint":
return -25.0
if heading == "Evidence Graph":
return 3.0
if heading == "Key Evidence":
return 5.0
return 0.0
def bm25_score(
counts: Counter[str],
doc_len: int,
doc_freq: Counter[str],
doc_count: int,
avg_len: float,
query_counts: Counter[str],
) -> float:
k1 = 1.5
b = 0.75
score = 0.0
for term, qf in query_counts.items():
tf = counts.get(term, 0)
if tf == 0:
continue
idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5))
denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1)))
score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3)
return score
def build_context_pack(
query: str,
selected: list[tuple[float, Chunk]],
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> dict[str, Any]:
intent = answer_intent(query, aliases, entries)
grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list)
for item in selected:
grouped[item[1].wikilink].append(item)
source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1]))
entry_primary = {item for entry in entries for item in entry.primary_sources}
primary_sources = [
source_context(wikilink, chunks)
for wikilink, chunks in source_items
if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary)
]
if not primary_sources:
primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]]
supporting_sources = [
source_context(wikilink, chunks, blocks_limit=1)
for wikilink, chunks in source_items
if wikilink not in {source["wikilink"] for source in primary_sources}
][:3]
coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete")
return {
"answer_intent": intent,
"coverage": coverage,
"answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.",
"primary_sources": primary_sources,
"supporting_sources": supporting_sources,
"answer_requirements": answer_requirements(intent, entries),
}
def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]:
top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit]
chunk = top[0][1]
return {
"title": chunk.title,
"wikilink": wikilink,
"source_path": str(chunk.source_path),
"reason": "Matched query terms and local metadata.",
"blocks": [
{
"block_id": f"{block.wikilink}#{block.index}",
"heading": block.heading,
"text": trim_block(block.text),
"score": score,
}
for score, block in top
],
}
def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str:
for entry in entries:
if entry.intent:
return entry.intent
for group in aliases:
if group.intent:
return group.intent
if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")):
return "procedure"
return "fact"
def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]:
requirements = unique([req for entry in entries for req in entry.answer_requirements])
if requirements:
return requirements
if intent == "procedure":
return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"]
return ["直接回答问题", "引用命中的来源"]
def unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
if value in seen:
continue
seen.add(value)
output.append(value)
return output
def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]:
top = selected[0][0] if selected else 0.0
return {
"confidence": "high" if top >= 8.0 else ("low" if selected else "none"),
"high_confidence": top >= 8.0,
"top_score": top,
"high_confidence_score": 8.0,
}
def wikilink_matches(candidate: str, expected: str) -> bool:
return candidate == expected or candidate == f"concepts/{expected}"
def source_summary(chunk: Chunk, score: float) -> dict[str, Any]:
return {
"title": chunk.title,
"wikilink": chunk.wikilink,
"source_path": str(chunk.source_path),
"heading": chunk.heading,
"snippet": trim_block(chunk.text, 320),
"score": score,
}
def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root
paths = source_paths(root)
chunks: list[Chunk] = []
for path in paths:
chunks.extend(chunks_for_path(path, root))
return chunks
def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
chunks: list[Chunk] = []
for path in source_paths(kb.root):
chunks.extend(chunks_for_path(path, kb.root))
return chunks
def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path:
return kb.root / ".d_wiki" / "wiki"
def source_paths(root: Path) -> list[Path]:
if (root / "_sources").is_dir():
return sorted(
path
for path in (root / "_sources").rglob("*.txt")
if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES)
)
generated = {"SCHEMA.md", "index.md", "log.md"}
generated_dirs = {".d_wiki", ".d_wiki_index"}
is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki"
return sorted(
path
for path in root.rglob("*.md")
if path.is_file()
and path.name not in generated
and (is_compiled_root or not any(part in generated_dirs for part in path.parts))
)
def chunks_for_path(path: Path, root: Path) -> list[Chunk]:
raw = path.read_text(encoding="utf-8", errors="ignore")
meta, body = frontmatter(raw)
title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title())
metadata_text = metadata_terms(meta)
wikilink = wikilink_for(path, root)
sections = split_sections(body, title)
chunks: list[Chunk] = []
for index, (heading, text) in enumerate(sections):
if heading in {"Source Map", "Related"}:
continue
if not text.strip():
continue
field_text = " ".join([title, heading, wikilink, metadata_text, text])
chunks.append(
Chunk(
title=title,
wikilink=wikilink,
source_path=path.resolve(),
page_type="sphinx_source" if "_sources" in path.parts else "markdown",
heading=heading,
text=text.strip(),
tokens=tuple(tokenize(field_text)),
index=index,
)
)
return chunks
def metadata_terms(meta: dict[str, Any]) -> str:
values: list[str] = []
for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"):
raw = meta.get(key)
if isinstance(raw, str):
values.append(raw)
elif isinstance(raw, list):
values.extend(str(item) for item in raw if str(item).strip())
return " ".join(values)
def split_sections(body: str, title: str) -> list[tuple[str, str]]:
heading = title
buffer: list[str] = []
sections: list[tuple[str, str]] = []
in_fence = False
for line in body.splitlines():
stripped = line.strip()
if stripped.startswith("```"):
in_fence = not in_fence
buffer.append(line)
continue
if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)):
if buffer:
sections.extend(paragraphs(heading, "\n".join(buffer)))
buffer = []
heading = match.group(1).strip() or title
continue
buffer.append(line)
if buffer:
sections.extend(paragraphs(heading, "\n".join(buffer)))
return sections or [(title, body)]
def paragraphs(heading: str, text: str) -> list[tuple[str, str]]:
blocks: list[tuple[str, str]] = []
for part in re.split(r"\n\s*\n", text):
cleaned = clean_block(part)
if cleaned:
blocks.append((heading, cleaned))
return blocks
def clean_block(text: str) -> str:
stripped = text.strip()
if not stripped:
return ""
if re.fullmatch(r"[-*_]{3,}", stripped):
return ""
compact = " ".join(stripped.split())
if "📷" in compact and len(compact) <= 80:
return ""
if compact.startswith("> 📷") and len(compact) <= 120:
return ""
return stripped
def first_heading(body: str) -> str | None:
in_fence = False
for line in body.splitlines():
stripped = line.strip()
if stripped.startswith("```"):
in_fence = not in_fence
continue
if in_fence:
continue
if match := re.match(r"^#{1,6}\s+(.+)$", line):
return match.group(1).strip()
return None
def wikilink_for(path: Path, root: Path) -> str:
try:
relative = path.relative_to(root)
except ValueError:
return path.stem
if relative.parts and relative.parts[0] == "_sources":
relative = Path(*relative.parts[1:])
value = relative.as_posix()
for suffix in SPHINX_SOURCE_SUFFIXES:
if value.endswith(suffix):
return value[: -len(suffix)]
return value[:-3] if value.endswith(".md") else path.stem
def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str:
compact = " ".join(text.split())
return compact if len(compact) <= limit else compact[:limit].rstrip()