forked from d-cloud/d-wiki
470 lines
16 KiB
Python
470 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import re
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .config import KnowledgeBaseConfig
|
|
from .metadata import (
|
|
AliasGroup,
|
|
QueryEntry,
|
|
frontmatter,
|
|
load_alias_groups,
|
|
load_query_entries,
|
|
normalize,
|
|
query_matches_entry,
|
|
)
|
|
from .dreams import load_dream_query_entries
|
|
from .tokenize import tokenize
|
|
|
|
SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt")
|
|
MAX_BLOCK_CHARS = 520
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Chunk:
|
|
title: str
|
|
wikilink: str
|
|
source_path: Path
|
|
page_type: str
|
|
heading: str
|
|
text: str
|
|
tokens: tuple[str, ...]
|
|
index: int
|
|
|
|
|
|
def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]:
|
|
return {
|
|
"status": "ok",
|
|
"config": str(config_path),
|
|
"knowledge_bases": {
|
|
name: {
|
|
"root": str(kb.root),
|
|
"source_type": kb.source_type,
|
|
"exists": kb.root.exists(),
|
|
"aliases": str(kb.aliases_path) if kb.aliases_path else None,
|
|
"queries": str(kb.queries_dir) if kb.queries_dir else None,
|
|
}
|
|
for name, kb in bases.items()
|
|
},
|
|
}
|
|
|
|
|
|
def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]:
|
|
chunks = load_chunks(kb)
|
|
index_dir = kb.root / ".d_wiki_index"
|
|
index_dir.mkdir(exist_ok=True)
|
|
payload = {
|
|
"kb": kb.name,
|
|
"root": str(kb.root),
|
|
"source_type": kb.source_type,
|
|
"compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None,
|
|
"chunks": len(chunks),
|
|
"pages": len({chunk.wikilink for chunk in chunks}),
|
|
"indexed_at": datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
(index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return {"status": "indexed", **payload}
|
|
|
|
|
|
def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]:
|
|
aliases = load_alias_groups(kb.aliases_path)
|
|
query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)]
|
|
matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)]
|
|
expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries)
|
|
chunks = load_chunks(kb)
|
|
ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries)
|
|
selected = ranked[: max(1, min(max_results, 8))]
|
|
context_pack = build_context_pack(query, selected, matched_aliases, matched_entries)
|
|
return {
|
|
"status": "found" if selected else "no_match",
|
|
"query": query,
|
|
"knowledge_base": kb.name,
|
|
"backend": "lightweight_bm25",
|
|
"quality": quality(selected),
|
|
"sources": [source_summary(item[1], item[0]) for item in selected],
|
|
"context_pack": context_pack,
|
|
}
|
|
|
|
|
|
def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]:
|
|
chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink]
|
|
if not chunks:
|
|
return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink}
|
|
first = chunks[0]
|
|
return {
|
|
"status": "found",
|
|
"knowledge_base": kb.name,
|
|
"id": wikilink,
|
|
"title": first.title,
|
|
"source_path": str(first.source_path),
|
|
"blocks": [
|
|
{"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"}
|
|
for chunk in chunks
|
|
],
|
|
}
|
|
|
|
|
|
def expand_query(
|
|
query: str,
|
|
aliases: list[AliasGroup],
|
|
entries: list[QueryEntry],
|
|
) -> tuple[list[str], list[AliasGroup]]:
|
|
terms = tokenize(query)
|
|
normalized_query = normalize(query)
|
|
matched: list[AliasGroup] = []
|
|
for group in aliases:
|
|
if any(normalize(term) in normalized_query for term in group.terms):
|
|
matched.append(group)
|
|
for term in group.terms:
|
|
terms.extend(tokenize(term))
|
|
for entry in entries:
|
|
terms.extend(tokenize(entry.title))
|
|
for alias in entry.aliases:
|
|
terms.extend(tokenize(alias))
|
|
return terms, matched
|
|
|
|
|
|
def rank_chunks(
|
|
chunks: list[Chunk],
|
|
terms: list[str],
|
|
aliases: list[AliasGroup],
|
|
entries: list[QueryEntry],
|
|
) -> list[tuple[float, Chunk]]:
|
|
if not chunks or not terms:
|
|
return []
|
|
term_counts = [Counter(chunk.tokens) for chunk in chunks]
|
|
doc_freq: Counter[str] = Counter()
|
|
for counts in term_counts:
|
|
for term in counts:
|
|
doc_freq[term] += 1
|
|
avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks))
|
|
query_counts = Counter(terms)
|
|
ranked: list[tuple[float, Chunk]] = []
|
|
related = {item for group in aliases for item in group.related}
|
|
primary = {item for entry in entries for item in entry.primary_sources}
|
|
supporting = {item for entry in entries for item in entry.supporting_sources}
|
|
for chunk, counts in zip(chunks, term_counts):
|
|
score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts)
|
|
if any(wikilink_matches(chunk.wikilink, item) for item in related):
|
|
score += 8.0
|
|
if any(wikilink_matches(chunk.wikilink, item) for item in primary):
|
|
score += 16.0
|
|
if any(wikilink_matches(chunk.wikilink, item) for item in supporting):
|
|
score += 5.0
|
|
score += heading_score_adjustment(chunk.heading)
|
|
if score > 0:
|
|
ranked.append((round(score, 3), chunk))
|
|
ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index))
|
|
return ranked
|
|
|
|
|
|
def heading_score_adjustment(heading: str) -> float:
|
|
if heading == "Retrieval Hints":
|
|
return -80.0
|
|
if heading == "Intent Model":
|
|
return -40.0
|
|
if heading == "Answer Blueprint":
|
|
return -25.0
|
|
if heading == "Evidence Graph":
|
|
return 3.0
|
|
if heading == "Key Evidence":
|
|
return 5.0
|
|
return 0.0
|
|
|
|
|
|
def bm25_score(
|
|
counts: Counter[str],
|
|
doc_len: int,
|
|
doc_freq: Counter[str],
|
|
doc_count: int,
|
|
avg_len: float,
|
|
query_counts: Counter[str],
|
|
) -> float:
|
|
k1 = 1.5
|
|
b = 0.75
|
|
score = 0.0
|
|
for term, qf in query_counts.items():
|
|
tf = counts.get(term, 0)
|
|
if tf == 0:
|
|
continue
|
|
idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5))
|
|
denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1)))
|
|
score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3)
|
|
return score
|
|
|
|
|
|
def build_context_pack(
|
|
query: str,
|
|
selected: list[tuple[float, Chunk]],
|
|
aliases: list[AliasGroup],
|
|
entries: list[QueryEntry],
|
|
) -> dict[str, Any]:
|
|
intent = answer_intent(query, aliases, entries)
|
|
grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list)
|
|
for item in selected:
|
|
grouped[item[1].wikilink].append(item)
|
|
source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1]))
|
|
entry_primary = {item for entry in entries for item in entry.primary_sources}
|
|
primary_sources = [
|
|
source_context(wikilink, chunks)
|
|
for wikilink, chunks in source_items
|
|
if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary)
|
|
]
|
|
if not primary_sources:
|
|
primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]]
|
|
supporting_sources = [
|
|
source_context(wikilink, chunks, blocks_limit=1)
|
|
for wikilink, chunks in source_items
|
|
if wikilink not in {source["wikilink"] for source in primary_sources}
|
|
][:3]
|
|
coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete")
|
|
return {
|
|
"answer_intent": intent,
|
|
"coverage": coverage,
|
|
"answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.",
|
|
"primary_sources": primary_sources,
|
|
"supporting_sources": supporting_sources,
|
|
"answer_requirements": answer_requirements(intent, entries),
|
|
}
|
|
|
|
|
|
def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]:
|
|
top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit]
|
|
chunk = top[0][1]
|
|
return {
|
|
"title": chunk.title,
|
|
"wikilink": wikilink,
|
|
"source_path": str(chunk.source_path),
|
|
"reason": "Matched query terms and local metadata.",
|
|
"blocks": [
|
|
{
|
|
"block_id": f"{block.wikilink}#{block.index}",
|
|
"heading": block.heading,
|
|
"text": trim_block(block.text),
|
|
"score": score,
|
|
}
|
|
for score, block in top
|
|
],
|
|
}
|
|
|
|
|
|
def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str:
|
|
for entry in entries:
|
|
if entry.intent:
|
|
return entry.intent
|
|
for group in aliases:
|
|
if group.intent:
|
|
return group.intent
|
|
if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")):
|
|
return "procedure"
|
|
return "fact"
|
|
|
|
|
|
def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]:
|
|
requirements = unique([req for entry in entries for req in entry.answer_requirements])
|
|
if requirements:
|
|
return requirements
|
|
if intent == "procedure":
|
|
return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"]
|
|
return ["直接回答问题", "引用命中的来源"]
|
|
|
|
|
|
def unique(values: list[str]) -> list[str]:
|
|
seen: set[str] = set()
|
|
output: list[str] = []
|
|
for value in values:
|
|
if value in seen:
|
|
continue
|
|
seen.add(value)
|
|
output.append(value)
|
|
return output
|
|
|
|
|
|
def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]:
|
|
top = selected[0][0] if selected else 0.0
|
|
return {
|
|
"confidence": "high" if top >= 8.0 else ("low" if selected else "none"),
|
|
"high_confidence": top >= 8.0,
|
|
"top_score": top,
|
|
"high_confidence_score": 8.0,
|
|
}
|
|
|
|
|
|
def wikilink_matches(candidate: str, expected: str) -> bool:
|
|
return candidate == expected or candidate == f"concepts/{expected}"
|
|
|
|
|
|
def source_summary(chunk: Chunk, score: float) -> dict[str, Any]:
|
|
return {
|
|
"title": chunk.title,
|
|
"wikilink": chunk.wikilink,
|
|
"source_path": str(chunk.source_path),
|
|
"heading": chunk.heading,
|
|
"snippet": trim_block(chunk.text, 320),
|
|
"score": score,
|
|
}
|
|
|
|
|
|
def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
|
|
root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root
|
|
paths = source_paths(root)
|
|
chunks: list[Chunk] = []
|
|
for path in paths:
|
|
chunks.extend(chunks_for_path(path, root))
|
|
return chunks
|
|
|
|
|
|
def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
|
|
chunks: list[Chunk] = []
|
|
for path in source_paths(kb.root):
|
|
chunks.extend(chunks_for_path(path, kb.root))
|
|
return chunks
|
|
|
|
|
|
def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path:
|
|
return kb.root / ".d_wiki" / "wiki"
|
|
|
|
|
|
def source_paths(root: Path) -> list[Path]:
|
|
if (root / "_sources").is_dir():
|
|
return sorted(
|
|
path
|
|
for path in (root / "_sources").rglob("*.txt")
|
|
if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES)
|
|
)
|
|
generated = {"SCHEMA.md", "index.md", "log.md"}
|
|
generated_dirs = {".d_wiki", ".d_wiki_index"}
|
|
is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki"
|
|
return sorted(
|
|
path
|
|
for path in root.rglob("*.md")
|
|
if path.is_file()
|
|
and path.name not in generated
|
|
and (is_compiled_root or not any(part in generated_dirs for part in path.parts))
|
|
)
|
|
|
|
|
|
def chunks_for_path(path: Path, root: Path) -> list[Chunk]:
|
|
raw = path.read_text(encoding="utf-8", errors="ignore")
|
|
meta, body = frontmatter(raw)
|
|
title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title())
|
|
metadata_text = metadata_terms(meta)
|
|
wikilink = wikilink_for(path, root)
|
|
sections = split_sections(body, title)
|
|
chunks: list[Chunk] = []
|
|
for index, (heading, text) in enumerate(sections):
|
|
if heading in {"Source Map", "Related"}:
|
|
continue
|
|
if not text.strip():
|
|
continue
|
|
field_text = " ".join([title, heading, wikilink, metadata_text, text])
|
|
chunks.append(
|
|
Chunk(
|
|
title=title,
|
|
wikilink=wikilink,
|
|
source_path=path.resolve(),
|
|
page_type="sphinx_source" if "_sources" in path.parts else "markdown",
|
|
heading=heading,
|
|
text=text.strip(),
|
|
tokens=tuple(tokenize(field_text)),
|
|
index=index,
|
|
)
|
|
)
|
|
return chunks
|
|
|
|
|
|
def metadata_terms(meta: dict[str, Any]) -> str:
|
|
values: list[str] = []
|
|
for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"):
|
|
raw = meta.get(key)
|
|
if isinstance(raw, str):
|
|
values.append(raw)
|
|
elif isinstance(raw, list):
|
|
values.extend(str(item) for item in raw if str(item).strip())
|
|
return " ".join(values)
|
|
|
|
|
|
def split_sections(body: str, title: str) -> list[tuple[str, str]]:
|
|
heading = title
|
|
buffer: list[str] = []
|
|
sections: list[tuple[str, str]] = []
|
|
in_fence = False
|
|
for line in body.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("```"):
|
|
in_fence = not in_fence
|
|
buffer.append(line)
|
|
continue
|
|
if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)):
|
|
if buffer:
|
|
sections.extend(paragraphs(heading, "\n".join(buffer)))
|
|
buffer = []
|
|
heading = match.group(1).strip() or title
|
|
continue
|
|
buffer.append(line)
|
|
if buffer:
|
|
sections.extend(paragraphs(heading, "\n".join(buffer)))
|
|
return sections or [(title, body)]
|
|
|
|
|
|
def paragraphs(heading: str, text: str) -> list[tuple[str, str]]:
|
|
blocks: list[tuple[str, str]] = []
|
|
for part in re.split(r"\n\s*\n", text):
|
|
cleaned = clean_block(part)
|
|
if cleaned:
|
|
blocks.append((heading, cleaned))
|
|
return blocks
|
|
|
|
|
|
def clean_block(text: str) -> str:
|
|
stripped = text.strip()
|
|
if not stripped:
|
|
return ""
|
|
if re.fullmatch(r"[-*_]{3,}", stripped):
|
|
return ""
|
|
compact = " ".join(stripped.split())
|
|
if "📷" in compact and len(compact) <= 80:
|
|
return ""
|
|
if compact.startswith("> 📷") and len(compact) <= 120:
|
|
return ""
|
|
return stripped
|
|
|
|
|
|
def first_heading(body: str) -> str | None:
|
|
in_fence = False
|
|
for line in body.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("```"):
|
|
in_fence = not in_fence
|
|
continue
|
|
if in_fence:
|
|
continue
|
|
if match := re.match(r"^#{1,6}\s+(.+)$", line):
|
|
return match.group(1).strip()
|
|
return None
|
|
|
|
|
|
def wikilink_for(path: Path, root: Path) -> str:
|
|
try:
|
|
relative = path.relative_to(root)
|
|
except ValueError:
|
|
return path.stem
|
|
if relative.parts and relative.parts[0] == "_sources":
|
|
relative = Path(*relative.parts[1:])
|
|
value = relative.as_posix()
|
|
for suffix in SPHINX_SOURCE_SUFFIXES:
|
|
if value.endswith(suffix):
|
|
return value[: -len(suffix)]
|
|
return value[:-3] if value.endswith(".md") else path.stem
|
|
|
|
|
|
def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str:
|
|
compact = " ".join(text.split())
|
|
return compact if len(compact) <= limit else compact[:limit].rstrip()
|