wdb/d_wiki/compile.py
2026-05-28 19:02:16 +08:00

591 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

from __future__ import annotations
import re
import shutil
import json
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .llm import LlmPageSynthesis, LlmPageSynthesizer
from .metadata import load_alias_groups, load_query_entries
from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block
from .tokenize import tokenize
@dataclass(frozen=True)
class CompiledPage:
wikilink: str
title: str
path: Path
@dataclass(frozen=True)
class _EvidenceProfile:
tokens: frozenset[str]
phrases: tuple[str, ...]
def compile_kb(
kb: KnowledgeBaseConfig,
*,
force: bool = False,
llm_mode: str = "off",
llm_scope: str = "queries",
) -> dict[str, Any]:
chunks = load_source_chunks(kb)
if not chunks:
raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS")
synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None
if llm_mode == "require" and synthesizer is None:
raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED")
wiki_root = compiled_wiki_root(kb)
concepts_dir = wiki_root / "concepts"
raw_dir = wiki_root / "raw"
if force:
shutil.rmtree(concepts_dir, ignore_errors=True)
concepts_dir.mkdir(parents=True, exist_ok=True)
raw_dir.mkdir(parents=True, exist_ok=True)
pages: list[CompiledPage] = []
grouped = _group_chunks(chunks)
query_entries = load_query_entries(kb.queries_dir)
alias_groups = load_alias_groups(kb.aliases_path)
query_by_source = {
source: entry
for entry in query_entries
for source in entry.primary_sources
}
aliases_by_source = {
related: group
for group in alias_groups
for related in group.related
}
for wikilink, page_chunks in sorted(grouped.items()):
page = _compile_page(
kb,
wikilink,
page_chunks,
query_entry=query_by_source.get(wikilink),
alias_group=aliases_by_source.get(wikilink),
synthesizer=(
synthesizer
if llm_scope == "all" or query_by_source.get(wikilink) is not None
else None
),
require_llm=llm_mode == "require",
)
if force or not page.path.exists():
page.path.write_text(page.content, encoding="utf-8")
pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path))
_write_schema(wiki_root, kb)
_write_index(wiki_root, pages)
_append_log(wiki_root, kb, len(chunks), len(pages))
return {
"status": "compiled",
"kb": kb.name,
"source_root": str(kb.root),
"wiki_root": str(wiki_root),
"source_chunks": len(chunks),
"pages": len(pages),
"llm_mode": llm_mode,
"llm_scope": llm_scope,
"llm_enabled": synthesizer is not None,
"compiled_at": datetime.now(timezone.utc).isoformat(),
}
@dataclass(frozen=True)
class _PageDraft:
title: str
path: Path
content: str
def _compile_page(
kb: KnowledgeBaseConfig,
wikilink: str,
chunks: list[Chunk],
*,
query_entry: Any | None,
alias_group: Any | None,
synthesizer: LlmPageSynthesizer | None,
require_llm: bool,
) -> _PageDraft:
first = chunks[0]
title = _title_for(first, query_entry)
slug = _slug(wikilink)
path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md"
evidence_profile = _evidence_profile(title, query_entry, alias_group)
tags = _tags(kb, first, alias_group)
aliases = _aliases(title, query_entry, alias_group)
source_ref = first.wikilink
source_path = str(first.source_path)
key_blocks = _key_blocks(chunks, evidence_profile)
intent = _intent(title, query_entry, alias_group)
summary = _summary(chunks, title, evidence_profile, intent=intent)
requirements = _answer_requirements(intent, query_entry)
synthesis = _llm_synthesis(
synthesizer,
title=title,
wikilink=wikilink,
query_entry=query_entry,
alias_group=alias_group,
key_blocks=key_blocks,
require_llm=require_llm,
)
if synthesis is not None:
summary = synthesis.summary
retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group)
blueprint = (
list(synthesis.answer_blueprint)
if synthesis and synthesis.answer_blueprint
else _deterministic_answer_blueprint(intent, requirements, key_blocks)
)
links = _related_links(query_entry, alias_group)
content = "\n".join(
[
"---",
f"llmWikiVersion: \"1\"",
f"conceptId: {_yaml_scalar(slug)}",
f"title: {_yaml_scalar(title)}",
f"summary: {_yaml_scalar(summary)}",
f"sources: {_yaml_list([source_ref])}",
f"source_paths: {_yaml_list([source_path])}",
f"kind: concept",
f"tags: {_yaml_list(tags)}",
f"aliases: {_yaml_list(aliases)}",
f"intents: {_yaml_list([intent])}",
f"retrieval_hints: {_yaml_list(retrieval_hints)}",
f"answer_requirements: {_yaml_list(requirements)}",
f"confidence: {synthesis.confidence if synthesis else 'medium'}",
f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}",
"---",
"",
f"# {title}",
"",
summary,
"",
"## Intent Model",
"",
f"- intent: {intent}",
f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}",
f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}",
"",
"## Retrieval Hints",
"",
*[f"- {hint}" for hint in retrieval_hints],
"",
"## Answer Blueprint",
"",
*[f"- {item}" for item in blueprint],
"",
"## Synthesized Evidence" if synthesis and synthesis.key_evidence else "",
"" if synthesis and synthesis.key_evidence else "",
*[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())],
"" if synthesis and synthesis.key_evidence else "",
"## Evidence Graph",
"",
*_evidence_graph_lines(key_blocks),
"",
"## Key Evidence",
"",
*[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks],
"",
"## Source Map",
"",
f"- 原始来源:`{source_ref}`",
f"- 文件路径:`{source_path}`",
"",
"## Related",
"",
*[f"- [[{link}]]" for link in links],
"",
]
)
return _PageDraft(title=title, path=path, content=content)
def _llm_synthesis(
synthesizer: LlmPageSynthesizer | None,
*,
title: str,
wikilink: str,
query_entry: Any | None,
alias_group: Any | None,
key_blocks: list[Chunk],
require_llm: bool,
) -> LlmPageSynthesis | None:
if synthesizer is None:
return None
prompt = _llm_compile_prompt(
title=title,
wikilink=wikilink,
query_entry=query_entry,
alias_group=alias_group,
key_blocks=key_blocks,
)
try:
return synthesizer.synthesize(prompt)
except Exception:
if require_llm:
raise
return None
def _llm_compile_prompt(
*,
title: str,
wikilink: str,
query_entry: Any | None,
alias_group: Any | None,
key_blocks: list[Chunk],
) -> str:
aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ())
terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ())
requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ())
intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None)
evidence = [
{
"block_id": f"{block.wikilink}#{block.index}",
"heading": block.heading,
"text": trim_block(block.text, 700),
}
for block in key_blocks[:8]
]
return "\n".join(
[
"Compile a d-wiki concept page from the local evidence below.",
"Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.",
"For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.",
"Return only JSON with this shape:",
'{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}',
"",
json_dumps(
{
"title": title,
"wikilink": wikilink,
"intent": intent,
"aliases": aliases,
"terms": terms,
"answer_requirements": requirements,
"evidence": evidence,
}
),
]
)
def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]:
if synthesis is None or not synthesis.answer_blueprint:
return []
return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""]
def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str:
if query_entry is not None and getattr(query_entry, "intent", ""):
return str(query_entry.intent)
if alias_group is not None and getattr(alias_group, "intent", ""):
return str(alias_group.intent)
lowered = title.lower()
if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")):
return "procedure"
return "fact"
def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]:
if query_entry is not None:
requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()]
if requirements:
return _unique(requirements)
if intent == "procedure":
return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"]
return ["直接给出结论", "保留关键数值和单位", "引用证据来源"]
def _retrieval_hints(
title: str,
aliases: list[str],
tags: list[str],
requirements: list[str],
alias_group: Any | None,
) -> list[str]:
values = [title, *aliases, *tags, *requirements]
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
hints: list[str] = []
for value in values:
hints.append(value)
hints.extend(tokenize(value))
return _unique(hints)[:32]
def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]:
if requirements:
return requirements[:6]
if intent == "procedure":
headings = _unique([block.heading for block in key_blocks if block.heading])
if headings:
return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]]
return ["按前提、配置、执行、验证的顺序回答"]
return ["先给出直接结论", "补充关键参数、限制和来源"]
def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]:
lines: list[str] = []
for block in key_blocks[:8]:
claim = trim_block(block.text, 180)
if not claim:
continue
lines.append(f"- claim: {claim}")
lines.append(f" source: {block.wikilink}#{block.index}")
if block.heading:
lines.append(f" heading: {block.heading}")
return lines or ["- claim: 暂无可用证据", " source: unknown"]
def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]:
grouped: dict[str, list[Chunk]] = {}
for chunk in chunks:
grouped.setdefault(chunk.wikilink, []).append(chunk)
return grouped
def _title_for(chunk: Chunk, query_entry: Any | None) -> str:
if query_entry is not None and getattr(query_entry, "title", ""):
return str(query_entry.title)
return chunk.title
def _summary(
chunks: list[Chunk],
title: str,
evidence_profile: _EvidenceProfile | None = None,
*,
intent: str = "fact",
) -> str:
if intent == "procedure":
for chunk in chunks[:12]:
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text):
return f"{title}:{text}"
ranked = _rank_evidence(chunks, evidence_profile)
for score, chunk in ranked:
if score <= 0:
continue
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text):
return f"{title}:{text}"
for chunk in chunks:
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text) and len(text) >= 20:
return f"{title}:{text}"
return f"{title} 的编译知识页,来源于本地文档。"
def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]:
preferred = [
chunk
for chunk in chunks
if _is_evidence_candidate(chunk.text)
]
ranked = _rank_evidence(preferred, evidence_profile)
if not ranked or ranked[0][0] <= 0:
return (preferred or chunks)[:6]
selected: list[Chunk] = []
seen: set[tuple[str, int]] = set()
for score, chunk in ranked:
if score <= 0:
continue
key = (chunk.wikilink, chunk.index)
if key in seen:
continue
selected.append(chunk)
seen.add(key)
if len(selected) >= 6:
return selected
for chunk in preferred:
key = (chunk.wikilink, chunk.index)
if key in seen:
continue
selected.append(chunk)
seen.add(key)
if len(selected) >= 6:
break
return selected
def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile:
values = [title]
if query_entry is not None:
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ()))
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
phrases = tuple(_unique([_normalize_phrase(value) for value in values if value]))
tokens: set[str] = set()
for value in values:
tokens.update(tokenize(str(value)))
return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases)
def _rank_evidence(
chunks: list[Chunk],
evidence_profile: _EvidenceProfile | None,
) -> list[tuple[float, Chunk]]:
if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases):
return []
ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks]
ranked.sort(key=lambda item: (-item[0], item[1].index))
return ranked
def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float:
text = trim_block(chunk.text, 1000)
normalized = _normalize_phrase(text)
chunk_tokens = set(chunk.tokens) | set(tokenize(text))
score = 0.0
score += len(chunk_tokens & evidence_profile.tokens) * 2.0
for phrase in evidence_profile.phrases:
if len(phrase) >= 2 and phrase in normalized:
score += 5.0
if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized):
score += 8.0
if "算力" in evidence_profile.tokens and "tops" in normalized:
score += 6.0
if "bpu" in evidence_profile.tokens and "bpu" in normalized:
score += 4.0
if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized):
score += 2.0
if _is_low_signal(text):
score -= 8.0
return score
def _is_evidence_candidate(text: str) -> bool:
stripped = text.lstrip()
return not stripped.startswith("|") and not stripped.startswith(">")
def _is_low_signal(text: str) -> bool:
stripped = text.strip()
without_marks = re.sub(r"[*_`#\-\s]+", "", stripped)
if len(without_marks) < 8:
return True
if stripped.startswith("```") or "```" in stripped:
return True
if stripped.startswith("|"):
return True
if "![](" in stripped:
return True
if re.search(r"(^|\n)\s*[├│└─]{2,}", stripped):
return True
return False
def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]:
tags = [kb.name]
if alias_group is not None:
tags.append(_slug(str(alias_group.name)).replace("--", "-"))
for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title):
lowered = token.lower()
if lowered not in tags:
tags.append(lowered)
return tags[:8]
def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]:
values = [title]
if query_entry is not None:
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
return _unique(values)[:12]
def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]:
links: list[str] = []
if query_entry is not None:
links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ()))
if alias_group is not None:
links.extend(str(item) for item in getattr(alias_group, "related", ()))
return _unique(links)[:10]
def _unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
cleaned = value.strip()
if not cleaned or cleaned in seen:
continue
seen.add(cleaned)
output.append(cleaned)
return output
def _normalize_phrase(value: str) -> str:
return re.sub(r"\s+", " ", value.lower()).strip()
def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None:
(wiki_root / "SCHEMA.md").write_text(
"\n".join(
[
"# Wiki Schema",
"",
"## Domain",
f"Compiled local knowledge for `{kb.name}`.",
"",
"## Conventions",
"- Pages live under `concepts/`.",
"- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.",
"- Claims in Key Evidence keep provenance markers pointing back to source block ids.",
"- Raw source files remain immutable; compiled pages are generated artifacts.",
"",
]
),
encoding="utf-8",
)
def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None:
lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""]
for page in sorted(pages, key=lambda item: item.title):
rel = page.path.relative_to(wiki_root).as_posix()
lines.append(f"- [[{rel[:-3]}]] — {page.title}")
lines.append("")
(wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8")
def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None:
line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n"
path = wiki_root / "log.md"
if not path.exists():
path.write_text("# Log\n\n", encoding="utf-8")
with path.open("a", encoding="utf-8") as handle:
handle.write(line)
def _slug(value: str) -> str:
slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower()
return slug or "page"
def _yaml_scalar(value: str) -> str:
import json
return json.dumps(value, ensure_ascii=False)
def _yaml_list(values: list[str]) -> str:
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, indent=2)