forked from d-cloud/d-wiki
591 lines
20 KiB
Python
591 lines
20 KiB
Python
from __future__ import annotations
|
||
|
||
import re
|
||
import shutil
|
||
import json
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from .config import KnowledgeBaseConfig
|
||
from .llm import LlmPageSynthesis, LlmPageSynthesizer
|
||
from .metadata import load_alias_groups, load_query_entries
|
||
from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block
|
||
from .tokenize import tokenize
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class CompiledPage:
|
||
wikilink: str
|
||
title: str
|
||
path: Path
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class _EvidenceProfile:
|
||
tokens: frozenset[str]
|
||
phrases: tuple[str, ...]
|
||
|
||
|
||
def compile_kb(
|
||
kb: KnowledgeBaseConfig,
|
||
*,
|
||
force: bool = False,
|
||
llm_mode: str = "off",
|
||
llm_scope: str = "queries",
|
||
) -> dict[str, Any]:
|
||
chunks = load_source_chunks(kb)
|
||
if not chunks:
|
||
raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS")
|
||
synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None
|
||
if llm_mode == "require" and synthesizer is None:
|
||
raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED")
|
||
wiki_root = compiled_wiki_root(kb)
|
||
concepts_dir = wiki_root / "concepts"
|
||
raw_dir = wiki_root / "raw"
|
||
if force:
|
||
shutil.rmtree(concepts_dir, ignore_errors=True)
|
||
concepts_dir.mkdir(parents=True, exist_ok=True)
|
||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
pages: list[CompiledPage] = []
|
||
grouped = _group_chunks(chunks)
|
||
query_entries = load_query_entries(kb.queries_dir)
|
||
alias_groups = load_alias_groups(kb.aliases_path)
|
||
query_by_source = {
|
||
source: entry
|
||
for entry in query_entries
|
||
for source in entry.primary_sources
|
||
}
|
||
aliases_by_source = {
|
||
related: group
|
||
for group in alias_groups
|
||
for related in group.related
|
||
}
|
||
|
||
for wikilink, page_chunks in sorted(grouped.items()):
|
||
page = _compile_page(
|
||
kb,
|
||
wikilink,
|
||
page_chunks,
|
||
query_entry=query_by_source.get(wikilink),
|
||
alias_group=aliases_by_source.get(wikilink),
|
||
synthesizer=(
|
||
synthesizer
|
||
if llm_scope == "all" or query_by_source.get(wikilink) is not None
|
||
else None
|
||
),
|
||
require_llm=llm_mode == "require",
|
||
)
|
||
if force or not page.path.exists():
|
||
page.path.write_text(page.content, encoding="utf-8")
|
||
pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path))
|
||
|
||
_write_schema(wiki_root, kb)
|
||
_write_index(wiki_root, pages)
|
||
_append_log(wiki_root, kb, len(chunks), len(pages))
|
||
return {
|
||
"status": "compiled",
|
||
"kb": kb.name,
|
||
"source_root": str(kb.root),
|
||
"wiki_root": str(wiki_root),
|
||
"source_chunks": len(chunks),
|
||
"pages": len(pages),
|
||
"llm_mode": llm_mode,
|
||
"llm_scope": llm_scope,
|
||
"llm_enabled": synthesizer is not None,
|
||
"compiled_at": datetime.now(timezone.utc).isoformat(),
|
||
}
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class _PageDraft:
|
||
title: str
|
||
path: Path
|
||
content: str
|
||
|
||
|
||
def _compile_page(
|
||
kb: KnowledgeBaseConfig,
|
||
wikilink: str,
|
||
chunks: list[Chunk],
|
||
*,
|
||
query_entry: Any | None,
|
||
alias_group: Any | None,
|
||
synthesizer: LlmPageSynthesizer | None,
|
||
require_llm: bool,
|
||
) -> _PageDraft:
|
||
first = chunks[0]
|
||
title = _title_for(first, query_entry)
|
||
slug = _slug(wikilink)
|
||
path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md"
|
||
evidence_profile = _evidence_profile(title, query_entry, alias_group)
|
||
tags = _tags(kb, first, alias_group)
|
||
aliases = _aliases(title, query_entry, alias_group)
|
||
source_ref = first.wikilink
|
||
source_path = str(first.source_path)
|
||
key_blocks = _key_blocks(chunks, evidence_profile)
|
||
intent = _intent(title, query_entry, alias_group)
|
||
summary = _summary(chunks, title, evidence_profile, intent=intent)
|
||
requirements = _answer_requirements(intent, query_entry)
|
||
synthesis = _llm_synthesis(
|
||
synthesizer,
|
||
title=title,
|
||
wikilink=wikilink,
|
||
query_entry=query_entry,
|
||
alias_group=alias_group,
|
||
key_blocks=key_blocks,
|
||
require_llm=require_llm,
|
||
)
|
||
if synthesis is not None:
|
||
summary = synthesis.summary
|
||
retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group)
|
||
blueprint = (
|
||
list(synthesis.answer_blueprint)
|
||
if synthesis and synthesis.answer_blueprint
|
||
else _deterministic_answer_blueprint(intent, requirements, key_blocks)
|
||
)
|
||
links = _related_links(query_entry, alias_group)
|
||
content = "\n".join(
|
||
[
|
||
"---",
|
||
f"llmWikiVersion: \"1\"",
|
||
f"conceptId: {_yaml_scalar(slug)}",
|
||
f"title: {_yaml_scalar(title)}",
|
||
f"summary: {_yaml_scalar(summary)}",
|
||
f"sources: {_yaml_list([source_ref])}",
|
||
f"source_paths: {_yaml_list([source_path])}",
|
||
f"kind: concept",
|
||
f"tags: {_yaml_list(tags)}",
|
||
f"aliases: {_yaml_list(aliases)}",
|
||
f"intents: {_yaml_list([intent])}",
|
||
f"retrieval_hints: {_yaml_list(retrieval_hints)}",
|
||
f"answer_requirements: {_yaml_list(requirements)}",
|
||
f"confidence: {synthesis.confidence if synthesis else 'medium'}",
|
||
f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}",
|
||
"---",
|
||
"",
|
||
f"# {title}",
|
||
"",
|
||
summary,
|
||
"",
|
||
"## Intent Model",
|
||
"",
|
||
f"- intent: {intent}",
|
||
f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}",
|
||
f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}",
|
||
"",
|
||
"## Retrieval Hints",
|
||
"",
|
||
*[f"- {hint}" for hint in retrieval_hints],
|
||
"",
|
||
"## Answer Blueprint",
|
||
"",
|
||
*[f"- {item}" for item in blueprint],
|
||
"",
|
||
"## Synthesized Evidence" if synthesis and synthesis.key_evidence else "",
|
||
"" if synthesis and synthesis.key_evidence else "",
|
||
*[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())],
|
||
"" if synthesis and synthesis.key_evidence else "",
|
||
"## Evidence Graph",
|
||
"",
|
||
*_evidence_graph_lines(key_blocks),
|
||
"",
|
||
"## Key Evidence",
|
||
"",
|
||
*[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks],
|
||
"",
|
||
"## Source Map",
|
||
"",
|
||
f"- 原始来源:`{source_ref}`",
|
||
f"- 文件路径:`{source_path}`",
|
||
"",
|
||
"## Related",
|
||
"",
|
||
*[f"- [[{link}]]" for link in links],
|
||
"",
|
||
]
|
||
)
|
||
return _PageDraft(title=title, path=path, content=content)
|
||
|
||
|
||
def _llm_synthesis(
|
||
synthesizer: LlmPageSynthesizer | None,
|
||
*,
|
||
title: str,
|
||
wikilink: str,
|
||
query_entry: Any | None,
|
||
alias_group: Any | None,
|
||
key_blocks: list[Chunk],
|
||
require_llm: bool,
|
||
) -> LlmPageSynthesis | None:
|
||
if synthesizer is None:
|
||
return None
|
||
prompt = _llm_compile_prompt(
|
||
title=title,
|
||
wikilink=wikilink,
|
||
query_entry=query_entry,
|
||
alias_group=alias_group,
|
||
key_blocks=key_blocks,
|
||
)
|
||
try:
|
||
return synthesizer.synthesize(prompt)
|
||
except Exception:
|
||
if require_llm:
|
||
raise
|
||
return None
|
||
|
||
|
||
def _llm_compile_prompt(
|
||
*,
|
||
title: str,
|
||
wikilink: str,
|
||
query_entry: Any | None,
|
||
alias_group: Any | None,
|
||
key_blocks: list[Chunk],
|
||
) -> str:
|
||
aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ())
|
||
terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ())
|
||
requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ())
|
||
intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None)
|
||
evidence = [
|
||
{
|
||
"block_id": f"{block.wikilink}#{block.index}",
|
||
"heading": block.heading,
|
||
"text": trim_block(block.text, 700),
|
||
}
|
||
for block in key_blocks[:8]
|
||
]
|
||
return "\n".join(
|
||
[
|
||
"Compile a d-wiki concept page from the local evidence below.",
|
||
"Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.",
|
||
"For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.",
|
||
"Return only JSON with this shape:",
|
||
'{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}',
|
||
"",
|
||
json_dumps(
|
||
{
|
||
"title": title,
|
||
"wikilink": wikilink,
|
||
"intent": intent,
|
||
"aliases": aliases,
|
||
"terms": terms,
|
||
"answer_requirements": requirements,
|
||
"evidence": evidence,
|
||
}
|
||
),
|
||
]
|
||
)
|
||
|
||
|
||
def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]:
|
||
if synthesis is None or not synthesis.answer_blueprint:
|
||
return []
|
||
return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""]
|
||
|
||
|
||
def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str:
|
||
if query_entry is not None and getattr(query_entry, "intent", ""):
|
||
return str(query_entry.intent)
|
||
if alias_group is not None and getattr(alias_group, "intent", ""):
|
||
return str(alias_group.intent)
|
||
lowered = title.lower()
|
||
if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")):
|
||
return "procedure"
|
||
return "fact"
|
||
|
||
|
||
def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]:
|
||
if query_entry is not None:
|
||
requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()]
|
||
if requirements:
|
||
return _unique(requirements)
|
||
if intent == "procedure":
|
||
return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"]
|
||
return ["直接给出结论", "保留关键数值和单位", "引用证据来源"]
|
||
|
||
|
||
def _retrieval_hints(
|
||
title: str,
|
||
aliases: list[str],
|
||
tags: list[str],
|
||
requirements: list[str],
|
||
alias_group: Any | None,
|
||
) -> list[str]:
|
||
values = [title, *aliases, *tags, *requirements]
|
||
if alias_group is not None:
|
||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||
hints: list[str] = []
|
||
for value in values:
|
||
hints.append(value)
|
||
hints.extend(tokenize(value))
|
||
return _unique(hints)[:32]
|
||
|
||
|
||
def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]:
|
||
if requirements:
|
||
return requirements[:6]
|
||
if intent == "procedure":
|
||
headings = _unique([block.heading for block in key_blocks if block.heading])
|
||
if headings:
|
||
return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]]
|
||
return ["按前提、配置、执行、验证的顺序回答"]
|
||
return ["先给出直接结论", "补充关键参数、限制和来源"]
|
||
|
||
|
||
def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]:
|
||
lines: list[str] = []
|
||
for block in key_blocks[:8]:
|
||
claim = trim_block(block.text, 180)
|
||
if not claim:
|
||
continue
|
||
lines.append(f"- claim: {claim}")
|
||
lines.append(f" source: {block.wikilink}#{block.index}")
|
||
if block.heading:
|
||
lines.append(f" heading: {block.heading}")
|
||
return lines or ["- claim: 暂无可用证据", " source: unknown"]
|
||
|
||
|
||
def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]:
|
||
grouped: dict[str, list[Chunk]] = {}
|
||
for chunk in chunks:
|
||
grouped.setdefault(chunk.wikilink, []).append(chunk)
|
||
return grouped
|
||
|
||
|
||
def _title_for(chunk: Chunk, query_entry: Any | None) -> str:
|
||
if query_entry is not None and getattr(query_entry, "title", ""):
|
||
return str(query_entry.title)
|
||
return chunk.title
|
||
|
||
|
||
def _summary(
|
||
chunks: list[Chunk],
|
||
title: str,
|
||
evidence_profile: _EvidenceProfile | None = None,
|
||
*,
|
||
intent: str = "fact",
|
||
) -> str:
|
||
if intent == "procedure":
|
||
for chunk in chunks[:12]:
|
||
text = trim_block(chunk.text, 220)
|
||
if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text):
|
||
return f"{title}:{text}"
|
||
ranked = _rank_evidence(chunks, evidence_profile)
|
||
for score, chunk in ranked:
|
||
if score <= 0:
|
||
continue
|
||
text = trim_block(chunk.text, 220)
|
||
if text and not _is_low_signal(text):
|
||
return f"{title}:{text}"
|
||
for chunk in chunks:
|
||
text = trim_block(chunk.text, 220)
|
||
if text and not _is_low_signal(text) and len(text) >= 20:
|
||
return f"{title}:{text}"
|
||
return f"{title} 的编译知识页,来源于本地文档。"
|
||
|
||
|
||
def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]:
|
||
preferred = [
|
||
chunk
|
||
for chunk in chunks
|
||
if _is_evidence_candidate(chunk.text)
|
||
]
|
||
ranked = _rank_evidence(preferred, evidence_profile)
|
||
if not ranked or ranked[0][0] <= 0:
|
||
return (preferred or chunks)[:6]
|
||
selected: list[Chunk] = []
|
||
seen: set[tuple[str, int]] = set()
|
||
for score, chunk in ranked:
|
||
if score <= 0:
|
||
continue
|
||
key = (chunk.wikilink, chunk.index)
|
||
if key in seen:
|
||
continue
|
||
selected.append(chunk)
|
||
seen.add(key)
|
||
if len(selected) >= 6:
|
||
return selected
|
||
for chunk in preferred:
|
||
key = (chunk.wikilink, chunk.index)
|
||
if key in seen:
|
||
continue
|
||
selected.append(chunk)
|
||
seen.add(key)
|
||
if len(selected) >= 6:
|
||
break
|
||
return selected
|
||
|
||
|
||
def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile:
|
||
values = [title]
|
||
if query_entry is not None:
|
||
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
|
||
values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ()))
|
||
if alias_group is not None:
|
||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||
phrases = tuple(_unique([_normalize_phrase(value) for value in values if value]))
|
||
tokens: set[str] = set()
|
||
for value in values:
|
||
tokens.update(tokenize(str(value)))
|
||
return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases)
|
||
|
||
|
||
def _rank_evidence(
|
||
chunks: list[Chunk],
|
||
evidence_profile: _EvidenceProfile | None,
|
||
) -> list[tuple[float, Chunk]]:
|
||
if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases):
|
||
return []
|
||
ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks]
|
||
ranked.sort(key=lambda item: (-item[0], item[1].index))
|
||
return ranked
|
||
|
||
|
||
def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float:
|
||
text = trim_block(chunk.text, 1000)
|
||
normalized = _normalize_phrase(text)
|
||
chunk_tokens = set(chunk.tokens) | set(tokenize(text))
|
||
score = 0.0
|
||
score += len(chunk_tokens & evidence_profile.tokens) * 2.0
|
||
for phrase in evidence_profile.phrases:
|
||
if len(phrase) >= 2 and phrase in normalized:
|
||
score += 5.0
|
||
if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized):
|
||
score += 8.0
|
||
if "算力" in evidence_profile.tokens and "tops" in normalized:
|
||
score += 6.0
|
||
if "bpu" in evidence_profile.tokens and "bpu" in normalized:
|
||
score += 4.0
|
||
if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized):
|
||
score += 2.0
|
||
if _is_low_signal(text):
|
||
score -= 8.0
|
||
return score
|
||
|
||
|
||
def _is_evidence_candidate(text: str) -> bool:
|
||
stripped = text.lstrip()
|
||
return not stripped.startswith("|") and not stripped.startswith(">")
|
||
|
||
|
||
def _is_low_signal(text: str) -> bool:
|
||
stripped = text.strip()
|
||
without_marks = re.sub(r"[*_`#\-\s]+", "", stripped)
|
||
if len(without_marks) < 8:
|
||
return True
|
||
if stripped.startswith("```") or "```" in stripped:
|
||
return True
|
||
if stripped.startswith("|"):
|
||
return True
|
||
if "\s*[├│└─]{2,}", stripped):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]:
|
||
tags = [kb.name]
|
||
if alias_group is not None:
|
||
tags.append(_slug(str(alias_group.name)).replace("--", "-"))
|
||
for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title):
|
||
lowered = token.lower()
|
||
if lowered not in tags:
|
||
tags.append(lowered)
|
||
return tags[:8]
|
||
|
||
|
||
def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]:
|
||
values = [title]
|
||
if query_entry is not None:
|
||
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
|
||
if alias_group is not None:
|
||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||
return _unique(values)[:12]
|
||
|
||
|
||
def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]:
|
||
links: list[str] = []
|
||
if query_entry is not None:
|
||
links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ()))
|
||
if alias_group is not None:
|
||
links.extend(str(item) for item in getattr(alias_group, "related", ()))
|
||
return _unique(links)[:10]
|
||
|
||
|
||
def _unique(values: list[str]) -> list[str]:
|
||
seen: set[str] = set()
|
||
output: list[str] = []
|
||
for value in values:
|
||
cleaned = value.strip()
|
||
if not cleaned or cleaned in seen:
|
||
continue
|
||
seen.add(cleaned)
|
||
output.append(cleaned)
|
||
return output
|
||
|
||
|
||
def _normalize_phrase(value: str) -> str:
|
||
return re.sub(r"\s+", " ", value.lower()).strip()
|
||
|
||
|
||
def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None:
|
||
(wiki_root / "SCHEMA.md").write_text(
|
||
"\n".join(
|
||
[
|
||
"# Wiki Schema",
|
||
"",
|
||
"## Domain",
|
||
f"Compiled local knowledge for `{kb.name}`.",
|
||
"",
|
||
"## Conventions",
|
||
"- Pages live under `concepts/`.",
|
||
"- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.",
|
||
"- Claims in Key Evidence keep provenance markers pointing back to source block ids.",
|
||
"- Raw source files remain immutable; compiled pages are generated artifacts.",
|
||
"",
|
||
]
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
|
||
def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None:
|
||
lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""]
|
||
for page in sorted(pages, key=lambda item: item.title):
|
||
rel = page.path.relative_to(wiki_root).as_posix()
|
||
lines.append(f"- [[{rel[:-3]}]] — {page.title}")
|
||
lines.append("")
|
||
(wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8")
|
||
|
||
|
||
def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None:
|
||
line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n"
|
||
path = wiki_root / "log.md"
|
||
if not path.exists():
|
||
path.write_text("# Log\n\n", encoding="utf-8")
|
||
with path.open("a", encoding="utf-8") as handle:
|
||
handle.write(line)
|
||
|
||
|
||
def _slug(value: str) -> str:
|
||
slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower()
|
||
return slug or "page"
|
||
|
||
|
||
def _yaml_scalar(value: str) -> str:
|
||
import json
|
||
|
||
return json.dumps(value, ensure_ascii=False)
|
||
|
||
|
||
def _yaml_list(values: list[str]) -> str:
|
||
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
|
||
|
||
|
||
def json_dumps(value: Any) -> str:
|
||
return json.dumps(value, ensure_ascii=False, indent=2)
|