forked from d-cloud/d-wiki
first commit
This commit is contained in:
parent
44de534185
commit
66eb67569e
3
build/lib/d_wiki/__init__.py
Normal file
3
build/lib/d_wiki/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
"""d-wiki lightweight local knowledge retrieval."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
4
build/lib/d_wiki/__main__.py
Normal file
4
build/lib/d_wiki/__main__.py
Normal file
@ -0,0 +1,4 @@
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
77
build/lib/d_wiki/cli.py
Normal file
77
build/lib/d_wiki/cli.py
Normal file
@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import DWikiConfig, load_config
|
||||
from .compile import compile_kb
|
||||
from .dreams import dream_kb
|
||||
from .search import get_page, index_payload, search, status_payload
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="d-wiki")
|
||||
parser.add_argument("--config", type=Path, default=None)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
status_parser = subparsers.add_parser("status")
|
||||
status_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
index_parser = subparsers.add_parser("index")
|
||||
index_parser.add_argument("--kb", required=True)
|
||||
index_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
compile_parser = subparsers.add_parser("compile")
|
||||
compile_parser.add_argument("--kb", required=True)
|
||||
compile_parser.add_argument("--force", action="store_true")
|
||||
compile_parser.add_argument("--llm", choices=["off", "auto", "require"], default="off")
|
||||
compile_parser.add_argument("--llm-scope", choices=["queries", "all"], default="queries")
|
||||
compile_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
dream_parser = subparsers.add_parser("dream")
|
||||
dream_parser.add_argument("--kb", required=True)
|
||||
dream_parser.add_argument("--force", action="store_true")
|
||||
dream_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
search_parser = subparsers.add_parser("search")
|
||||
search_parser.add_argument("--kb", required=True)
|
||||
search_parser.add_argument("--query", required=True)
|
||||
search_parser.add_argument("--max-results", type=int, default=5)
|
||||
search_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
get_parser = subparsers.add_parser("get")
|
||||
get_parser.add_argument("--kb", required=True)
|
||||
get_parser.add_argument("--id", required=True)
|
||||
get_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
payload = dispatch(args, config)
|
||||
except Exception as exc:
|
||||
print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def dispatch(args: argparse.Namespace, config: DWikiConfig) -> dict[str, Any]:
|
||||
if args.command == "status":
|
||||
return status_payload(config.path, config.knowledge_bases)
|
||||
kb = config.knowledge_bases.get(args.kb)
|
||||
if kb is None:
|
||||
raise ValueError(f"D_WIKI_KB_NOT_FOUND: {args.kb}")
|
||||
if args.command == "index":
|
||||
return index_payload(kb)
|
||||
if args.command == "compile":
|
||||
return compile_kb(kb, force=args.force, llm_mode=args.llm, llm_scope=args.llm_scope)
|
||||
if args.command == "dream":
|
||||
return dream_kb(kb, force=args.force)
|
||||
if args.command == "search":
|
||||
return search(kb, args.query, args.max_results)
|
||||
if args.command == "get":
|
||||
return get_page(kb, args.id)
|
||||
raise ValueError(f"D_WIKI_UNKNOWN_COMMAND: {args.command}")
|
||||
590
build/lib/d_wiki/compile.py
Normal file
590
build/lib/d_wiki/compile.py
Normal file
@ -0,0 +1,590 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import KnowledgeBaseConfig
|
||||
from .llm import LlmPageSynthesis, LlmPageSynthesizer
|
||||
from .metadata import load_alias_groups, load_query_entries
|
||||
from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block
|
||||
from .tokenize import tokenize
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompiledPage:
|
||||
wikilink: str
|
||||
title: str
|
||||
path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _EvidenceProfile:
|
||||
tokens: frozenset[str]
|
||||
phrases: tuple[str, ...]
|
||||
|
||||
|
||||
def compile_kb(
|
||||
kb: KnowledgeBaseConfig,
|
||||
*,
|
||||
force: bool = False,
|
||||
llm_mode: str = "off",
|
||||
llm_scope: str = "queries",
|
||||
) -> dict[str, Any]:
|
||||
chunks = load_source_chunks(kb)
|
||||
if not chunks:
|
||||
raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS")
|
||||
synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None
|
||||
if llm_mode == "require" and synthesizer is None:
|
||||
raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED")
|
||||
wiki_root = compiled_wiki_root(kb)
|
||||
concepts_dir = wiki_root / "concepts"
|
||||
raw_dir = wiki_root / "raw"
|
||||
if force:
|
||||
shutil.rmtree(concepts_dir, ignore_errors=True)
|
||||
concepts_dir.mkdir(parents=True, exist_ok=True)
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pages: list[CompiledPage] = []
|
||||
grouped = _group_chunks(chunks)
|
||||
query_entries = load_query_entries(kb.queries_dir)
|
||||
alias_groups = load_alias_groups(kb.aliases_path)
|
||||
query_by_source = {
|
||||
source: entry
|
||||
for entry in query_entries
|
||||
for source in entry.primary_sources
|
||||
}
|
||||
aliases_by_source = {
|
||||
related: group
|
||||
for group in alias_groups
|
||||
for related in group.related
|
||||
}
|
||||
|
||||
for wikilink, page_chunks in sorted(grouped.items()):
|
||||
page = _compile_page(
|
||||
kb,
|
||||
wikilink,
|
||||
page_chunks,
|
||||
query_entry=query_by_source.get(wikilink),
|
||||
alias_group=aliases_by_source.get(wikilink),
|
||||
synthesizer=(
|
||||
synthesizer
|
||||
if llm_scope == "all" or query_by_source.get(wikilink) is not None
|
||||
else None
|
||||
),
|
||||
require_llm=llm_mode == "require",
|
||||
)
|
||||
if force or not page.path.exists():
|
||||
page.path.write_text(page.content, encoding="utf-8")
|
||||
pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path))
|
||||
|
||||
_write_schema(wiki_root, kb)
|
||||
_write_index(wiki_root, pages)
|
||||
_append_log(wiki_root, kb, len(chunks), len(pages))
|
||||
return {
|
||||
"status": "compiled",
|
||||
"kb": kb.name,
|
||||
"source_root": str(kb.root),
|
||||
"wiki_root": str(wiki_root),
|
||||
"source_chunks": len(chunks),
|
||||
"pages": len(pages),
|
||||
"llm_mode": llm_mode,
|
||||
"llm_scope": llm_scope,
|
||||
"llm_enabled": synthesizer is not None,
|
||||
"compiled_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PageDraft:
|
||||
title: str
|
||||
path: Path
|
||||
content: str
|
||||
|
||||
|
||||
def _compile_page(
|
||||
kb: KnowledgeBaseConfig,
|
||||
wikilink: str,
|
||||
chunks: list[Chunk],
|
||||
*,
|
||||
query_entry: Any | None,
|
||||
alias_group: Any | None,
|
||||
synthesizer: LlmPageSynthesizer | None,
|
||||
require_llm: bool,
|
||||
) -> _PageDraft:
|
||||
first = chunks[0]
|
||||
title = _title_for(first, query_entry)
|
||||
slug = _slug(wikilink)
|
||||
path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md"
|
||||
evidence_profile = _evidence_profile(title, query_entry, alias_group)
|
||||
tags = _tags(kb, first, alias_group)
|
||||
aliases = _aliases(title, query_entry, alias_group)
|
||||
source_ref = first.wikilink
|
||||
source_path = str(first.source_path)
|
||||
key_blocks = _key_blocks(chunks, evidence_profile)
|
||||
intent = _intent(title, query_entry, alias_group)
|
||||
summary = _summary(chunks, title, evidence_profile, intent=intent)
|
||||
requirements = _answer_requirements(intent, query_entry)
|
||||
synthesis = _llm_synthesis(
|
||||
synthesizer,
|
||||
title=title,
|
||||
wikilink=wikilink,
|
||||
query_entry=query_entry,
|
||||
alias_group=alias_group,
|
||||
key_blocks=key_blocks,
|
||||
require_llm=require_llm,
|
||||
)
|
||||
if synthesis is not None:
|
||||
summary = synthesis.summary
|
||||
retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group)
|
||||
blueprint = (
|
||||
list(synthesis.answer_blueprint)
|
||||
if synthesis and synthesis.answer_blueprint
|
||||
else _deterministic_answer_blueprint(intent, requirements, key_blocks)
|
||||
)
|
||||
links = _related_links(query_entry, alias_group)
|
||||
content = "\n".join(
|
||||
[
|
||||
"---",
|
||||
f"llmWikiVersion: \"1\"",
|
||||
f"conceptId: {_yaml_scalar(slug)}",
|
||||
f"title: {_yaml_scalar(title)}",
|
||||
f"summary: {_yaml_scalar(summary)}",
|
||||
f"sources: {_yaml_list([source_ref])}",
|
||||
f"source_paths: {_yaml_list([source_path])}",
|
||||
f"kind: concept",
|
||||
f"tags: {_yaml_list(tags)}",
|
||||
f"aliases: {_yaml_list(aliases)}",
|
||||
f"intents: {_yaml_list([intent])}",
|
||||
f"retrieval_hints: {_yaml_list(retrieval_hints)}",
|
||||
f"answer_requirements: {_yaml_list(requirements)}",
|
||||
f"confidence: {synthesis.confidence if synthesis else 'medium'}",
|
||||
f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}",
|
||||
"---",
|
||||
"",
|
||||
f"# {title}",
|
||||
"",
|
||||
summary,
|
||||
"",
|
||||
"## Intent Model",
|
||||
"",
|
||||
f"- intent: {intent}",
|
||||
f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}",
|
||||
f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}",
|
||||
"",
|
||||
"## Retrieval Hints",
|
||||
"",
|
||||
*[f"- {hint}" for hint in retrieval_hints],
|
||||
"",
|
||||
"## Answer Blueprint",
|
||||
"",
|
||||
*[f"- {item}" for item in blueprint],
|
||||
"",
|
||||
"## Synthesized Evidence" if synthesis and synthesis.key_evidence else "",
|
||||
"" if synthesis and synthesis.key_evidence else "",
|
||||
*[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())],
|
||||
"" if synthesis and synthesis.key_evidence else "",
|
||||
"## Evidence Graph",
|
||||
"",
|
||||
*_evidence_graph_lines(key_blocks),
|
||||
"",
|
||||
"## Key Evidence",
|
||||
"",
|
||||
*[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks],
|
||||
"",
|
||||
"## Source Map",
|
||||
"",
|
||||
f"- 原始来源:`{source_ref}`",
|
||||
f"- 文件路径:`{source_path}`",
|
||||
"",
|
||||
"## Related",
|
||||
"",
|
||||
*[f"- [[{link}]]" for link in links],
|
||||
"",
|
||||
]
|
||||
)
|
||||
return _PageDraft(title=title, path=path, content=content)
|
||||
|
||||
|
||||
def _llm_synthesis(
|
||||
synthesizer: LlmPageSynthesizer | None,
|
||||
*,
|
||||
title: str,
|
||||
wikilink: str,
|
||||
query_entry: Any | None,
|
||||
alias_group: Any | None,
|
||||
key_blocks: list[Chunk],
|
||||
require_llm: bool,
|
||||
) -> LlmPageSynthesis | None:
|
||||
if synthesizer is None:
|
||||
return None
|
||||
prompt = _llm_compile_prompt(
|
||||
title=title,
|
||||
wikilink=wikilink,
|
||||
query_entry=query_entry,
|
||||
alias_group=alias_group,
|
||||
key_blocks=key_blocks,
|
||||
)
|
||||
try:
|
||||
return synthesizer.synthesize(prompt)
|
||||
except Exception:
|
||||
if require_llm:
|
||||
raise
|
||||
return None
|
||||
|
||||
|
||||
def _llm_compile_prompt(
|
||||
*,
|
||||
title: str,
|
||||
wikilink: str,
|
||||
query_entry: Any | None,
|
||||
alias_group: Any | None,
|
||||
key_blocks: list[Chunk],
|
||||
) -> str:
|
||||
aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ())
|
||||
terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ())
|
||||
requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ())
|
||||
intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None)
|
||||
evidence = [
|
||||
{
|
||||
"block_id": f"{block.wikilink}#{block.index}",
|
||||
"heading": block.heading,
|
||||
"text": trim_block(block.text, 700),
|
||||
}
|
||||
for block in key_blocks[:8]
|
||||
]
|
||||
return "\n".join(
|
||||
[
|
||||
"Compile a d-wiki concept page from the local evidence below.",
|
||||
"Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.",
|
||||
"For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.",
|
||||
"Return only JSON with this shape:",
|
||||
'{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}',
|
||||
"",
|
||||
json_dumps(
|
||||
{
|
||||
"title": title,
|
||||
"wikilink": wikilink,
|
||||
"intent": intent,
|
||||
"aliases": aliases,
|
||||
"terms": terms,
|
||||
"answer_requirements": requirements,
|
||||
"evidence": evidence,
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]:
|
||||
if synthesis is None or not synthesis.answer_blueprint:
|
||||
return []
|
||||
return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""]
|
||||
|
||||
|
||||
def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str:
|
||||
if query_entry is not None and getattr(query_entry, "intent", ""):
|
||||
return str(query_entry.intent)
|
||||
if alias_group is not None and getattr(alias_group, "intent", ""):
|
||||
return str(alias_group.intent)
|
||||
lowered = title.lower()
|
||||
if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")):
|
||||
return "procedure"
|
||||
return "fact"
|
||||
|
||||
|
||||
def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]:
|
||||
if query_entry is not None:
|
||||
requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()]
|
||||
if requirements:
|
||||
return _unique(requirements)
|
||||
if intent == "procedure":
|
||||
return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"]
|
||||
return ["直接给出结论", "保留关键数值和单位", "引用证据来源"]
|
||||
|
||||
|
||||
def _retrieval_hints(
|
||||
title: str,
|
||||
aliases: list[str],
|
||||
tags: list[str],
|
||||
requirements: list[str],
|
||||
alias_group: Any | None,
|
||||
) -> list[str]:
|
||||
values = [title, *aliases, *tags, *requirements]
|
||||
if alias_group is not None:
|
||||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||||
hints: list[str] = []
|
||||
for value in values:
|
||||
hints.append(value)
|
||||
hints.extend(tokenize(value))
|
||||
return _unique(hints)[:32]
|
||||
|
||||
|
||||
def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]:
|
||||
if requirements:
|
||||
return requirements[:6]
|
||||
if intent == "procedure":
|
||||
headings = _unique([block.heading for block in key_blocks if block.heading])
|
||||
if headings:
|
||||
return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]]
|
||||
return ["按前提、配置、执行、验证的顺序回答"]
|
||||
return ["先给出直接结论", "补充关键参数、限制和来源"]
|
||||
|
||||
|
||||
def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for block in key_blocks[:8]:
|
||||
claim = trim_block(block.text, 180)
|
||||
if not claim:
|
||||
continue
|
||||
lines.append(f"- claim: {claim}")
|
||||
lines.append(f" source: {block.wikilink}#{block.index}")
|
||||
if block.heading:
|
||||
lines.append(f" heading: {block.heading}")
|
||||
return lines or ["- claim: 暂无可用证据", " source: unknown"]
|
||||
|
||||
|
||||
def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]:
|
||||
grouped: dict[str, list[Chunk]] = {}
|
||||
for chunk in chunks:
|
||||
grouped.setdefault(chunk.wikilink, []).append(chunk)
|
||||
return grouped
|
||||
|
||||
|
||||
def _title_for(chunk: Chunk, query_entry: Any | None) -> str:
|
||||
if query_entry is not None and getattr(query_entry, "title", ""):
|
||||
return str(query_entry.title)
|
||||
return chunk.title
|
||||
|
||||
|
||||
def _summary(
|
||||
chunks: list[Chunk],
|
||||
title: str,
|
||||
evidence_profile: _EvidenceProfile | None = None,
|
||||
*,
|
||||
intent: str = "fact",
|
||||
) -> str:
|
||||
if intent == "procedure":
|
||||
for chunk in chunks[:12]:
|
||||
text = trim_block(chunk.text, 220)
|
||||
if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text):
|
||||
return f"{title}:{text}"
|
||||
ranked = _rank_evidence(chunks, evidence_profile)
|
||||
for score, chunk in ranked:
|
||||
if score <= 0:
|
||||
continue
|
||||
text = trim_block(chunk.text, 220)
|
||||
if text and not _is_low_signal(text):
|
||||
return f"{title}:{text}"
|
||||
for chunk in chunks:
|
||||
text = trim_block(chunk.text, 220)
|
||||
if text and not _is_low_signal(text) and len(text) >= 20:
|
||||
return f"{title}:{text}"
|
||||
return f"{title} 的编译知识页,来源于本地文档。"
|
||||
|
||||
|
||||
def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]:
|
||||
preferred = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if _is_evidence_candidate(chunk.text)
|
||||
]
|
||||
ranked = _rank_evidence(preferred, evidence_profile)
|
||||
if not ranked or ranked[0][0] <= 0:
|
||||
return (preferred or chunks)[:6]
|
||||
selected: list[Chunk] = []
|
||||
seen: set[tuple[str, int]] = set()
|
||||
for score, chunk in ranked:
|
||||
if score <= 0:
|
||||
continue
|
||||
key = (chunk.wikilink, chunk.index)
|
||||
if key in seen:
|
||||
continue
|
||||
selected.append(chunk)
|
||||
seen.add(key)
|
||||
if len(selected) >= 6:
|
||||
return selected
|
||||
for chunk in preferred:
|
||||
key = (chunk.wikilink, chunk.index)
|
||||
if key in seen:
|
||||
continue
|
||||
selected.append(chunk)
|
||||
seen.add(key)
|
||||
if len(selected) >= 6:
|
||||
break
|
||||
return selected
|
||||
|
||||
|
||||
def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile:
|
||||
values = [title]
|
||||
if query_entry is not None:
|
||||
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
|
||||
values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ()))
|
||||
if alias_group is not None:
|
||||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||||
phrases = tuple(_unique([_normalize_phrase(value) for value in values if value]))
|
||||
tokens: set[str] = set()
|
||||
for value in values:
|
||||
tokens.update(tokenize(str(value)))
|
||||
return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases)
|
||||
|
||||
|
||||
def _rank_evidence(
|
||||
chunks: list[Chunk],
|
||||
evidence_profile: _EvidenceProfile | None,
|
||||
) -> list[tuple[float, Chunk]]:
|
||||
if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases):
|
||||
return []
|
||||
ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks]
|
||||
ranked.sort(key=lambda item: (-item[0], item[1].index))
|
||||
return ranked
|
||||
|
||||
|
||||
def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float:
|
||||
text = trim_block(chunk.text, 1000)
|
||||
normalized = _normalize_phrase(text)
|
||||
chunk_tokens = set(chunk.tokens) | set(tokenize(text))
|
||||
score = 0.0
|
||||
score += len(chunk_tokens & evidence_profile.tokens) * 2.0
|
||||
for phrase in evidence_profile.phrases:
|
||||
if len(phrase) >= 2 and phrase in normalized:
|
||||
score += 5.0
|
||||
if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized):
|
||||
score += 8.0
|
||||
if "算力" in evidence_profile.tokens and "tops" in normalized:
|
||||
score += 6.0
|
||||
if "bpu" in evidence_profile.tokens and "bpu" in normalized:
|
||||
score += 4.0
|
||||
if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized):
|
||||
score += 2.0
|
||||
if _is_low_signal(text):
|
||||
score -= 8.0
|
||||
return score
|
||||
|
||||
|
||||
def _is_evidence_candidate(text: str) -> bool:
|
||||
stripped = text.lstrip()
|
||||
return not stripped.startswith("|") and not stripped.startswith(">")
|
||||
|
||||
|
||||
def _is_low_signal(text: str) -> bool:
|
||||
stripped = text.strip()
|
||||
without_marks = re.sub(r"[*_`#\-\s]+", "", stripped)
|
||||
if len(without_marks) < 8:
|
||||
return True
|
||||
if stripped.startswith("```") or "```" in stripped:
|
||||
return True
|
||||
if stripped.startswith("|"):
|
||||
return True
|
||||
if "\s*[├│└─]{2,}", stripped):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]:
|
||||
tags = [kb.name]
|
||||
if alias_group is not None:
|
||||
tags.append(_slug(str(alias_group.name)).replace("--", "-"))
|
||||
for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title):
|
||||
lowered = token.lower()
|
||||
if lowered not in tags:
|
||||
tags.append(lowered)
|
||||
return tags[:8]
|
||||
|
||||
|
||||
def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]:
|
||||
values = [title]
|
||||
if query_entry is not None:
|
||||
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
|
||||
if alias_group is not None:
|
||||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||||
return _unique(values)[:12]
|
||||
|
||||
|
||||
def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]:
|
||||
links: list[str] = []
|
||||
if query_entry is not None:
|
||||
links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ()))
|
||||
if alias_group is not None:
|
||||
links.extend(str(item) for item in getattr(alias_group, "related", ()))
|
||||
return _unique(links)[:10]
|
||||
|
||||
|
||||
def _unique(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
output: list[str] = []
|
||||
for value in values:
|
||||
cleaned = value.strip()
|
||||
if not cleaned or cleaned in seen:
|
||||
continue
|
||||
seen.add(cleaned)
|
||||
output.append(cleaned)
|
||||
return output
|
||||
|
||||
|
||||
def _normalize_phrase(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", value.lower()).strip()
|
||||
|
||||
|
||||
def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None:
|
||||
(wiki_root / "SCHEMA.md").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"# Wiki Schema",
|
||||
"",
|
||||
"## Domain",
|
||||
f"Compiled local knowledge for `{kb.name}`.",
|
||||
"",
|
||||
"## Conventions",
|
||||
"- Pages live under `concepts/`.",
|
||||
"- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.",
|
||||
"- Claims in Key Evidence keep provenance markers pointing back to source block ids.",
|
||||
"- Raw source files remain immutable; compiled pages are generated artifacts.",
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None:
|
||||
lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""]
|
||||
for page in sorted(pages, key=lambda item: item.title):
|
||||
rel = page.path.relative_to(wiki_root).as_posix()
|
||||
lines.append(f"- [[{rel[:-3]}]] — {page.title}")
|
||||
lines.append("")
|
||||
(wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None:
|
||||
line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n"
|
||||
path = wiki_root / "log.md"
|
||||
if not path.exists():
|
||||
path.write_text("# Log\n\n", encoding="utf-8")
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(line)
|
||||
|
||||
|
||||
def _slug(value: str) -> str:
|
||||
slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower()
|
||||
return slug or "page"
|
||||
|
||||
|
||||
def _yaml_scalar(value: str) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _yaml_list(values: list[str]) -> str:
|
||||
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
59
build/lib/d_wiki/config.py
Normal file
59
build/lib/d_wiki/config.py
Normal file
@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .yaml_compat import safe_load
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KnowledgeBaseConfig:
|
||||
name: str
|
||||
root: Path
|
||||
source_type: str
|
||||
aliases_path: Path | None
|
||||
queries_dir: Path | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DWikiConfig:
|
||||
path: Path
|
||||
knowledge_bases: dict[str, KnowledgeBaseConfig]
|
||||
|
||||
|
||||
def default_config_path() -> Path:
|
||||
return Path(os.environ.get("D_WIKI_CONFIG", Path.cwd() / "d-wiki.yml")).expanduser()
|
||||
|
||||
|
||||
def load_config(path: Path | None = None) -> DWikiConfig:
|
||||
config_path = (path or default_config_path()).expanduser().resolve()
|
||||
data = safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||||
raw_kbs = data.get("knowledge_bases")
|
||||
if not isinstance(raw_kbs, dict):
|
||||
raise ValueError("D_WIKI_CONFIG_MISSING_KNOWLEDGE_BASES")
|
||||
bases: dict[str, KnowledgeBaseConfig] = {}
|
||||
for name, raw in raw_kbs.items():
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"D_WIKI_KB_INVALID: {name}")
|
||||
root = Path(str(raw.get("root", ""))).expanduser()
|
||||
if not root.is_absolute():
|
||||
root = (config_path.parent / root).resolve()
|
||||
aliases_path = _optional_path(config_path.parent, raw.get("aliases"))
|
||||
queries_dir = _optional_path(config_path.parent, raw.get("queries"))
|
||||
bases[str(name)] = KnowledgeBaseConfig(
|
||||
name=str(name),
|
||||
root=root,
|
||||
source_type=str(raw.get("source_type", "markdown")),
|
||||
aliases_path=aliases_path,
|
||||
queries_dir=queries_dir,
|
||||
)
|
||||
return DWikiConfig(path=config_path, knowledge_bases=bases)
|
||||
|
||||
|
||||
def _optional_path(base: Path, value: Any) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
path = Path(str(value)).expanduser()
|
||||
return path if path.is_absolute() else (base / path).resolve()
|
||||
227
build/lib/d_wiki/dreams.py
Normal file
227
build/lib/d_wiki/dreams.py
Normal file
@ -0,0 +1,227 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import KnowledgeBaseConfig
|
||||
from .metadata import frontmatter, load_query_entries
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DreamedQuery:
|
||||
slug: str
|
||||
title: str
|
||||
aliases: tuple[str, ...]
|
||||
intent: str
|
||||
primary_sources: tuple[str, ...]
|
||||
supporting_sources: tuple[str, ...]
|
||||
answer_requirements: tuple[str, ...]
|
||||
|
||||
|
||||
def dream_kb(kb: KnowledgeBaseConfig, *, force: bool = False) -> dict[str, Any]:
|
||||
wiki_root = kb.root / ".d_wiki" / "wiki"
|
||||
if not wiki_root.is_dir():
|
||||
raise ValueError("D_WIKI_DREAM_REQUIRES_COMPILED_WIKI")
|
||||
output_root = dream_root(kb)
|
||||
queries_dir = output_root / "queries"
|
||||
if force:
|
||||
shutil.rmtree(output_root, ignore_errors=True)
|
||||
queries_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pages = sorted((wiki_root / "concepts").glob("*.md"))
|
||||
dreamed = [_dream_page(path, wiki_root) for path in pages]
|
||||
dreamed = [item for item in dreamed if item is not None]
|
||||
for item in dreamed:
|
||||
(queries_dir / f"{item.slug}.md").write_text(_query_markdown(item), encoding="utf-8")
|
||||
(output_root / "README.md").write_text(_readme(kb, len(pages), len(dreamed)), encoding="utf-8")
|
||||
return {
|
||||
"status": "dreamed",
|
||||
"kb": kb.name,
|
||||
"input_wiki": str(wiki_root),
|
||||
"output_store": str(output_root),
|
||||
"queries": len(dreamed),
|
||||
"dreamed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def load_dream_query_entries(kb: KnowledgeBaseConfig) -> list[Any]:
|
||||
return load_query_entries(dream_root(kb) / "queries")
|
||||
|
||||
|
||||
def dream_root(kb: KnowledgeBaseConfig) -> Path:
|
||||
return kb.root / ".d_wiki" / "dreams" / "latest"
|
||||
|
||||
|
||||
def _dream_page(path: Path, wiki_root: Path) -> DreamedQuery | None:
|
||||
meta, body = frontmatter(path.read_text(encoding="utf-8", errors="ignore"))
|
||||
title = str(meta.get("title") or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
curation_body = _curation_body(body)
|
||||
source = _first(meta.get("sources")) or _wikilink_for_compiled(path, wiki_root)
|
||||
aliases = _unique([title, *_dream_aliases(title, curation_body)])
|
||||
intent = _first(meta.get("intents")) or _dream_intent(title, curation_body)
|
||||
requirements = _as_list(meta.get("answer_requirements")) or _dream_requirements(title, curation_body, intent)
|
||||
supporting = _related_sources(body, source)
|
||||
return DreamedQuery(
|
||||
slug=path.stem,
|
||||
title=title,
|
||||
aliases=tuple(aliases[:18]),
|
||||
intent=intent,
|
||||
primary_sources=(source,),
|
||||
supporting_sources=tuple(supporting[:6]),
|
||||
answer_requirements=tuple(requirements[:8]),
|
||||
)
|
||||
|
||||
|
||||
def _dream_aliases(title: str, body: str) -> list[str]:
|
||||
text = f"{title}\n{body}".lower()
|
||||
aliases = [
|
||||
f"{title}怎么用",
|
||||
f"如何使用{title}",
|
||||
]
|
||||
if _looks_like_3d_generation(title, text):
|
||||
aliases.extend(
|
||||
[
|
||||
"3D资产生成怎么用",
|
||||
"如何使用 3D 资产生成功能",
|
||||
"我想知道如何使用3d资产生成功能",
|
||||
"图生3D怎么用",
|
||||
"如何生成3D模型",
|
||||
"如何把图片生成3D资产",
|
||||
]
|
||||
)
|
||||
if "camera" in text and any(term in text for term in ("点亮", "bring up", "sensor")):
|
||||
aliases.extend(["如何在X5平台上点亮camera", "X5 camera 点亮", "X5 摄像头 bring up"])
|
||||
if any(term in text for term in ("算力", "tops", "bpu")):
|
||||
aliases.extend(["x5的算力是多少", "X5 BPU 算力"])
|
||||
return aliases
|
||||
|
||||
|
||||
def _dream_intent(title: str, body: str) -> str:
|
||||
text = f"{title}\n{body}".lower()
|
||||
if any(term in text for term in ("如何", "怎么", "步骤", "使用", "创建", "配置", "点亮", "生成", "调试", "bring up")):
|
||||
return "procedure"
|
||||
return "fact"
|
||||
|
||||
|
||||
def _dream_requirements(title: str, body: str, intent: str) -> list[str]:
|
||||
text = f"{title}\n{body}".lower()
|
||||
if _looks_like_3d_generation(title, text):
|
||||
return [
|
||||
"说明 3D 生成是图生 3D 工具,不是镜像资产",
|
||||
"按准备参考图、生成、保存到 3D 资产的流程回答",
|
||||
"说明输出默认 glb,生成结果可保存到 3D 资产",
|
||||
"提醒暂不支持文生 3D,文字描述可先用图像生成产图",
|
||||
]
|
||||
if intent == "procedure":
|
||||
return ["按步骤回答", "说明关键前提和入口", "引用证据来源"]
|
||||
return ["直接回答问题", "引用证据来源"]
|
||||
|
||||
|
||||
def _looks_like_3d_generation(title: str, text: str) -> bool:
|
||||
title_text = title.lower()
|
||||
return (
|
||||
"3d 生成" in title_text
|
||||
or "3d生成" in title_text
|
||||
or "3d 资产生成" in title_text
|
||||
or "3d资产生成" in title_text
|
||||
or "embodygen" in text
|
||||
or "ai 图生 3d 工具" in text
|
||||
or "ai 图生3d 工具" in text
|
||||
)
|
||||
|
||||
|
||||
def _curation_body(body: str) -> str:
|
||||
lines: list[str] = []
|
||||
skip = False
|
||||
for line in body.splitlines():
|
||||
if line.startswith("## "):
|
||||
skip = line.strip() in {"## Retrieval Hints", "## Related"}
|
||||
if skip:
|
||||
continue
|
||||
if not skip:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _related_sources(body: str, primary: str) -> list[str]:
|
||||
links = re.findall(r"\[\[([^\]]+)\]\]", body)
|
||||
refs = re.findall(r"\^\[([^#\]]+)(?:#[^\]]+)?\]", body)
|
||||
return _unique([item for item in [*links, *refs] if item and item != primary])
|
||||
|
||||
|
||||
def _query_markdown(item: DreamedQuery) -> str:
|
||||
lines = [
|
||||
"---",
|
||||
f"title: {_yaml_scalar(item.title)}",
|
||||
f"aliases: {_yaml_list(item.aliases)}",
|
||||
f"intent: {item.intent}",
|
||||
f"primary_sources: {_yaml_list(item.primary_sources)}",
|
||||
f"supporting_sources: {_yaml_list(item.supporting_sources)}",
|
||||
f"answer_requirements: {_yaml_list(item.answer_requirements)}",
|
||||
"provenance: dreamed",
|
||||
"---",
|
||||
"",
|
||||
f"Dreamed query memory for {item.title}.",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _readme(kb: KnowledgeBaseConfig, pages: int, queries: int) -> str:
|
||||
return "\n".join(
|
||||
[
|
||||
"# d-wiki dream output",
|
||||
"",
|
||||
"This store is generated from the compiled wiki. It does not modify source documents or compiled concept pages.",
|
||||
"",
|
||||
f"- kb: `{kb.name}`",
|
||||
f"- input_pages: {pages}",
|
||||
f"- dreamed_queries: {queries}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _first(value: Any) -> str:
|
||||
items = _as_list(value)
|
||||
return items[0] if items else ""
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
return [value]
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
return []
|
||||
|
||||
|
||||
def _unique(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
output: list[str] = []
|
||||
for value in values:
|
||||
cleaned = re.sub(r"\s+", " ", str(value)).strip()
|
||||
if not cleaned or cleaned in seen:
|
||||
continue
|
||||
seen.add(cleaned)
|
||||
output.append(cleaned)
|
||||
return output
|
||||
|
||||
|
||||
def _wikilink_for_compiled(path: Path, wiki_root: Path) -> str:
|
||||
return path.relative_to(wiki_root).with_suffix("").as_posix()
|
||||
|
||||
|
||||
def _yaml_scalar(value: str) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _yaml_list(values: tuple[str, ...] | list[str]) -> str:
|
||||
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
|
||||
156
build/lib/d_wiki/llm.py
Normal file
156
build/lib/d_wiki/llm.py
Normal file
@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LlmSettings:
|
||||
base_url: str
|
||||
model: str
|
||||
api_key: str
|
||||
api_style: str = "anthropic"
|
||||
max_tokens: int = 900
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LlmPageSynthesis:
|
||||
summary: str
|
||||
answer_blueprint: tuple[str, ...]
|
||||
key_evidence: tuple[str, ...]
|
||||
confidence: str = "medium"
|
||||
|
||||
|
||||
class LlmPageSynthesizer:
|
||||
def __init__(self, settings: LlmSettings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
@classmethod
|
||||
def from_environment(cls) -> "LlmPageSynthesizer | None":
|
||||
base_url = os.environ.get("D_WIKI_LLM_BASE_URL") or os.environ.get("ROBOGO_LLM_BASE_URL")
|
||||
model = os.environ.get("D_WIKI_LLM_MODEL") or os.environ.get("ROBOGO_LLM_MODEL")
|
||||
api_key = os.environ.get("D_WIKI_LLM_API_KEY") or os.environ.get("ROBOGO_LLM_API_KEY")
|
||||
if not base_url or not model or not api_key:
|
||||
return None
|
||||
api_style = os.environ.get("D_WIKI_LLM_API_STYLE") or os.environ.get("ROBOGO_LLM_API_STYLE") or "anthropic"
|
||||
max_tokens = int(os.environ.get("D_WIKI_LLM_MAX_TOKENS", "900"))
|
||||
return cls(LlmSettings(base_url=base_url, model=model, api_key=api_key, api_style=api_style, max_tokens=max_tokens))
|
||||
|
||||
def synthesize(self, prompt: str) -> LlmPageSynthesis | None:
|
||||
text = self._complete(prompt).strip()
|
||||
payload = _extract_json(text)
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
summary = str(payload.get("summary") or "").strip()
|
||||
if not summary:
|
||||
return None
|
||||
answer_blueprint = _string_tuple(payload.get("answer_blueprint"))
|
||||
key_evidence = _string_tuple(payload.get("key_evidence"))
|
||||
confidence = str(payload.get("confidence") or "medium").strip().lower()
|
||||
if confidence not in {"low", "medium", "high"}:
|
||||
confidence = "medium"
|
||||
return LlmPageSynthesis(
|
||||
summary=summary,
|
||||
answer_blueprint=answer_blueprint,
|
||||
key_evidence=key_evidence,
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
def _complete(self, prompt: str) -> str:
|
||||
if self.settings.api_style == "openai":
|
||||
return self._complete_openai(prompt)
|
||||
return self._complete_anthropic(prompt)
|
||||
|
||||
def _complete_anthropic(self, prompt: str) -> str:
|
||||
payload = {
|
||||
"model": self.settings.model,
|
||||
"max_tokens": self.settings.max_tokens,
|
||||
"temperature": 0,
|
||||
"system": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
data = self._post_json(_join_url(self.settings.base_url, "v1/messages"), payload)
|
||||
content = data.get("content")
|
||||
if isinstance(content, list):
|
||||
return "\n".join(str(item.get("text", "")) for item in content if isinstance(item, dict))
|
||||
return str(data.get("text") or "")
|
||||
|
||||
def _complete_openai(self, prompt: str) -> str:
|
||||
payload = {
|
||||
"model": self.settings.model,
|
||||
"temperature": 0,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
|
||||
},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
}
|
||||
data = self._post_json(_join_url(self.settings.base_url, "v1/chat/completions"), payload)
|
||||
choices = data.get("choices")
|
||||
if isinstance(choices, list) and choices:
|
||||
first = choices[0]
|
||||
if isinstance(first, dict):
|
||||
message = first.get("message")
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("content") or "")
|
||||
return ""
|
||||
|
||||
def _post_json(self, url: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={
|
||||
"content-type": "application/json",
|
||||
"accept": "application/json",
|
||||
"x-api-key": self.settings.api_key,
|
||||
"authorization": f"Bearer {self.settings.api_key}",
|
||||
"anthropic-version": "2023-06-01",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", errors="ignore")
|
||||
raise RuntimeError(f"D_WIKI_LLM_HTTP_ERROR: {exc.code} {detail[:300]}") from exc
|
||||
|
||||
|
||||
def _join_url(base_url: str, suffix: str) -> str:
|
||||
base = base_url.rstrip("/")
|
||||
if base.endswith("/v1/messages") or base.endswith("/v1/chat/completions"):
|
||||
return base
|
||||
if base.endswith("/v1"):
|
||||
return f"{base}/{suffix.removeprefix('v1/')}"
|
||||
return f"{base}/{suffix}"
|
||||
|
||||
|
||||
def _extract_json(text: str) -> Any:
|
||||
cleaned = text.strip()
|
||||
if cleaned.startswith("```"):
|
||||
cleaned = cleaned.strip("`")
|
||||
if cleaned.lower().startswith("json"):
|
||||
cleaned = cleaned[4:].strip()
|
||||
decoder = json.JSONDecoder()
|
||||
for index, char in enumerate(cleaned):
|
||||
if char != "{":
|
||||
continue
|
||||
try:
|
||||
value, _end = decoder.raw_decode(cleaned[index:])
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _string_tuple(value: Any) -> tuple[str, ...]:
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
return tuple(str(item).strip() for item in value if str(item).strip())
|
||||
119
build/lib/d_wiki/metadata.py
Normal file
119
build/lib/d_wiki/metadata.py
Normal file
@ -0,0 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .yaml_compat import safe_load
|
||||
from .tokenize import tokenize
|
||||
|
||||
QUERY_MATCH_STOP_TOKENS = {
|
||||
"如何",
|
||||
"怎么",
|
||||
"使用",
|
||||
"创建",
|
||||
"知道",
|
||||
"功能",
|
||||
"资产",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AliasGroup:
|
||||
name: str
|
||||
terms: tuple[str, ...]
|
||||
intent: str | None
|
||||
related: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueryEntry:
|
||||
path: Path
|
||||
title: str
|
||||
aliases: tuple[str, ...]
|
||||
intent: str | None
|
||||
primary_sources: tuple[str, ...]
|
||||
supporting_sources: tuple[str, ...]
|
||||
answer_requirements: tuple[str, ...]
|
||||
|
||||
|
||||
def load_alias_groups(path: Path | None) -> list[AliasGroup]:
|
||||
if path is None or not path.exists():
|
||||
return []
|
||||
data = safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
raw_aliases = data.get("aliases", {})
|
||||
if not isinstance(raw_aliases, dict):
|
||||
return []
|
||||
groups: list[AliasGroup] = []
|
||||
for name, raw in raw_aliases.items():
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
groups.append(
|
||||
AliasGroup(
|
||||
name=str(name),
|
||||
terms=tuple(str(term).strip() for term in raw.get("terms", []) if str(term).strip()),
|
||||
intent=str(raw["intent"]) if raw.get("intent") else None,
|
||||
related=tuple(str(item).strip() for item in raw.get("related", []) if str(item).strip()),
|
||||
)
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def load_query_entries(directory: Path | None) -> list[QueryEntry]:
|
||||
if directory is None or not directory.exists():
|
||||
return []
|
||||
return [_query_entry(path) for path in sorted(directory.rglob("*.md")) if path.is_file()]
|
||||
|
||||
|
||||
def frontmatter(text: str) -> tuple[dict[str, Any], str]:
|
||||
if not text.startswith("---\n"):
|
||||
return {}, text
|
||||
end = text.find("\n---", 4)
|
||||
if end < 0:
|
||||
return {}, text
|
||||
meta = safe_load(text[4:end]) or {}
|
||||
if not isinstance(meta, dict):
|
||||
meta = {}
|
||||
return meta, text[end + 4 :].lstrip("\n")
|
||||
|
||||
|
||||
def query_matches_entry(query: str, entry: QueryEntry) -> bool:
|
||||
normalized = normalize(query)
|
||||
if any(normalize(alias) in normalized or normalized in normalize(alias) for alias in entry.aliases):
|
||||
return True
|
||||
query_terms = _meaningful_query_terms(query)
|
||||
for candidate in (entry.title, *entry.aliases):
|
||||
candidate_terms = _meaningful_query_terms(candidate)
|
||||
if len(query_terms & candidate_terms) >= 2:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _meaningful_query_terms(value: str) -> set[str]:
|
||||
return {term for term in tokenize(value) if term not in QUERY_MATCH_STOP_TOKENS and len(term.strip()) >= 2}
|
||||
|
||||
|
||||
def normalize(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", value.lower()).strip()
|
||||
|
||||
|
||||
def _query_entry(path: Path) -> QueryEntry:
|
||||
meta, body = frontmatter(path.read_text(encoding="utf-8"))
|
||||
title = str(meta.get("title") or _first_heading(body) or path.stem)
|
||||
return QueryEntry(
|
||||
path=path,
|
||||
title=title,
|
||||
aliases=tuple(str(item).strip() for item in meta.get("aliases", []) if str(item).strip()),
|
||||
intent=str(meta["intent"]) if meta.get("intent") else None,
|
||||
primary_sources=tuple(str(item).strip() for item in meta.get("primary_sources", []) if str(item).strip()),
|
||||
supporting_sources=tuple(str(item).strip() for item in meta.get("supporting_sources", []) if str(item).strip()),
|
||||
answer_requirements=tuple(str(item).strip() for item in meta.get("answer_requirements", []) if str(item).strip()),
|
||||
)
|
||||
|
||||
|
||||
def _first_heading(text: str) -> str | None:
|
||||
for line in text.splitlines():
|
||||
if match := re.match(r"^#{1,6}\s+(.+)$", line):
|
||||
return match.group(1).strip()
|
||||
return None
|
||||
469
build/lib/d_wiki/search.py
Normal file
469
build/lib/d_wiki/search.py
Normal file
@ -0,0 +1,469 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import KnowledgeBaseConfig
|
||||
from .metadata import (
|
||||
AliasGroup,
|
||||
QueryEntry,
|
||||
frontmatter,
|
||||
load_alias_groups,
|
||||
load_query_entries,
|
||||
normalize,
|
||||
query_matches_entry,
|
||||
)
|
||||
from .dreams import load_dream_query_entries
|
||||
from .tokenize import tokenize
|
||||
|
||||
SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt")
|
||||
MAX_BLOCK_CHARS = 520
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Chunk:
|
||||
title: str
|
||||
wikilink: str
|
||||
source_path: Path
|
||||
page_type: str
|
||||
heading: str
|
||||
text: str
|
||||
tokens: tuple[str, ...]
|
||||
index: int
|
||||
|
||||
|
||||
def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "ok",
|
||||
"config": str(config_path),
|
||||
"knowledge_bases": {
|
||||
name: {
|
||||
"root": str(kb.root),
|
||||
"source_type": kb.source_type,
|
||||
"exists": kb.root.exists(),
|
||||
"aliases": str(kb.aliases_path) if kb.aliases_path else None,
|
||||
"queries": str(kb.queries_dir) if kb.queries_dir else None,
|
||||
}
|
||||
for name, kb in bases.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]:
|
||||
chunks = load_chunks(kb)
|
||||
index_dir = kb.root / ".d_wiki_index"
|
||||
index_dir.mkdir(exist_ok=True)
|
||||
payload = {
|
||||
"kb": kb.name,
|
||||
"root": str(kb.root),
|
||||
"source_type": kb.source_type,
|
||||
"compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None,
|
||||
"chunks": len(chunks),
|
||||
"pages": len({chunk.wikilink for chunk in chunks}),
|
||||
"indexed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
(index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return {"status": "indexed", **payload}
|
||||
|
||||
|
||||
def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]:
|
||||
aliases = load_alias_groups(kb.aliases_path)
|
||||
query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)]
|
||||
matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)]
|
||||
expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries)
|
||||
chunks = load_chunks(kb)
|
||||
ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries)
|
||||
selected = ranked[: max(1, min(max_results, 8))]
|
||||
context_pack = build_context_pack(query, selected, matched_aliases, matched_entries)
|
||||
return {
|
||||
"status": "found" if selected else "no_match",
|
||||
"query": query,
|
||||
"knowledge_base": kb.name,
|
||||
"backend": "lightweight_bm25",
|
||||
"quality": quality(selected),
|
||||
"sources": [source_summary(item[1], item[0]) for item in selected],
|
||||
"context_pack": context_pack,
|
||||
}
|
||||
|
||||
|
||||
def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]:
|
||||
chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink]
|
||||
if not chunks:
|
||||
return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink}
|
||||
first = chunks[0]
|
||||
return {
|
||||
"status": "found",
|
||||
"knowledge_base": kb.name,
|
||||
"id": wikilink,
|
||||
"title": first.title,
|
||||
"source_path": str(first.source_path),
|
||||
"blocks": [
|
||||
{"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"}
|
||||
for chunk in chunks
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def expand_query(
|
||||
query: str,
|
||||
aliases: list[AliasGroup],
|
||||
entries: list[QueryEntry],
|
||||
) -> tuple[list[str], list[AliasGroup]]:
|
||||
terms = tokenize(query)
|
||||
normalized_query = normalize(query)
|
||||
matched: list[AliasGroup] = []
|
||||
for group in aliases:
|
||||
if any(normalize(term) in normalized_query for term in group.terms):
|
||||
matched.append(group)
|
||||
for term in group.terms:
|
||||
terms.extend(tokenize(term))
|
||||
for entry in entries:
|
||||
terms.extend(tokenize(entry.title))
|
||||
for alias in entry.aliases:
|
||||
terms.extend(tokenize(alias))
|
||||
return terms, matched
|
||||
|
||||
|
||||
def rank_chunks(
|
||||
chunks: list[Chunk],
|
||||
terms: list[str],
|
||||
aliases: list[AliasGroup],
|
||||
entries: list[QueryEntry],
|
||||
) -> list[tuple[float, Chunk]]:
|
||||
if not chunks or not terms:
|
||||
return []
|
||||
term_counts = [Counter(chunk.tokens) for chunk in chunks]
|
||||
doc_freq: Counter[str] = Counter()
|
||||
for counts in term_counts:
|
||||
for term in counts:
|
||||
doc_freq[term] += 1
|
||||
avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks))
|
||||
query_counts = Counter(terms)
|
||||
ranked: list[tuple[float, Chunk]] = []
|
||||
related = {item for group in aliases for item in group.related}
|
||||
primary = {item for entry in entries for item in entry.primary_sources}
|
||||
supporting = {item for entry in entries for item in entry.supporting_sources}
|
||||
for chunk, counts in zip(chunks, term_counts):
|
||||
score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts)
|
||||
if any(wikilink_matches(chunk.wikilink, item) for item in related):
|
||||
score += 8.0
|
||||
if any(wikilink_matches(chunk.wikilink, item) for item in primary):
|
||||
score += 16.0
|
||||
if any(wikilink_matches(chunk.wikilink, item) for item in supporting):
|
||||
score += 5.0
|
||||
score += heading_score_adjustment(chunk.heading)
|
||||
if score > 0:
|
||||
ranked.append((round(score, 3), chunk))
|
||||
ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index))
|
||||
return ranked
|
||||
|
||||
|
||||
def heading_score_adjustment(heading: str) -> float:
|
||||
if heading == "Retrieval Hints":
|
||||
return -80.0
|
||||
if heading == "Intent Model":
|
||||
return -40.0
|
||||
if heading == "Answer Blueprint":
|
||||
return -25.0
|
||||
if heading == "Evidence Graph":
|
||||
return 3.0
|
||||
if heading == "Key Evidence":
|
||||
return 5.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def bm25_score(
|
||||
counts: Counter[str],
|
||||
doc_len: int,
|
||||
doc_freq: Counter[str],
|
||||
doc_count: int,
|
||||
avg_len: float,
|
||||
query_counts: Counter[str],
|
||||
) -> float:
|
||||
k1 = 1.5
|
||||
b = 0.75
|
||||
score = 0.0
|
||||
for term, qf in query_counts.items():
|
||||
tf = counts.get(term, 0)
|
||||
if tf == 0:
|
||||
continue
|
||||
idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5))
|
||||
denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1)))
|
||||
score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3)
|
||||
return score
|
||||
|
||||
|
||||
def build_context_pack(
|
||||
query: str,
|
||||
selected: list[tuple[float, Chunk]],
|
||||
aliases: list[AliasGroup],
|
||||
entries: list[QueryEntry],
|
||||
) -> dict[str, Any]:
|
||||
intent = answer_intent(query, aliases, entries)
|
||||
grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list)
|
||||
for item in selected:
|
||||
grouped[item[1].wikilink].append(item)
|
||||
source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1]))
|
||||
entry_primary = {item for entry in entries for item in entry.primary_sources}
|
||||
primary_sources = [
|
||||
source_context(wikilink, chunks)
|
||||
for wikilink, chunks in source_items
|
||||
if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary)
|
||||
]
|
||||
if not primary_sources:
|
||||
primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]]
|
||||
supporting_sources = [
|
||||
source_context(wikilink, chunks, blocks_limit=1)
|
||||
for wikilink, chunks in source_items
|
||||
if wikilink not in {source["wikilink"] for source in primary_sources}
|
||||
][:3]
|
||||
coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete")
|
||||
return {
|
||||
"answer_intent": intent,
|
||||
"coverage": coverage,
|
||||
"answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.",
|
||||
"primary_sources": primary_sources,
|
||||
"supporting_sources": supporting_sources,
|
||||
"answer_requirements": answer_requirements(intent, entries),
|
||||
}
|
||||
|
||||
|
||||
def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]:
|
||||
top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit]
|
||||
chunk = top[0][1]
|
||||
return {
|
||||
"title": chunk.title,
|
||||
"wikilink": wikilink,
|
||||
"source_path": str(chunk.source_path),
|
||||
"reason": "Matched query terms and local metadata.",
|
||||
"blocks": [
|
||||
{
|
||||
"block_id": f"{block.wikilink}#{block.index}",
|
||||
"heading": block.heading,
|
||||
"text": trim_block(block.text),
|
||||
"score": score,
|
||||
}
|
||||
for score, block in top
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str:
|
||||
for entry in entries:
|
||||
if entry.intent:
|
||||
return entry.intent
|
||||
for group in aliases:
|
||||
if group.intent:
|
||||
return group.intent
|
||||
if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")):
|
||||
return "procedure"
|
||||
return "fact"
|
||||
|
||||
|
||||
def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]:
|
||||
requirements = unique([req for entry in entries for req in entry.answer_requirements])
|
||||
if requirements:
|
||||
return requirements
|
||||
if intent == "procedure":
|
||||
return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"]
|
||||
return ["直接回答问题", "引用命中的来源"]
|
||||
|
||||
|
||||
def unique(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
output: list[str] = []
|
||||
for value in values:
|
||||
if value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
output.append(value)
|
||||
return output
|
||||
|
||||
|
||||
def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]:
|
||||
top = selected[0][0] if selected else 0.0
|
||||
return {
|
||||
"confidence": "high" if top >= 8.0 else ("low" if selected else "none"),
|
||||
"high_confidence": top >= 8.0,
|
||||
"top_score": top,
|
||||
"high_confidence_score": 8.0,
|
||||
}
|
||||
|
||||
|
||||
def wikilink_matches(candidate: str, expected: str) -> bool:
|
||||
return candidate == expected or candidate == f"concepts/{expected}"
|
||||
|
||||
|
||||
def source_summary(chunk: Chunk, score: float) -> dict[str, Any]:
|
||||
return {
|
||||
"title": chunk.title,
|
||||
"wikilink": chunk.wikilink,
|
||||
"source_path": str(chunk.source_path),
|
||||
"heading": chunk.heading,
|
||||
"snippet": trim_block(chunk.text, 320),
|
||||
"score": score,
|
||||
}
|
||||
|
||||
|
||||
def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
|
||||
root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root
|
||||
paths = source_paths(root)
|
||||
chunks: list[Chunk] = []
|
||||
for path in paths:
|
||||
chunks.extend(chunks_for_path(path, root))
|
||||
return chunks
|
||||
|
||||
|
||||
def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
|
||||
chunks: list[Chunk] = []
|
||||
for path in source_paths(kb.root):
|
||||
chunks.extend(chunks_for_path(path, kb.root))
|
||||
return chunks
|
||||
|
||||
|
||||
def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path:
|
||||
return kb.root / ".d_wiki" / "wiki"
|
||||
|
||||
|
||||
def source_paths(root: Path) -> list[Path]:
|
||||
if (root / "_sources").is_dir():
|
||||
return sorted(
|
||||
path
|
||||
for path in (root / "_sources").rglob("*.txt")
|
||||
if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES)
|
||||
)
|
||||
generated = {"SCHEMA.md", "index.md", "log.md"}
|
||||
generated_dirs = {".d_wiki", ".d_wiki_index"}
|
||||
is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki"
|
||||
return sorted(
|
||||
path
|
||||
for path in root.rglob("*.md")
|
||||
if path.is_file()
|
||||
and path.name not in generated
|
||||
and (is_compiled_root or not any(part in generated_dirs for part in path.parts))
|
||||
)
|
||||
|
||||
|
||||
def chunks_for_path(path: Path, root: Path) -> list[Chunk]:
|
||||
raw = path.read_text(encoding="utf-8", errors="ignore")
|
||||
meta, body = frontmatter(raw)
|
||||
title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title())
|
||||
metadata_text = metadata_terms(meta)
|
||||
wikilink = wikilink_for(path, root)
|
||||
sections = split_sections(body, title)
|
||||
chunks: list[Chunk] = []
|
||||
for index, (heading, text) in enumerate(sections):
|
||||
if heading in {"Source Map", "Related"}:
|
||||
continue
|
||||
if not text.strip():
|
||||
continue
|
||||
field_text = " ".join([title, heading, wikilink, metadata_text, text])
|
||||
chunks.append(
|
||||
Chunk(
|
||||
title=title,
|
||||
wikilink=wikilink,
|
||||
source_path=path.resolve(),
|
||||
page_type="sphinx_source" if "_sources" in path.parts else "markdown",
|
||||
heading=heading,
|
||||
text=text.strip(),
|
||||
tokens=tuple(tokenize(field_text)),
|
||||
index=index,
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def metadata_terms(meta: dict[str, Any]) -> str:
|
||||
values: list[str] = []
|
||||
for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"):
|
||||
raw = meta.get(key)
|
||||
if isinstance(raw, str):
|
||||
values.append(raw)
|
||||
elif isinstance(raw, list):
|
||||
values.extend(str(item) for item in raw if str(item).strip())
|
||||
return " ".join(values)
|
||||
|
||||
|
||||
def split_sections(body: str, title: str) -> list[tuple[str, str]]:
|
||||
heading = title
|
||||
buffer: list[str] = []
|
||||
sections: list[tuple[str, str]] = []
|
||||
in_fence = False
|
||||
for line in body.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("```"):
|
||||
in_fence = not in_fence
|
||||
buffer.append(line)
|
||||
continue
|
||||
if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)):
|
||||
if buffer:
|
||||
sections.extend(paragraphs(heading, "\n".join(buffer)))
|
||||
buffer = []
|
||||
heading = match.group(1).strip() or title
|
||||
continue
|
||||
buffer.append(line)
|
||||
if buffer:
|
||||
sections.extend(paragraphs(heading, "\n".join(buffer)))
|
||||
return sections or [(title, body)]
|
||||
|
||||
|
||||
def paragraphs(heading: str, text: str) -> list[tuple[str, str]]:
|
||||
blocks: list[tuple[str, str]] = []
|
||||
for part in re.split(r"\n\s*\n", text):
|
||||
cleaned = clean_block(part)
|
||||
if cleaned:
|
||||
blocks.append((heading, cleaned))
|
||||
return blocks
|
||||
|
||||
|
||||
def clean_block(text: str) -> str:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return ""
|
||||
if re.fullmatch(r"[-*_]{3,}", stripped):
|
||||
return ""
|
||||
compact = " ".join(stripped.split())
|
||||
if "📷" in compact and len(compact) <= 80:
|
||||
return ""
|
||||
if compact.startswith("> 📷") and len(compact) <= 120:
|
||||
return ""
|
||||
return stripped
|
||||
|
||||
|
||||
def first_heading(body: str) -> str | None:
|
||||
in_fence = False
|
||||
for line in body.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("```"):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
if in_fence:
|
||||
continue
|
||||
if match := re.match(r"^#{1,6}\s+(.+)$", line):
|
||||
return match.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def wikilink_for(path: Path, root: Path) -> str:
|
||||
try:
|
||||
relative = path.relative_to(root)
|
||||
except ValueError:
|
||||
return path.stem
|
||||
if relative.parts and relative.parts[0] == "_sources":
|
||||
relative = Path(*relative.parts[1:])
|
||||
value = relative.as_posix()
|
||||
for suffix in SPHINX_SOURCE_SUFFIXES:
|
||||
if value.endswith(suffix):
|
||||
return value[: -len(suffix)]
|
||||
return value[:-3] if value.endswith(".md") else path.stem
|
||||
|
||||
|
||||
def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str:
|
||||
compact = " ".join(text.split())
|
||||
return compact if len(compact) <= limit else compact[:limit].rstrip()
|
||||
22
build/lib/d_wiki/tokenize.py
Normal file
22
build/lib/d_wiki/tokenize.py
Normal file
@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上")
|
||||
STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"}
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
lowered = text.lower()
|
||||
terms = re.findall(r"[a-z0-9_./+-]+", lowered)
|
||||
cjk_chars = re.findall(r"[\u4e00-\u9fff]", text)
|
||||
terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1)))
|
||||
return [term for term in terms if _useful(term)]
|
||||
|
||||
|
||||
def _useful(term: str) -> bool:
|
||||
if not term or term in STOP_TERMS:
|
||||
return False
|
||||
if re.search(r"[\u4e00-\u9fff]", term):
|
||||
return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term))
|
||||
return True
|
||||
70
build/lib/d_wiki/yaml_compat.py
Normal file
70
build/lib/d_wiki/yaml_compat.py
Normal file
@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml as _yaml # type: ignore
|
||||
except Exception: # pragma: no cover - exercised in environments without PyYAML
|
||||
_yaml = None
|
||||
|
||||
|
||||
def safe_load(text: str) -> Any:
|
||||
if _yaml is not None:
|
||||
return _yaml.safe_load(text)
|
||||
lines = [
|
||||
(len(line) - len(line.lstrip(" ")), line.strip())
|
||||
for line in text.splitlines()
|
||||
if line.strip() and not line.strip().startswith("#")
|
||||
]
|
||||
value, _index = _parse_node(lines, 0, 0)
|
||||
return value
|
||||
|
||||
|
||||
def _parse_node(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]:
|
||||
if index >= len(lines):
|
||||
return {}, index
|
||||
if lines[index][0] < indent:
|
||||
return {}, index
|
||||
if lines[index][1].startswith("- "):
|
||||
values: list[Any] = []
|
||||
while index < len(lines) and lines[index][0] == indent and lines[index][1].startswith("- "):
|
||||
values.append(_scalar(lines[index][1][2:].strip()))
|
||||
index += 1
|
||||
return values, index
|
||||
mapping: dict[str, Any] = {}
|
||||
while index < len(lines):
|
||||
current_indent, stripped = lines[index]
|
||||
if current_indent < indent:
|
||||
break
|
||||
if current_indent > indent:
|
||||
break
|
||||
if ":" not in stripped:
|
||||
index += 1
|
||||
continue
|
||||
key, raw_value = stripped.split(":", 1)
|
||||
key = key.strip()
|
||||
raw_value = raw_value.strip()
|
||||
index += 1
|
||||
if raw_value:
|
||||
mapping[key] = _scalar(raw_value)
|
||||
continue
|
||||
child, index = _parse_node(lines, index, indent + 2)
|
||||
mapping[key] = child
|
||||
return mapping, index
|
||||
|
||||
|
||||
def _scalar(value: str) -> Any:
|
||||
if value in {"null", "None", "~"}:
|
||||
return None
|
||||
if value in {"true", "True"}:
|
||||
return True
|
||||
if value in {"false", "False"}:
|
||||
return False
|
||||
if value.startswith("[") and value.endswith("]"):
|
||||
inner = value[1:-1].strip()
|
||||
if not inner:
|
||||
return []
|
||||
return [_scalar(item.strip()) for item in inner.split(",")]
|
||||
if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
|
||||
return value[1:-1]
|
||||
return value
|
||||
5
d-wiki
Executable file
5
d-wiki
Executable file
@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
from d_wiki.cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
11
d-wiki.yml
Normal file
11
d-wiki.yml
Normal file
@ -0,0 +1,11 @@
|
||||
knowledge_bases:
|
||||
x5:
|
||||
root: "/Users/d-robotics/Desktop/evb/sdk_management_system/x5_user_manual/latest"
|
||||
source_type: "sphinx"
|
||||
aliases: "knowledge/x5/aliases.yml"
|
||||
queries: "knowledge/x5/queries"
|
||||
robogo:
|
||||
root: "/Users/d-robotics/Documents/New project/read/robogo-user-guide"
|
||||
source_type: "markdown"
|
||||
aliases: "knowledge/robogo/aliases.yml"
|
||||
queries: "knowledge/robogo/queries"
|
||||
5
d_wiki.egg-info/PKG-INFO
Normal file
5
d_wiki.egg-info/PKG-INFO
Normal file
@ -0,0 +1,5 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: d-wiki
|
||||
Version: 0.1.0
|
||||
Summary: Lightweight local knowledge retrieval with BM25 and metadata context packs.
|
||||
Requires-Python: >=3.10
|
||||
19
d_wiki.egg-info/SOURCES.txt
Normal file
19
d_wiki.egg-info/SOURCES.txt
Normal file
@ -0,0 +1,19 @@
|
||||
README.md
|
||||
pyproject.toml
|
||||
d_wiki/__init__.py
|
||||
d_wiki/__main__.py
|
||||
d_wiki/cli.py
|
||||
d_wiki/compile.py
|
||||
d_wiki/config.py
|
||||
d_wiki/dreams.py
|
||||
d_wiki/llm.py
|
||||
d_wiki/metadata.py
|
||||
d_wiki/search.py
|
||||
d_wiki/tokenize.py
|
||||
d_wiki/yaml_compat.py
|
||||
d_wiki.egg-info/PKG-INFO
|
||||
d_wiki.egg-info/SOURCES.txt
|
||||
d_wiki.egg-info/dependency_links.txt
|
||||
d_wiki.egg-info/entry_points.txt
|
||||
d_wiki.egg-info/top_level.txt
|
||||
tests/test_d_wiki.py
|
||||
1
d_wiki.egg-info/dependency_links.txt
Normal file
1
d_wiki.egg-info/dependency_links.txt
Normal file
@ -0,0 +1 @@
|
||||
|
||||
2
d_wiki.egg-info/entry_points.txt
Normal file
2
d_wiki.egg-info/entry_points.txt
Normal file
@ -0,0 +1,2 @@
|
||||
[console_scripts]
|
||||
d-wiki = d_wiki.cli:main
|
||||
1
d_wiki.egg-info/top_level.txt
Normal file
1
d_wiki.egg-info/top_level.txt
Normal file
@ -0,0 +1 @@
|
||||
d_wiki
|
||||
3
d_wiki/__init__.py
Normal file
3
d_wiki/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
"""d-wiki lightweight local knowledge retrieval."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
4
d_wiki/__main__.py
Normal file
4
d_wiki/__main__.py
Normal file
@ -0,0 +1,4 @@
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
BIN
d_wiki/__pycache__/__init__.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/__init__.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/__init__.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/__init__.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/__main__.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/__main__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/__main__.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/__main__.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/cli.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/cli.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/cli.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/cli.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/cli.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/cli.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/compile.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/compile.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/compile.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/compile.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/compile.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/compile.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/config.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/config.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/config.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/config.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/config.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/config.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/dreams.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/dreams.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/dreams.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/dreams.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/dreams.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/dreams.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/llm.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/llm.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/llm.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/llm.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/llm.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/llm.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/metadata.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/metadata.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/metadata.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/metadata.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/metadata.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/metadata.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/search.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/search.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/search.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/search.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/search.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/search.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/tokenize.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/tokenize.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/tokenize.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/tokenize.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/tokenize.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/tokenize.cpython-314.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/yaml_compat.cpython-311.pyc
Normal file
BIN
d_wiki/__pycache__/yaml_compat.cpython-311.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/yaml_compat.cpython-312.pyc
Normal file
BIN
d_wiki/__pycache__/yaml_compat.cpython-312.pyc
Normal file
Binary file not shown.
BIN
d_wiki/__pycache__/yaml_compat.cpython-314.pyc
Normal file
BIN
d_wiki/__pycache__/yaml_compat.cpython-314.pyc
Normal file
Binary file not shown.
77
d_wiki/cli.py
Normal file
77
d_wiki/cli.py
Normal file
@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import DWikiConfig, load_config
|
||||
from .compile import compile_kb
|
||||
from .dreams import dream_kb
|
||||
from .search import get_page, index_payload, search, status_payload
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="d-wiki")
|
||||
parser.add_argument("--config", type=Path, default=None)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
status_parser = subparsers.add_parser("status")
|
||||
status_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
index_parser = subparsers.add_parser("index")
|
||||
index_parser.add_argument("--kb", required=True)
|
||||
index_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
compile_parser = subparsers.add_parser("compile")
|
||||
compile_parser.add_argument("--kb", required=True)
|
||||
compile_parser.add_argument("--force", action="store_true")
|
||||
compile_parser.add_argument("--llm", choices=["off", "auto", "require"], default="off")
|
||||
compile_parser.add_argument("--llm-scope", choices=["queries", "all"], default="queries")
|
||||
compile_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
dream_parser = subparsers.add_parser("dream")
|
||||
dream_parser.add_argument("--kb", required=True)
|
||||
dream_parser.add_argument("--force", action="store_true")
|
||||
dream_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
search_parser = subparsers.add_parser("search")
|
||||
search_parser.add_argument("--kb", required=True)
|
||||
search_parser.add_argument("--query", required=True)
|
||||
search_parser.add_argument("--max-results", type=int, default=5)
|
||||
search_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
get_parser = subparsers.add_parser("get")
|
||||
get_parser.add_argument("--kb", required=True)
|
||||
get_parser.add_argument("--id", required=True)
|
||||
get_parser.add_argument("--format", choices=["json"], default="json")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
payload = dispatch(args, config)
|
||||
except Exception as exc:
|
||||
print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def dispatch(args: argparse.Namespace, config: DWikiConfig) -> dict[str, Any]:
|
||||
if args.command == "status":
|
||||
return status_payload(config.path, config.knowledge_bases)
|
||||
kb = config.knowledge_bases.get(args.kb)
|
||||
if kb is None:
|
||||
raise ValueError(f"D_WIKI_KB_NOT_FOUND: {args.kb}")
|
||||
if args.command == "index":
|
||||
return index_payload(kb)
|
||||
if args.command == "compile":
|
||||
return compile_kb(kb, force=args.force, llm_mode=args.llm, llm_scope=args.llm_scope)
|
||||
if args.command == "dream":
|
||||
return dream_kb(kb, force=args.force)
|
||||
if args.command == "search":
|
||||
return search(kb, args.query, args.max_results)
|
||||
if args.command == "get":
|
||||
return get_page(kb, args.id)
|
||||
raise ValueError(f"D_WIKI_UNKNOWN_COMMAND: {args.command}")
|
||||
590
d_wiki/compile.py
Normal file
590
d_wiki/compile.py
Normal file
@ -0,0 +1,590 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import KnowledgeBaseConfig
|
||||
from .llm import LlmPageSynthesis, LlmPageSynthesizer
|
||||
from .metadata import load_alias_groups, load_query_entries
|
||||
from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block
|
||||
from .tokenize import tokenize
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompiledPage:
|
||||
wikilink: str
|
||||
title: str
|
||||
path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _EvidenceProfile:
|
||||
tokens: frozenset[str]
|
||||
phrases: tuple[str, ...]
|
||||
|
||||
|
||||
def compile_kb(
|
||||
kb: KnowledgeBaseConfig,
|
||||
*,
|
||||
force: bool = False,
|
||||
llm_mode: str = "off",
|
||||
llm_scope: str = "queries",
|
||||
) -> dict[str, Any]:
|
||||
chunks = load_source_chunks(kb)
|
||||
if not chunks:
|
||||
raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS")
|
||||
synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None
|
||||
if llm_mode == "require" and synthesizer is None:
|
||||
raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED")
|
||||
wiki_root = compiled_wiki_root(kb)
|
||||
concepts_dir = wiki_root / "concepts"
|
||||
raw_dir = wiki_root / "raw"
|
||||
if force:
|
||||
shutil.rmtree(concepts_dir, ignore_errors=True)
|
||||
concepts_dir.mkdir(parents=True, exist_ok=True)
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pages: list[CompiledPage] = []
|
||||
grouped = _group_chunks(chunks)
|
||||
query_entries = load_query_entries(kb.queries_dir)
|
||||
alias_groups = load_alias_groups(kb.aliases_path)
|
||||
query_by_source = {
|
||||
source: entry
|
||||
for entry in query_entries
|
||||
for source in entry.primary_sources
|
||||
}
|
||||
aliases_by_source = {
|
||||
related: group
|
||||
for group in alias_groups
|
||||
for related in group.related
|
||||
}
|
||||
|
||||
for wikilink, page_chunks in sorted(grouped.items()):
|
||||
page = _compile_page(
|
||||
kb,
|
||||
wikilink,
|
||||
page_chunks,
|
||||
query_entry=query_by_source.get(wikilink),
|
||||
alias_group=aliases_by_source.get(wikilink),
|
||||
synthesizer=(
|
||||
synthesizer
|
||||
if llm_scope == "all" or query_by_source.get(wikilink) is not None
|
||||
else None
|
||||
),
|
||||
require_llm=llm_mode == "require",
|
||||
)
|
||||
if force or not page.path.exists():
|
||||
page.path.write_text(page.content, encoding="utf-8")
|
||||
pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path))
|
||||
|
||||
_write_schema(wiki_root, kb)
|
||||
_write_index(wiki_root, pages)
|
||||
_append_log(wiki_root, kb, len(chunks), len(pages))
|
||||
return {
|
||||
"status": "compiled",
|
||||
"kb": kb.name,
|
||||
"source_root": str(kb.root),
|
||||
"wiki_root": str(wiki_root),
|
||||
"source_chunks": len(chunks),
|
||||
"pages": len(pages),
|
||||
"llm_mode": llm_mode,
|
||||
"llm_scope": llm_scope,
|
||||
"llm_enabled": synthesizer is not None,
|
||||
"compiled_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PageDraft:
|
||||
title: str
|
||||
path: Path
|
||||
content: str
|
||||
|
||||
|
||||
def _compile_page(
|
||||
kb: KnowledgeBaseConfig,
|
||||
wikilink: str,
|
||||
chunks: list[Chunk],
|
||||
*,
|
||||
query_entry: Any | None,
|
||||
alias_group: Any | None,
|
||||
synthesizer: LlmPageSynthesizer | None,
|
||||
require_llm: bool,
|
||||
) -> _PageDraft:
|
||||
first = chunks[0]
|
||||
title = _title_for(first, query_entry)
|
||||
slug = _slug(wikilink)
|
||||
path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md"
|
||||
evidence_profile = _evidence_profile(title, query_entry, alias_group)
|
||||
tags = _tags(kb, first, alias_group)
|
||||
aliases = _aliases(title, query_entry, alias_group)
|
||||
source_ref = first.wikilink
|
||||
source_path = str(first.source_path)
|
||||
key_blocks = _key_blocks(chunks, evidence_profile)
|
||||
intent = _intent(title, query_entry, alias_group)
|
||||
summary = _summary(chunks, title, evidence_profile, intent=intent)
|
||||
requirements = _answer_requirements(intent, query_entry)
|
||||
synthesis = _llm_synthesis(
|
||||
synthesizer,
|
||||
title=title,
|
||||
wikilink=wikilink,
|
||||
query_entry=query_entry,
|
||||
alias_group=alias_group,
|
||||
key_blocks=key_blocks,
|
||||
require_llm=require_llm,
|
||||
)
|
||||
if synthesis is not None:
|
||||
summary = synthesis.summary
|
||||
retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group)
|
||||
blueprint = (
|
||||
list(synthesis.answer_blueprint)
|
||||
if synthesis and synthesis.answer_blueprint
|
||||
else _deterministic_answer_blueprint(intent, requirements, key_blocks)
|
||||
)
|
||||
links = _related_links(query_entry, alias_group)
|
||||
content = "\n".join(
|
||||
[
|
||||
"---",
|
||||
f"llmWikiVersion: \"1\"",
|
||||
f"conceptId: {_yaml_scalar(slug)}",
|
||||
f"title: {_yaml_scalar(title)}",
|
||||
f"summary: {_yaml_scalar(summary)}",
|
||||
f"sources: {_yaml_list([source_ref])}",
|
||||
f"source_paths: {_yaml_list([source_path])}",
|
||||
f"kind: concept",
|
||||
f"tags: {_yaml_list(tags)}",
|
||||
f"aliases: {_yaml_list(aliases)}",
|
||||
f"intents: {_yaml_list([intent])}",
|
||||
f"retrieval_hints: {_yaml_list(retrieval_hints)}",
|
||||
f"answer_requirements: {_yaml_list(requirements)}",
|
||||
f"confidence: {synthesis.confidence if synthesis else 'medium'}",
|
||||
f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}",
|
||||
"---",
|
||||
"",
|
||||
f"# {title}",
|
||||
"",
|
||||
summary,
|
||||
"",
|
||||
"## Intent Model",
|
||||
"",
|
||||
f"- intent: {intent}",
|
||||
f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}",
|
||||
f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}",
|
||||
"",
|
||||
"## Retrieval Hints",
|
||||
"",
|
||||
*[f"- {hint}" for hint in retrieval_hints],
|
||||
"",
|
||||
"## Answer Blueprint",
|
||||
"",
|
||||
*[f"- {item}" for item in blueprint],
|
||||
"",
|
||||
"## Synthesized Evidence" if synthesis and synthesis.key_evidence else "",
|
||||
"" if synthesis and synthesis.key_evidence else "",
|
||||
*[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())],
|
||||
"" if synthesis and synthesis.key_evidence else "",
|
||||
"## Evidence Graph",
|
||||
"",
|
||||
*_evidence_graph_lines(key_blocks),
|
||||
"",
|
||||
"## Key Evidence",
|
||||
"",
|
||||
*[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks],
|
||||
"",
|
||||
"## Source Map",
|
||||
"",
|
||||
f"- 原始来源:`{source_ref}`",
|
||||
f"- 文件路径:`{source_path}`",
|
||||
"",
|
||||
"## Related",
|
||||
"",
|
||||
*[f"- [[{link}]]" for link in links],
|
||||
"",
|
||||
]
|
||||
)
|
||||
return _PageDraft(title=title, path=path, content=content)
|
||||
|
||||
|
||||
def _llm_synthesis(
|
||||
synthesizer: LlmPageSynthesizer | None,
|
||||
*,
|
||||
title: str,
|
||||
wikilink: str,
|
||||
query_entry: Any | None,
|
||||
alias_group: Any | None,
|
||||
key_blocks: list[Chunk],
|
||||
require_llm: bool,
|
||||
) -> LlmPageSynthesis | None:
|
||||
if synthesizer is None:
|
||||
return None
|
||||
prompt = _llm_compile_prompt(
|
||||
title=title,
|
||||
wikilink=wikilink,
|
||||
query_entry=query_entry,
|
||||
alias_group=alias_group,
|
||||
key_blocks=key_blocks,
|
||||
)
|
||||
try:
|
||||
return synthesizer.synthesize(prompt)
|
||||
except Exception:
|
||||
if require_llm:
|
||||
raise
|
||||
return None
|
||||
|
||||
|
||||
def _llm_compile_prompt(
|
||||
*,
|
||||
title: str,
|
||||
wikilink: str,
|
||||
query_entry: Any | None,
|
||||
alias_group: Any | None,
|
||||
key_blocks: list[Chunk],
|
||||
) -> str:
|
||||
aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ())
|
||||
terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ())
|
||||
requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ())
|
||||
intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None)
|
||||
evidence = [
|
||||
{
|
||||
"block_id": f"{block.wikilink}#{block.index}",
|
||||
"heading": block.heading,
|
||||
"text": trim_block(block.text, 700),
|
||||
}
|
||||
for block in key_blocks[:8]
|
||||
]
|
||||
return "\n".join(
|
||||
[
|
||||
"Compile a d-wiki concept page from the local evidence below.",
|
||||
"Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.",
|
||||
"For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.",
|
||||
"Return only JSON with this shape:",
|
||||
'{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}',
|
||||
"",
|
||||
json_dumps(
|
||||
{
|
||||
"title": title,
|
||||
"wikilink": wikilink,
|
||||
"intent": intent,
|
||||
"aliases": aliases,
|
||||
"terms": terms,
|
||||
"answer_requirements": requirements,
|
||||
"evidence": evidence,
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]:
|
||||
if synthesis is None or not synthesis.answer_blueprint:
|
||||
return []
|
||||
return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""]
|
||||
|
||||
|
||||
def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str:
|
||||
if query_entry is not None and getattr(query_entry, "intent", ""):
|
||||
return str(query_entry.intent)
|
||||
if alias_group is not None and getattr(alias_group, "intent", ""):
|
||||
return str(alias_group.intent)
|
||||
lowered = title.lower()
|
||||
if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")):
|
||||
return "procedure"
|
||||
return "fact"
|
||||
|
||||
|
||||
def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]:
|
||||
if query_entry is not None:
|
||||
requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()]
|
||||
if requirements:
|
||||
return _unique(requirements)
|
||||
if intent == "procedure":
|
||||
return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"]
|
||||
return ["直接给出结论", "保留关键数值和单位", "引用证据来源"]
|
||||
|
||||
|
||||
def _retrieval_hints(
|
||||
title: str,
|
||||
aliases: list[str],
|
||||
tags: list[str],
|
||||
requirements: list[str],
|
||||
alias_group: Any | None,
|
||||
) -> list[str]:
|
||||
values = [title, *aliases, *tags, *requirements]
|
||||
if alias_group is not None:
|
||||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||||
hints: list[str] = []
|
||||
for value in values:
|
||||
hints.append(value)
|
||||
hints.extend(tokenize(value))
|
||||
return _unique(hints)[:32]
|
||||
|
||||
|
||||
def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]:
|
||||
if requirements:
|
||||
return requirements[:6]
|
||||
if intent == "procedure":
|
||||
headings = _unique([block.heading for block in key_blocks if block.heading])
|
||||
if headings:
|
||||
return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]]
|
||||
return ["按前提、配置、执行、验证的顺序回答"]
|
||||
return ["先给出直接结论", "补充关键参数、限制和来源"]
|
||||
|
||||
|
||||
def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for block in key_blocks[:8]:
|
||||
claim = trim_block(block.text, 180)
|
||||
if not claim:
|
||||
continue
|
||||
lines.append(f"- claim: {claim}")
|
||||
lines.append(f" source: {block.wikilink}#{block.index}")
|
||||
if block.heading:
|
||||
lines.append(f" heading: {block.heading}")
|
||||
return lines or ["- claim: 暂无可用证据", " source: unknown"]
|
||||
|
||||
|
||||
def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]:
|
||||
grouped: dict[str, list[Chunk]] = {}
|
||||
for chunk in chunks:
|
||||
grouped.setdefault(chunk.wikilink, []).append(chunk)
|
||||
return grouped
|
||||
|
||||
|
||||
def _title_for(chunk: Chunk, query_entry: Any | None) -> str:
|
||||
if query_entry is not None and getattr(query_entry, "title", ""):
|
||||
return str(query_entry.title)
|
||||
return chunk.title
|
||||
|
||||
|
||||
def _summary(
|
||||
chunks: list[Chunk],
|
||||
title: str,
|
||||
evidence_profile: _EvidenceProfile | None = None,
|
||||
*,
|
||||
intent: str = "fact",
|
||||
) -> str:
|
||||
if intent == "procedure":
|
||||
for chunk in chunks[:12]:
|
||||
text = trim_block(chunk.text, 220)
|
||||
if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text):
|
||||
return f"{title}:{text}"
|
||||
ranked = _rank_evidence(chunks, evidence_profile)
|
||||
for score, chunk in ranked:
|
||||
if score <= 0:
|
||||
continue
|
||||
text = trim_block(chunk.text, 220)
|
||||
if text and not _is_low_signal(text):
|
||||
return f"{title}:{text}"
|
||||
for chunk in chunks:
|
||||
text = trim_block(chunk.text, 220)
|
||||
if text and not _is_low_signal(text) and len(text) >= 20:
|
||||
return f"{title}:{text}"
|
||||
return f"{title} 的编译知识页,来源于本地文档。"
|
||||
|
||||
|
||||
def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]:
|
||||
preferred = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if _is_evidence_candidate(chunk.text)
|
||||
]
|
||||
ranked = _rank_evidence(preferred, evidence_profile)
|
||||
if not ranked or ranked[0][0] <= 0:
|
||||
return (preferred or chunks)[:6]
|
||||
selected: list[Chunk] = []
|
||||
seen: set[tuple[str, int]] = set()
|
||||
for score, chunk in ranked:
|
||||
if score <= 0:
|
||||
continue
|
||||
key = (chunk.wikilink, chunk.index)
|
||||
if key in seen:
|
||||
continue
|
||||
selected.append(chunk)
|
||||
seen.add(key)
|
||||
if len(selected) >= 6:
|
||||
return selected
|
||||
for chunk in preferred:
|
||||
key = (chunk.wikilink, chunk.index)
|
||||
if key in seen:
|
||||
continue
|
||||
selected.append(chunk)
|
||||
seen.add(key)
|
||||
if len(selected) >= 6:
|
||||
break
|
||||
return selected
|
||||
|
||||
|
||||
def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile:
|
||||
values = [title]
|
||||
if query_entry is not None:
|
||||
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
|
||||
values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ()))
|
||||
if alias_group is not None:
|
||||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||||
phrases = tuple(_unique([_normalize_phrase(value) for value in values if value]))
|
||||
tokens: set[str] = set()
|
||||
for value in values:
|
||||
tokens.update(tokenize(str(value)))
|
||||
return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases)
|
||||
|
||||
|
||||
def _rank_evidence(
|
||||
chunks: list[Chunk],
|
||||
evidence_profile: _EvidenceProfile | None,
|
||||
) -> list[tuple[float, Chunk]]:
|
||||
if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases):
|
||||
return []
|
||||
ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks]
|
||||
ranked.sort(key=lambda item: (-item[0], item[1].index))
|
||||
return ranked
|
||||
|
||||
|
||||
def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float:
|
||||
text = trim_block(chunk.text, 1000)
|
||||
normalized = _normalize_phrase(text)
|
||||
chunk_tokens = set(chunk.tokens) | set(tokenize(text))
|
||||
score = 0.0
|
||||
score += len(chunk_tokens & evidence_profile.tokens) * 2.0
|
||||
for phrase in evidence_profile.phrases:
|
||||
if len(phrase) >= 2 and phrase in normalized:
|
||||
score += 5.0
|
||||
if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized):
|
||||
score += 8.0
|
||||
if "算力" in evidence_profile.tokens and "tops" in normalized:
|
||||
score += 6.0
|
||||
if "bpu" in evidence_profile.tokens and "bpu" in normalized:
|
||||
score += 4.0
|
||||
if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized):
|
||||
score += 2.0
|
||||
if _is_low_signal(text):
|
||||
score -= 8.0
|
||||
return score
|
||||
|
||||
|
||||
def _is_evidence_candidate(text: str) -> bool:
|
||||
stripped = text.lstrip()
|
||||
return not stripped.startswith("|") and not stripped.startswith(">")
|
||||
|
||||
|
||||
def _is_low_signal(text: str) -> bool:
|
||||
stripped = text.strip()
|
||||
without_marks = re.sub(r"[*_`#\-\s]+", "", stripped)
|
||||
if len(without_marks) < 8:
|
||||
return True
|
||||
if stripped.startswith("```") or "```" in stripped:
|
||||
return True
|
||||
if stripped.startswith("|"):
|
||||
return True
|
||||
if "\s*[├│└─]{2,}", stripped):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]:
|
||||
tags = [kb.name]
|
||||
if alias_group is not None:
|
||||
tags.append(_slug(str(alias_group.name)).replace("--", "-"))
|
||||
for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title):
|
||||
lowered = token.lower()
|
||||
if lowered not in tags:
|
||||
tags.append(lowered)
|
||||
return tags[:8]
|
||||
|
||||
|
||||
def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]:
|
||||
values = [title]
|
||||
if query_entry is not None:
|
||||
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
|
||||
if alias_group is not None:
|
||||
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
|
||||
return _unique(values)[:12]
|
||||
|
||||
|
||||
def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]:
|
||||
links: list[str] = []
|
||||
if query_entry is not None:
|
||||
links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ()))
|
||||
if alias_group is not None:
|
||||
links.extend(str(item) for item in getattr(alias_group, "related", ()))
|
||||
return _unique(links)[:10]
|
||||
|
||||
|
||||
def _unique(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
output: list[str] = []
|
||||
for value in values:
|
||||
cleaned = value.strip()
|
||||
if not cleaned or cleaned in seen:
|
||||
continue
|
||||
seen.add(cleaned)
|
||||
output.append(cleaned)
|
||||
return output
|
||||
|
||||
|
||||
def _normalize_phrase(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", value.lower()).strip()
|
||||
|
||||
|
||||
def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None:
|
||||
(wiki_root / "SCHEMA.md").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"# Wiki Schema",
|
||||
"",
|
||||
"## Domain",
|
||||
f"Compiled local knowledge for `{kb.name}`.",
|
||||
"",
|
||||
"## Conventions",
|
||||
"- Pages live under `concepts/`.",
|
||||
"- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.",
|
||||
"- Claims in Key Evidence keep provenance markers pointing back to source block ids.",
|
||||
"- Raw source files remain immutable; compiled pages are generated artifacts.",
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None:
|
||||
lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""]
|
||||
for page in sorted(pages, key=lambda item: item.title):
|
||||
rel = page.path.relative_to(wiki_root).as_posix()
|
||||
lines.append(f"- [[{rel[:-3]}]] — {page.title}")
|
||||
lines.append("")
|
||||
(wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None:
|
||||
line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n"
|
||||
path = wiki_root / "log.md"
|
||||
if not path.exists():
|
||||
path.write_text("# Log\n\n", encoding="utf-8")
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(line)
|
||||
|
||||
|
||||
def _slug(value: str) -> str:
|
||||
slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower()
|
||||
return slug or "page"
|
||||
|
||||
|
||||
def _yaml_scalar(value: str) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _yaml_list(values: list[str]) -> str:
|
||||
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
59
d_wiki/config.py
Normal file
59
d_wiki/config.py
Normal file
@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .yaml_compat import safe_load
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KnowledgeBaseConfig:
|
||||
name: str
|
||||
root: Path
|
||||
source_type: str
|
||||
aliases_path: Path | None
|
||||
queries_dir: Path | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DWikiConfig:
|
||||
path: Path
|
||||
knowledge_bases: dict[str, KnowledgeBaseConfig]
|
||||
|
||||
|
||||
def default_config_path() -> Path:
|
||||
return Path(os.environ.get("D_WIKI_CONFIG", Path.cwd() / "d-wiki.yml")).expanduser()
|
||||
|
||||
|
||||
def load_config(path: Path | None = None) -> DWikiConfig:
|
||||
config_path = (path or default_config_path()).expanduser().resolve()
|
||||
data = safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||||
raw_kbs = data.get("knowledge_bases")
|
||||
if not isinstance(raw_kbs, dict):
|
||||
raise ValueError("D_WIKI_CONFIG_MISSING_KNOWLEDGE_BASES")
|
||||
bases: dict[str, KnowledgeBaseConfig] = {}
|
||||
for name, raw in raw_kbs.items():
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"D_WIKI_KB_INVALID: {name}")
|
||||
root = Path(str(raw.get("root", ""))).expanduser()
|
||||
if not root.is_absolute():
|
||||
root = (config_path.parent / root).resolve()
|
||||
aliases_path = _optional_path(config_path.parent, raw.get("aliases"))
|
||||
queries_dir = _optional_path(config_path.parent, raw.get("queries"))
|
||||
bases[str(name)] = KnowledgeBaseConfig(
|
||||
name=str(name),
|
||||
root=root,
|
||||
source_type=str(raw.get("source_type", "markdown")),
|
||||
aliases_path=aliases_path,
|
||||
queries_dir=queries_dir,
|
||||
)
|
||||
return DWikiConfig(path=config_path, knowledge_bases=bases)
|
||||
|
||||
|
||||
def _optional_path(base: Path, value: Any) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
path = Path(str(value)).expanduser()
|
||||
return path if path.is_absolute() else (base / path).resolve()
|
||||
227
d_wiki/dreams.py
Normal file
227
d_wiki/dreams.py
Normal file
@ -0,0 +1,227 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import KnowledgeBaseConfig
|
||||
from .metadata import frontmatter, load_query_entries
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DreamedQuery:
|
||||
slug: str
|
||||
title: str
|
||||
aliases: tuple[str, ...]
|
||||
intent: str
|
||||
primary_sources: tuple[str, ...]
|
||||
supporting_sources: tuple[str, ...]
|
||||
answer_requirements: tuple[str, ...]
|
||||
|
||||
|
||||
def dream_kb(kb: KnowledgeBaseConfig, *, force: bool = False) -> dict[str, Any]:
|
||||
wiki_root = kb.root / ".d_wiki" / "wiki"
|
||||
if not wiki_root.is_dir():
|
||||
raise ValueError("D_WIKI_DREAM_REQUIRES_COMPILED_WIKI")
|
||||
output_root = dream_root(kb)
|
||||
queries_dir = output_root / "queries"
|
||||
if force:
|
||||
shutil.rmtree(output_root, ignore_errors=True)
|
||||
queries_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pages = sorted((wiki_root / "concepts").glob("*.md"))
|
||||
dreamed = [_dream_page(path, wiki_root) for path in pages]
|
||||
dreamed = [item for item in dreamed if item is not None]
|
||||
for item in dreamed:
|
||||
(queries_dir / f"{item.slug}.md").write_text(_query_markdown(item), encoding="utf-8")
|
||||
(output_root / "README.md").write_text(_readme(kb, len(pages), len(dreamed)), encoding="utf-8")
|
||||
return {
|
||||
"status": "dreamed",
|
||||
"kb": kb.name,
|
||||
"input_wiki": str(wiki_root),
|
||||
"output_store": str(output_root),
|
||||
"queries": len(dreamed),
|
||||
"dreamed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def load_dream_query_entries(kb: KnowledgeBaseConfig) -> list[Any]:
|
||||
return load_query_entries(dream_root(kb) / "queries")
|
||||
|
||||
|
||||
def dream_root(kb: KnowledgeBaseConfig) -> Path:
|
||||
return kb.root / ".d_wiki" / "dreams" / "latest"
|
||||
|
||||
|
||||
def _dream_page(path: Path, wiki_root: Path) -> DreamedQuery | None:
|
||||
meta, body = frontmatter(path.read_text(encoding="utf-8", errors="ignore"))
|
||||
title = str(meta.get("title") or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
curation_body = _curation_body(body)
|
||||
source = _first(meta.get("sources")) or _wikilink_for_compiled(path, wiki_root)
|
||||
aliases = _unique([title, *_dream_aliases(title, curation_body)])
|
||||
intent = _first(meta.get("intents")) or _dream_intent(title, curation_body)
|
||||
requirements = _as_list(meta.get("answer_requirements")) or _dream_requirements(title, curation_body, intent)
|
||||
supporting = _related_sources(body, source)
|
||||
return DreamedQuery(
|
||||
slug=path.stem,
|
||||
title=title,
|
||||
aliases=tuple(aliases[:18]),
|
||||
intent=intent,
|
||||
primary_sources=(source,),
|
||||
supporting_sources=tuple(supporting[:6]),
|
||||
answer_requirements=tuple(requirements[:8]),
|
||||
)
|
||||
|
||||
|
||||
def _dream_aliases(title: str, body: str) -> list[str]:
|
||||
text = f"{title}\n{body}".lower()
|
||||
aliases = [
|
||||
f"{title}怎么用",
|
||||
f"如何使用{title}",
|
||||
]
|
||||
if _looks_like_3d_generation(title, text):
|
||||
aliases.extend(
|
||||
[
|
||||
"3D资产生成怎么用",
|
||||
"如何使用 3D 资产生成功能",
|
||||
"我想知道如何使用3d资产生成功能",
|
||||
"图生3D怎么用",
|
||||
"如何生成3D模型",
|
||||
"如何把图片生成3D资产",
|
||||
]
|
||||
)
|
||||
if "camera" in text and any(term in text for term in ("点亮", "bring up", "sensor")):
|
||||
aliases.extend(["如何在X5平台上点亮camera", "X5 camera 点亮", "X5 摄像头 bring up"])
|
||||
if any(term in text for term in ("算力", "tops", "bpu")):
|
||||
aliases.extend(["x5的算力是多少", "X5 BPU 算力"])
|
||||
return aliases
|
||||
|
||||
|
||||
def _dream_intent(title: str, body: str) -> str:
|
||||
text = f"{title}\n{body}".lower()
|
||||
if any(term in text for term in ("如何", "怎么", "步骤", "使用", "创建", "配置", "点亮", "生成", "调试", "bring up")):
|
||||
return "procedure"
|
||||
return "fact"
|
||||
|
||||
|
||||
def _dream_requirements(title: str, body: str, intent: str) -> list[str]:
|
||||
text = f"{title}\n{body}".lower()
|
||||
if _looks_like_3d_generation(title, text):
|
||||
return [
|
||||
"说明 3D 生成是图生 3D 工具,不是镜像资产",
|
||||
"按准备参考图、生成、保存到 3D 资产的流程回答",
|
||||
"说明输出默认 glb,生成结果可保存到 3D 资产",
|
||||
"提醒暂不支持文生 3D,文字描述可先用图像生成产图",
|
||||
]
|
||||
if intent == "procedure":
|
||||
return ["按步骤回答", "说明关键前提和入口", "引用证据来源"]
|
||||
return ["直接回答问题", "引用证据来源"]
|
||||
|
||||
|
||||
def _looks_like_3d_generation(title: str, text: str) -> bool:
|
||||
title_text = title.lower()
|
||||
return (
|
||||
"3d 生成" in title_text
|
||||
or "3d生成" in title_text
|
||||
or "3d 资产生成" in title_text
|
||||
or "3d资产生成" in title_text
|
||||
or "embodygen" in text
|
||||
or "ai 图生 3d 工具" in text
|
||||
or "ai 图生3d 工具" in text
|
||||
)
|
||||
|
||||
|
||||
def _curation_body(body: str) -> str:
|
||||
lines: list[str] = []
|
||||
skip = False
|
||||
for line in body.splitlines():
|
||||
if line.startswith("## "):
|
||||
skip = line.strip() in {"## Retrieval Hints", "## Related"}
|
||||
if skip:
|
||||
continue
|
||||
if not skip:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _related_sources(body: str, primary: str) -> list[str]:
|
||||
links = re.findall(r"\[\[([^\]]+)\]\]", body)
|
||||
refs = re.findall(r"\^\[([^#\]]+)(?:#[^\]]+)?\]", body)
|
||||
return _unique([item for item in [*links, *refs] if item and item != primary])
|
||||
|
||||
|
||||
def _query_markdown(item: DreamedQuery) -> str:
|
||||
lines = [
|
||||
"---",
|
||||
f"title: {_yaml_scalar(item.title)}",
|
||||
f"aliases: {_yaml_list(item.aliases)}",
|
||||
f"intent: {item.intent}",
|
||||
f"primary_sources: {_yaml_list(item.primary_sources)}",
|
||||
f"supporting_sources: {_yaml_list(item.supporting_sources)}",
|
||||
f"answer_requirements: {_yaml_list(item.answer_requirements)}",
|
||||
"provenance: dreamed",
|
||||
"---",
|
||||
"",
|
||||
f"Dreamed query memory for {item.title}.",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _readme(kb: KnowledgeBaseConfig, pages: int, queries: int) -> str:
|
||||
return "\n".join(
|
||||
[
|
||||
"# d-wiki dream output",
|
||||
"",
|
||||
"This store is generated from the compiled wiki. It does not modify source documents or compiled concept pages.",
|
||||
"",
|
||||
f"- kb: `{kb.name}`",
|
||||
f"- input_pages: {pages}",
|
||||
f"- dreamed_queries: {queries}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _first(value: Any) -> str:
|
||||
items = _as_list(value)
|
||||
return items[0] if items else ""
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
return [value]
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
return []
|
||||
|
||||
|
||||
def _unique(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
output: list[str] = []
|
||||
for value in values:
|
||||
cleaned = re.sub(r"\s+", " ", str(value)).strip()
|
||||
if not cleaned or cleaned in seen:
|
||||
continue
|
||||
seen.add(cleaned)
|
||||
output.append(cleaned)
|
||||
return output
|
||||
|
||||
|
||||
def _wikilink_for_compiled(path: Path, wiki_root: Path) -> str:
|
||||
return path.relative_to(wiki_root).with_suffix("").as_posix()
|
||||
|
||||
|
||||
def _yaml_scalar(value: str) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _yaml_list(values: tuple[str, ...] | list[str]) -> str:
|
||||
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
|
||||
156
d_wiki/llm.py
Normal file
156
d_wiki/llm.py
Normal file
@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LlmSettings:
|
||||
base_url: str
|
||||
model: str
|
||||
api_key: str
|
||||
api_style: str = "anthropic"
|
||||
max_tokens: int = 900
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LlmPageSynthesis:
|
||||
summary: str
|
||||
answer_blueprint: tuple[str, ...]
|
||||
key_evidence: tuple[str, ...]
|
||||
confidence: str = "medium"
|
||||
|
||||
|
||||
class LlmPageSynthesizer:
|
||||
def __init__(self, settings: LlmSettings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
@classmethod
|
||||
def from_environment(cls) -> "LlmPageSynthesizer | None":
|
||||
base_url = os.environ.get("D_WIKI_LLM_BASE_URL") or os.environ.get("ROBOGO_LLM_BASE_URL")
|
||||
model = os.environ.get("D_WIKI_LLM_MODEL") or os.environ.get("ROBOGO_LLM_MODEL")
|
||||
api_key = os.environ.get("D_WIKI_LLM_API_KEY") or os.environ.get("ROBOGO_LLM_API_KEY")
|
||||
if not base_url or not model or not api_key:
|
||||
return None
|
||||
api_style = os.environ.get("D_WIKI_LLM_API_STYLE") or os.environ.get("ROBOGO_LLM_API_STYLE") or "anthropic"
|
||||
max_tokens = int(os.environ.get("D_WIKI_LLM_MAX_TOKENS", "900"))
|
||||
return cls(LlmSettings(base_url=base_url, model=model, api_key=api_key, api_style=api_style, max_tokens=max_tokens))
|
||||
|
||||
def synthesize(self, prompt: str) -> LlmPageSynthesis | None:
|
||||
text = self._complete(prompt).strip()
|
||||
payload = _extract_json(text)
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
summary = str(payload.get("summary") or "").strip()
|
||||
if not summary:
|
||||
return None
|
||||
answer_blueprint = _string_tuple(payload.get("answer_blueprint"))
|
||||
key_evidence = _string_tuple(payload.get("key_evidence"))
|
||||
confidence = str(payload.get("confidence") or "medium").strip().lower()
|
||||
if confidence not in {"low", "medium", "high"}:
|
||||
confidence = "medium"
|
||||
return LlmPageSynthesis(
|
||||
summary=summary,
|
||||
answer_blueprint=answer_blueprint,
|
||||
key_evidence=key_evidence,
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
def _complete(self, prompt: str) -> str:
|
||||
if self.settings.api_style == "openai":
|
||||
return self._complete_openai(prompt)
|
||||
return self._complete_anthropic(prompt)
|
||||
|
||||
def _complete_anthropic(self, prompt: str) -> str:
|
||||
payload = {
|
||||
"model": self.settings.model,
|
||||
"max_tokens": self.settings.max_tokens,
|
||||
"temperature": 0,
|
||||
"system": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
data = self._post_json(_join_url(self.settings.base_url, "v1/messages"), payload)
|
||||
content = data.get("content")
|
||||
if isinstance(content, list):
|
||||
return "\n".join(str(item.get("text", "")) for item in content if isinstance(item, dict))
|
||||
return str(data.get("text") or "")
|
||||
|
||||
def _complete_openai(self, prompt: str) -> str:
|
||||
payload = {
|
||||
"model": self.settings.model,
|
||||
"temperature": 0,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
|
||||
},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
}
|
||||
data = self._post_json(_join_url(self.settings.base_url, "v1/chat/completions"), payload)
|
||||
choices = data.get("choices")
|
||||
if isinstance(choices, list) and choices:
|
||||
first = choices[0]
|
||||
if isinstance(first, dict):
|
||||
message = first.get("message")
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("content") or "")
|
||||
return ""
|
||||
|
||||
def _post_json(self, url: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={
|
||||
"content-type": "application/json",
|
||||
"accept": "application/json",
|
||||
"x-api-key": self.settings.api_key,
|
||||
"authorization": f"Bearer {self.settings.api_key}",
|
||||
"anthropic-version": "2023-06-01",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", errors="ignore")
|
||||
raise RuntimeError(f"D_WIKI_LLM_HTTP_ERROR: {exc.code} {detail[:300]}") from exc
|
||||
|
||||
|
||||
def _join_url(base_url: str, suffix: str) -> str:
|
||||
base = base_url.rstrip("/")
|
||||
if base.endswith("/v1/messages") or base.endswith("/v1/chat/completions"):
|
||||
return base
|
||||
if base.endswith("/v1"):
|
||||
return f"{base}/{suffix.removeprefix('v1/')}"
|
||||
return f"{base}/{suffix}"
|
||||
|
||||
|
||||
def _extract_json(text: str) -> Any:
|
||||
cleaned = text.strip()
|
||||
if cleaned.startswith("```"):
|
||||
cleaned = cleaned.strip("`")
|
||||
if cleaned.lower().startswith("json"):
|
||||
cleaned = cleaned[4:].strip()
|
||||
decoder = json.JSONDecoder()
|
||||
for index, char in enumerate(cleaned):
|
||||
if char != "{":
|
||||
continue
|
||||
try:
|
||||
value, _end = decoder.raw_decode(cleaned[index:])
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _string_tuple(value: Any) -> tuple[str, ...]:
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
return tuple(str(item).strip() for item in value if str(item).strip())
|
||||
119
d_wiki/metadata.py
Normal file
119
d_wiki/metadata.py
Normal file
@ -0,0 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .yaml_compat import safe_load
|
||||
from .tokenize import tokenize
|
||||
|
||||
QUERY_MATCH_STOP_TOKENS = {
|
||||
"如何",
|
||||
"怎么",
|
||||
"使用",
|
||||
"创建",
|
||||
"知道",
|
||||
"功能",
|
||||
"资产",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AliasGroup:
|
||||
name: str
|
||||
terms: tuple[str, ...]
|
||||
intent: str | None
|
||||
related: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueryEntry:
|
||||
path: Path
|
||||
title: str
|
||||
aliases: tuple[str, ...]
|
||||
intent: str | None
|
||||
primary_sources: tuple[str, ...]
|
||||
supporting_sources: tuple[str, ...]
|
||||
answer_requirements: tuple[str, ...]
|
||||
|
||||
|
||||
def load_alias_groups(path: Path | None) -> list[AliasGroup]:
|
||||
if path is None or not path.exists():
|
||||
return []
|
||||
data = safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
raw_aliases = data.get("aliases", {})
|
||||
if not isinstance(raw_aliases, dict):
|
||||
return []
|
||||
groups: list[AliasGroup] = []
|
||||
for name, raw in raw_aliases.items():
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
groups.append(
|
||||
AliasGroup(
|
||||
name=str(name),
|
||||
terms=tuple(str(term).strip() for term in raw.get("terms", []) if str(term).strip()),
|
||||
intent=str(raw["intent"]) if raw.get("intent") else None,
|
||||
related=tuple(str(item).strip() for item in raw.get("related", []) if str(item).strip()),
|
||||
)
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def load_query_entries(directory: Path | None) -> list[QueryEntry]:
|
||||
if directory is None or not directory.exists():
|
||||
return []
|
||||
return [_query_entry(path) for path in sorted(directory.rglob("*.md")) if path.is_file()]
|
||||
|
||||
|
||||
def frontmatter(text: str) -> tuple[dict[str, Any], str]:
|
||||
if not text.startswith("---\n"):
|
||||
return {}, text
|
||||
end = text.find("\n---", 4)
|
||||
if end < 0:
|
||||
return {}, text
|
||||
meta = safe_load(text[4:end]) or {}
|
||||
if not isinstance(meta, dict):
|
||||
meta = {}
|
||||
return meta, text[end + 4 :].lstrip("\n")
|
||||
|
||||
|
||||
def query_matches_entry(query: str, entry: QueryEntry) -> bool:
|
||||
normalized = normalize(query)
|
||||
if any(normalize(alias) in normalized or normalized in normalize(alias) for alias in entry.aliases):
|
||||
return True
|
||||
query_terms = _meaningful_query_terms(query)
|
||||
for candidate in (entry.title, *entry.aliases):
|
||||
candidate_terms = _meaningful_query_terms(candidate)
|
||||
if len(query_terms & candidate_terms) >= 2:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _meaningful_query_terms(value: str) -> set[str]:
|
||||
return {term for term in tokenize(value) if term not in QUERY_MATCH_STOP_TOKENS and len(term.strip()) >= 2}
|
||||
|
||||
|
||||
def normalize(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", value.lower()).strip()
|
||||
|
||||
|
||||
def _query_entry(path: Path) -> QueryEntry:
|
||||
meta, body = frontmatter(path.read_text(encoding="utf-8"))
|
||||
title = str(meta.get("title") or _first_heading(body) or path.stem)
|
||||
return QueryEntry(
|
||||
path=path,
|
||||
title=title,
|
||||
aliases=tuple(str(item).strip() for item in meta.get("aliases", []) if str(item).strip()),
|
||||
intent=str(meta["intent"]) if meta.get("intent") else None,
|
||||
primary_sources=tuple(str(item).strip() for item in meta.get("primary_sources", []) if str(item).strip()),
|
||||
supporting_sources=tuple(str(item).strip() for item in meta.get("supporting_sources", []) if str(item).strip()),
|
||||
answer_requirements=tuple(str(item).strip() for item in meta.get("answer_requirements", []) if str(item).strip()),
|
||||
)
|
||||
|
||||
|
||||
def _first_heading(text: str) -> str | None:
|
||||
for line in text.splitlines():
|
||||
if match := re.match(r"^#{1,6}\s+(.+)$", line):
|
||||
return match.group(1).strip()
|
||||
return None
|
||||
469
d_wiki/search.py
Normal file
469
d_wiki/search.py
Normal file
@ -0,0 +1,469 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import KnowledgeBaseConfig
|
||||
from .metadata import (
|
||||
AliasGroup,
|
||||
QueryEntry,
|
||||
frontmatter,
|
||||
load_alias_groups,
|
||||
load_query_entries,
|
||||
normalize,
|
||||
query_matches_entry,
|
||||
)
|
||||
from .dreams import load_dream_query_entries
|
||||
from .tokenize import tokenize
|
||||
|
||||
SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt")
|
||||
MAX_BLOCK_CHARS = 520
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Chunk:
|
||||
title: str
|
||||
wikilink: str
|
||||
source_path: Path
|
||||
page_type: str
|
||||
heading: str
|
||||
text: str
|
||||
tokens: tuple[str, ...]
|
||||
index: int
|
||||
|
||||
|
||||
def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "ok",
|
||||
"config": str(config_path),
|
||||
"knowledge_bases": {
|
||||
name: {
|
||||
"root": str(kb.root),
|
||||
"source_type": kb.source_type,
|
||||
"exists": kb.root.exists(),
|
||||
"aliases": str(kb.aliases_path) if kb.aliases_path else None,
|
||||
"queries": str(kb.queries_dir) if kb.queries_dir else None,
|
||||
}
|
||||
for name, kb in bases.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]:
|
||||
chunks = load_chunks(kb)
|
||||
index_dir = kb.root / ".d_wiki_index"
|
||||
index_dir.mkdir(exist_ok=True)
|
||||
payload = {
|
||||
"kb": kb.name,
|
||||
"root": str(kb.root),
|
||||
"source_type": kb.source_type,
|
||||
"compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None,
|
||||
"chunks": len(chunks),
|
||||
"pages": len({chunk.wikilink for chunk in chunks}),
|
||||
"indexed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
(index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return {"status": "indexed", **payload}
|
||||
|
||||
|
||||
def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]:
|
||||
aliases = load_alias_groups(kb.aliases_path)
|
||||
query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)]
|
||||
matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)]
|
||||
expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries)
|
||||
chunks = load_chunks(kb)
|
||||
ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries)
|
||||
selected = ranked[: max(1, min(max_results, 8))]
|
||||
context_pack = build_context_pack(query, selected, matched_aliases, matched_entries)
|
||||
return {
|
||||
"status": "found" if selected else "no_match",
|
||||
"query": query,
|
||||
"knowledge_base": kb.name,
|
||||
"backend": "lightweight_bm25",
|
||||
"quality": quality(selected),
|
||||
"sources": [source_summary(item[1], item[0]) for item in selected],
|
||||
"context_pack": context_pack,
|
||||
}
|
||||
|
||||
|
||||
def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]:
|
||||
chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink]
|
||||
if not chunks:
|
||||
return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink}
|
||||
first = chunks[0]
|
||||
return {
|
||||
"status": "found",
|
||||
"knowledge_base": kb.name,
|
||||
"id": wikilink,
|
||||
"title": first.title,
|
||||
"source_path": str(first.source_path),
|
||||
"blocks": [
|
||||
{"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"}
|
||||
for chunk in chunks
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def expand_query(
|
||||
query: str,
|
||||
aliases: list[AliasGroup],
|
||||
entries: list[QueryEntry],
|
||||
) -> tuple[list[str], list[AliasGroup]]:
|
||||
terms = tokenize(query)
|
||||
normalized_query = normalize(query)
|
||||
matched: list[AliasGroup] = []
|
||||
for group in aliases:
|
||||
if any(normalize(term) in normalized_query for term in group.terms):
|
||||
matched.append(group)
|
||||
for term in group.terms:
|
||||
terms.extend(tokenize(term))
|
||||
for entry in entries:
|
||||
terms.extend(tokenize(entry.title))
|
||||
for alias in entry.aliases:
|
||||
terms.extend(tokenize(alias))
|
||||
return terms, matched
|
||||
|
||||
|
||||
def rank_chunks(
|
||||
chunks: list[Chunk],
|
||||
terms: list[str],
|
||||
aliases: list[AliasGroup],
|
||||
entries: list[QueryEntry],
|
||||
) -> list[tuple[float, Chunk]]:
|
||||
if not chunks or not terms:
|
||||
return []
|
||||
term_counts = [Counter(chunk.tokens) for chunk in chunks]
|
||||
doc_freq: Counter[str] = Counter()
|
||||
for counts in term_counts:
|
||||
for term in counts:
|
||||
doc_freq[term] += 1
|
||||
avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks))
|
||||
query_counts = Counter(terms)
|
||||
ranked: list[tuple[float, Chunk]] = []
|
||||
related = {item for group in aliases for item in group.related}
|
||||
primary = {item for entry in entries for item in entry.primary_sources}
|
||||
supporting = {item for entry in entries for item in entry.supporting_sources}
|
||||
for chunk, counts in zip(chunks, term_counts):
|
||||
score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts)
|
||||
if any(wikilink_matches(chunk.wikilink, item) for item in related):
|
||||
score += 8.0
|
||||
if any(wikilink_matches(chunk.wikilink, item) for item in primary):
|
||||
score += 16.0
|
||||
if any(wikilink_matches(chunk.wikilink, item) for item in supporting):
|
||||
score += 5.0
|
||||
score += heading_score_adjustment(chunk.heading)
|
||||
if score > 0:
|
||||
ranked.append((round(score, 3), chunk))
|
||||
ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index))
|
||||
return ranked
|
||||
|
||||
|
||||
def heading_score_adjustment(heading: str) -> float:
|
||||
if heading == "Retrieval Hints":
|
||||
return -80.0
|
||||
if heading == "Intent Model":
|
||||
return -40.0
|
||||
if heading == "Answer Blueprint":
|
||||
return -25.0
|
||||
if heading == "Evidence Graph":
|
||||
return 3.0
|
||||
if heading == "Key Evidence":
|
||||
return 5.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def bm25_score(
|
||||
counts: Counter[str],
|
||||
doc_len: int,
|
||||
doc_freq: Counter[str],
|
||||
doc_count: int,
|
||||
avg_len: float,
|
||||
query_counts: Counter[str],
|
||||
) -> float:
|
||||
k1 = 1.5
|
||||
b = 0.75
|
||||
score = 0.0
|
||||
for term, qf in query_counts.items():
|
||||
tf = counts.get(term, 0)
|
||||
if tf == 0:
|
||||
continue
|
||||
idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5))
|
||||
denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1)))
|
||||
score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3)
|
||||
return score
|
||||
|
||||
|
||||
def build_context_pack(
|
||||
query: str,
|
||||
selected: list[tuple[float, Chunk]],
|
||||
aliases: list[AliasGroup],
|
||||
entries: list[QueryEntry],
|
||||
) -> dict[str, Any]:
|
||||
intent = answer_intent(query, aliases, entries)
|
||||
grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list)
|
||||
for item in selected:
|
||||
grouped[item[1].wikilink].append(item)
|
||||
source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1]))
|
||||
entry_primary = {item for entry in entries for item in entry.primary_sources}
|
||||
primary_sources = [
|
||||
source_context(wikilink, chunks)
|
||||
for wikilink, chunks in source_items
|
||||
if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary)
|
||||
]
|
||||
if not primary_sources:
|
||||
primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]]
|
||||
supporting_sources = [
|
||||
source_context(wikilink, chunks, blocks_limit=1)
|
||||
for wikilink, chunks in source_items
|
||||
if wikilink not in {source["wikilink"] for source in primary_sources}
|
||||
][:3]
|
||||
coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete")
|
||||
return {
|
||||
"answer_intent": intent,
|
||||
"coverage": coverage,
|
||||
"answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.",
|
||||
"primary_sources": primary_sources,
|
||||
"supporting_sources": supporting_sources,
|
||||
"answer_requirements": answer_requirements(intent, entries),
|
||||
}
|
||||
|
||||
|
||||
def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]:
|
||||
top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit]
|
||||
chunk = top[0][1]
|
||||
return {
|
||||
"title": chunk.title,
|
||||
"wikilink": wikilink,
|
||||
"source_path": str(chunk.source_path),
|
||||
"reason": "Matched query terms and local metadata.",
|
||||
"blocks": [
|
||||
{
|
||||
"block_id": f"{block.wikilink}#{block.index}",
|
||||
"heading": block.heading,
|
||||
"text": trim_block(block.text),
|
||||
"score": score,
|
||||
}
|
||||
for score, block in top
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str:
|
||||
for entry in entries:
|
||||
if entry.intent:
|
||||
return entry.intent
|
||||
for group in aliases:
|
||||
if group.intent:
|
||||
return group.intent
|
||||
if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")):
|
||||
return "procedure"
|
||||
return "fact"
|
||||
|
||||
|
||||
def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]:
|
||||
requirements = unique([req for entry in entries for req in entry.answer_requirements])
|
||||
if requirements:
|
||||
return requirements
|
||||
if intent == "procedure":
|
||||
return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"]
|
||||
return ["直接回答问题", "引用命中的来源"]
|
||||
|
||||
|
||||
def unique(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
output: list[str] = []
|
||||
for value in values:
|
||||
if value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
output.append(value)
|
||||
return output
|
||||
|
||||
|
||||
def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]:
|
||||
top = selected[0][0] if selected else 0.0
|
||||
return {
|
||||
"confidence": "high" if top >= 8.0 else ("low" if selected else "none"),
|
||||
"high_confidence": top >= 8.0,
|
||||
"top_score": top,
|
||||
"high_confidence_score": 8.0,
|
||||
}
|
||||
|
||||
|
||||
def wikilink_matches(candidate: str, expected: str) -> bool:
|
||||
return candidate == expected or candidate == f"concepts/{expected}"
|
||||
|
||||
|
||||
def source_summary(chunk: Chunk, score: float) -> dict[str, Any]:
|
||||
return {
|
||||
"title": chunk.title,
|
||||
"wikilink": chunk.wikilink,
|
||||
"source_path": str(chunk.source_path),
|
||||
"heading": chunk.heading,
|
||||
"snippet": trim_block(chunk.text, 320),
|
||||
"score": score,
|
||||
}
|
||||
|
||||
|
||||
def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
|
||||
root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root
|
||||
paths = source_paths(root)
|
||||
chunks: list[Chunk] = []
|
||||
for path in paths:
|
||||
chunks.extend(chunks_for_path(path, root))
|
||||
return chunks
|
||||
|
||||
|
||||
def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
|
||||
chunks: list[Chunk] = []
|
||||
for path in source_paths(kb.root):
|
||||
chunks.extend(chunks_for_path(path, kb.root))
|
||||
return chunks
|
||||
|
||||
|
||||
def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path:
|
||||
return kb.root / ".d_wiki" / "wiki"
|
||||
|
||||
|
||||
def source_paths(root: Path) -> list[Path]:
|
||||
if (root / "_sources").is_dir():
|
||||
return sorted(
|
||||
path
|
||||
for path in (root / "_sources").rglob("*.txt")
|
||||
if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES)
|
||||
)
|
||||
generated = {"SCHEMA.md", "index.md", "log.md"}
|
||||
generated_dirs = {".d_wiki", ".d_wiki_index"}
|
||||
is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki"
|
||||
return sorted(
|
||||
path
|
||||
for path in root.rglob("*.md")
|
||||
if path.is_file()
|
||||
and path.name not in generated
|
||||
and (is_compiled_root or not any(part in generated_dirs for part in path.parts))
|
||||
)
|
||||
|
||||
|
||||
def chunks_for_path(path: Path, root: Path) -> list[Chunk]:
|
||||
raw = path.read_text(encoding="utf-8", errors="ignore")
|
||||
meta, body = frontmatter(raw)
|
||||
title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title())
|
||||
metadata_text = metadata_terms(meta)
|
||||
wikilink = wikilink_for(path, root)
|
||||
sections = split_sections(body, title)
|
||||
chunks: list[Chunk] = []
|
||||
for index, (heading, text) in enumerate(sections):
|
||||
if heading in {"Source Map", "Related"}:
|
||||
continue
|
||||
if not text.strip():
|
||||
continue
|
||||
field_text = " ".join([title, heading, wikilink, metadata_text, text])
|
||||
chunks.append(
|
||||
Chunk(
|
||||
title=title,
|
||||
wikilink=wikilink,
|
||||
source_path=path.resolve(),
|
||||
page_type="sphinx_source" if "_sources" in path.parts else "markdown",
|
||||
heading=heading,
|
||||
text=text.strip(),
|
||||
tokens=tuple(tokenize(field_text)),
|
||||
index=index,
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def metadata_terms(meta: dict[str, Any]) -> str:
|
||||
values: list[str] = []
|
||||
for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"):
|
||||
raw = meta.get(key)
|
||||
if isinstance(raw, str):
|
||||
values.append(raw)
|
||||
elif isinstance(raw, list):
|
||||
values.extend(str(item) for item in raw if str(item).strip())
|
||||
return " ".join(values)
|
||||
|
||||
|
||||
def split_sections(body: str, title: str) -> list[tuple[str, str]]:
|
||||
heading = title
|
||||
buffer: list[str] = []
|
||||
sections: list[tuple[str, str]] = []
|
||||
in_fence = False
|
||||
for line in body.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("```"):
|
||||
in_fence = not in_fence
|
||||
buffer.append(line)
|
||||
continue
|
||||
if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)):
|
||||
if buffer:
|
||||
sections.extend(paragraphs(heading, "\n".join(buffer)))
|
||||
buffer = []
|
||||
heading = match.group(1).strip() or title
|
||||
continue
|
||||
buffer.append(line)
|
||||
if buffer:
|
||||
sections.extend(paragraphs(heading, "\n".join(buffer)))
|
||||
return sections or [(title, body)]
|
||||
|
||||
|
||||
def paragraphs(heading: str, text: str) -> list[tuple[str, str]]:
|
||||
blocks: list[tuple[str, str]] = []
|
||||
for part in re.split(r"\n\s*\n", text):
|
||||
cleaned = clean_block(part)
|
||||
if cleaned:
|
||||
blocks.append((heading, cleaned))
|
||||
return blocks
|
||||
|
||||
|
||||
def clean_block(text: str) -> str:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return ""
|
||||
if re.fullmatch(r"[-*_]{3,}", stripped):
|
||||
return ""
|
||||
compact = " ".join(stripped.split())
|
||||
if "📷" in compact and len(compact) <= 80:
|
||||
return ""
|
||||
if compact.startswith("> 📷") and len(compact) <= 120:
|
||||
return ""
|
||||
return stripped
|
||||
|
||||
|
||||
def first_heading(body: str) -> str | None:
|
||||
in_fence = False
|
||||
for line in body.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("```"):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
if in_fence:
|
||||
continue
|
||||
if match := re.match(r"^#{1,6}\s+(.+)$", line):
|
||||
return match.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def wikilink_for(path: Path, root: Path) -> str:
|
||||
try:
|
||||
relative = path.relative_to(root)
|
||||
except ValueError:
|
||||
return path.stem
|
||||
if relative.parts and relative.parts[0] == "_sources":
|
||||
relative = Path(*relative.parts[1:])
|
||||
value = relative.as_posix()
|
||||
for suffix in SPHINX_SOURCE_SUFFIXES:
|
||||
if value.endswith(suffix):
|
||||
return value[: -len(suffix)]
|
||||
return value[:-3] if value.endswith(".md") else path.stem
|
||||
|
||||
|
||||
def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str:
|
||||
compact = " ".join(text.split())
|
||||
return compact if len(compact) <= limit else compact[:limit].rstrip()
|
||||
22
d_wiki/tokenize.py
Normal file
22
d_wiki/tokenize.py
Normal file
@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上")
|
||||
STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"}
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
lowered = text.lower()
|
||||
terms = re.findall(r"[a-z0-9_./+-]+", lowered)
|
||||
cjk_chars = re.findall(r"[\u4e00-\u9fff]", text)
|
||||
terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1)))
|
||||
return [term for term in terms if _useful(term)]
|
||||
|
||||
|
||||
def _useful(term: str) -> bool:
|
||||
if not term or term in STOP_TERMS:
|
||||
return False
|
||||
if re.search(r"[\u4e00-\u9fff]", term):
|
||||
return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term))
|
||||
return True
|
||||
70
d_wiki/yaml_compat.py
Normal file
70
d_wiki/yaml_compat.py
Normal file
@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml as _yaml # type: ignore
|
||||
except Exception: # pragma: no cover - exercised in environments without PyYAML
|
||||
_yaml = None
|
||||
|
||||
|
||||
def safe_load(text: str) -> Any:
|
||||
if _yaml is not None:
|
||||
return _yaml.safe_load(text)
|
||||
lines = [
|
||||
(len(line) - len(line.lstrip(" ")), line.strip())
|
||||
for line in text.splitlines()
|
||||
if line.strip() and not line.strip().startswith("#")
|
||||
]
|
||||
value, _index = _parse_node(lines, 0, 0)
|
||||
return value
|
||||
|
||||
|
||||
def _parse_node(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]:
|
||||
if index >= len(lines):
|
||||
return {}, index
|
||||
if lines[index][0] < indent:
|
||||
return {}, index
|
||||
if lines[index][1].startswith("- "):
|
||||
values: list[Any] = []
|
||||
while index < len(lines) and lines[index][0] == indent and lines[index][1].startswith("- "):
|
||||
values.append(_scalar(lines[index][1][2:].strip()))
|
||||
index += 1
|
||||
return values, index
|
||||
mapping: dict[str, Any] = {}
|
||||
while index < len(lines):
|
||||
current_indent, stripped = lines[index]
|
||||
if current_indent < indent:
|
||||
break
|
||||
if current_indent > indent:
|
||||
break
|
||||
if ":" not in stripped:
|
||||
index += 1
|
||||
continue
|
||||
key, raw_value = stripped.split(":", 1)
|
||||
key = key.strip()
|
||||
raw_value = raw_value.strip()
|
||||
index += 1
|
||||
if raw_value:
|
||||
mapping[key] = _scalar(raw_value)
|
||||
continue
|
||||
child, index = _parse_node(lines, index, indent + 2)
|
||||
mapping[key] = child
|
||||
return mapping, index
|
||||
|
||||
|
||||
def _scalar(value: str) -> Any:
|
||||
if value in {"null", "None", "~"}:
|
||||
return None
|
||||
if value in {"true", "True"}:
|
||||
return True
|
||||
if value in {"false", "False"}:
|
||||
return False
|
||||
if value.startswith("[") and value.endswith("]"):
|
||||
inner = value[1:-1].strip()
|
||||
if not inner:
|
||||
return []
|
||||
return [_scalar(item.strip()) for item in inner.split(",")]
|
||||
if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
|
||||
return value[1:-1]
|
||||
return value
|
||||
56
docs/language-decision.md
Normal file
56
docs/language-decision.md
Normal file
@ -0,0 +1,56 @@
|
||||
# d-wiki CLI language decision
|
||||
|
||||
## Decision
|
||||
|
||||
Use Python for the first implementation of `d-wiki`.
|
||||
|
||||
The CLI is a local knowledge retrieval component that needs predictable file
|
||||
system traversal, Markdown/Sphinx source parsing, YAML metadata, BM25 ranking,
|
||||
JSON output, and straightforward integration with `robogo_assistant_service`.
|
||||
Python is the lower-risk choice for this first version because the surrounding
|
||||
service code and tests are already Python, and the implementation intentionally
|
||||
avoids qmd, embeddings, and Node-specific runtime requirements.
|
||||
|
||||
## Python pros
|
||||
|
||||
- Matches the existing `robogo_assistant_service` stack, test style, and local
|
||||
developer workflow.
|
||||
- Strong standard library support for CLI parsing, paths, subprocess-friendly
|
||||
JSON output, dataclasses, and text processing.
|
||||
- Easy to keep dependency-light; the current package has no required runtime
|
||||
dependencies and can optionally use PyYAML when present.
|
||||
- Good fit for future retrieval upgrades such as ranking experiments,
|
||||
evaluation scripts, corpus analysis, and possible embedding backends.
|
||||
- Lower integration friction for service tests that call the CLI from Python.
|
||||
|
||||
## Python cons
|
||||
|
||||
- Single-file executable distribution is less natural than Node/npm CLIs unless
|
||||
extra packaging tools are introduced.
|
||||
- Startup can be slower than a bundled JS runtime for very small commands,
|
||||
though this is not a bottleneck for local document search.
|
||||
- Type safety depends on discipline and tools rather than the language runtime.
|
||||
|
||||
## TypeScript pros
|
||||
|
||||
- Excellent CLI packaging through npm and strong cross-platform installation
|
||||
conventions for JS-heavy teams.
|
||||
- Static types are first-class and make API shape evolution explicit.
|
||||
- Easier future alignment if `d-wiki` becomes a qmd-native or MCP-first Node
|
||||
package.
|
||||
|
||||
## TypeScript cons
|
||||
|
||||
- Adds a second runtime stack beside the Python service for the first version.
|
||||
- The plan explicitly avoids qmd/Node runtime complexity in v1; starting with
|
||||
TypeScript would pull the project toward the complexity it is trying to defer.
|
||||
- Requires build tooling, lockfiles, module-format decisions, and generated
|
||||
output conventions before the retrieval logic itself benefits from them.
|
||||
- Python service integration would still happen through subprocess JSON, so the
|
||||
main service does not gain much from a TypeScript implementation today.
|
||||
|
||||
## Revisit triggers
|
||||
|
||||
Reconsider TypeScript if `d-wiki` becomes a distributed npm CLI, adopts qmd as
|
||||
the default backend, ships an MCP server primarily for Node-based agents, or
|
||||
needs to share substantial code with a TypeScript UI or SDK.
|
||||
25
knowledge/robogo/aliases.yml
Normal file
25
knowledge/robogo/aliases.yml
Normal file
@ -0,0 +1,25 @@
|
||||
aliases:
|
||||
dev_machine:
|
||||
terms: [开发机, 云主机, SSH, Web 应用, 申请开发机, 创建开发机, 远程开发]
|
||||
intent: procedure
|
||||
related: [21-dev-machine]
|
||||
guagan:
|
||||
terms: [地瓜干, 计费, 充值, 余额, 消费, 价格, 账户]
|
||||
intent: fact
|
||||
related: [03-account-and-guagan]
|
||||
mirror:
|
||||
terms: [镜像, 镜像资产, 创建镜像, 保存镜像, fork, 公开镜像, 群组镜像]
|
||||
intent: procedure
|
||||
related: [11-mirror-center, 20-cloud-desktop]
|
||||
three_d_generation:
|
||||
terms: [3D生成, 3D 生成, 图生3D, 图生 3D, 3D资产生成, 3D 资产生成, 资产生成, 3d资产生成功能, EmbodyGen, glb]
|
||||
intent: procedure
|
||||
related: [40-3d-generation, 12-3d-assert, 41-image-generation, 42-urdf-studio]
|
||||
rdk_agent:
|
||||
terms: [RDK Agent, VS Code 插件, 插件安装]
|
||||
intent: procedure
|
||||
related: [51-rdk-agent, 50-openclaw]
|
||||
openclaw:
|
||||
terms: [OpenClaw, MagicBox, openclaw-magicbox, rdkclaw, 智能控制, AI 代理, Agent Platform, 设备配对, Skills 管理, 飞书]
|
||||
intent: procedure
|
||||
related: [50-openclaw, 51-rdk-agent]
|
||||
25
knowledge/robogo/queries/3d-generation.md
Normal file
25
knowledge/robogo/queries/3d-generation.md
Normal file
@ -0,0 +1,25 @@
|
||||
---
|
||||
title: 3D 资产生成
|
||||
aliases:
|
||||
- 我想知道如何使用3d资产生成功能
|
||||
- 如何使用 3D 资产生成功能
|
||||
- 3D资产生成怎么用
|
||||
- 3D 生成怎么用
|
||||
- 图生3D怎么用
|
||||
- 如何生成3D模型
|
||||
- 如何把图片生成3D资产
|
||||
intent: procedure
|
||||
primary_sources:
|
||||
- 40-3d-generation
|
||||
supporting_sources:
|
||||
- 12-3d-assert
|
||||
- 41-image-generation
|
||||
- 42-urdf-studio
|
||||
answer_requirements:
|
||||
- 说明 3D 生成是图生 3D 工具,不是镜像资产
|
||||
- 按准备参考图、生成、保存到 3D 资产的流程回答
|
||||
- 说明输出默认 glb,生成结果可保存到 3D 资产
|
||||
- 提醒暂不支持文生 3D,文字描述可先用图像生成产图
|
||||
---
|
||||
|
||||
RoboGo 3D 资产生成 / 图生 3D 的入口问题。
|
||||
7
knowledge/robogo/queries/dev-machine.md
Normal file
7
knowledge/robogo/queries/dev-machine.md
Normal file
@ -0,0 +1,7 @@
|
||||
---
|
||||
title: 开发机使用
|
||||
aliases: [开发机怎么用, 如何使用开发机, 怎么创建开发机, 开发机 SSH, 开发机 Web 应用]
|
||||
intent: procedure
|
||||
primary_sources: [21-dev-machine]
|
||||
answer_requirements: [说明开发机定位, 按创建和登录步骤回答, 说明 Web 应用和 SSH 入口]
|
||||
---
|
||||
7
knowledge/robogo/queries/guagan.md
Normal file
7
knowledge/robogo/queries/guagan.md
Normal file
@ -0,0 +1,7 @@
|
||||
---
|
||||
title: 地瓜干计费
|
||||
aliases: [地瓜干怎么计费, 地瓜干是什么, 怎么充值, 余额在哪里看]
|
||||
intent: fact
|
||||
primary_sources: [03-account-and-guagan]
|
||||
answer_requirements: [说明地瓜干用途, 说明充值和消费路径, 引用计费口径以平台实际页面为准]
|
||||
---
|
||||
8
knowledge/robogo/queries/mirror.md
Normal file
8
knowledge/robogo/queries/mirror.md
Normal file
@ -0,0 +1,8 @@
|
||||
---
|
||||
title: 镜像资产使用
|
||||
aliases: [镜像怎么使用, 如何创建镜像, 怎么保存镜像, 镜像资产怎么用]
|
||||
intent: procedure
|
||||
primary_sources: [11-mirror-center]
|
||||
supporting_sources: [20-cloud-desktop]
|
||||
answer_requirements: [区分镜像地址导入和地瓜桌面保存, 说明 Fork 和分享, 引用镜像资产来源]
|
||||
---
|
||||
8
knowledge/robogo/queries/openclaw.md
Normal file
8
knowledge/robogo/queries/openclaw.md
Normal file
@ -0,0 +1,8 @@
|
||||
---
|
||||
title: OpenClaw 使用
|
||||
aliases: [如何使用openclaw, OpenClaw 怎么用, 如何创建 OpenClaw, MagicBox 怎么用, OpenClaw 设备配对, OpenClaw Skills]
|
||||
intent: procedure
|
||||
primary_sources: [50-openclaw]
|
||||
supporting_sources: [51-rdk-agent]
|
||||
answer_requirements: [说明 OpenClaw 定位, 说明实例类型和创建入口, 说明 Skills/飞书/设备配对的使用路径]
|
||||
---
|
||||
8
knowledge/robogo/queries/rdk-agent.md
Normal file
8
knowledge/robogo/queries/rdk-agent.md
Normal file
@ -0,0 +1,8 @@
|
||||
---
|
||||
title: RDK Agent 使用
|
||||
aliases: [RDK Agent 怎么用, 如何安装 RDK Agent, RDK Agent 插件, RDK 设备怎么接入]
|
||||
intent: procedure
|
||||
primary_sources: [51-rdk-agent]
|
||||
supporting_sources: [50-openclaw]
|
||||
answer_requirements: [说明 RDK Agent 是 VS Code 插件入口, 区分设备接入在 OpenClaw MagicBox, 引用页面注意事项]
|
||||
---
|
||||
33
knowledge/x5/aliases.yml
Normal file
33
knowledge/x5/aliases.yml
Normal file
@ -0,0 +1,33 @@
|
||||
aliases:
|
||||
camera-bring-up:
|
||||
terms:
|
||||
- camera
|
||||
- 摄像头
|
||||
- sensor
|
||||
- mipi
|
||||
- csi
|
||||
- vin
|
||||
- isp
|
||||
- camsys
|
||||
- 点亮
|
||||
- bring up
|
||||
- bring-up
|
||||
- bringup
|
||||
- 调试
|
||||
- 配置
|
||||
- 预览
|
||||
intent: procedure
|
||||
related:
|
||||
- multimedia_development/7-camsys_debug
|
||||
- samples/sunrise_camera_user_guide
|
||||
- samples/sample_pipeline
|
||||
compute:
|
||||
terms:
|
||||
- 算力
|
||||
- tops
|
||||
- bpu
|
||||
- bayes
|
||||
- bayers
|
||||
intent: fact
|
||||
related:
|
||||
- preface/overview
|
||||
19
knowledge/x5/queries/camera-bring-up.md
Normal file
19
knowledge/x5/queries/camera-bring-up.md
Normal file
@ -0,0 +1,19 @@
|
||||
---
|
||||
title: X5 camera bring up
|
||||
aliases:
|
||||
- 如何在X5平台上点亮camera
|
||||
- X5 camera 点亮
|
||||
- X5 摄像头 bring up
|
||||
intent: procedure
|
||||
primary_sources:
|
||||
- multimedia_development/7-camsys_debug
|
||||
supporting_sources:
|
||||
- samples/sunrise_camera_user_guide
|
||||
- samples/sample_pipeline
|
||||
answer_requirements:
|
||||
- 说明这是 X5 camera bring up / 点亮流程
|
||||
- 按硬件连接、DTS/GPIO、sensor 驱动、MIPI 配置、运行验证组织
|
||||
- 引用 Camera调试指南 和 Sunrise camera 用户使用说明等来源
|
||||
---
|
||||
|
||||
X5 平台点亮 camera 的入口问题。
|
||||
14
knowledge/x5/queries/compute.md
Normal file
14
knowledge/x5/queries/compute.md
Normal file
@ -0,0 +1,14 @@
|
||||
---
|
||||
title: X5 算力
|
||||
aliases:
|
||||
- x5的算力是多少
|
||||
- X5 BPU 算力
|
||||
intent: fact
|
||||
primary_sources:
|
||||
- preface/overview
|
||||
answer_requirements:
|
||||
- 直接给出 X5 BPU 算力
|
||||
- 保留 TOPS 和 BPU 信息
|
||||
---
|
||||
|
||||
X5 算力参数入口问题。
|
||||
17
pyproject.toml
Normal file
17
pyproject.toml
Normal file
@ -0,0 +1,17 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "d-wiki"
|
||||
version = "0.1.0"
|
||||
description = "Lightweight local knowledge retrieval with BM25 and metadata context packs."
|
||||
requires-python = ">=3.10"
|
||||
dependencies = []
|
||||
|
||||
[project.scripts]
|
||||
d-wiki = "d_wiki.cli:main"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["d_wiki*"]
|
||||
BIN
tests/__pycache__/test_d_wiki.cpython-311-pytest-9.0.3.pyc
Normal file
BIN
tests/__pycache__/test_d_wiki.cpython-311-pytest-9.0.3.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_d_wiki.cpython-312.pyc
Normal file
BIN
tests/__pycache__/test_d_wiki.cpython-312.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_d_wiki.cpython-314.pyc
Normal file
BIN
tests/__pycache__/test_d_wiki.cpython-314.pyc
Normal file
Binary file not shown.
509
tests/test_d_wiki.py
Normal file
509
tests/test_d_wiki.py
Normal file
@ -0,0 +1,509 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from d_wiki.cli import main
|
||||
from d_wiki.metadata import QueryEntry, query_matches_entry
|
||||
|
||||
|
||||
class DWikiTest(unittest.TestCase):
|
||||
def test_query_entry_matches_rewritten_alias_by_token_overlap(self) -> None:
|
||||
entry = QueryEntry(
|
||||
path=Path("camera.md"),
|
||||
title="camera bring up",
|
||||
aliases=("如何在X5平台上点亮camera",),
|
||||
intent="procedure",
|
||||
primary_sources=("multimedia_development/7-camsys_debug",),
|
||||
supporting_sources=(),
|
||||
answer_requirements=(),
|
||||
)
|
||||
|
||||
self.assertTrue(query_matches_entry("X5平台 点亮 camera 使能", entry))
|
||||
|
||||
def test_bm25_metadata_context_pack_for_camera_bring_up(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
config = root / "d-wiki.yml"
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"如何在X5平台上点亮camera",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(output["status"], "found")
|
||||
self.assertEqual(output["context_pack"]["answer_intent"], "procedure")
|
||||
self.assertEqual(output["context_pack"]["coverage"], "partial_procedure")
|
||||
first = output["context_pack"]["primary_sources"][0]
|
||||
self.assertEqual(first["wikilink"], "multimedia_development/7-camsys_debug")
|
||||
self.assertGreaterEqual(len(first["blocks"]), 2)
|
||||
self.assertIn("点亮 camera", json.dumps(first, ensure_ascii=False))
|
||||
|
||||
def test_rewritten_camera_query_still_uses_query_entry_primary_source(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"X5平台 点亮 camera 使能",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(output["status"], "found")
|
||||
first = output["context_pack"]["primary_sources"][0]
|
||||
self.assertEqual(first["wikilink"], "multimedia_development/7-camsys_debug")
|
||||
|
||||
def test_fact_query_for_compute(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"x5的算力是多少",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(output["status"], "found")
|
||||
self.assertEqual(output["context_pack"]["answer_intent"], "fact")
|
||||
self.assertIn("10 TOPS", json.dumps(output["context_pack"], ensure_ascii=False))
|
||||
|
||||
def test_unrelated_query_is_insufficient(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"火星咖啡机怎么维修",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(output["status"], "no_match")
|
||||
self.assertEqual(output["context_pack"]["coverage"], "insufficient")
|
||||
|
||||
def test_page_frontmatter_metadata_participates_in_retrieval(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"x5 coredump breadcrumbs",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(output["status"], "found")
|
||||
self.assertEqual(output["sources"][0]["wikilink"], "misc/pstore")
|
||||
|
||||
def test_markdown_cleaning_filters_thematic_breaks_and_screenshot_only_blocks(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
docs = root / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "dev-machine.md").write_text(
|
||||
"# 开发机\n\n"
|
||||
"> 📷 截图:开发机列表页\n\n"
|
||||
"---\n\n"
|
||||
"开发机支持 SSH 登录和 Web 应用访问。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "d-wiki.yml").write_text(
|
||||
"knowledge_bases:\n"
|
||||
" robogo:\n"
|
||||
f" root: {docs}\n"
|
||||
" source_type: markdown\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"search",
|
||||
"--kb",
|
||||
"robogo",
|
||||
"--query",
|
||||
"开发机怎么用",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
snippets = [str(source["snippet"]) for source in output["sources"]]
|
||||
self.assertNotIn("---", snippets)
|
||||
self.assertFalse(any("📷" in snippet for snippet in snippets))
|
||||
self.assertIn("开发机支持 SSH 登录", snippets[0])
|
||||
|
||||
def test_compile_builds_wiki_pages_and_search_prefers_compiled_wiki(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
config = root / "d-wiki.yml"
|
||||
compiled = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"compile",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--force",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
indexed = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"index",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"x5的算力是多少",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
wiki_root = Path(str(compiled["wiki_root"]))
|
||||
self.assertEqual(compiled["status"], "compiled")
|
||||
self.assertTrue((wiki_root / "SCHEMA.md").exists())
|
||||
self.assertTrue((wiki_root / "index.md").exists())
|
||||
self.assertTrue((wiki_root / "concepts" / "preface-overview.md").exists())
|
||||
self.assertEqual(indexed["compiled_wiki"], str(wiki_root))
|
||||
self.assertEqual(output["status"], "found")
|
||||
self.assertIn(".d_wiki/wiki/concepts", output["sources"][0]["source_path"])
|
||||
self.assertIn("10 TOPS", json.dumps(output["context_pack"], ensure_ascii=False))
|
||||
|
||||
def test_compile_emits_llm_wiki_structure_without_breaking_retrieval(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
config = root / "d-wiki.yml"
|
||||
compiled = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"compile",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--force",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"index",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
compute = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"x5的算力是多少",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
camera = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(config),
|
||||
"search",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--query",
|
||||
"如何在X5平台上点亮camera",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
wiki_root = Path(str(compiled["wiki_root"]))
|
||||
compute_page = (wiki_root / "concepts" / "preface-overview.md").read_text(encoding="utf-8")
|
||||
camera_page = (
|
||||
wiki_root / "concepts" / "multimedia-development-7-camsys-debug.md"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("llmWikiVersion: \"1\"", compute_page)
|
||||
self.assertIn("## Intent Model", compute_page)
|
||||
self.assertIn("## Retrieval Hints", compute_page)
|
||||
self.assertIn("## Answer Blueprint", compute_page)
|
||||
self.assertIn("## Evidence Graph", compute_page)
|
||||
self.assertIn("## Key Evidence", compute_page)
|
||||
self.assertIn("X5 算力", compute_page)
|
||||
self.assertIn("10 TOPS", compute_page)
|
||||
self.assertIn("- intent: fact", compute_page)
|
||||
self.assertIn("- intent: procedure", camera_page)
|
||||
self.assertIn("按步骤回答", camera_page)
|
||||
self.assertEqual(compute["status"], "found")
|
||||
self.assertIn("10 TOPS", json.dumps(compute["context_pack"], ensure_ascii=False))
|
||||
self.assertIn(
|
||||
"Key Evidence",
|
||||
[block["heading"] for block in compute["context_pack"]["primary_sources"][0]["blocks"]],
|
||||
)
|
||||
self.assertEqual(camera["context_pack"]["answer_intent"], "procedure")
|
||||
self.assertEqual(
|
||||
camera["context_pack"]["primary_sources"][0]["wikilink"],
|
||||
"concepts/multimedia-development-7-camsys-debug",
|
||||
)
|
||||
self.assertNotEqual(camera["sources"][0]["heading"], "Retrieval Hints")
|
||||
|
||||
def test_dream_queries_route_without_manual_query_entry(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
docs = root / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "11-mirror-center.md").write_text(
|
||||
"# 镜像资产使用\n\n"
|
||||
"镜像资产是 RoboGo 的容器环境仓库,用于保存和复用开发环境。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(docs / "40-3d-generation.md").write_text(
|
||||
"# 40 · 3D 生成\n\n"
|
||||
"3D 生成是平台提供的 AI 图生 3D 工具,基于 EmbodyGen,用一张参考图片生成 3D 模型,默认 glb。\n\n"
|
||||
"生成结果可以保存到 3D 资产继续使用。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "knowledge" / "robogo").mkdir(parents=True)
|
||||
(root / "knowledge" / "robogo" / "aliases.yml").write_text(
|
||||
"aliases:\n"
|
||||
" mirror:\n"
|
||||
" terms: [镜像, 镜像资产, 创建镜像]\n"
|
||||
" intent: procedure\n"
|
||||
" related: [11-mirror-center]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "d-wiki.yml").write_text(
|
||||
"knowledge_bases:\n"
|
||||
" robogo:\n"
|
||||
f" root: {docs}\n"
|
||||
" source_type: markdown\n"
|
||||
" aliases: knowledge/robogo/aliases.yml\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
compiled = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"compile",
|
||||
"--kb",
|
||||
"robogo",
|
||||
"--force",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
dreamed = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"dream",
|
||||
"--kb",
|
||||
"robogo",
|
||||
"--force",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"index",
|
||||
"--kb",
|
||||
"robogo",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
output = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"search",
|
||||
"--kb",
|
||||
"robogo",
|
||||
"--query",
|
||||
"我想知道如何使用3d资产生成功能",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
page = Path(str(compiled["wiki_root"])) / "concepts" / "40-3d-generation.md"
|
||||
page_content = page.read_text(encoding="utf-8")
|
||||
dreamed_query = Path(str(dreamed["output_store"])) / "queries" / "40-3d-generation.md"
|
||||
dreamed_content = dreamed_query.read_text(encoding="utf-8")
|
||||
|
||||
self.assertNotIn("dream_queries:", page_content)
|
||||
self.assertIn("3D资产生成怎么用", dreamed_content)
|
||||
self.assertIn("provenance: dreamed", dreamed_content)
|
||||
self.assertEqual(output["status"], "found")
|
||||
self.assertEqual(output["context_pack"]["primary_sources"][0]["wikilink"], "concepts/40-3d-generation")
|
||||
self.assertNotIn("镜像资产使用", json.dumps(output["context_pack"]["answer_requirements"], ensure_ascii=False))
|
||||
|
||||
def test_compile_llm_auto_falls_back_without_llm_config(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = self._fixture(Path(tmp))
|
||||
compiled = self._run_json(
|
||||
[
|
||||
"--config",
|
||||
str(root / "d-wiki.yml"),
|
||||
"compile",
|
||||
"--kb",
|
||||
"x5",
|
||||
"--force",
|
||||
"--llm",
|
||||
"auto",
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
page = Path(str(compiled["wiki_root"])) / "concepts" / "preface-overview.md"
|
||||
page_content = page.read_text(encoding="utf-8")
|
||||
|
||||
self.assertEqual(compiled["status"], "compiled")
|
||||
self.assertEqual(compiled["llm_mode"], "auto")
|
||||
self.assertEqual(compiled["llm_scope"], "queries")
|
||||
self.assertFalse(compiled["llm_enabled"])
|
||||
self.assertIn("provenanceState: compiled", page_content)
|
||||
|
||||
def _run_json(self, argv: list[str]) -> dict[str, object]:
|
||||
from contextlib import redirect_stdout
|
||||
from io import StringIO
|
||||
|
||||
out = StringIO()
|
||||
with redirect_stdout(out):
|
||||
code = main(argv)
|
||||
self.assertEqual(code, 0, out.getvalue())
|
||||
return json.loads(out.getvalue())
|
||||
|
||||
def _fixture(self, root: Path) -> Path:
|
||||
sources = root / "manual" / "_sources"
|
||||
(sources / "multimedia_development").mkdir(parents=True)
|
||||
(sources / "preface").mkdir(parents=True)
|
||||
(sources / "misc").mkdir(parents=True)
|
||||
(sources / "multimedia_development" / "7-camsys_debug.md.txt").write_text(
|
||||
"# Camera调试指南\n\n"
|
||||
"## 一 范围\n\n"
|
||||
"本章节概述了 camera bring up 的过程,用于帮助读者快速了解并掌握 X5 camera 框架,"
|
||||
"如何快速的新增camera 配置,并点亮 camera。\n\n"
|
||||
"## 1.1 sensor gpio 配置\n\n"
|
||||
"vcon 是 X5 camera 用于管理 sensor 硬件相关的 dts 节点。\n\n"
|
||||
"## 2 sensor 驱动文件添加\n\n"
|
||||
"需要将原厂 sensor 驱动转换成 X5 camera 驱动代码,并编译生成 so 库。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(sources / "preface" / "overview.md.txt").write_text(
|
||||
"# 前言\n\n## 关键特性\n\n- 贝叶斯架构BPU,提供最高 10 TOPS 算力。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(sources / "misc" / "pstore.md.txt").write_text(
|
||||
"---\n"
|
||||
"title: pstore\n"
|
||||
"aliases: [coredump breadcrumbs]\n"
|
||||
"tags: [debug, reboot]\n"
|
||||
"---\n"
|
||||
"# pstore\n\n在X5平台上,当系统异常重启时会保存日志。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "knowledge" / "x5" / "queries").mkdir(parents=True)
|
||||
(root / "knowledge" / "x5" / "aliases.yml").write_text(
|
||||
"aliases:\n"
|
||||
" camera:\n"
|
||||
" terms: [camera, 摄像头, sensor, mipi, csi, vin, isp, 点亮, bring up, 调试, 配置]\n"
|
||||
" intent: procedure\n"
|
||||
" related: [multimedia_development/7-camsys_debug]\n"
|
||||
" compute:\n"
|
||||
" terms: [算力, tops, bpu]\n"
|
||||
" intent: fact\n"
|
||||
" related: [preface/overview]\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "knowledge" / "x5" / "queries" / "camera.md").write_text(
|
||||
"---\n"
|
||||
"title: camera bring up\n"
|
||||
"aliases: [如何在X5平台上点亮camera]\n"
|
||||
"intent: procedure\n"
|
||||
"primary_sources: [multimedia_development/7-camsys_debug]\n"
|
||||
"answer_requirements: [按步骤回答]\n"
|
||||
"---\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "knowledge" / "x5" / "queries" / "compute.md").write_text(
|
||||
"---\n"
|
||||
"title: X5 算力\n"
|
||||
"aliases: [x5的算力是多少]\n"
|
||||
"intent: fact\n"
|
||||
"primary_sources: [preface/overview]\n"
|
||||
"---\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "d-wiki.yml").write_text(
|
||||
"knowledge_bases:\n"
|
||||
" x5:\n"
|
||||
f" root: {root / 'manual'}\n"
|
||||
" source_type: sphinx\n"
|
||||
" aliases: knowledge/x5/aliases.yml\n"
|
||||
" queries: knowledge/x5/queries\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user