first commit

This commit is contained in:
yue02.sun 2026-05-28 19:02:16 +08:00
parent 44de534185
commit 66eb67569e
77 changed files with 4372 additions and 0 deletions

View File

@ -0,0 +1,3 @@
"""d-wiki lightweight local knowledge retrieval."""
__version__ = "0.1.0"

View File

@ -0,0 +1,4 @@
from .cli import main
if __name__ == "__main__":
raise SystemExit(main())

77
build/lib/d_wiki/cli.py Normal file
View File

@ -0,0 +1,77 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
from .config import DWikiConfig, load_config
from .compile import compile_kb
from .dreams import dream_kb
from .search import get_page, index_payload, search, status_payload
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="d-wiki")
parser.add_argument("--config", type=Path, default=None)
subparsers = parser.add_subparsers(dest="command", required=True)
status_parser = subparsers.add_parser("status")
status_parser.add_argument("--format", choices=["json"], default="json")
index_parser = subparsers.add_parser("index")
index_parser.add_argument("--kb", required=True)
index_parser.add_argument("--format", choices=["json"], default="json")
compile_parser = subparsers.add_parser("compile")
compile_parser.add_argument("--kb", required=True)
compile_parser.add_argument("--force", action="store_true")
compile_parser.add_argument("--llm", choices=["off", "auto", "require"], default="off")
compile_parser.add_argument("--llm-scope", choices=["queries", "all"], default="queries")
compile_parser.add_argument("--format", choices=["json"], default="json")
dream_parser = subparsers.add_parser("dream")
dream_parser.add_argument("--kb", required=True)
dream_parser.add_argument("--force", action="store_true")
dream_parser.add_argument("--format", choices=["json"], default="json")
search_parser = subparsers.add_parser("search")
search_parser.add_argument("--kb", required=True)
search_parser.add_argument("--query", required=True)
search_parser.add_argument("--max-results", type=int, default=5)
search_parser.add_argument("--format", choices=["json"], default="json")
get_parser = subparsers.add_parser("get")
get_parser.add_argument("--kb", required=True)
get_parser.add_argument("--id", required=True)
get_parser.add_argument("--format", choices=["json"], default="json")
args = parser.parse_args(argv)
try:
config = load_config(args.config)
payload = dispatch(args, config)
except Exception as exc:
print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 2
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
return 0
def dispatch(args: argparse.Namespace, config: DWikiConfig) -> dict[str, Any]:
if args.command == "status":
return status_payload(config.path, config.knowledge_bases)
kb = config.knowledge_bases.get(args.kb)
if kb is None:
raise ValueError(f"D_WIKI_KB_NOT_FOUND: {args.kb}")
if args.command == "index":
return index_payload(kb)
if args.command == "compile":
return compile_kb(kb, force=args.force, llm_mode=args.llm, llm_scope=args.llm_scope)
if args.command == "dream":
return dream_kb(kb, force=args.force)
if args.command == "search":
return search(kb, args.query, args.max_results)
if args.command == "get":
return get_page(kb, args.id)
raise ValueError(f"D_WIKI_UNKNOWN_COMMAND: {args.command}")

590
build/lib/d_wiki/compile.py Normal file
View File

@ -0,0 +1,590 @@
from __future__ import annotations
import re
import shutil
import json
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .llm import LlmPageSynthesis, LlmPageSynthesizer
from .metadata import load_alias_groups, load_query_entries
from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block
from .tokenize import tokenize
@dataclass(frozen=True)
class CompiledPage:
wikilink: str
title: str
path: Path
@dataclass(frozen=True)
class _EvidenceProfile:
tokens: frozenset[str]
phrases: tuple[str, ...]
def compile_kb(
kb: KnowledgeBaseConfig,
*,
force: bool = False,
llm_mode: str = "off",
llm_scope: str = "queries",
) -> dict[str, Any]:
chunks = load_source_chunks(kb)
if not chunks:
raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS")
synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None
if llm_mode == "require" and synthesizer is None:
raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED")
wiki_root = compiled_wiki_root(kb)
concepts_dir = wiki_root / "concepts"
raw_dir = wiki_root / "raw"
if force:
shutil.rmtree(concepts_dir, ignore_errors=True)
concepts_dir.mkdir(parents=True, exist_ok=True)
raw_dir.mkdir(parents=True, exist_ok=True)
pages: list[CompiledPage] = []
grouped = _group_chunks(chunks)
query_entries = load_query_entries(kb.queries_dir)
alias_groups = load_alias_groups(kb.aliases_path)
query_by_source = {
source: entry
for entry in query_entries
for source in entry.primary_sources
}
aliases_by_source = {
related: group
for group in alias_groups
for related in group.related
}
for wikilink, page_chunks in sorted(grouped.items()):
page = _compile_page(
kb,
wikilink,
page_chunks,
query_entry=query_by_source.get(wikilink),
alias_group=aliases_by_source.get(wikilink),
synthesizer=(
synthesizer
if llm_scope == "all" or query_by_source.get(wikilink) is not None
else None
),
require_llm=llm_mode == "require",
)
if force or not page.path.exists():
page.path.write_text(page.content, encoding="utf-8")
pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path))
_write_schema(wiki_root, kb)
_write_index(wiki_root, pages)
_append_log(wiki_root, kb, len(chunks), len(pages))
return {
"status": "compiled",
"kb": kb.name,
"source_root": str(kb.root),
"wiki_root": str(wiki_root),
"source_chunks": len(chunks),
"pages": len(pages),
"llm_mode": llm_mode,
"llm_scope": llm_scope,
"llm_enabled": synthesizer is not None,
"compiled_at": datetime.now(timezone.utc).isoformat(),
}
@dataclass(frozen=True)
class _PageDraft:
title: str
path: Path
content: str
def _compile_page(
kb: KnowledgeBaseConfig,
wikilink: str,
chunks: list[Chunk],
*,
query_entry: Any | None,
alias_group: Any | None,
synthesizer: LlmPageSynthesizer | None,
require_llm: bool,
) -> _PageDraft:
first = chunks[0]
title = _title_for(first, query_entry)
slug = _slug(wikilink)
path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md"
evidence_profile = _evidence_profile(title, query_entry, alias_group)
tags = _tags(kb, first, alias_group)
aliases = _aliases(title, query_entry, alias_group)
source_ref = first.wikilink
source_path = str(first.source_path)
key_blocks = _key_blocks(chunks, evidence_profile)
intent = _intent(title, query_entry, alias_group)
summary = _summary(chunks, title, evidence_profile, intent=intent)
requirements = _answer_requirements(intent, query_entry)
synthesis = _llm_synthesis(
synthesizer,
title=title,
wikilink=wikilink,
query_entry=query_entry,
alias_group=alias_group,
key_blocks=key_blocks,
require_llm=require_llm,
)
if synthesis is not None:
summary = synthesis.summary
retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group)
blueprint = (
list(synthesis.answer_blueprint)
if synthesis and synthesis.answer_blueprint
else _deterministic_answer_blueprint(intent, requirements, key_blocks)
)
links = _related_links(query_entry, alias_group)
content = "\n".join(
[
"---",
f"llmWikiVersion: \"1\"",
f"conceptId: {_yaml_scalar(slug)}",
f"title: {_yaml_scalar(title)}",
f"summary: {_yaml_scalar(summary)}",
f"sources: {_yaml_list([source_ref])}",
f"source_paths: {_yaml_list([source_path])}",
f"kind: concept",
f"tags: {_yaml_list(tags)}",
f"aliases: {_yaml_list(aliases)}",
f"intents: {_yaml_list([intent])}",
f"retrieval_hints: {_yaml_list(retrieval_hints)}",
f"answer_requirements: {_yaml_list(requirements)}",
f"confidence: {synthesis.confidence if synthesis else 'medium'}",
f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}",
"---",
"",
f"# {title}",
"",
summary,
"",
"## Intent Model",
"",
f"- intent: {intent}",
f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}",
f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}",
"",
"## Retrieval Hints",
"",
*[f"- {hint}" for hint in retrieval_hints],
"",
"## Answer Blueprint",
"",
*[f"- {item}" for item in blueprint],
"",
"## Synthesized Evidence" if synthesis and synthesis.key_evidence else "",
"" if synthesis and synthesis.key_evidence else "",
*[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())],
"" if synthesis and synthesis.key_evidence else "",
"## Evidence Graph",
"",
*_evidence_graph_lines(key_blocks),
"",
"## Key Evidence",
"",
*[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks],
"",
"## Source Map",
"",
f"- 原始来源:`{source_ref}`",
f"- 文件路径:`{source_path}`",
"",
"## Related",
"",
*[f"- [[{link}]]" for link in links],
"",
]
)
return _PageDraft(title=title, path=path, content=content)
def _llm_synthesis(
synthesizer: LlmPageSynthesizer | None,
*,
title: str,
wikilink: str,
query_entry: Any | None,
alias_group: Any | None,
key_blocks: list[Chunk],
require_llm: bool,
) -> LlmPageSynthesis | None:
if synthesizer is None:
return None
prompt = _llm_compile_prompt(
title=title,
wikilink=wikilink,
query_entry=query_entry,
alias_group=alias_group,
key_blocks=key_blocks,
)
try:
return synthesizer.synthesize(prompt)
except Exception:
if require_llm:
raise
return None
def _llm_compile_prompt(
*,
title: str,
wikilink: str,
query_entry: Any | None,
alias_group: Any | None,
key_blocks: list[Chunk],
) -> str:
aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ())
terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ())
requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ())
intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None)
evidence = [
{
"block_id": f"{block.wikilink}#{block.index}",
"heading": block.heading,
"text": trim_block(block.text, 700),
}
for block in key_blocks[:8]
]
return "\n".join(
[
"Compile a d-wiki concept page from the local evidence below.",
"Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.",
"For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.",
"Return only JSON with this shape:",
'{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}',
"",
json_dumps(
{
"title": title,
"wikilink": wikilink,
"intent": intent,
"aliases": aliases,
"terms": terms,
"answer_requirements": requirements,
"evidence": evidence,
}
),
]
)
def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]:
if synthesis is None or not synthesis.answer_blueprint:
return []
return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""]
def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str:
if query_entry is not None and getattr(query_entry, "intent", ""):
return str(query_entry.intent)
if alias_group is not None and getattr(alias_group, "intent", ""):
return str(alias_group.intent)
lowered = title.lower()
if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")):
return "procedure"
return "fact"
def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]:
if query_entry is not None:
requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()]
if requirements:
return _unique(requirements)
if intent == "procedure":
return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"]
return ["直接给出结论", "保留关键数值和单位", "引用证据来源"]
def _retrieval_hints(
title: str,
aliases: list[str],
tags: list[str],
requirements: list[str],
alias_group: Any | None,
) -> list[str]:
values = [title, *aliases, *tags, *requirements]
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
hints: list[str] = []
for value in values:
hints.append(value)
hints.extend(tokenize(value))
return _unique(hints)[:32]
def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]:
if requirements:
return requirements[:6]
if intent == "procedure":
headings = _unique([block.heading for block in key_blocks if block.heading])
if headings:
return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]]
return ["按前提、配置、执行、验证的顺序回答"]
return ["先给出直接结论", "补充关键参数、限制和来源"]
def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]:
lines: list[str] = []
for block in key_blocks[:8]:
claim = trim_block(block.text, 180)
if not claim:
continue
lines.append(f"- claim: {claim}")
lines.append(f" source: {block.wikilink}#{block.index}")
if block.heading:
lines.append(f" heading: {block.heading}")
return lines or ["- claim: 暂无可用证据", " source: unknown"]
def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]:
grouped: dict[str, list[Chunk]] = {}
for chunk in chunks:
grouped.setdefault(chunk.wikilink, []).append(chunk)
return grouped
def _title_for(chunk: Chunk, query_entry: Any | None) -> str:
if query_entry is not None and getattr(query_entry, "title", ""):
return str(query_entry.title)
return chunk.title
def _summary(
chunks: list[Chunk],
title: str,
evidence_profile: _EvidenceProfile | None = None,
*,
intent: str = "fact",
) -> str:
if intent == "procedure":
for chunk in chunks[:12]:
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text):
return f"{title}:{text}"
ranked = _rank_evidence(chunks, evidence_profile)
for score, chunk in ranked:
if score <= 0:
continue
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text):
return f"{title}:{text}"
for chunk in chunks:
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text) and len(text) >= 20:
return f"{title}:{text}"
return f"{title} 的编译知识页,来源于本地文档。"
def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]:
preferred = [
chunk
for chunk in chunks
if _is_evidence_candidate(chunk.text)
]
ranked = _rank_evidence(preferred, evidence_profile)
if not ranked or ranked[0][0] <= 0:
return (preferred or chunks)[:6]
selected: list[Chunk] = []
seen: set[tuple[str, int]] = set()
for score, chunk in ranked:
if score <= 0:
continue
key = (chunk.wikilink, chunk.index)
if key in seen:
continue
selected.append(chunk)
seen.add(key)
if len(selected) >= 6:
return selected
for chunk in preferred:
key = (chunk.wikilink, chunk.index)
if key in seen:
continue
selected.append(chunk)
seen.add(key)
if len(selected) >= 6:
break
return selected
def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile:
values = [title]
if query_entry is not None:
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ()))
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
phrases = tuple(_unique([_normalize_phrase(value) for value in values if value]))
tokens: set[str] = set()
for value in values:
tokens.update(tokenize(str(value)))
return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases)
def _rank_evidence(
chunks: list[Chunk],
evidence_profile: _EvidenceProfile | None,
) -> list[tuple[float, Chunk]]:
if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases):
return []
ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks]
ranked.sort(key=lambda item: (-item[0], item[1].index))
return ranked
def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float:
text = trim_block(chunk.text, 1000)
normalized = _normalize_phrase(text)
chunk_tokens = set(chunk.tokens) | set(tokenize(text))
score = 0.0
score += len(chunk_tokens & evidence_profile.tokens) * 2.0
for phrase in evidence_profile.phrases:
if len(phrase) >= 2 and phrase in normalized:
score += 5.0
if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized):
score += 8.0
if "算力" in evidence_profile.tokens and "tops" in normalized:
score += 6.0
if "bpu" in evidence_profile.tokens and "bpu" in normalized:
score += 4.0
if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized):
score += 2.0
if _is_low_signal(text):
score -= 8.0
return score
def _is_evidence_candidate(text: str) -> bool:
stripped = text.lstrip()
return not stripped.startswith("|") and not stripped.startswith(">")
def _is_low_signal(text: str) -> bool:
stripped = text.strip()
without_marks = re.sub(r"[*_`#\-\s]+", "", stripped)
if len(without_marks) < 8:
return True
if stripped.startswith("```") or "```" in stripped:
return True
if stripped.startswith("|"):
return True
if "![](" in stripped:
return True
if re.search(r"(^|\n)\s*[├│└─]{2,}", stripped):
return True
return False
def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]:
tags = [kb.name]
if alias_group is not None:
tags.append(_slug(str(alias_group.name)).replace("--", "-"))
for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title):
lowered = token.lower()
if lowered not in tags:
tags.append(lowered)
return tags[:8]
def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]:
values = [title]
if query_entry is not None:
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
return _unique(values)[:12]
def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]:
links: list[str] = []
if query_entry is not None:
links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ()))
if alias_group is not None:
links.extend(str(item) for item in getattr(alias_group, "related", ()))
return _unique(links)[:10]
def _unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
cleaned = value.strip()
if not cleaned or cleaned in seen:
continue
seen.add(cleaned)
output.append(cleaned)
return output
def _normalize_phrase(value: str) -> str:
return re.sub(r"\s+", " ", value.lower()).strip()
def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None:
(wiki_root / "SCHEMA.md").write_text(
"\n".join(
[
"# Wiki Schema",
"",
"## Domain",
f"Compiled local knowledge for `{kb.name}`.",
"",
"## Conventions",
"- Pages live under `concepts/`.",
"- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.",
"- Claims in Key Evidence keep provenance markers pointing back to source block ids.",
"- Raw source files remain immutable; compiled pages are generated artifacts.",
"",
]
),
encoding="utf-8",
)
def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None:
lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""]
for page in sorted(pages, key=lambda item: item.title):
rel = page.path.relative_to(wiki_root).as_posix()
lines.append(f"- [[{rel[:-3]}]] — {page.title}")
lines.append("")
(wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8")
def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None:
line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n"
path = wiki_root / "log.md"
if not path.exists():
path.write_text("# Log\n\n", encoding="utf-8")
with path.open("a", encoding="utf-8") as handle:
handle.write(line)
def _slug(value: str) -> str:
slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower()
return slug or "page"
def _yaml_scalar(value: str) -> str:
import json
return json.dumps(value, ensure_ascii=False)
def _yaml_list(values: list[str]) -> str:
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, indent=2)

View File

@ -0,0 +1,59 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .yaml_compat import safe_load
@dataclass(frozen=True)
class KnowledgeBaseConfig:
name: str
root: Path
source_type: str
aliases_path: Path | None
queries_dir: Path | None
@dataclass(frozen=True)
class DWikiConfig:
path: Path
knowledge_bases: dict[str, KnowledgeBaseConfig]
def default_config_path() -> Path:
return Path(os.environ.get("D_WIKI_CONFIG", Path.cwd() / "d-wiki.yml")).expanduser()
def load_config(path: Path | None = None) -> DWikiConfig:
config_path = (path or default_config_path()).expanduser().resolve()
data = safe_load(config_path.read_text(encoding="utf-8")) or {}
raw_kbs = data.get("knowledge_bases")
if not isinstance(raw_kbs, dict):
raise ValueError("D_WIKI_CONFIG_MISSING_KNOWLEDGE_BASES")
bases: dict[str, KnowledgeBaseConfig] = {}
for name, raw in raw_kbs.items():
if not isinstance(raw, dict):
raise ValueError(f"D_WIKI_KB_INVALID: {name}")
root = Path(str(raw.get("root", ""))).expanduser()
if not root.is_absolute():
root = (config_path.parent / root).resolve()
aliases_path = _optional_path(config_path.parent, raw.get("aliases"))
queries_dir = _optional_path(config_path.parent, raw.get("queries"))
bases[str(name)] = KnowledgeBaseConfig(
name=str(name),
root=root,
source_type=str(raw.get("source_type", "markdown")),
aliases_path=aliases_path,
queries_dir=queries_dir,
)
return DWikiConfig(path=config_path, knowledge_bases=bases)
def _optional_path(base: Path, value: Any) -> Path | None:
if not value:
return None
path = Path(str(value)).expanduser()
return path if path.is_absolute() else (base / path).resolve()

227
build/lib/d_wiki/dreams.py Normal file
View File

@ -0,0 +1,227 @@
from __future__ import annotations
import re
import shutil
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .metadata import frontmatter, load_query_entries
@dataclass(frozen=True)
class DreamedQuery:
slug: str
title: str
aliases: tuple[str, ...]
intent: str
primary_sources: tuple[str, ...]
supporting_sources: tuple[str, ...]
answer_requirements: tuple[str, ...]
def dream_kb(kb: KnowledgeBaseConfig, *, force: bool = False) -> dict[str, Any]:
wiki_root = kb.root / ".d_wiki" / "wiki"
if not wiki_root.is_dir():
raise ValueError("D_WIKI_DREAM_REQUIRES_COMPILED_WIKI")
output_root = dream_root(kb)
queries_dir = output_root / "queries"
if force:
shutil.rmtree(output_root, ignore_errors=True)
queries_dir.mkdir(parents=True, exist_ok=True)
pages = sorted((wiki_root / "concepts").glob("*.md"))
dreamed = [_dream_page(path, wiki_root) for path in pages]
dreamed = [item for item in dreamed if item is not None]
for item in dreamed:
(queries_dir / f"{item.slug}.md").write_text(_query_markdown(item), encoding="utf-8")
(output_root / "README.md").write_text(_readme(kb, len(pages), len(dreamed)), encoding="utf-8")
return {
"status": "dreamed",
"kb": kb.name,
"input_wiki": str(wiki_root),
"output_store": str(output_root),
"queries": len(dreamed),
"dreamed_at": datetime.now(timezone.utc).isoformat(),
}
def load_dream_query_entries(kb: KnowledgeBaseConfig) -> list[Any]:
return load_query_entries(dream_root(kb) / "queries")
def dream_root(kb: KnowledgeBaseConfig) -> Path:
return kb.root / ".d_wiki" / "dreams" / "latest"
def _dream_page(path: Path, wiki_root: Path) -> DreamedQuery | None:
meta, body = frontmatter(path.read_text(encoding="utf-8", errors="ignore"))
title = str(meta.get("title") or "").strip()
if not title:
return None
curation_body = _curation_body(body)
source = _first(meta.get("sources")) or _wikilink_for_compiled(path, wiki_root)
aliases = _unique([title, *_dream_aliases(title, curation_body)])
intent = _first(meta.get("intents")) or _dream_intent(title, curation_body)
requirements = _as_list(meta.get("answer_requirements")) or _dream_requirements(title, curation_body, intent)
supporting = _related_sources(body, source)
return DreamedQuery(
slug=path.stem,
title=title,
aliases=tuple(aliases[:18]),
intent=intent,
primary_sources=(source,),
supporting_sources=tuple(supporting[:6]),
answer_requirements=tuple(requirements[:8]),
)
def _dream_aliases(title: str, body: str) -> list[str]:
text = f"{title}\n{body}".lower()
aliases = [
f"{title}怎么用",
f"如何使用{title}",
]
if _looks_like_3d_generation(title, text):
aliases.extend(
[
"3D资产生成怎么用",
"如何使用 3D 资产生成功能",
"我想知道如何使用3d资产生成功能",
"图生3D怎么用",
"如何生成3D模型",
"如何把图片生成3D资产",
]
)
if "camera" in text and any(term in text for term in ("点亮", "bring up", "sensor")):
aliases.extend(["如何在X5平台上点亮camera", "X5 camera 点亮", "X5 摄像头 bring up"])
if any(term in text for term in ("算力", "tops", "bpu")):
aliases.extend(["x5的算力是多少", "X5 BPU 算力"])
return aliases
def _dream_intent(title: str, body: str) -> str:
text = f"{title}\n{body}".lower()
if any(term in text for term in ("如何", "怎么", "步骤", "使用", "创建", "配置", "点亮", "生成", "调试", "bring up")):
return "procedure"
return "fact"
def _dream_requirements(title: str, body: str, intent: str) -> list[str]:
text = f"{title}\n{body}".lower()
if _looks_like_3d_generation(title, text):
return [
"说明 3D 生成是图生 3D 工具,不是镜像资产",
"按准备参考图、生成、保存到 3D 资产的流程回答",
"说明输出默认 glb,生成结果可保存到 3D 资产",
"提醒暂不支持文生 3D,文字描述可先用图像生成产图",
]
if intent == "procedure":
return ["按步骤回答", "说明关键前提和入口", "引用证据来源"]
return ["直接回答问题", "引用证据来源"]
def _looks_like_3d_generation(title: str, text: str) -> bool:
title_text = title.lower()
return (
"3d 生成" in title_text
or "3d生成" in title_text
or "3d 资产生成" in title_text
or "3d资产生成" in title_text
or "embodygen" in text
or "ai 图生 3d 工具" in text
or "ai 图生3d 工具" in text
)
def _curation_body(body: str) -> str:
lines: list[str] = []
skip = False
for line in body.splitlines():
if line.startswith("## "):
skip = line.strip() in {"## Retrieval Hints", "## Related"}
if skip:
continue
if not skip:
lines.append(line)
return "\n".join(lines)
def _related_sources(body: str, primary: str) -> list[str]:
links = re.findall(r"\[\[([^\]]+)\]\]", body)
refs = re.findall(r"\^\[([^#\]]+)(?:#[^\]]+)?\]", body)
return _unique([item for item in [*links, *refs] if item and item != primary])
def _query_markdown(item: DreamedQuery) -> str:
lines = [
"---",
f"title: {_yaml_scalar(item.title)}",
f"aliases: {_yaml_list(item.aliases)}",
f"intent: {item.intent}",
f"primary_sources: {_yaml_list(item.primary_sources)}",
f"supporting_sources: {_yaml_list(item.supporting_sources)}",
f"answer_requirements: {_yaml_list(item.answer_requirements)}",
"provenance: dreamed",
"---",
"",
f"Dreamed query memory for {item.title}.",
"",
]
return "\n".join(lines)
def _readme(kb: KnowledgeBaseConfig, pages: int, queries: int) -> str:
return "\n".join(
[
"# d-wiki dream output",
"",
"This store is generated from the compiled wiki. It does not modify source documents or compiled concept pages.",
"",
f"- kb: `{kb.name}`",
f"- input_pages: {pages}",
f"- dreamed_queries: {queries}",
"",
]
)
def _first(value: Any) -> str:
items = _as_list(value)
return items[0] if items else ""
def _as_list(value: Any) -> list[str]:
if isinstance(value, str):
return [value]
if isinstance(value, list):
return [str(item).strip() for item in value if str(item).strip()]
return []
def _unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
cleaned = re.sub(r"\s+", " ", str(value)).strip()
if not cleaned or cleaned in seen:
continue
seen.add(cleaned)
output.append(cleaned)
return output
def _wikilink_for_compiled(path: Path, wiki_root: Path) -> str:
return path.relative_to(wiki_root).with_suffix("").as_posix()
def _yaml_scalar(value: str) -> str:
import json
return json.dumps(value, ensure_ascii=False)
def _yaml_list(values: tuple[str, ...] | list[str]) -> str:
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"

156
build/lib/d_wiki/llm.py Normal file
View File

@ -0,0 +1,156 @@
from __future__ import annotations
import json
import os
import urllib.error
import urllib.request
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class LlmSettings:
base_url: str
model: str
api_key: str
api_style: str = "anthropic"
max_tokens: int = 900
@dataclass(frozen=True)
class LlmPageSynthesis:
summary: str
answer_blueprint: tuple[str, ...]
key_evidence: tuple[str, ...]
confidence: str = "medium"
class LlmPageSynthesizer:
def __init__(self, settings: LlmSettings) -> None:
self.settings = settings
@classmethod
def from_environment(cls) -> "LlmPageSynthesizer | None":
base_url = os.environ.get("D_WIKI_LLM_BASE_URL") or os.environ.get("ROBOGO_LLM_BASE_URL")
model = os.environ.get("D_WIKI_LLM_MODEL") or os.environ.get("ROBOGO_LLM_MODEL")
api_key = os.environ.get("D_WIKI_LLM_API_KEY") or os.environ.get("ROBOGO_LLM_API_KEY")
if not base_url or not model or not api_key:
return None
api_style = os.environ.get("D_WIKI_LLM_API_STYLE") or os.environ.get("ROBOGO_LLM_API_STYLE") or "anthropic"
max_tokens = int(os.environ.get("D_WIKI_LLM_MAX_TOKENS", "900"))
return cls(LlmSettings(base_url=base_url, model=model, api_key=api_key, api_style=api_style, max_tokens=max_tokens))
def synthesize(self, prompt: str) -> LlmPageSynthesis | None:
text = self._complete(prompt).strip()
payload = _extract_json(text)
if not isinstance(payload, dict):
return None
summary = str(payload.get("summary") or "").strip()
if not summary:
return None
answer_blueprint = _string_tuple(payload.get("answer_blueprint"))
key_evidence = _string_tuple(payload.get("key_evidence"))
confidence = str(payload.get("confidence") or "medium").strip().lower()
if confidence not in {"low", "medium", "high"}:
confidence = "medium"
return LlmPageSynthesis(
summary=summary,
answer_blueprint=answer_blueprint,
key_evidence=key_evidence,
confidence=confidence,
)
def _complete(self, prompt: str) -> str:
if self.settings.api_style == "openai":
return self._complete_openai(prompt)
return self._complete_anthropic(prompt)
def _complete_anthropic(self, prompt: str) -> str:
payload = {
"model": self.settings.model,
"max_tokens": self.settings.max_tokens,
"temperature": 0,
"system": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
"messages": [{"role": "user", "content": prompt}],
}
data = self._post_json(_join_url(self.settings.base_url, "v1/messages"), payload)
content = data.get("content")
if isinstance(content, list):
return "\n".join(str(item.get("text", "")) for item in content if isinstance(item, dict))
return str(data.get("text") or "")
def _complete_openai(self, prompt: str) -> str:
payload = {
"model": self.settings.model,
"temperature": 0,
"messages": [
{
"role": "system",
"content": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
},
{"role": "user", "content": prompt},
],
}
data = self._post_json(_join_url(self.settings.base_url, "v1/chat/completions"), payload)
choices = data.get("choices")
if isinstance(choices, list) and choices:
first = choices[0]
if isinstance(first, dict):
message = first.get("message")
if isinstance(message, dict):
return str(message.get("content") or "")
return ""
def _post_json(self, url: str, payload: dict[str, Any]) -> dict[str, Any]:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
request = urllib.request.Request(
url,
data=body,
method="POST",
headers={
"content-type": "application/json",
"accept": "application/json",
"x-api-key": self.settings.api_key,
"authorization": f"Bearer {self.settings.api_key}",
"anthropic-version": "2023-06-01",
},
)
try:
with urllib.request.urlopen(request, timeout=60) as response:
return json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="ignore")
raise RuntimeError(f"D_WIKI_LLM_HTTP_ERROR: {exc.code} {detail[:300]}") from exc
def _join_url(base_url: str, suffix: str) -> str:
base = base_url.rstrip("/")
if base.endswith("/v1/messages") or base.endswith("/v1/chat/completions"):
return base
if base.endswith("/v1"):
return f"{base}/{suffix.removeprefix('v1/')}"
return f"{base}/{suffix}"
def _extract_json(text: str) -> Any:
cleaned = text.strip()
if cleaned.startswith("```"):
cleaned = cleaned.strip("`")
if cleaned.lower().startswith("json"):
cleaned = cleaned[4:].strip()
decoder = json.JSONDecoder()
for index, char in enumerate(cleaned):
if char != "{":
continue
try:
value, _end = decoder.raw_decode(cleaned[index:])
except json.JSONDecodeError:
continue
return value
return None
def _string_tuple(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
return ()
return tuple(str(item).strip() for item in value if str(item).strip())

View File

@ -0,0 +1,119 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .yaml_compat import safe_load
from .tokenize import tokenize
QUERY_MATCH_STOP_TOKENS = {
"如何",
"怎么",
"使用",
"创建",
"知道",
"功能",
"资产",
}
@dataclass(frozen=True)
class AliasGroup:
name: str
terms: tuple[str, ...]
intent: str | None
related: tuple[str, ...]
@dataclass(frozen=True)
class QueryEntry:
path: Path
title: str
aliases: tuple[str, ...]
intent: str | None
primary_sources: tuple[str, ...]
supporting_sources: tuple[str, ...]
answer_requirements: tuple[str, ...]
def load_alias_groups(path: Path | None) -> list[AliasGroup]:
if path is None or not path.exists():
return []
data = safe_load(path.read_text(encoding="utf-8")) or {}
raw_aliases = data.get("aliases", {})
if not isinstance(raw_aliases, dict):
return []
groups: list[AliasGroup] = []
for name, raw in raw_aliases.items():
if not isinstance(raw, dict):
continue
groups.append(
AliasGroup(
name=str(name),
terms=tuple(str(term).strip() for term in raw.get("terms", []) if str(term).strip()),
intent=str(raw["intent"]) if raw.get("intent") else None,
related=tuple(str(item).strip() for item in raw.get("related", []) if str(item).strip()),
)
)
return groups
def load_query_entries(directory: Path | None) -> list[QueryEntry]:
if directory is None or not directory.exists():
return []
return [_query_entry(path) for path in sorted(directory.rglob("*.md")) if path.is_file()]
def frontmatter(text: str) -> tuple[dict[str, Any], str]:
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---", 4)
if end < 0:
return {}, text
meta = safe_load(text[4:end]) or {}
if not isinstance(meta, dict):
meta = {}
return meta, text[end + 4 :].lstrip("\n")
def query_matches_entry(query: str, entry: QueryEntry) -> bool:
normalized = normalize(query)
if any(normalize(alias) in normalized or normalized in normalize(alias) for alias in entry.aliases):
return True
query_terms = _meaningful_query_terms(query)
for candidate in (entry.title, *entry.aliases):
candidate_terms = _meaningful_query_terms(candidate)
if len(query_terms & candidate_terms) >= 2:
return True
return False
def _meaningful_query_terms(value: str) -> set[str]:
return {term for term in tokenize(value) if term not in QUERY_MATCH_STOP_TOKENS and len(term.strip()) >= 2}
def normalize(value: str) -> str:
return re.sub(r"\s+", " ", value.lower()).strip()
def _query_entry(path: Path) -> QueryEntry:
meta, body = frontmatter(path.read_text(encoding="utf-8"))
title = str(meta.get("title") or _first_heading(body) or path.stem)
return QueryEntry(
path=path,
title=title,
aliases=tuple(str(item).strip() for item in meta.get("aliases", []) if str(item).strip()),
intent=str(meta["intent"]) if meta.get("intent") else None,
primary_sources=tuple(str(item).strip() for item in meta.get("primary_sources", []) if str(item).strip()),
supporting_sources=tuple(str(item).strip() for item in meta.get("supporting_sources", []) if str(item).strip()),
answer_requirements=tuple(str(item).strip() for item in meta.get("answer_requirements", []) if str(item).strip()),
)
def _first_heading(text: str) -> str | None:
for line in text.splitlines():
if match := re.match(r"^#{1,6}\s+(.+)$", line):
return match.group(1).strip()
return None

469
build/lib/d_wiki/search.py Normal file
View File

@ -0,0 +1,469 @@
from __future__ import annotations
import json
import math
import re
from collections import Counter, defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .metadata import (
AliasGroup,
QueryEntry,
frontmatter,
load_alias_groups,
load_query_entries,
normalize,
query_matches_entry,
)
from .dreams import load_dream_query_entries
from .tokenize import tokenize
SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt")
MAX_BLOCK_CHARS = 520
@dataclass(frozen=True)
class Chunk:
title: str
wikilink: str
source_path: Path
page_type: str
heading: str
text: str
tokens: tuple[str, ...]
index: int
def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]:
return {
"status": "ok",
"config": str(config_path),
"knowledge_bases": {
name: {
"root": str(kb.root),
"source_type": kb.source_type,
"exists": kb.root.exists(),
"aliases": str(kb.aliases_path) if kb.aliases_path else None,
"queries": str(kb.queries_dir) if kb.queries_dir else None,
}
for name, kb in bases.items()
},
}
def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]:
chunks = load_chunks(kb)
index_dir = kb.root / ".d_wiki_index"
index_dir.mkdir(exist_ok=True)
payload = {
"kb": kb.name,
"root": str(kb.root),
"source_type": kb.source_type,
"compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None,
"chunks": len(chunks),
"pages": len({chunk.wikilink for chunk in chunks}),
"indexed_at": datetime.now(timezone.utc).isoformat(),
}
(index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return {"status": "indexed", **payload}
def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]:
aliases = load_alias_groups(kb.aliases_path)
query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)]
matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)]
expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries)
chunks = load_chunks(kb)
ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries)
selected = ranked[: max(1, min(max_results, 8))]
context_pack = build_context_pack(query, selected, matched_aliases, matched_entries)
return {
"status": "found" if selected else "no_match",
"query": query,
"knowledge_base": kb.name,
"backend": "lightweight_bm25",
"quality": quality(selected),
"sources": [source_summary(item[1], item[0]) for item in selected],
"context_pack": context_pack,
}
def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]:
chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink]
if not chunks:
return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink}
first = chunks[0]
return {
"status": "found",
"knowledge_base": kb.name,
"id": wikilink,
"title": first.title,
"source_path": str(first.source_path),
"blocks": [
{"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"}
for chunk in chunks
],
}
def expand_query(
query: str,
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> tuple[list[str], list[AliasGroup]]:
terms = tokenize(query)
normalized_query = normalize(query)
matched: list[AliasGroup] = []
for group in aliases:
if any(normalize(term) in normalized_query for term in group.terms):
matched.append(group)
for term in group.terms:
terms.extend(tokenize(term))
for entry in entries:
terms.extend(tokenize(entry.title))
for alias in entry.aliases:
terms.extend(tokenize(alias))
return terms, matched
def rank_chunks(
chunks: list[Chunk],
terms: list[str],
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> list[tuple[float, Chunk]]:
if not chunks or not terms:
return []
term_counts = [Counter(chunk.tokens) for chunk in chunks]
doc_freq: Counter[str] = Counter()
for counts in term_counts:
for term in counts:
doc_freq[term] += 1
avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks))
query_counts = Counter(terms)
ranked: list[tuple[float, Chunk]] = []
related = {item for group in aliases for item in group.related}
primary = {item for entry in entries for item in entry.primary_sources}
supporting = {item for entry in entries for item in entry.supporting_sources}
for chunk, counts in zip(chunks, term_counts):
score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts)
if any(wikilink_matches(chunk.wikilink, item) for item in related):
score += 8.0
if any(wikilink_matches(chunk.wikilink, item) for item in primary):
score += 16.0
if any(wikilink_matches(chunk.wikilink, item) for item in supporting):
score += 5.0
score += heading_score_adjustment(chunk.heading)
if score > 0:
ranked.append((round(score, 3), chunk))
ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index))
return ranked
def heading_score_adjustment(heading: str) -> float:
if heading == "Retrieval Hints":
return -80.0
if heading == "Intent Model":
return -40.0
if heading == "Answer Blueprint":
return -25.0
if heading == "Evidence Graph":
return 3.0
if heading == "Key Evidence":
return 5.0
return 0.0
def bm25_score(
counts: Counter[str],
doc_len: int,
doc_freq: Counter[str],
doc_count: int,
avg_len: float,
query_counts: Counter[str],
) -> float:
k1 = 1.5
b = 0.75
score = 0.0
for term, qf in query_counts.items():
tf = counts.get(term, 0)
if tf == 0:
continue
idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5))
denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1)))
score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3)
return score
def build_context_pack(
query: str,
selected: list[tuple[float, Chunk]],
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> dict[str, Any]:
intent = answer_intent(query, aliases, entries)
grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list)
for item in selected:
grouped[item[1].wikilink].append(item)
source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1]))
entry_primary = {item for entry in entries for item in entry.primary_sources}
primary_sources = [
source_context(wikilink, chunks)
for wikilink, chunks in source_items
if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary)
]
if not primary_sources:
primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]]
supporting_sources = [
source_context(wikilink, chunks, blocks_limit=1)
for wikilink, chunks in source_items
if wikilink not in {source["wikilink"] for source in primary_sources}
][:3]
coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete")
return {
"answer_intent": intent,
"coverage": coverage,
"answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.",
"primary_sources": primary_sources,
"supporting_sources": supporting_sources,
"answer_requirements": answer_requirements(intent, entries),
}
def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]:
top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit]
chunk = top[0][1]
return {
"title": chunk.title,
"wikilink": wikilink,
"source_path": str(chunk.source_path),
"reason": "Matched query terms and local metadata.",
"blocks": [
{
"block_id": f"{block.wikilink}#{block.index}",
"heading": block.heading,
"text": trim_block(block.text),
"score": score,
}
for score, block in top
],
}
def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str:
for entry in entries:
if entry.intent:
return entry.intent
for group in aliases:
if group.intent:
return group.intent
if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")):
return "procedure"
return "fact"
def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]:
requirements = unique([req for entry in entries for req in entry.answer_requirements])
if requirements:
return requirements
if intent == "procedure":
return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"]
return ["直接回答问题", "引用命中的来源"]
def unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
if value in seen:
continue
seen.add(value)
output.append(value)
return output
def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]:
top = selected[0][0] if selected else 0.0
return {
"confidence": "high" if top >= 8.0 else ("low" if selected else "none"),
"high_confidence": top >= 8.0,
"top_score": top,
"high_confidence_score": 8.0,
}
def wikilink_matches(candidate: str, expected: str) -> bool:
return candidate == expected or candidate == f"concepts/{expected}"
def source_summary(chunk: Chunk, score: float) -> dict[str, Any]:
return {
"title": chunk.title,
"wikilink": chunk.wikilink,
"source_path": str(chunk.source_path),
"heading": chunk.heading,
"snippet": trim_block(chunk.text, 320),
"score": score,
}
def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root
paths = source_paths(root)
chunks: list[Chunk] = []
for path in paths:
chunks.extend(chunks_for_path(path, root))
return chunks
def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
chunks: list[Chunk] = []
for path in source_paths(kb.root):
chunks.extend(chunks_for_path(path, kb.root))
return chunks
def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path:
return kb.root / ".d_wiki" / "wiki"
def source_paths(root: Path) -> list[Path]:
if (root / "_sources").is_dir():
return sorted(
path
for path in (root / "_sources").rglob("*.txt")
if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES)
)
generated = {"SCHEMA.md", "index.md", "log.md"}
generated_dirs = {".d_wiki", ".d_wiki_index"}
is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki"
return sorted(
path
for path in root.rglob("*.md")
if path.is_file()
and path.name not in generated
and (is_compiled_root or not any(part in generated_dirs for part in path.parts))
)
def chunks_for_path(path: Path, root: Path) -> list[Chunk]:
raw = path.read_text(encoding="utf-8", errors="ignore")
meta, body = frontmatter(raw)
title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title())
metadata_text = metadata_terms(meta)
wikilink = wikilink_for(path, root)
sections = split_sections(body, title)
chunks: list[Chunk] = []
for index, (heading, text) in enumerate(sections):
if heading in {"Source Map", "Related"}:
continue
if not text.strip():
continue
field_text = " ".join([title, heading, wikilink, metadata_text, text])
chunks.append(
Chunk(
title=title,
wikilink=wikilink,
source_path=path.resolve(),
page_type="sphinx_source" if "_sources" in path.parts else "markdown",
heading=heading,
text=text.strip(),
tokens=tuple(tokenize(field_text)),
index=index,
)
)
return chunks
def metadata_terms(meta: dict[str, Any]) -> str:
values: list[str] = []
for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"):
raw = meta.get(key)
if isinstance(raw, str):
values.append(raw)
elif isinstance(raw, list):
values.extend(str(item) for item in raw if str(item).strip())
return " ".join(values)
def split_sections(body: str, title: str) -> list[tuple[str, str]]:
heading = title
buffer: list[str] = []
sections: list[tuple[str, str]] = []
in_fence = False
for line in body.splitlines():
stripped = line.strip()
if stripped.startswith("```"):
in_fence = not in_fence
buffer.append(line)
continue
if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)):
if buffer:
sections.extend(paragraphs(heading, "\n".join(buffer)))
buffer = []
heading = match.group(1).strip() or title
continue
buffer.append(line)
if buffer:
sections.extend(paragraphs(heading, "\n".join(buffer)))
return sections or [(title, body)]
def paragraphs(heading: str, text: str) -> list[tuple[str, str]]:
blocks: list[tuple[str, str]] = []
for part in re.split(r"\n\s*\n", text):
cleaned = clean_block(part)
if cleaned:
blocks.append((heading, cleaned))
return blocks
def clean_block(text: str) -> str:
stripped = text.strip()
if not stripped:
return ""
if re.fullmatch(r"[-*_]{3,}", stripped):
return ""
compact = " ".join(stripped.split())
if "📷" in compact and len(compact) <= 80:
return ""
if compact.startswith("> 📷") and len(compact) <= 120:
return ""
return stripped
def first_heading(body: str) -> str | None:
in_fence = False
for line in body.splitlines():
stripped = line.strip()
if stripped.startswith("```"):
in_fence = not in_fence
continue
if in_fence:
continue
if match := re.match(r"^#{1,6}\s+(.+)$", line):
return match.group(1).strip()
return None
def wikilink_for(path: Path, root: Path) -> str:
try:
relative = path.relative_to(root)
except ValueError:
return path.stem
if relative.parts and relative.parts[0] == "_sources":
relative = Path(*relative.parts[1:])
value = relative.as_posix()
for suffix in SPHINX_SOURCE_SUFFIXES:
if value.endswith(suffix):
return value[: -len(suffix)]
return value[:-3] if value.endswith(".md") else path.stem
def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str:
compact = " ".join(text.split())
return compact if len(compact) <= limit else compact[:limit].rstrip()

View File

@ -0,0 +1,22 @@
from __future__ import annotations
import re
STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上")
STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"}
def tokenize(text: str) -> list[str]:
lowered = text.lower()
terms = re.findall(r"[a-z0-9_./+-]+", lowered)
cjk_chars = re.findall(r"[\u4e00-\u9fff]", text)
terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1)))
return [term for term in terms if _useful(term)]
def _useful(term: str) -> bool:
if not term or term in STOP_TERMS:
return False
if re.search(r"[\u4e00-\u9fff]", term):
return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term))
return True

View File

@ -0,0 +1,70 @@
from __future__ import annotations
from typing import Any
try:
import yaml as _yaml # type: ignore
except Exception: # pragma: no cover - exercised in environments without PyYAML
_yaml = None
def safe_load(text: str) -> Any:
if _yaml is not None:
return _yaml.safe_load(text)
lines = [
(len(line) - len(line.lstrip(" ")), line.strip())
for line in text.splitlines()
if line.strip() and not line.strip().startswith("#")
]
value, _index = _parse_node(lines, 0, 0)
return value
def _parse_node(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]:
if index >= len(lines):
return {}, index
if lines[index][0] < indent:
return {}, index
if lines[index][1].startswith("- "):
values: list[Any] = []
while index < len(lines) and lines[index][0] == indent and lines[index][1].startswith("- "):
values.append(_scalar(lines[index][1][2:].strip()))
index += 1
return values, index
mapping: dict[str, Any] = {}
while index < len(lines):
current_indent, stripped = lines[index]
if current_indent < indent:
break
if current_indent > indent:
break
if ":" not in stripped:
index += 1
continue
key, raw_value = stripped.split(":", 1)
key = key.strip()
raw_value = raw_value.strip()
index += 1
if raw_value:
mapping[key] = _scalar(raw_value)
continue
child, index = _parse_node(lines, index, indent + 2)
mapping[key] = child
return mapping, index
def _scalar(value: str) -> Any:
if value in {"null", "None", "~"}:
return None
if value in {"true", "True"}:
return True
if value in {"false", "False"}:
return False
if value.startswith("[") and value.endswith("]"):
inner = value[1:-1].strip()
if not inner:
return []
return [_scalar(item.strip()) for item in inner.split(",")]
if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
return value[1:-1]
return value

5
d-wiki Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env python3
from d_wiki.cli import main
if __name__ == "__main__":
raise SystemExit(main())

11
d-wiki.yml Normal file
View File

@ -0,0 +1,11 @@
knowledge_bases:
x5:
root: "/Users/d-robotics/Desktop/evb/sdk_management_system/x5_user_manual/latest"
source_type: "sphinx"
aliases: "knowledge/x5/aliases.yml"
queries: "knowledge/x5/queries"
robogo:
root: "/Users/d-robotics/Documents/New project/read/robogo-user-guide"
source_type: "markdown"
aliases: "knowledge/robogo/aliases.yml"
queries: "knowledge/robogo/queries"

5
d_wiki.egg-info/PKG-INFO Normal file
View File

@ -0,0 +1,5 @@
Metadata-Version: 2.4
Name: d-wiki
Version: 0.1.0
Summary: Lightweight local knowledge retrieval with BM25 and metadata context packs.
Requires-Python: >=3.10

View File

@ -0,0 +1,19 @@
README.md
pyproject.toml
d_wiki/__init__.py
d_wiki/__main__.py
d_wiki/cli.py
d_wiki/compile.py
d_wiki/config.py
d_wiki/dreams.py
d_wiki/llm.py
d_wiki/metadata.py
d_wiki/search.py
d_wiki/tokenize.py
d_wiki/yaml_compat.py
d_wiki.egg-info/PKG-INFO
d_wiki.egg-info/SOURCES.txt
d_wiki.egg-info/dependency_links.txt
d_wiki.egg-info/entry_points.txt
d_wiki.egg-info/top_level.txt
tests/test_d_wiki.py

View File

@ -0,0 +1 @@

View File

@ -0,0 +1,2 @@
[console_scripts]
d-wiki = d_wiki.cli:main

View File

@ -0,0 +1 @@
d_wiki

3
d_wiki/__init__.py Normal file
View File

@ -0,0 +1,3 @@
"""d-wiki lightweight local knowledge retrieval."""
__version__ = "0.1.0"

4
d_wiki/__main__.py Normal file
View File

@ -0,0 +1,4 @@
from .cli import main
if __name__ == "__main__":
raise SystemExit(main())

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

77
d_wiki/cli.py Normal file
View File

@ -0,0 +1,77 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
from .config import DWikiConfig, load_config
from .compile import compile_kb
from .dreams import dream_kb
from .search import get_page, index_payload, search, status_payload
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="d-wiki")
parser.add_argument("--config", type=Path, default=None)
subparsers = parser.add_subparsers(dest="command", required=True)
status_parser = subparsers.add_parser("status")
status_parser.add_argument("--format", choices=["json"], default="json")
index_parser = subparsers.add_parser("index")
index_parser.add_argument("--kb", required=True)
index_parser.add_argument("--format", choices=["json"], default="json")
compile_parser = subparsers.add_parser("compile")
compile_parser.add_argument("--kb", required=True)
compile_parser.add_argument("--force", action="store_true")
compile_parser.add_argument("--llm", choices=["off", "auto", "require"], default="off")
compile_parser.add_argument("--llm-scope", choices=["queries", "all"], default="queries")
compile_parser.add_argument("--format", choices=["json"], default="json")
dream_parser = subparsers.add_parser("dream")
dream_parser.add_argument("--kb", required=True)
dream_parser.add_argument("--force", action="store_true")
dream_parser.add_argument("--format", choices=["json"], default="json")
search_parser = subparsers.add_parser("search")
search_parser.add_argument("--kb", required=True)
search_parser.add_argument("--query", required=True)
search_parser.add_argument("--max-results", type=int, default=5)
search_parser.add_argument("--format", choices=["json"], default="json")
get_parser = subparsers.add_parser("get")
get_parser.add_argument("--kb", required=True)
get_parser.add_argument("--id", required=True)
get_parser.add_argument("--format", choices=["json"], default="json")
args = parser.parse_args(argv)
try:
config = load_config(args.config)
payload = dispatch(args, config)
except Exception as exc:
print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 2
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
return 0
def dispatch(args: argparse.Namespace, config: DWikiConfig) -> dict[str, Any]:
if args.command == "status":
return status_payload(config.path, config.knowledge_bases)
kb = config.knowledge_bases.get(args.kb)
if kb is None:
raise ValueError(f"D_WIKI_KB_NOT_FOUND: {args.kb}")
if args.command == "index":
return index_payload(kb)
if args.command == "compile":
return compile_kb(kb, force=args.force, llm_mode=args.llm, llm_scope=args.llm_scope)
if args.command == "dream":
return dream_kb(kb, force=args.force)
if args.command == "search":
return search(kb, args.query, args.max_results)
if args.command == "get":
return get_page(kb, args.id)
raise ValueError(f"D_WIKI_UNKNOWN_COMMAND: {args.command}")

590
d_wiki/compile.py Normal file
View File

@ -0,0 +1,590 @@
from __future__ import annotations
import re
import shutil
import json
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .llm import LlmPageSynthesis, LlmPageSynthesizer
from .metadata import load_alias_groups, load_query_entries
from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block
from .tokenize import tokenize
@dataclass(frozen=True)
class CompiledPage:
wikilink: str
title: str
path: Path
@dataclass(frozen=True)
class _EvidenceProfile:
tokens: frozenset[str]
phrases: tuple[str, ...]
def compile_kb(
kb: KnowledgeBaseConfig,
*,
force: bool = False,
llm_mode: str = "off",
llm_scope: str = "queries",
) -> dict[str, Any]:
chunks = load_source_chunks(kb)
if not chunks:
raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS")
synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None
if llm_mode == "require" and synthesizer is None:
raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED")
wiki_root = compiled_wiki_root(kb)
concepts_dir = wiki_root / "concepts"
raw_dir = wiki_root / "raw"
if force:
shutil.rmtree(concepts_dir, ignore_errors=True)
concepts_dir.mkdir(parents=True, exist_ok=True)
raw_dir.mkdir(parents=True, exist_ok=True)
pages: list[CompiledPage] = []
grouped = _group_chunks(chunks)
query_entries = load_query_entries(kb.queries_dir)
alias_groups = load_alias_groups(kb.aliases_path)
query_by_source = {
source: entry
for entry in query_entries
for source in entry.primary_sources
}
aliases_by_source = {
related: group
for group in alias_groups
for related in group.related
}
for wikilink, page_chunks in sorted(grouped.items()):
page = _compile_page(
kb,
wikilink,
page_chunks,
query_entry=query_by_source.get(wikilink),
alias_group=aliases_by_source.get(wikilink),
synthesizer=(
synthesizer
if llm_scope == "all" or query_by_source.get(wikilink) is not None
else None
),
require_llm=llm_mode == "require",
)
if force or not page.path.exists():
page.path.write_text(page.content, encoding="utf-8")
pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path))
_write_schema(wiki_root, kb)
_write_index(wiki_root, pages)
_append_log(wiki_root, kb, len(chunks), len(pages))
return {
"status": "compiled",
"kb": kb.name,
"source_root": str(kb.root),
"wiki_root": str(wiki_root),
"source_chunks": len(chunks),
"pages": len(pages),
"llm_mode": llm_mode,
"llm_scope": llm_scope,
"llm_enabled": synthesizer is not None,
"compiled_at": datetime.now(timezone.utc).isoformat(),
}
@dataclass(frozen=True)
class _PageDraft:
title: str
path: Path
content: str
def _compile_page(
kb: KnowledgeBaseConfig,
wikilink: str,
chunks: list[Chunk],
*,
query_entry: Any | None,
alias_group: Any | None,
synthesizer: LlmPageSynthesizer | None,
require_llm: bool,
) -> _PageDraft:
first = chunks[0]
title = _title_for(first, query_entry)
slug = _slug(wikilink)
path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md"
evidence_profile = _evidence_profile(title, query_entry, alias_group)
tags = _tags(kb, first, alias_group)
aliases = _aliases(title, query_entry, alias_group)
source_ref = first.wikilink
source_path = str(first.source_path)
key_blocks = _key_blocks(chunks, evidence_profile)
intent = _intent(title, query_entry, alias_group)
summary = _summary(chunks, title, evidence_profile, intent=intent)
requirements = _answer_requirements(intent, query_entry)
synthesis = _llm_synthesis(
synthesizer,
title=title,
wikilink=wikilink,
query_entry=query_entry,
alias_group=alias_group,
key_blocks=key_blocks,
require_llm=require_llm,
)
if synthesis is not None:
summary = synthesis.summary
retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group)
blueprint = (
list(synthesis.answer_blueprint)
if synthesis and synthesis.answer_blueprint
else _deterministic_answer_blueprint(intent, requirements, key_blocks)
)
links = _related_links(query_entry, alias_group)
content = "\n".join(
[
"---",
f"llmWikiVersion: \"1\"",
f"conceptId: {_yaml_scalar(slug)}",
f"title: {_yaml_scalar(title)}",
f"summary: {_yaml_scalar(summary)}",
f"sources: {_yaml_list([source_ref])}",
f"source_paths: {_yaml_list([source_path])}",
f"kind: concept",
f"tags: {_yaml_list(tags)}",
f"aliases: {_yaml_list(aliases)}",
f"intents: {_yaml_list([intent])}",
f"retrieval_hints: {_yaml_list(retrieval_hints)}",
f"answer_requirements: {_yaml_list(requirements)}",
f"confidence: {synthesis.confidence if synthesis else 'medium'}",
f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}",
"---",
"",
f"# {title}",
"",
summary,
"",
"## Intent Model",
"",
f"- intent: {intent}",
f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}",
f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}",
"",
"## Retrieval Hints",
"",
*[f"- {hint}" for hint in retrieval_hints],
"",
"## Answer Blueprint",
"",
*[f"- {item}" for item in blueprint],
"",
"## Synthesized Evidence" if synthesis and synthesis.key_evidence else "",
"" if synthesis and synthesis.key_evidence else "",
*[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())],
"" if synthesis and synthesis.key_evidence else "",
"## Evidence Graph",
"",
*_evidence_graph_lines(key_blocks),
"",
"## Key Evidence",
"",
*[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks],
"",
"## Source Map",
"",
f"- 原始来源:`{source_ref}`",
f"- 文件路径:`{source_path}`",
"",
"## Related",
"",
*[f"- [[{link}]]" for link in links],
"",
]
)
return _PageDraft(title=title, path=path, content=content)
def _llm_synthesis(
synthesizer: LlmPageSynthesizer | None,
*,
title: str,
wikilink: str,
query_entry: Any | None,
alias_group: Any | None,
key_blocks: list[Chunk],
require_llm: bool,
) -> LlmPageSynthesis | None:
if synthesizer is None:
return None
prompt = _llm_compile_prompt(
title=title,
wikilink=wikilink,
query_entry=query_entry,
alias_group=alias_group,
key_blocks=key_blocks,
)
try:
return synthesizer.synthesize(prompt)
except Exception:
if require_llm:
raise
return None
def _llm_compile_prompt(
*,
title: str,
wikilink: str,
query_entry: Any | None,
alias_group: Any | None,
key_blocks: list[Chunk],
) -> str:
aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ())
terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ())
requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ())
intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None)
evidence = [
{
"block_id": f"{block.wikilink}#{block.index}",
"heading": block.heading,
"text": trim_block(block.text, 700),
}
for block in key_blocks[:8]
]
return "\n".join(
[
"Compile a d-wiki concept page from the local evidence below.",
"Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.",
"For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.",
"Return only JSON with this shape:",
'{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}',
"",
json_dumps(
{
"title": title,
"wikilink": wikilink,
"intent": intent,
"aliases": aliases,
"terms": terms,
"answer_requirements": requirements,
"evidence": evidence,
}
),
]
)
def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]:
if synthesis is None or not synthesis.answer_blueprint:
return []
return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""]
def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str:
if query_entry is not None and getattr(query_entry, "intent", ""):
return str(query_entry.intent)
if alias_group is not None and getattr(alias_group, "intent", ""):
return str(alias_group.intent)
lowered = title.lower()
if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")):
return "procedure"
return "fact"
def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]:
if query_entry is not None:
requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()]
if requirements:
return _unique(requirements)
if intent == "procedure":
return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"]
return ["直接给出结论", "保留关键数值和单位", "引用证据来源"]
def _retrieval_hints(
title: str,
aliases: list[str],
tags: list[str],
requirements: list[str],
alias_group: Any | None,
) -> list[str]:
values = [title, *aliases, *tags, *requirements]
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
hints: list[str] = []
for value in values:
hints.append(value)
hints.extend(tokenize(value))
return _unique(hints)[:32]
def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]:
if requirements:
return requirements[:6]
if intent == "procedure":
headings = _unique([block.heading for block in key_blocks if block.heading])
if headings:
return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]]
return ["按前提、配置、执行、验证的顺序回答"]
return ["先给出直接结论", "补充关键参数、限制和来源"]
def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]:
lines: list[str] = []
for block in key_blocks[:8]:
claim = trim_block(block.text, 180)
if not claim:
continue
lines.append(f"- claim: {claim}")
lines.append(f" source: {block.wikilink}#{block.index}")
if block.heading:
lines.append(f" heading: {block.heading}")
return lines or ["- claim: 暂无可用证据", " source: unknown"]
def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]:
grouped: dict[str, list[Chunk]] = {}
for chunk in chunks:
grouped.setdefault(chunk.wikilink, []).append(chunk)
return grouped
def _title_for(chunk: Chunk, query_entry: Any | None) -> str:
if query_entry is not None and getattr(query_entry, "title", ""):
return str(query_entry.title)
return chunk.title
def _summary(
chunks: list[Chunk],
title: str,
evidence_profile: _EvidenceProfile | None = None,
*,
intent: str = "fact",
) -> str:
if intent == "procedure":
for chunk in chunks[:12]:
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text):
return f"{title}:{text}"
ranked = _rank_evidence(chunks, evidence_profile)
for score, chunk in ranked:
if score <= 0:
continue
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text):
return f"{title}:{text}"
for chunk in chunks:
text = trim_block(chunk.text, 220)
if text and not _is_low_signal(text) and len(text) >= 20:
return f"{title}:{text}"
return f"{title} 的编译知识页,来源于本地文档。"
def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]:
preferred = [
chunk
for chunk in chunks
if _is_evidence_candidate(chunk.text)
]
ranked = _rank_evidence(preferred, evidence_profile)
if not ranked or ranked[0][0] <= 0:
return (preferred or chunks)[:6]
selected: list[Chunk] = []
seen: set[tuple[str, int]] = set()
for score, chunk in ranked:
if score <= 0:
continue
key = (chunk.wikilink, chunk.index)
if key in seen:
continue
selected.append(chunk)
seen.add(key)
if len(selected) >= 6:
return selected
for chunk in preferred:
key = (chunk.wikilink, chunk.index)
if key in seen:
continue
selected.append(chunk)
seen.add(key)
if len(selected) >= 6:
break
return selected
def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile:
values = [title]
if query_entry is not None:
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ()))
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
phrases = tuple(_unique([_normalize_phrase(value) for value in values if value]))
tokens: set[str] = set()
for value in values:
tokens.update(tokenize(str(value)))
return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases)
def _rank_evidence(
chunks: list[Chunk],
evidence_profile: _EvidenceProfile | None,
) -> list[tuple[float, Chunk]]:
if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases):
return []
ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks]
ranked.sort(key=lambda item: (-item[0], item[1].index))
return ranked
def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float:
text = trim_block(chunk.text, 1000)
normalized = _normalize_phrase(text)
chunk_tokens = set(chunk.tokens) | set(tokenize(text))
score = 0.0
score += len(chunk_tokens & evidence_profile.tokens) * 2.0
for phrase in evidence_profile.phrases:
if len(phrase) >= 2 and phrase in normalized:
score += 5.0
if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized):
score += 8.0
if "算力" in evidence_profile.tokens and "tops" in normalized:
score += 6.0
if "bpu" in evidence_profile.tokens and "bpu" in normalized:
score += 4.0
if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized):
score += 2.0
if _is_low_signal(text):
score -= 8.0
return score
def _is_evidence_candidate(text: str) -> bool:
stripped = text.lstrip()
return not stripped.startswith("|") and not stripped.startswith(">")
def _is_low_signal(text: str) -> bool:
stripped = text.strip()
without_marks = re.sub(r"[*_`#\-\s]+", "", stripped)
if len(without_marks) < 8:
return True
if stripped.startswith("```") or "```" in stripped:
return True
if stripped.startswith("|"):
return True
if "![](" in stripped:
return True
if re.search(r"(^|\n)\s*[├│└─]{2,}", stripped):
return True
return False
def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]:
tags = [kb.name]
if alias_group is not None:
tags.append(_slug(str(alias_group.name)).replace("--", "-"))
for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title):
lowered = token.lower()
if lowered not in tags:
tags.append(lowered)
return tags[:8]
def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]:
values = [title]
if query_entry is not None:
values.extend(str(item) for item in getattr(query_entry, "aliases", ()))
if alias_group is not None:
values.extend(str(item) for item in getattr(alias_group, "terms", ()))
return _unique(values)[:12]
def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]:
links: list[str] = []
if query_entry is not None:
links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ()))
if alias_group is not None:
links.extend(str(item) for item in getattr(alias_group, "related", ()))
return _unique(links)[:10]
def _unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
cleaned = value.strip()
if not cleaned or cleaned in seen:
continue
seen.add(cleaned)
output.append(cleaned)
return output
def _normalize_phrase(value: str) -> str:
return re.sub(r"\s+", " ", value.lower()).strip()
def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None:
(wiki_root / "SCHEMA.md").write_text(
"\n".join(
[
"# Wiki Schema",
"",
"## Domain",
f"Compiled local knowledge for `{kb.name}`.",
"",
"## Conventions",
"- Pages live under `concepts/`.",
"- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.",
"- Claims in Key Evidence keep provenance markers pointing back to source block ids.",
"- Raw source files remain immutable; compiled pages are generated artifacts.",
"",
]
),
encoding="utf-8",
)
def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None:
lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""]
for page in sorted(pages, key=lambda item: item.title):
rel = page.path.relative_to(wiki_root).as_posix()
lines.append(f"- [[{rel[:-3]}]] — {page.title}")
lines.append("")
(wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8")
def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None:
line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n"
path = wiki_root / "log.md"
if not path.exists():
path.write_text("# Log\n\n", encoding="utf-8")
with path.open("a", encoding="utf-8") as handle:
handle.write(line)
def _slug(value: str) -> str:
slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower()
return slug or "page"
def _yaml_scalar(value: str) -> str:
import json
return json.dumps(value, ensure_ascii=False)
def _yaml_list(values: list[str]) -> str:
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, indent=2)

59
d_wiki/config.py Normal file
View File

@ -0,0 +1,59 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .yaml_compat import safe_load
@dataclass(frozen=True)
class KnowledgeBaseConfig:
name: str
root: Path
source_type: str
aliases_path: Path | None
queries_dir: Path | None
@dataclass(frozen=True)
class DWikiConfig:
path: Path
knowledge_bases: dict[str, KnowledgeBaseConfig]
def default_config_path() -> Path:
return Path(os.environ.get("D_WIKI_CONFIG", Path.cwd() / "d-wiki.yml")).expanduser()
def load_config(path: Path | None = None) -> DWikiConfig:
config_path = (path or default_config_path()).expanduser().resolve()
data = safe_load(config_path.read_text(encoding="utf-8")) or {}
raw_kbs = data.get("knowledge_bases")
if not isinstance(raw_kbs, dict):
raise ValueError("D_WIKI_CONFIG_MISSING_KNOWLEDGE_BASES")
bases: dict[str, KnowledgeBaseConfig] = {}
for name, raw in raw_kbs.items():
if not isinstance(raw, dict):
raise ValueError(f"D_WIKI_KB_INVALID: {name}")
root = Path(str(raw.get("root", ""))).expanduser()
if not root.is_absolute():
root = (config_path.parent / root).resolve()
aliases_path = _optional_path(config_path.parent, raw.get("aliases"))
queries_dir = _optional_path(config_path.parent, raw.get("queries"))
bases[str(name)] = KnowledgeBaseConfig(
name=str(name),
root=root,
source_type=str(raw.get("source_type", "markdown")),
aliases_path=aliases_path,
queries_dir=queries_dir,
)
return DWikiConfig(path=config_path, knowledge_bases=bases)
def _optional_path(base: Path, value: Any) -> Path | None:
if not value:
return None
path = Path(str(value)).expanduser()
return path if path.is_absolute() else (base / path).resolve()

227
d_wiki/dreams.py Normal file
View File

@ -0,0 +1,227 @@
from __future__ import annotations
import re
import shutil
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .metadata import frontmatter, load_query_entries
@dataclass(frozen=True)
class DreamedQuery:
slug: str
title: str
aliases: tuple[str, ...]
intent: str
primary_sources: tuple[str, ...]
supporting_sources: tuple[str, ...]
answer_requirements: tuple[str, ...]
def dream_kb(kb: KnowledgeBaseConfig, *, force: bool = False) -> dict[str, Any]:
wiki_root = kb.root / ".d_wiki" / "wiki"
if not wiki_root.is_dir():
raise ValueError("D_WIKI_DREAM_REQUIRES_COMPILED_WIKI")
output_root = dream_root(kb)
queries_dir = output_root / "queries"
if force:
shutil.rmtree(output_root, ignore_errors=True)
queries_dir.mkdir(parents=True, exist_ok=True)
pages = sorted((wiki_root / "concepts").glob("*.md"))
dreamed = [_dream_page(path, wiki_root) for path in pages]
dreamed = [item for item in dreamed if item is not None]
for item in dreamed:
(queries_dir / f"{item.slug}.md").write_text(_query_markdown(item), encoding="utf-8")
(output_root / "README.md").write_text(_readme(kb, len(pages), len(dreamed)), encoding="utf-8")
return {
"status": "dreamed",
"kb": kb.name,
"input_wiki": str(wiki_root),
"output_store": str(output_root),
"queries": len(dreamed),
"dreamed_at": datetime.now(timezone.utc).isoformat(),
}
def load_dream_query_entries(kb: KnowledgeBaseConfig) -> list[Any]:
return load_query_entries(dream_root(kb) / "queries")
def dream_root(kb: KnowledgeBaseConfig) -> Path:
return kb.root / ".d_wiki" / "dreams" / "latest"
def _dream_page(path: Path, wiki_root: Path) -> DreamedQuery | None:
meta, body = frontmatter(path.read_text(encoding="utf-8", errors="ignore"))
title = str(meta.get("title") or "").strip()
if not title:
return None
curation_body = _curation_body(body)
source = _first(meta.get("sources")) or _wikilink_for_compiled(path, wiki_root)
aliases = _unique([title, *_dream_aliases(title, curation_body)])
intent = _first(meta.get("intents")) or _dream_intent(title, curation_body)
requirements = _as_list(meta.get("answer_requirements")) or _dream_requirements(title, curation_body, intent)
supporting = _related_sources(body, source)
return DreamedQuery(
slug=path.stem,
title=title,
aliases=tuple(aliases[:18]),
intent=intent,
primary_sources=(source,),
supporting_sources=tuple(supporting[:6]),
answer_requirements=tuple(requirements[:8]),
)
def _dream_aliases(title: str, body: str) -> list[str]:
text = f"{title}\n{body}".lower()
aliases = [
f"{title}怎么用",
f"如何使用{title}",
]
if _looks_like_3d_generation(title, text):
aliases.extend(
[
"3D资产生成怎么用",
"如何使用 3D 资产生成功能",
"我想知道如何使用3d资产生成功能",
"图生3D怎么用",
"如何生成3D模型",
"如何把图片生成3D资产",
]
)
if "camera" in text and any(term in text for term in ("点亮", "bring up", "sensor")):
aliases.extend(["如何在X5平台上点亮camera", "X5 camera 点亮", "X5 摄像头 bring up"])
if any(term in text for term in ("算力", "tops", "bpu")):
aliases.extend(["x5的算力是多少", "X5 BPU 算力"])
return aliases
def _dream_intent(title: str, body: str) -> str:
text = f"{title}\n{body}".lower()
if any(term in text for term in ("如何", "怎么", "步骤", "使用", "创建", "配置", "点亮", "生成", "调试", "bring up")):
return "procedure"
return "fact"
def _dream_requirements(title: str, body: str, intent: str) -> list[str]:
text = f"{title}\n{body}".lower()
if _looks_like_3d_generation(title, text):
return [
"说明 3D 生成是图生 3D 工具,不是镜像资产",
"按准备参考图、生成、保存到 3D 资产的流程回答",
"说明输出默认 glb,生成结果可保存到 3D 资产",
"提醒暂不支持文生 3D,文字描述可先用图像生成产图",
]
if intent == "procedure":
return ["按步骤回答", "说明关键前提和入口", "引用证据来源"]
return ["直接回答问题", "引用证据来源"]
def _looks_like_3d_generation(title: str, text: str) -> bool:
title_text = title.lower()
return (
"3d 生成" in title_text
or "3d生成" in title_text
or "3d 资产生成" in title_text
or "3d资产生成" in title_text
or "embodygen" in text
or "ai 图生 3d 工具" in text
or "ai 图生3d 工具" in text
)
def _curation_body(body: str) -> str:
lines: list[str] = []
skip = False
for line in body.splitlines():
if line.startswith("## "):
skip = line.strip() in {"## Retrieval Hints", "## Related"}
if skip:
continue
if not skip:
lines.append(line)
return "\n".join(lines)
def _related_sources(body: str, primary: str) -> list[str]:
links = re.findall(r"\[\[([^\]]+)\]\]", body)
refs = re.findall(r"\^\[([^#\]]+)(?:#[^\]]+)?\]", body)
return _unique([item for item in [*links, *refs] if item and item != primary])
def _query_markdown(item: DreamedQuery) -> str:
lines = [
"---",
f"title: {_yaml_scalar(item.title)}",
f"aliases: {_yaml_list(item.aliases)}",
f"intent: {item.intent}",
f"primary_sources: {_yaml_list(item.primary_sources)}",
f"supporting_sources: {_yaml_list(item.supporting_sources)}",
f"answer_requirements: {_yaml_list(item.answer_requirements)}",
"provenance: dreamed",
"---",
"",
f"Dreamed query memory for {item.title}.",
"",
]
return "\n".join(lines)
def _readme(kb: KnowledgeBaseConfig, pages: int, queries: int) -> str:
return "\n".join(
[
"# d-wiki dream output",
"",
"This store is generated from the compiled wiki. It does not modify source documents or compiled concept pages.",
"",
f"- kb: `{kb.name}`",
f"- input_pages: {pages}",
f"- dreamed_queries: {queries}",
"",
]
)
def _first(value: Any) -> str:
items = _as_list(value)
return items[0] if items else ""
def _as_list(value: Any) -> list[str]:
if isinstance(value, str):
return [value]
if isinstance(value, list):
return [str(item).strip() for item in value if str(item).strip()]
return []
def _unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
cleaned = re.sub(r"\s+", " ", str(value)).strip()
if not cleaned or cleaned in seen:
continue
seen.add(cleaned)
output.append(cleaned)
return output
def _wikilink_for_compiled(path: Path, wiki_root: Path) -> str:
return path.relative_to(wiki_root).with_suffix("").as_posix()
def _yaml_scalar(value: str) -> str:
import json
return json.dumps(value, ensure_ascii=False)
def _yaml_list(values: tuple[str, ...] | list[str]) -> str:
return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]"

156
d_wiki/llm.py Normal file
View File

@ -0,0 +1,156 @@
from __future__ import annotations
import json
import os
import urllib.error
import urllib.request
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class LlmSettings:
base_url: str
model: str
api_key: str
api_style: str = "anthropic"
max_tokens: int = 900
@dataclass(frozen=True)
class LlmPageSynthesis:
summary: str
answer_blueprint: tuple[str, ...]
key_evidence: tuple[str, ...]
confidence: str = "medium"
class LlmPageSynthesizer:
def __init__(self, settings: LlmSettings) -> None:
self.settings = settings
@classmethod
def from_environment(cls) -> "LlmPageSynthesizer | None":
base_url = os.environ.get("D_WIKI_LLM_BASE_URL") or os.environ.get("ROBOGO_LLM_BASE_URL")
model = os.environ.get("D_WIKI_LLM_MODEL") or os.environ.get("ROBOGO_LLM_MODEL")
api_key = os.environ.get("D_WIKI_LLM_API_KEY") or os.environ.get("ROBOGO_LLM_API_KEY")
if not base_url or not model or not api_key:
return None
api_style = os.environ.get("D_WIKI_LLM_API_STYLE") or os.environ.get("ROBOGO_LLM_API_STYLE") or "anthropic"
max_tokens = int(os.environ.get("D_WIKI_LLM_MAX_TOKENS", "900"))
return cls(LlmSettings(base_url=base_url, model=model, api_key=api_key, api_style=api_style, max_tokens=max_tokens))
def synthesize(self, prompt: str) -> LlmPageSynthesis | None:
text = self._complete(prompt).strip()
payload = _extract_json(text)
if not isinstance(payload, dict):
return None
summary = str(payload.get("summary") or "").strip()
if not summary:
return None
answer_blueprint = _string_tuple(payload.get("answer_blueprint"))
key_evidence = _string_tuple(payload.get("key_evidence"))
confidence = str(payload.get("confidence") or "medium").strip().lower()
if confidence not in {"low", "medium", "high"}:
confidence = "medium"
return LlmPageSynthesis(
summary=summary,
answer_blueprint=answer_blueprint,
key_evidence=key_evidence,
confidence=confidence,
)
def _complete(self, prompt: str) -> str:
if self.settings.api_style == "openai":
return self._complete_openai(prompt)
return self._complete_anthropic(prompt)
def _complete_anthropic(self, prompt: str) -> str:
payload = {
"model": self.settings.model,
"max_tokens": self.settings.max_tokens,
"temperature": 0,
"system": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
"messages": [{"role": "user", "content": prompt}],
}
data = self._post_json(_join_url(self.settings.base_url, "v1/messages"), payload)
content = data.get("content")
if isinstance(content, list):
return "\n".join(str(item.get("text", "")) for item in content if isinstance(item, dict))
return str(data.get("text") or "")
def _complete_openai(self, prompt: str) -> str:
payload = {
"model": self.settings.model,
"temperature": 0,
"messages": [
{
"role": "system",
"content": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.",
},
{"role": "user", "content": prompt},
],
}
data = self._post_json(_join_url(self.settings.base_url, "v1/chat/completions"), payload)
choices = data.get("choices")
if isinstance(choices, list) and choices:
first = choices[0]
if isinstance(first, dict):
message = first.get("message")
if isinstance(message, dict):
return str(message.get("content") or "")
return ""
def _post_json(self, url: str, payload: dict[str, Any]) -> dict[str, Any]:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
request = urllib.request.Request(
url,
data=body,
method="POST",
headers={
"content-type": "application/json",
"accept": "application/json",
"x-api-key": self.settings.api_key,
"authorization": f"Bearer {self.settings.api_key}",
"anthropic-version": "2023-06-01",
},
)
try:
with urllib.request.urlopen(request, timeout=60) as response:
return json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="ignore")
raise RuntimeError(f"D_WIKI_LLM_HTTP_ERROR: {exc.code} {detail[:300]}") from exc
def _join_url(base_url: str, suffix: str) -> str:
base = base_url.rstrip("/")
if base.endswith("/v1/messages") or base.endswith("/v1/chat/completions"):
return base
if base.endswith("/v1"):
return f"{base}/{suffix.removeprefix('v1/')}"
return f"{base}/{suffix}"
def _extract_json(text: str) -> Any:
cleaned = text.strip()
if cleaned.startswith("```"):
cleaned = cleaned.strip("`")
if cleaned.lower().startswith("json"):
cleaned = cleaned[4:].strip()
decoder = json.JSONDecoder()
for index, char in enumerate(cleaned):
if char != "{":
continue
try:
value, _end = decoder.raw_decode(cleaned[index:])
except json.JSONDecodeError:
continue
return value
return None
def _string_tuple(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
return ()
return tuple(str(item).strip() for item in value if str(item).strip())

119
d_wiki/metadata.py Normal file
View File

@ -0,0 +1,119 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from .yaml_compat import safe_load
from .tokenize import tokenize
QUERY_MATCH_STOP_TOKENS = {
"如何",
"怎么",
"使用",
"创建",
"知道",
"功能",
"资产",
}
@dataclass(frozen=True)
class AliasGroup:
name: str
terms: tuple[str, ...]
intent: str | None
related: tuple[str, ...]
@dataclass(frozen=True)
class QueryEntry:
path: Path
title: str
aliases: tuple[str, ...]
intent: str | None
primary_sources: tuple[str, ...]
supporting_sources: tuple[str, ...]
answer_requirements: tuple[str, ...]
def load_alias_groups(path: Path | None) -> list[AliasGroup]:
if path is None or not path.exists():
return []
data = safe_load(path.read_text(encoding="utf-8")) or {}
raw_aliases = data.get("aliases", {})
if not isinstance(raw_aliases, dict):
return []
groups: list[AliasGroup] = []
for name, raw in raw_aliases.items():
if not isinstance(raw, dict):
continue
groups.append(
AliasGroup(
name=str(name),
terms=tuple(str(term).strip() for term in raw.get("terms", []) if str(term).strip()),
intent=str(raw["intent"]) if raw.get("intent") else None,
related=tuple(str(item).strip() for item in raw.get("related", []) if str(item).strip()),
)
)
return groups
def load_query_entries(directory: Path | None) -> list[QueryEntry]:
if directory is None or not directory.exists():
return []
return [_query_entry(path) for path in sorted(directory.rglob("*.md")) if path.is_file()]
def frontmatter(text: str) -> tuple[dict[str, Any], str]:
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---", 4)
if end < 0:
return {}, text
meta = safe_load(text[4:end]) or {}
if not isinstance(meta, dict):
meta = {}
return meta, text[end + 4 :].lstrip("\n")
def query_matches_entry(query: str, entry: QueryEntry) -> bool:
normalized = normalize(query)
if any(normalize(alias) in normalized or normalized in normalize(alias) for alias in entry.aliases):
return True
query_terms = _meaningful_query_terms(query)
for candidate in (entry.title, *entry.aliases):
candidate_terms = _meaningful_query_terms(candidate)
if len(query_terms & candidate_terms) >= 2:
return True
return False
def _meaningful_query_terms(value: str) -> set[str]:
return {term for term in tokenize(value) if term not in QUERY_MATCH_STOP_TOKENS and len(term.strip()) >= 2}
def normalize(value: str) -> str:
return re.sub(r"\s+", " ", value.lower()).strip()
def _query_entry(path: Path) -> QueryEntry:
meta, body = frontmatter(path.read_text(encoding="utf-8"))
title = str(meta.get("title") or _first_heading(body) or path.stem)
return QueryEntry(
path=path,
title=title,
aliases=tuple(str(item).strip() for item in meta.get("aliases", []) if str(item).strip()),
intent=str(meta["intent"]) if meta.get("intent") else None,
primary_sources=tuple(str(item).strip() for item in meta.get("primary_sources", []) if str(item).strip()),
supporting_sources=tuple(str(item).strip() for item in meta.get("supporting_sources", []) if str(item).strip()),
answer_requirements=tuple(str(item).strip() for item in meta.get("answer_requirements", []) if str(item).strip()),
)
def _first_heading(text: str) -> str | None:
for line in text.splitlines():
if match := re.match(r"^#{1,6}\s+(.+)$", line):
return match.group(1).strip()
return None

469
d_wiki/search.py Normal file
View File

@ -0,0 +1,469 @@
from __future__ import annotations
import json
import math
import re
from collections import Counter, defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from .config import KnowledgeBaseConfig
from .metadata import (
AliasGroup,
QueryEntry,
frontmatter,
load_alias_groups,
load_query_entries,
normalize,
query_matches_entry,
)
from .dreams import load_dream_query_entries
from .tokenize import tokenize
SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt")
MAX_BLOCK_CHARS = 520
@dataclass(frozen=True)
class Chunk:
title: str
wikilink: str
source_path: Path
page_type: str
heading: str
text: str
tokens: tuple[str, ...]
index: int
def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]:
return {
"status": "ok",
"config": str(config_path),
"knowledge_bases": {
name: {
"root": str(kb.root),
"source_type": kb.source_type,
"exists": kb.root.exists(),
"aliases": str(kb.aliases_path) if kb.aliases_path else None,
"queries": str(kb.queries_dir) if kb.queries_dir else None,
}
for name, kb in bases.items()
},
}
def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]:
chunks = load_chunks(kb)
index_dir = kb.root / ".d_wiki_index"
index_dir.mkdir(exist_ok=True)
payload = {
"kb": kb.name,
"root": str(kb.root),
"source_type": kb.source_type,
"compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None,
"chunks": len(chunks),
"pages": len({chunk.wikilink for chunk in chunks}),
"indexed_at": datetime.now(timezone.utc).isoformat(),
}
(index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return {"status": "indexed", **payload}
def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]:
aliases = load_alias_groups(kb.aliases_path)
query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)]
matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)]
expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries)
chunks = load_chunks(kb)
ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries)
selected = ranked[: max(1, min(max_results, 8))]
context_pack = build_context_pack(query, selected, matched_aliases, matched_entries)
return {
"status": "found" if selected else "no_match",
"query": query,
"knowledge_base": kb.name,
"backend": "lightweight_bm25",
"quality": quality(selected),
"sources": [source_summary(item[1], item[0]) for item in selected],
"context_pack": context_pack,
}
def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]:
chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink]
if not chunks:
return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink}
first = chunks[0]
return {
"status": "found",
"knowledge_base": kb.name,
"id": wikilink,
"title": first.title,
"source_path": str(first.source_path),
"blocks": [
{"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"}
for chunk in chunks
],
}
def expand_query(
query: str,
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> tuple[list[str], list[AliasGroup]]:
terms = tokenize(query)
normalized_query = normalize(query)
matched: list[AliasGroup] = []
for group in aliases:
if any(normalize(term) in normalized_query for term in group.terms):
matched.append(group)
for term in group.terms:
terms.extend(tokenize(term))
for entry in entries:
terms.extend(tokenize(entry.title))
for alias in entry.aliases:
terms.extend(tokenize(alias))
return terms, matched
def rank_chunks(
chunks: list[Chunk],
terms: list[str],
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> list[tuple[float, Chunk]]:
if not chunks or not terms:
return []
term_counts = [Counter(chunk.tokens) for chunk in chunks]
doc_freq: Counter[str] = Counter()
for counts in term_counts:
for term in counts:
doc_freq[term] += 1
avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks))
query_counts = Counter(terms)
ranked: list[tuple[float, Chunk]] = []
related = {item for group in aliases for item in group.related}
primary = {item for entry in entries for item in entry.primary_sources}
supporting = {item for entry in entries for item in entry.supporting_sources}
for chunk, counts in zip(chunks, term_counts):
score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts)
if any(wikilink_matches(chunk.wikilink, item) for item in related):
score += 8.0
if any(wikilink_matches(chunk.wikilink, item) for item in primary):
score += 16.0
if any(wikilink_matches(chunk.wikilink, item) for item in supporting):
score += 5.0
score += heading_score_adjustment(chunk.heading)
if score > 0:
ranked.append((round(score, 3), chunk))
ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index))
return ranked
def heading_score_adjustment(heading: str) -> float:
if heading == "Retrieval Hints":
return -80.0
if heading == "Intent Model":
return -40.0
if heading == "Answer Blueprint":
return -25.0
if heading == "Evidence Graph":
return 3.0
if heading == "Key Evidence":
return 5.0
return 0.0
def bm25_score(
counts: Counter[str],
doc_len: int,
doc_freq: Counter[str],
doc_count: int,
avg_len: float,
query_counts: Counter[str],
) -> float:
k1 = 1.5
b = 0.75
score = 0.0
for term, qf in query_counts.items():
tf = counts.get(term, 0)
if tf == 0:
continue
idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5))
denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1)))
score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3)
return score
def build_context_pack(
query: str,
selected: list[tuple[float, Chunk]],
aliases: list[AliasGroup],
entries: list[QueryEntry],
) -> dict[str, Any]:
intent = answer_intent(query, aliases, entries)
grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list)
for item in selected:
grouped[item[1].wikilink].append(item)
source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1]))
entry_primary = {item for entry in entries for item in entry.primary_sources}
primary_sources = [
source_context(wikilink, chunks)
for wikilink, chunks in source_items
if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary)
]
if not primary_sources:
primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]]
supporting_sources = [
source_context(wikilink, chunks, blocks_limit=1)
for wikilink, chunks in source_items
if wikilink not in {source["wikilink"] for source in primary_sources}
][:3]
coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete")
return {
"answer_intent": intent,
"coverage": coverage,
"answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.",
"primary_sources": primary_sources,
"supporting_sources": supporting_sources,
"answer_requirements": answer_requirements(intent, entries),
}
def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]:
top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit]
chunk = top[0][1]
return {
"title": chunk.title,
"wikilink": wikilink,
"source_path": str(chunk.source_path),
"reason": "Matched query terms and local metadata.",
"blocks": [
{
"block_id": f"{block.wikilink}#{block.index}",
"heading": block.heading,
"text": trim_block(block.text),
"score": score,
}
for score, block in top
],
}
def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str:
for entry in entries:
if entry.intent:
return entry.intent
for group in aliases:
if group.intent:
return group.intent
if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")):
return "procedure"
return "fact"
def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]:
requirements = unique([req for entry in entries for req in entry.answer_requirements])
if requirements:
return requirements
if intent == "procedure":
return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"]
return ["直接回答问题", "引用命中的来源"]
def unique(values: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for value in values:
if value in seen:
continue
seen.add(value)
output.append(value)
return output
def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]:
top = selected[0][0] if selected else 0.0
return {
"confidence": "high" if top >= 8.0 else ("low" if selected else "none"),
"high_confidence": top >= 8.0,
"top_score": top,
"high_confidence_score": 8.0,
}
def wikilink_matches(candidate: str, expected: str) -> bool:
return candidate == expected or candidate == f"concepts/{expected}"
def source_summary(chunk: Chunk, score: float) -> dict[str, Any]:
return {
"title": chunk.title,
"wikilink": chunk.wikilink,
"source_path": str(chunk.source_path),
"heading": chunk.heading,
"snippet": trim_block(chunk.text, 320),
"score": score,
}
def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root
paths = source_paths(root)
chunks: list[Chunk] = []
for path in paths:
chunks.extend(chunks_for_path(path, root))
return chunks
def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]:
chunks: list[Chunk] = []
for path in source_paths(kb.root):
chunks.extend(chunks_for_path(path, kb.root))
return chunks
def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path:
return kb.root / ".d_wiki" / "wiki"
def source_paths(root: Path) -> list[Path]:
if (root / "_sources").is_dir():
return sorted(
path
for path in (root / "_sources").rglob("*.txt")
if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES)
)
generated = {"SCHEMA.md", "index.md", "log.md"}
generated_dirs = {".d_wiki", ".d_wiki_index"}
is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki"
return sorted(
path
for path in root.rglob("*.md")
if path.is_file()
and path.name not in generated
and (is_compiled_root or not any(part in generated_dirs for part in path.parts))
)
def chunks_for_path(path: Path, root: Path) -> list[Chunk]:
raw = path.read_text(encoding="utf-8", errors="ignore")
meta, body = frontmatter(raw)
title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title())
metadata_text = metadata_terms(meta)
wikilink = wikilink_for(path, root)
sections = split_sections(body, title)
chunks: list[Chunk] = []
for index, (heading, text) in enumerate(sections):
if heading in {"Source Map", "Related"}:
continue
if not text.strip():
continue
field_text = " ".join([title, heading, wikilink, metadata_text, text])
chunks.append(
Chunk(
title=title,
wikilink=wikilink,
source_path=path.resolve(),
page_type="sphinx_source" if "_sources" in path.parts else "markdown",
heading=heading,
text=text.strip(),
tokens=tuple(tokenize(field_text)),
index=index,
)
)
return chunks
def metadata_terms(meta: dict[str, Any]) -> str:
values: list[str] = []
for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"):
raw = meta.get(key)
if isinstance(raw, str):
values.append(raw)
elif isinstance(raw, list):
values.extend(str(item) for item in raw if str(item).strip())
return " ".join(values)
def split_sections(body: str, title: str) -> list[tuple[str, str]]:
heading = title
buffer: list[str] = []
sections: list[tuple[str, str]] = []
in_fence = False
for line in body.splitlines():
stripped = line.strip()
if stripped.startswith("```"):
in_fence = not in_fence
buffer.append(line)
continue
if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)):
if buffer:
sections.extend(paragraphs(heading, "\n".join(buffer)))
buffer = []
heading = match.group(1).strip() or title
continue
buffer.append(line)
if buffer:
sections.extend(paragraphs(heading, "\n".join(buffer)))
return sections or [(title, body)]
def paragraphs(heading: str, text: str) -> list[tuple[str, str]]:
blocks: list[tuple[str, str]] = []
for part in re.split(r"\n\s*\n", text):
cleaned = clean_block(part)
if cleaned:
blocks.append((heading, cleaned))
return blocks
def clean_block(text: str) -> str:
stripped = text.strip()
if not stripped:
return ""
if re.fullmatch(r"[-*_]{3,}", stripped):
return ""
compact = " ".join(stripped.split())
if "📷" in compact and len(compact) <= 80:
return ""
if compact.startswith("> 📷") and len(compact) <= 120:
return ""
return stripped
def first_heading(body: str) -> str | None:
in_fence = False
for line in body.splitlines():
stripped = line.strip()
if stripped.startswith("```"):
in_fence = not in_fence
continue
if in_fence:
continue
if match := re.match(r"^#{1,6}\s+(.+)$", line):
return match.group(1).strip()
return None
def wikilink_for(path: Path, root: Path) -> str:
try:
relative = path.relative_to(root)
except ValueError:
return path.stem
if relative.parts and relative.parts[0] == "_sources":
relative = Path(*relative.parts[1:])
value = relative.as_posix()
for suffix in SPHINX_SOURCE_SUFFIXES:
if value.endswith(suffix):
return value[: -len(suffix)]
return value[:-3] if value.endswith(".md") else path.stem
def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str:
compact = " ".join(text.split())
return compact if len(compact) <= limit else compact[:limit].rstrip()

22
d_wiki/tokenize.py Normal file
View File

@ -0,0 +1,22 @@
from __future__ import annotations
import re
STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上")
STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"}
def tokenize(text: str) -> list[str]:
lowered = text.lower()
terms = re.findall(r"[a-z0-9_./+-]+", lowered)
cjk_chars = re.findall(r"[\u4e00-\u9fff]", text)
terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1)))
return [term for term in terms if _useful(term)]
def _useful(term: str) -> bool:
if not term or term in STOP_TERMS:
return False
if re.search(r"[\u4e00-\u9fff]", term):
return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term))
return True

70
d_wiki/yaml_compat.py Normal file
View File

@ -0,0 +1,70 @@
from __future__ import annotations
from typing import Any
try:
import yaml as _yaml # type: ignore
except Exception: # pragma: no cover - exercised in environments without PyYAML
_yaml = None
def safe_load(text: str) -> Any:
if _yaml is not None:
return _yaml.safe_load(text)
lines = [
(len(line) - len(line.lstrip(" ")), line.strip())
for line in text.splitlines()
if line.strip() and not line.strip().startswith("#")
]
value, _index = _parse_node(lines, 0, 0)
return value
def _parse_node(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]:
if index >= len(lines):
return {}, index
if lines[index][0] < indent:
return {}, index
if lines[index][1].startswith("- "):
values: list[Any] = []
while index < len(lines) and lines[index][0] == indent and lines[index][1].startswith("- "):
values.append(_scalar(lines[index][1][2:].strip()))
index += 1
return values, index
mapping: dict[str, Any] = {}
while index < len(lines):
current_indent, stripped = lines[index]
if current_indent < indent:
break
if current_indent > indent:
break
if ":" not in stripped:
index += 1
continue
key, raw_value = stripped.split(":", 1)
key = key.strip()
raw_value = raw_value.strip()
index += 1
if raw_value:
mapping[key] = _scalar(raw_value)
continue
child, index = _parse_node(lines, index, indent + 2)
mapping[key] = child
return mapping, index
def _scalar(value: str) -> Any:
if value in {"null", "None", "~"}:
return None
if value in {"true", "True"}:
return True
if value in {"false", "False"}:
return False
if value.startswith("[") and value.endswith("]"):
inner = value[1:-1].strip()
if not inner:
return []
return [_scalar(item.strip()) for item in inner.split(",")]
if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
return value[1:-1]
return value

56
docs/language-decision.md Normal file
View File

@ -0,0 +1,56 @@
# d-wiki CLI language decision
## Decision
Use Python for the first implementation of `d-wiki`.
The CLI is a local knowledge retrieval component that needs predictable file
system traversal, Markdown/Sphinx source parsing, YAML metadata, BM25 ranking,
JSON output, and straightforward integration with `robogo_assistant_service`.
Python is the lower-risk choice for this first version because the surrounding
service code and tests are already Python, and the implementation intentionally
avoids qmd, embeddings, and Node-specific runtime requirements.
## Python pros
- Matches the existing `robogo_assistant_service` stack, test style, and local
developer workflow.
- Strong standard library support for CLI parsing, paths, subprocess-friendly
JSON output, dataclasses, and text processing.
- Easy to keep dependency-light; the current package has no required runtime
dependencies and can optionally use PyYAML when present.
- Good fit for future retrieval upgrades such as ranking experiments,
evaluation scripts, corpus analysis, and possible embedding backends.
- Lower integration friction for service tests that call the CLI from Python.
## Python cons
- Single-file executable distribution is less natural than Node/npm CLIs unless
extra packaging tools are introduced.
- Startup can be slower than a bundled JS runtime for very small commands,
though this is not a bottleneck for local document search.
- Type safety depends on discipline and tools rather than the language runtime.
## TypeScript pros
- Excellent CLI packaging through npm and strong cross-platform installation
conventions for JS-heavy teams.
- Static types are first-class and make API shape evolution explicit.
- Easier future alignment if `d-wiki` becomes a qmd-native or MCP-first Node
package.
## TypeScript cons
- Adds a second runtime stack beside the Python service for the first version.
- The plan explicitly avoids qmd/Node runtime complexity in v1; starting with
TypeScript would pull the project toward the complexity it is trying to defer.
- Requires build tooling, lockfiles, module-format decisions, and generated
output conventions before the retrieval logic itself benefits from them.
- Python service integration would still happen through subprocess JSON, so the
main service does not gain much from a TypeScript implementation today.
## Revisit triggers
Reconsider TypeScript if `d-wiki` becomes a distributed npm CLI, adopts qmd as
the default backend, ships an MCP server primarily for Node-based agents, or
needs to share substantial code with a TypeScript UI or SDK.

View File

@ -0,0 +1,25 @@
aliases:
dev_machine:
terms: [开发机, 云主机, SSH, Web 应用, 申请开发机, 创建开发机, 远程开发]
intent: procedure
related: [21-dev-machine]
guagan:
terms: [地瓜干, 计费, 充值, 余额, 消费, 价格, 账户]
intent: fact
related: [03-account-and-guagan]
mirror:
terms: [镜像, 镜像资产, 创建镜像, 保存镜像, fork, 公开镜像, 群组镜像]
intent: procedure
related: [11-mirror-center, 20-cloud-desktop]
three_d_generation:
terms: [3D生成, 3D 生成, 图生3D, 图生 3D, 3D资产生成, 3D 资产生成, 资产生成, 3d资产生成功能, EmbodyGen, glb]
intent: procedure
related: [40-3d-generation, 12-3d-assert, 41-image-generation, 42-urdf-studio]
rdk_agent:
terms: [RDK Agent, VS Code 插件, 插件安装]
intent: procedure
related: [51-rdk-agent, 50-openclaw]
openclaw:
terms: [OpenClaw, MagicBox, openclaw-magicbox, rdkclaw, 智能控制, AI 代理, Agent Platform, 设备配对, Skills 管理, 飞书]
intent: procedure
related: [50-openclaw, 51-rdk-agent]

View File

@ -0,0 +1,25 @@
---
title: 3D 资产生成
aliases:
- 我想知道如何使用3d资产生成功能
- 如何使用 3D 资产生成功能
- 3D资产生成怎么用
- 3D 生成怎么用
- 图生3D怎么用
- 如何生成3D模型
- 如何把图片生成3D资产
intent: procedure
primary_sources:
- 40-3d-generation
supporting_sources:
- 12-3d-assert
- 41-image-generation
- 42-urdf-studio
answer_requirements:
- 说明 3D 生成是图生 3D 工具,不是镜像资产
- 按准备参考图、生成、保存到 3D 资产的流程回答
- 说明输出默认 glb,生成结果可保存到 3D 资产
- 提醒暂不支持文生 3D,文字描述可先用图像生成产图
---
RoboGo 3D 资产生成 / 图生 3D 的入口问题。

View File

@ -0,0 +1,7 @@
---
title: 开发机使用
aliases: [开发机怎么用, 如何使用开发机, 怎么创建开发机, 开发机 SSH, 开发机 Web 应用]
intent: procedure
primary_sources: [21-dev-machine]
answer_requirements: [说明开发机定位, 按创建和登录步骤回答, 说明 Web 应用和 SSH 入口]
---

View File

@ -0,0 +1,7 @@
---
title: 地瓜干计费
aliases: [地瓜干怎么计费, 地瓜干是什么, 怎么充值, 余额在哪里看]
intent: fact
primary_sources: [03-account-and-guagan]
answer_requirements: [说明地瓜干用途, 说明充值和消费路径, 引用计费口径以平台实际页面为准]
---

View File

@ -0,0 +1,8 @@
---
title: 镜像资产使用
aliases: [镜像怎么使用, 如何创建镜像, 怎么保存镜像, 镜像资产怎么用]
intent: procedure
primary_sources: [11-mirror-center]
supporting_sources: [20-cloud-desktop]
answer_requirements: [区分镜像地址导入和地瓜桌面保存, 说明 Fork 和分享, 引用镜像资产来源]
---

View File

@ -0,0 +1,8 @@
---
title: OpenClaw 使用
aliases: [如何使用openclaw, OpenClaw 怎么用, 如何创建 OpenClaw, MagicBox 怎么用, OpenClaw 设备配对, OpenClaw Skills]
intent: procedure
primary_sources: [50-openclaw]
supporting_sources: [51-rdk-agent]
answer_requirements: [说明 OpenClaw 定位, 说明实例类型和创建入口, 说明 Skills/飞书/设备配对的使用路径]
---

View File

@ -0,0 +1,8 @@
---
title: RDK Agent 使用
aliases: [RDK Agent 怎么用, 如何安装 RDK Agent, RDK Agent 插件, RDK 设备怎么接入]
intent: procedure
primary_sources: [51-rdk-agent]
supporting_sources: [50-openclaw]
answer_requirements: [说明 RDK Agent 是 VS Code 插件入口, 区分设备接入在 OpenClaw MagicBox, 引用页面注意事项]
---

33
knowledge/x5/aliases.yml Normal file
View File

@ -0,0 +1,33 @@
aliases:
camera-bring-up:
terms:
- camera
- 摄像头
- sensor
- mipi
- csi
- vin
- isp
- camsys
- 点亮
- bring up
- bring-up
- bringup
- 调试
- 配置
- 预览
intent: procedure
related:
- multimedia_development/7-camsys_debug
- samples/sunrise_camera_user_guide
- samples/sample_pipeline
compute:
terms:
- 算力
- tops
- bpu
- bayes
- bayers
intent: fact
related:
- preface/overview

View File

@ -0,0 +1,19 @@
---
title: X5 camera bring up
aliases:
- 如何在X5平台上点亮camera
- X5 camera 点亮
- X5 摄像头 bring up
intent: procedure
primary_sources:
- multimedia_development/7-camsys_debug
supporting_sources:
- samples/sunrise_camera_user_guide
- samples/sample_pipeline
answer_requirements:
- 说明这是 X5 camera bring up / 点亮流程
- 按硬件连接、DTS/GPIO、sensor 驱动、MIPI 配置、运行验证组织
- 引用 Camera调试指南 和 Sunrise camera 用户使用说明等来源
---
X5 平台点亮 camera 的入口问题。

View File

@ -0,0 +1,14 @@
---
title: X5 算力
aliases:
- x5的算力是多少
- X5 BPU 算力
intent: fact
primary_sources:
- preface/overview
answer_requirements:
- 直接给出 X5 BPU 算力
- 保留 TOPS 和 BPU 信息
---
X5 算力参数入口问题。

17
pyproject.toml Normal file
View File

@ -0,0 +1,17 @@
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
[project]
name = "d-wiki"
version = "0.1.0"
description = "Lightweight local knowledge retrieval with BM25 and metadata context packs."
requires-python = ">=3.10"
dependencies = []
[project.scripts]
d-wiki = "d_wiki.cli:main"
[tool.setuptools.packages.find]
where = ["."]
include = ["d_wiki*"]

Binary file not shown.

Binary file not shown.

509
tests/test_d_wiki.py Normal file
View File

@ -0,0 +1,509 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from d_wiki.cli import main
from d_wiki.metadata import QueryEntry, query_matches_entry
class DWikiTest(unittest.TestCase):
def test_query_entry_matches_rewritten_alias_by_token_overlap(self) -> None:
entry = QueryEntry(
path=Path("camera.md"),
title="camera bring up",
aliases=("如何在X5平台上点亮camera",),
intent="procedure",
primary_sources=("multimedia_development/7-camsys_debug",),
supporting_sources=(),
answer_requirements=(),
)
self.assertTrue(query_matches_entry("X5平台 点亮 camera 使能", entry))
def test_bm25_metadata_context_pack_for_camera_bring_up(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
config = root / "d-wiki.yml"
output = self._run_json(
[
"--config",
str(config),
"search",
"--kb",
"x5",
"--query",
"如何在X5平台上点亮camera",
"--format",
"json",
]
)
self.assertEqual(output["status"], "found")
self.assertEqual(output["context_pack"]["answer_intent"], "procedure")
self.assertEqual(output["context_pack"]["coverage"], "partial_procedure")
first = output["context_pack"]["primary_sources"][0]
self.assertEqual(first["wikilink"], "multimedia_development/7-camsys_debug")
self.assertGreaterEqual(len(first["blocks"]), 2)
self.assertIn("点亮 camera", json.dumps(first, ensure_ascii=False))
def test_rewritten_camera_query_still_uses_query_entry_primary_source(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
output = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"search",
"--kb",
"x5",
"--query",
"X5平台 点亮 camera 使能",
"--format",
"json",
]
)
self.assertEqual(output["status"], "found")
first = output["context_pack"]["primary_sources"][0]
self.assertEqual(first["wikilink"], "multimedia_development/7-camsys_debug")
def test_fact_query_for_compute(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
output = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"search",
"--kb",
"x5",
"--query",
"x5的算力是多少",
"--format",
"json",
]
)
self.assertEqual(output["status"], "found")
self.assertEqual(output["context_pack"]["answer_intent"], "fact")
self.assertIn("10 TOPS", json.dumps(output["context_pack"], ensure_ascii=False))
def test_unrelated_query_is_insufficient(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
output = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"search",
"--kb",
"x5",
"--query",
"火星咖啡机怎么维修",
"--format",
"json",
]
)
self.assertEqual(output["status"], "no_match")
self.assertEqual(output["context_pack"]["coverage"], "insufficient")
def test_page_frontmatter_metadata_participates_in_retrieval(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
output = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"search",
"--kb",
"x5",
"--query",
"x5 coredump breadcrumbs",
"--format",
"json",
]
)
self.assertEqual(output["status"], "found")
self.assertEqual(output["sources"][0]["wikilink"], "misc/pstore")
def test_markdown_cleaning_filters_thematic_breaks_and_screenshot_only_blocks(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
docs = root / "docs"
docs.mkdir()
(docs / "dev-machine.md").write_text(
"# 开发机\n\n"
"> 📷 截图:开发机列表页\n\n"
"---\n\n"
"开发机支持 SSH 登录和 Web 应用访问。\n",
encoding="utf-8",
)
(root / "d-wiki.yml").write_text(
"knowledge_bases:\n"
" robogo:\n"
f" root: {docs}\n"
" source_type: markdown\n",
encoding="utf-8",
)
output = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"search",
"--kb",
"robogo",
"--query",
"开发机怎么用",
"--format",
"json",
]
)
snippets = [str(source["snippet"]) for source in output["sources"]]
self.assertNotIn("---", snippets)
self.assertFalse(any("📷" in snippet for snippet in snippets))
self.assertIn("开发机支持 SSH 登录", snippets[0])
def test_compile_builds_wiki_pages_and_search_prefers_compiled_wiki(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
config = root / "d-wiki.yml"
compiled = self._run_json(
[
"--config",
str(config),
"compile",
"--kb",
"x5",
"--force",
"--format",
"json",
]
)
indexed = self._run_json(
[
"--config",
str(config),
"index",
"--kb",
"x5",
"--format",
"json",
]
)
output = self._run_json(
[
"--config",
str(config),
"search",
"--kb",
"x5",
"--query",
"x5的算力是多少",
"--format",
"json",
]
)
wiki_root = Path(str(compiled["wiki_root"]))
self.assertEqual(compiled["status"], "compiled")
self.assertTrue((wiki_root / "SCHEMA.md").exists())
self.assertTrue((wiki_root / "index.md").exists())
self.assertTrue((wiki_root / "concepts" / "preface-overview.md").exists())
self.assertEqual(indexed["compiled_wiki"], str(wiki_root))
self.assertEqual(output["status"], "found")
self.assertIn(".d_wiki/wiki/concepts", output["sources"][0]["source_path"])
self.assertIn("10 TOPS", json.dumps(output["context_pack"], ensure_ascii=False))
def test_compile_emits_llm_wiki_structure_without_breaking_retrieval(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
config = root / "d-wiki.yml"
compiled = self._run_json(
[
"--config",
str(config),
"compile",
"--kb",
"x5",
"--force",
"--format",
"json",
]
)
self._run_json(
[
"--config",
str(config),
"index",
"--kb",
"x5",
"--format",
"json",
]
)
compute = self._run_json(
[
"--config",
str(config),
"search",
"--kb",
"x5",
"--query",
"x5的算力是多少",
"--format",
"json",
]
)
camera = self._run_json(
[
"--config",
str(config),
"search",
"--kb",
"x5",
"--query",
"如何在X5平台上点亮camera",
"--format",
"json",
]
)
wiki_root = Path(str(compiled["wiki_root"]))
compute_page = (wiki_root / "concepts" / "preface-overview.md").read_text(encoding="utf-8")
camera_page = (
wiki_root / "concepts" / "multimedia-development-7-camsys-debug.md"
).read_text(encoding="utf-8")
self.assertIn("llmWikiVersion: \"1\"", compute_page)
self.assertIn("## Intent Model", compute_page)
self.assertIn("## Retrieval Hints", compute_page)
self.assertIn("## Answer Blueprint", compute_page)
self.assertIn("## Evidence Graph", compute_page)
self.assertIn("## Key Evidence", compute_page)
self.assertIn("X5 算力", compute_page)
self.assertIn("10 TOPS", compute_page)
self.assertIn("- intent: fact", compute_page)
self.assertIn("- intent: procedure", camera_page)
self.assertIn("按步骤回答", camera_page)
self.assertEqual(compute["status"], "found")
self.assertIn("10 TOPS", json.dumps(compute["context_pack"], ensure_ascii=False))
self.assertIn(
"Key Evidence",
[block["heading"] for block in compute["context_pack"]["primary_sources"][0]["blocks"]],
)
self.assertEqual(camera["context_pack"]["answer_intent"], "procedure")
self.assertEqual(
camera["context_pack"]["primary_sources"][0]["wikilink"],
"concepts/multimedia-development-7-camsys-debug",
)
self.assertNotEqual(camera["sources"][0]["heading"], "Retrieval Hints")
def test_dream_queries_route_without_manual_query_entry(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
docs = root / "docs"
docs.mkdir()
(docs / "11-mirror-center.md").write_text(
"# 镜像资产使用\n\n"
"镜像资产是 RoboGo 的容器环境仓库,用于保存和复用开发环境。\n",
encoding="utf-8",
)
(docs / "40-3d-generation.md").write_text(
"# 40 · 3D 生成\n\n"
"3D 生成是平台提供的 AI 图生 3D 工具,基于 EmbodyGen,用一张参考图片生成 3D 模型,默认 glb。\n\n"
"生成结果可以保存到 3D 资产继续使用。\n",
encoding="utf-8",
)
(root / "knowledge" / "robogo").mkdir(parents=True)
(root / "knowledge" / "robogo" / "aliases.yml").write_text(
"aliases:\n"
" mirror:\n"
" terms: [镜像, 镜像资产, 创建镜像]\n"
" intent: procedure\n"
" related: [11-mirror-center]\n",
encoding="utf-8",
)
(root / "d-wiki.yml").write_text(
"knowledge_bases:\n"
" robogo:\n"
f" root: {docs}\n"
" source_type: markdown\n"
" aliases: knowledge/robogo/aliases.yml\n",
encoding="utf-8",
)
compiled = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"compile",
"--kb",
"robogo",
"--force",
"--format",
"json",
]
)
dreamed = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"dream",
"--kb",
"robogo",
"--force",
"--format",
"json",
]
)
self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"index",
"--kb",
"robogo",
"--format",
"json",
]
)
output = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"search",
"--kb",
"robogo",
"--query",
"我想知道如何使用3d资产生成功能",
"--format",
"json",
]
)
page = Path(str(compiled["wiki_root"])) / "concepts" / "40-3d-generation.md"
page_content = page.read_text(encoding="utf-8")
dreamed_query = Path(str(dreamed["output_store"])) / "queries" / "40-3d-generation.md"
dreamed_content = dreamed_query.read_text(encoding="utf-8")
self.assertNotIn("dream_queries:", page_content)
self.assertIn("3D资产生成怎么用", dreamed_content)
self.assertIn("provenance: dreamed", dreamed_content)
self.assertEqual(output["status"], "found")
self.assertEqual(output["context_pack"]["primary_sources"][0]["wikilink"], "concepts/40-3d-generation")
self.assertNotIn("镜像资产使用", json.dumps(output["context_pack"]["answer_requirements"], ensure_ascii=False))
def test_compile_llm_auto_falls_back_without_llm_config(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = self._fixture(Path(tmp))
compiled = self._run_json(
[
"--config",
str(root / "d-wiki.yml"),
"compile",
"--kb",
"x5",
"--force",
"--llm",
"auto",
"--format",
"json",
]
)
page = Path(str(compiled["wiki_root"])) / "concepts" / "preface-overview.md"
page_content = page.read_text(encoding="utf-8")
self.assertEqual(compiled["status"], "compiled")
self.assertEqual(compiled["llm_mode"], "auto")
self.assertEqual(compiled["llm_scope"], "queries")
self.assertFalse(compiled["llm_enabled"])
self.assertIn("provenanceState: compiled", page_content)
def _run_json(self, argv: list[str]) -> dict[str, object]:
from contextlib import redirect_stdout
from io import StringIO
out = StringIO()
with redirect_stdout(out):
code = main(argv)
self.assertEqual(code, 0, out.getvalue())
return json.loads(out.getvalue())
def _fixture(self, root: Path) -> Path:
sources = root / "manual" / "_sources"
(sources / "multimedia_development").mkdir(parents=True)
(sources / "preface").mkdir(parents=True)
(sources / "misc").mkdir(parents=True)
(sources / "multimedia_development" / "7-camsys_debug.md.txt").write_text(
"# Camera调试指南\n\n"
"## 一 范围\n\n"
"本章节概述了 camera bring up 的过程,用于帮助读者快速了解并掌握 X5 camera 框架,"
"如何快速的新增camera 配置,并点亮 camera。\n\n"
"## 1.1 sensor gpio 配置\n\n"
"vcon 是 X5 camera 用于管理 sensor 硬件相关的 dts 节点。\n\n"
"## 2 sensor 驱动文件添加\n\n"
"需要将原厂 sensor 驱动转换成 X5 camera 驱动代码,并编译生成 so 库。\n",
encoding="utf-8",
)
(sources / "preface" / "overview.md.txt").write_text(
"# 前言\n\n## 关键特性\n\n- 贝叶斯架构BPU,提供最高 10 TOPS 算力。\n",
encoding="utf-8",
)
(sources / "misc" / "pstore.md.txt").write_text(
"---\n"
"title: pstore\n"
"aliases: [coredump breadcrumbs]\n"
"tags: [debug, reboot]\n"
"---\n"
"# pstore\n\n在X5平台上,当系统异常重启时会保存日志。\n",
encoding="utf-8",
)
(root / "knowledge" / "x5" / "queries").mkdir(parents=True)
(root / "knowledge" / "x5" / "aliases.yml").write_text(
"aliases:\n"
" camera:\n"
" terms: [camera, 摄像头, sensor, mipi, csi, vin, isp, 点亮, bring up, 调试, 配置]\n"
" intent: procedure\n"
" related: [multimedia_development/7-camsys_debug]\n"
" compute:\n"
" terms: [算力, tops, bpu]\n"
" intent: fact\n"
" related: [preface/overview]\n",
encoding="utf-8",
)
(root / "knowledge" / "x5" / "queries" / "camera.md").write_text(
"---\n"
"title: camera bring up\n"
"aliases: [如何在X5平台上点亮camera]\n"
"intent: procedure\n"
"primary_sources: [multimedia_development/7-camsys_debug]\n"
"answer_requirements: [按步骤回答]\n"
"---\n",
encoding="utf-8",
)
(root / "knowledge" / "x5" / "queries" / "compute.md").write_text(
"---\n"
"title: X5 算力\n"
"aliases: [x5的算力是多少]\n"
"intent: fact\n"
"primary_sources: [preface/overview]\n"
"---\n",
encoding="utf-8",
)
(root / "d-wiki.yml").write_text(
"knowledge_bases:\n"
" x5:\n"
f" root: {root / 'manual'}\n"
" source_type: sphinx\n"
" aliases: knowledge/x5/aliases.yml\n"
" queries: knowledge/x5/queries\n",
encoding="utf-8",
)
return root
if __name__ == "__main__":
unittest.main()