diff --git a/build/lib/d_wiki/__init__.py b/build/lib/d_wiki/__init__.py new file mode 100644 index 0000000..365f5d0 --- /dev/null +++ b/build/lib/d_wiki/__init__.py @@ -0,0 +1,3 @@ +"""d-wiki lightweight local knowledge retrieval.""" + +__version__ = "0.1.0" diff --git a/build/lib/d_wiki/__main__.py b/build/lib/d_wiki/__main__.py new file mode 100644 index 0000000..bfdcd0c --- /dev/null +++ b/build/lib/d_wiki/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/build/lib/d_wiki/cli.py b/build/lib/d_wiki/cli.py new file mode 100644 index 0000000..92d81fc --- /dev/null +++ b/build/lib/d_wiki/cli.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from .config import DWikiConfig, load_config +from .compile import compile_kb +from .dreams import dream_kb +from .search import get_page, index_payload, search, status_payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="d-wiki") + parser.add_argument("--config", type=Path, default=None) + subparsers = parser.add_subparsers(dest="command", required=True) + + status_parser = subparsers.add_parser("status") + status_parser.add_argument("--format", choices=["json"], default="json") + + index_parser = subparsers.add_parser("index") + index_parser.add_argument("--kb", required=True) + index_parser.add_argument("--format", choices=["json"], default="json") + + compile_parser = subparsers.add_parser("compile") + compile_parser.add_argument("--kb", required=True) + compile_parser.add_argument("--force", action="store_true") + compile_parser.add_argument("--llm", choices=["off", "auto", "require"], default="off") + compile_parser.add_argument("--llm-scope", choices=["queries", "all"], default="queries") + compile_parser.add_argument("--format", choices=["json"], default="json") + + dream_parser = subparsers.add_parser("dream") + dream_parser.add_argument("--kb", required=True) + dream_parser.add_argument("--force", action="store_true") + dream_parser.add_argument("--format", choices=["json"], default="json") + + search_parser = subparsers.add_parser("search") + search_parser.add_argument("--kb", required=True) + search_parser.add_argument("--query", required=True) + search_parser.add_argument("--max-results", type=int, default=5) + search_parser.add_argument("--format", choices=["json"], default="json") + + get_parser = subparsers.add_parser("get") + get_parser.add_argument("--kb", required=True) + get_parser.add_argument("--id", required=True) + get_parser.add_argument("--format", choices=["json"], default="json") + + args = parser.parse_args(argv) + try: + config = load_config(args.config) + payload = dispatch(args, config) + except Exception as exc: + print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 2 + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +def dispatch(args: argparse.Namespace, config: DWikiConfig) -> dict[str, Any]: + if args.command == "status": + return status_payload(config.path, config.knowledge_bases) + kb = config.knowledge_bases.get(args.kb) + if kb is None: + raise ValueError(f"D_WIKI_KB_NOT_FOUND: {args.kb}") + if args.command == "index": + return index_payload(kb) + if args.command == "compile": + return compile_kb(kb, force=args.force, llm_mode=args.llm, llm_scope=args.llm_scope) + if args.command == "dream": + return dream_kb(kb, force=args.force) + if args.command == "search": + return search(kb, args.query, args.max_results) + if args.command == "get": + return get_page(kb, args.id) + raise ValueError(f"D_WIKI_UNKNOWN_COMMAND: {args.command}") diff --git a/build/lib/d_wiki/compile.py b/build/lib/d_wiki/compile.py new file mode 100644 index 0000000..bd90cae --- /dev/null +++ b/build/lib/d_wiki/compile.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +import re +import shutil +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .config import KnowledgeBaseConfig +from .llm import LlmPageSynthesis, LlmPageSynthesizer +from .metadata import load_alias_groups, load_query_entries +from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block +from .tokenize import tokenize + + +@dataclass(frozen=True) +class CompiledPage: + wikilink: str + title: str + path: Path + + +@dataclass(frozen=True) +class _EvidenceProfile: + tokens: frozenset[str] + phrases: tuple[str, ...] + + +def compile_kb( + kb: KnowledgeBaseConfig, + *, + force: bool = False, + llm_mode: str = "off", + llm_scope: str = "queries", +) -> dict[str, Any]: + chunks = load_source_chunks(kb) + if not chunks: + raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS") + synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None + if llm_mode == "require" and synthesizer is None: + raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED") + wiki_root = compiled_wiki_root(kb) + concepts_dir = wiki_root / "concepts" + raw_dir = wiki_root / "raw" + if force: + shutil.rmtree(concepts_dir, ignore_errors=True) + concepts_dir.mkdir(parents=True, exist_ok=True) + raw_dir.mkdir(parents=True, exist_ok=True) + + pages: list[CompiledPage] = [] + grouped = _group_chunks(chunks) + query_entries = load_query_entries(kb.queries_dir) + alias_groups = load_alias_groups(kb.aliases_path) + query_by_source = { + source: entry + for entry in query_entries + for source in entry.primary_sources + } + aliases_by_source = { + related: group + for group in alias_groups + for related in group.related + } + + for wikilink, page_chunks in sorted(grouped.items()): + page = _compile_page( + kb, + wikilink, + page_chunks, + query_entry=query_by_source.get(wikilink), + alias_group=aliases_by_source.get(wikilink), + synthesizer=( + synthesizer + if llm_scope == "all" or query_by_source.get(wikilink) is not None + else None + ), + require_llm=llm_mode == "require", + ) + if force or not page.path.exists(): + page.path.write_text(page.content, encoding="utf-8") + pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path)) + + _write_schema(wiki_root, kb) + _write_index(wiki_root, pages) + _append_log(wiki_root, kb, len(chunks), len(pages)) + return { + "status": "compiled", + "kb": kb.name, + "source_root": str(kb.root), + "wiki_root": str(wiki_root), + "source_chunks": len(chunks), + "pages": len(pages), + "llm_mode": llm_mode, + "llm_scope": llm_scope, + "llm_enabled": synthesizer is not None, + "compiled_at": datetime.now(timezone.utc).isoformat(), + } + + +@dataclass(frozen=True) +class _PageDraft: + title: str + path: Path + content: str + + +def _compile_page( + kb: KnowledgeBaseConfig, + wikilink: str, + chunks: list[Chunk], + *, + query_entry: Any | None, + alias_group: Any | None, + synthesizer: LlmPageSynthesizer | None, + require_llm: bool, +) -> _PageDraft: + first = chunks[0] + title = _title_for(first, query_entry) + slug = _slug(wikilink) + path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md" + evidence_profile = _evidence_profile(title, query_entry, alias_group) + tags = _tags(kb, first, alias_group) + aliases = _aliases(title, query_entry, alias_group) + source_ref = first.wikilink + source_path = str(first.source_path) + key_blocks = _key_blocks(chunks, evidence_profile) + intent = _intent(title, query_entry, alias_group) + summary = _summary(chunks, title, evidence_profile, intent=intent) + requirements = _answer_requirements(intent, query_entry) + synthesis = _llm_synthesis( + synthesizer, + title=title, + wikilink=wikilink, + query_entry=query_entry, + alias_group=alias_group, + key_blocks=key_blocks, + require_llm=require_llm, + ) + if synthesis is not None: + summary = synthesis.summary + retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group) + blueprint = ( + list(synthesis.answer_blueprint) + if synthesis and synthesis.answer_blueprint + else _deterministic_answer_blueprint(intent, requirements, key_blocks) + ) + links = _related_links(query_entry, alias_group) + content = "\n".join( + [ + "---", + f"llmWikiVersion: \"1\"", + f"conceptId: {_yaml_scalar(slug)}", + f"title: {_yaml_scalar(title)}", + f"summary: {_yaml_scalar(summary)}", + f"sources: {_yaml_list([source_ref])}", + f"source_paths: {_yaml_list([source_path])}", + f"kind: concept", + f"tags: {_yaml_list(tags)}", + f"aliases: {_yaml_list(aliases)}", + f"intents: {_yaml_list([intent])}", + f"retrieval_hints: {_yaml_list(retrieval_hints)}", + f"answer_requirements: {_yaml_list(requirements)}", + f"confidence: {synthesis.confidence if synthesis else 'medium'}", + f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}", + "---", + "", + f"# {title}", + "", + summary, + "", + "## Intent Model", + "", + f"- intent: {intent}", + f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}", + f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}", + "", + "## Retrieval Hints", + "", + *[f"- {hint}" for hint in retrieval_hints], + "", + "## Answer Blueprint", + "", + *[f"- {item}" for item in blueprint], + "", + "## Synthesized Evidence" if synthesis and synthesis.key_evidence else "", + "" if synthesis and synthesis.key_evidence else "", + *[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())], + "" if synthesis and synthesis.key_evidence else "", + "## Evidence Graph", + "", + *_evidence_graph_lines(key_blocks), + "", + "## Key Evidence", + "", + *[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks], + "", + "## Source Map", + "", + f"- 原始来源:`{source_ref}`", + f"- 文件路径:`{source_path}`", + "", + "## Related", + "", + *[f"- [[{link}]]" for link in links], + "", + ] + ) + return _PageDraft(title=title, path=path, content=content) + + +def _llm_synthesis( + synthesizer: LlmPageSynthesizer | None, + *, + title: str, + wikilink: str, + query_entry: Any | None, + alias_group: Any | None, + key_blocks: list[Chunk], + require_llm: bool, +) -> LlmPageSynthesis | None: + if synthesizer is None: + return None + prompt = _llm_compile_prompt( + title=title, + wikilink=wikilink, + query_entry=query_entry, + alias_group=alias_group, + key_blocks=key_blocks, + ) + try: + return synthesizer.synthesize(prompt) + except Exception: + if require_llm: + raise + return None + + +def _llm_compile_prompt( + *, + title: str, + wikilink: str, + query_entry: Any | None, + alias_group: Any | None, + key_blocks: list[Chunk], +) -> str: + aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ()) + terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ()) + requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ()) + intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None) + evidence = [ + { + "block_id": f"{block.wikilink}#{block.index}", + "heading": block.heading, + "text": trim_block(block.text, 700), + } + for block in key_blocks[:8] + ] + return "\n".join( + [ + "Compile a d-wiki concept page from the local evidence below.", + "Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.", + "For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.", + "Return only JSON with this shape:", + '{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}', + "", + json_dumps( + { + "title": title, + "wikilink": wikilink, + "intent": intent, + "aliases": aliases, + "terms": terms, + "answer_requirements": requirements, + "evidence": evidence, + } + ), + ] + ) + + +def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]: + if synthesis is None or not synthesis.answer_blueprint: + return [] + return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""] + + +def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str: + if query_entry is not None and getattr(query_entry, "intent", ""): + return str(query_entry.intent) + if alias_group is not None and getattr(alias_group, "intent", ""): + return str(alias_group.intent) + lowered = title.lower() + if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")): + return "procedure" + return "fact" + + +def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]: + if query_entry is not None: + requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()] + if requirements: + return _unique(requirements) + if intent == "procedure": + return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"] + return ["直接给出结论", "保留关键数值和单位", "引用证据来源"] + + +def _retrieval_hints( + title: str, + aliases: list[str], + tags: list[str], + requirements: list[str], + alias_group: Any | None, +) -> list[str]: + values = [title, *aliases, *tags, *requirements] + if alias_group is not None: + values.extend(str(item) for item in getattr(alias_group, "terms", ())) + hints: list[str] = [] + for value in values: + hints.append(value) + hints.extend(tokenize(value)) + return _unique(hints)[:32] + + +def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]: + if requirements: + return requirements[:6] + if intent == "procedure": + headings = _unique([block.heading for block in key_blocks if block.heading]) + if headings: + return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]] + return ["按前提、配置、执行、验证的顺序回答"] + return ["先给出直接结论", "补充关键参数、限制和来源"] + + +def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]: + lines: list[str] = [] + for block in key_blocks[:8]: + claim = trim_block(block.text, 180) + if not claim: + continue + lines.append(f"- claim: {claim}") + lines.append(f" source: {block.wikilink}#{block.index}") + if block.heading: + lines.append(f" heading: {block.heading}") + return lines or ["- claim: 暂无可用证据", " source: unknown"] + + +def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]: + grouped: dict[str, list[Chunk]] = {} + for chunk in chunks: + grouped.setdefault(chunk.wikilink, []).append(chunk) + return grouped + + +def _title_for(chunk: Chunk, query_entry: Any | None) -> str: + if query_entry is not None and getattr(query_entry, "title", ""): + return str(query_entry.title) + return chunk.title + + +def _summary( + chunks: list[Chunk], + title: str, + evidence_profile: _EvidenceProfile | None = None, + *, + intent: str = "fact", +) -> str: + if intent == "procedure": + for chunk in chunks[:12]: + text = trim_block(chunk.text, 220) + if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text): + return f"{title}:{text}" + ranked = _rank_evidence(chunks, evidence_profile) + for score, chunk in ranked: + if score <= 0: + continue + text = trim_block(chunk.text, 220) + if text and not _is_low_signal(text): + return f"{title}:{text}" + for chunk in chunks: + text = trim_block(chunk.text, 220) + if text and not _is_low_signal(text) and len(text) >= 20: + return f"{title}:{text}" + return f"{title} 的编译知识页,来源于本地文档。" + + +def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]: + preferred = [ + chunk + for chunk in chunks + if _is_evidence_candidate(chunk.text) + ] + ranked = _rank_evidence(preferred, evidence_profile) + if not ranked or ranked[0][0] <= 0: + return (preferred or chunks)[:6] + selected: list[Chunk] = [] + seen: set[tuple[str, int]] = set() + for score, chunk in ranked: + if score <= 0: + continue + key = (chunk.wikilink, chunk.index) + if key in seen: + continue + selected.append(chunk) + seen.add(key) + if len(selected) >= 6: + return selected + for chunk in preferred: + key = (chunk.wikilink, chunk.index) + if key in seen: + continue + selected.append(chunk) + seen.add(key) + if len(selected) >= 6: + break + return selected + + +def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile: + values = [title] + if query_entry is not None: + values.extend(str(item) for item in getattr(query_entry, "aliases", ())) + values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ())) + if alias_group is not None: + values.extend(str(item) for item in getattr(alias_group, "terms", ())) + phrases = tuple(_unique([_normalize_phrase(value) for value in values if value])) + tokens: set[str] = set() + for value in values: + tokens.update(tokenize(str(value))) + return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases) + + +def _rank_evidence( + chunks: list[Chunk], + evidence_profile: _EvidenceProfile | None, +) -> list[tuple[float, Chunk]]: + if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases): + return [] + ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks] + ranked.sort(key=lambda item: (-item[0], item[1].index)) + return ranked + + +def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float: + text = trim_block(chunk.text, 1000) + normalized = _normalize_phrase(text) + chunk_tokens = set(chunk.tokens) | set(tokenize(text)) + score = 0.0 + score += len(chunk_tokens & evidence_profile.tokens) * 2.0 + for phrase in evidence_profile.phrases: + if len(phrase) >= 2 and phrase in normalized: + score += 5.0 + if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized): + score += 8.0 + if "算力" in evidence_profile.tokens and "tops" in normalized: + score += 6.0 + if "bpu" in evidence_profile.tokens and "bpu" in normalized: + score += 4.0 + if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized): + score += 2.0 + if _is_low_signal(text): + score -= 8.0 + return score + + +def _is_evidence_candidate(text: str) -> bool: + stripped = text.lstrip() + return not stripped.startswith("|") and not stripped.startswith(">") + + +def _is_low_signal(text: str) -> bool: + stripped = text.strip() + without_marks = re.sub(r"[*_`#\-\s]+", "", stripped) + if len(without_marks) < 8: + return True + if stripped.startswith("```") or "```" in stripped: + return True + if stripped.startswith("|"): + return True + if "![](" in stripped: + return True + if re.search(r"(^|\n)\s*[├│└─]{2,}", stripped): + return True + return False + + +def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]: + tags = [kb.name] + if alias_group is not None: + tags.append(_slug(str(alias_group.name)).replace("--", "-")) + for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title): + lowered = token.lower() + if lowered not in tags: + tags.append(lowered) + return tags[:8] + + +def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]: + values = [title] + if query_entry is not None: + values.extend(str(item) for item in getattr(query_entry, "aliases", ())) + if alias_group is not None: + values.extend(str(item) for item in getattr(alias_group, "terms", ())) + return _unique(values)[:12] + + +def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]: + links: list[str] = [] + if query_entry is not None: + links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ())) + if alias_group is not None: + links.extend(str(item) for item in getattr(alias_group, "related", ())) + return _unique(links)[:10] + + +def _unique(values: list[str]) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for value in values: + cleaned = value.strip() + if not cleaned or cleaned in seen: + continue + seen.add(cleaned) + output.append(cleaned) + return output + + +def _normalize_phrase(value: str) -> str: + return re.sub(r"\s+", " ", value.lower()).strip() + + +def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None: + (wiki_root / "SCHEMA.md").write_text( + "\n".join( + [ + "# Wiki Schema", + "", + "## Domain", + f"Compiled local knowledge for `{kb.name}`.", + "", + "## Conventions", + "- Pages live under `concepts/`.", + "- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.", + "- Claims in Key Evidence keep provenance markers pointing back to source block ids.", + "- Raw source files remain immutable; compiled pages are generated artifacts.", + "", + ] + ), + encoding="utf-8", + ) + + +def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None: + lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""] + for page in sorted(pages, key=lambda item: item.title): + rel = page.path.relative_to(wiki_root).as_posix() + lines.append(f"- [[{rel[:-3]}]] — {page.title}") + lines.append("") + (wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8") + + +def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None: + line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n" + path = wiki_root / "log.md" + if not path.exists(): + path.write_text("# Log\n\n", encoding="utf-8") + with path.open("a", encoding="utf-8") as handle: + handle.write(line) + + +def _slug(value: str) -> str: + slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower() + return slug or "page" + + +def _yaml_scalar(value: str) -> str: + import json + + return json.dumps(value, ensure_ascii=False) + + +def _yaml_list(values: list[str]) -> str: + return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]" + + +def json_dumps(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, indent=2) diff --git a/build/lib/d_wiki/config.py b/build/lib/d_wiki/config.py new file mode 100644 index 0000000..b8bee1e --- /dev/null +++ b/build/lib/d_wiki/config.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .yaml_compat import safe_load + + +@dataclass(frozen=True) +class KnowledgeBaseConfig: + name: str + root: Path + source_type: str + aliases_path: Path | None + queries_dir: Path | None + + +@dataclass(frozen=True) +class DWikiConfig: + path: Path + knowledge_bases: dict[str, KnowledgeBaseConfig] + + +def default_config_path() -> Path: + return Path(os.environ.get("D_WIKI_CONFIG", Path.cwd() / "d-wiki.yml")).expanduser() + + +def load_config(path: Path | None = None) -> DWikiConfig: + config_path = (path or default_config_path()).expanduser().resolve() + data = safe_load(config_path.read_text(encoding="utf-8")) or {} + raw_kbs = data.get("knowledge_bases") + if not isinstance(raw_kbs, dict): + raise ValueError("D_WIKI_CONFIG_MISSING_KNOWLEDGE_BASES") + bases: dict[str, KnowledgeBaseConfig] = {} + for name, raw in raw_kbs.items(): + if not isinstance(raw, dict): + raise ValueError(f"D_WIKI_KB_INVALID: {name}") + root = Path(str(raw.get("root", ""))).expanduser() + if not root.is_absolute(): + root = (config_path.parent / root).resolve() + aliases_path = _optional_path(config_path.parent, raw.get("aliases")) + queries_dir = _optional_path(config_path.parent, raw.get("queries")) + bases[str(name)] = KnowledgeBaseConfig( + name=str(name), + root=root, + source_type=str(raw.get("source_type", "markdown")), + aliases_path=aliases_path, + queries_dir=queries_dir, + ) + return DWikiConfig(path=config_path, knowledge_bases=bases) + + +def _optional_path(base: Path, value: Any) -> Path | None: + if not value: + return None + path = Path(str(value)).expanduser() + return path if path.is_absolute() else (base / path).resolve() diff --git a/build/lib/d_wiki/dreams.py b/build/lib/d_wiki/dreams.py new file mode 100644 index 0000000..ffec505 --- /dev/null +++ b/build/lib/d_wiki/dreams.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import re +import shutil +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .config import KnowledgeBaseConfig +from .metadata import frontmatter, load_query_entries + + +@dataclass(frozen=True) +class DreamedQuery: + slug: str + title: str + aliases: tuple[str, ...] + intent: str + primary_sources: tuple[str, ...] + supporting_sources: tuple[str, ...] + answer_requirements: tuple[str, ...] + + +def dream_kb(kb: KnowledgeBaseConfig, *, force: bool = False) -> dict[str, Any]: + wiki_root = kb.root / ".d_wiki" / "wiki" + if not wiki_root.is_dir(): + raise ValueError("D_WIKI_DREAM_REQUIRES_COMPILED_WIKI") + output_root = dream_root(kb) + queries_dir = output_root / "queries" + if force: + shutil.rmtree(output_root, ignore_errors=True) + queries_dir.mkdir(parents=True, exist_ok=True) + + pages = sorted((wiki_root / "concepts").glob("*.md")) + dreamed = [_dream_page(path, wiki_root) for path in pages] + dreamed = [item for item in dreamed if item is not None] + for item in dreamed: + (queries_dir / f"{item.slug}.md").write_text(_query_markdown(item), encoding="utf-8") + (output_root / "README.md").write_text(_readme(kb, len(pages), len(dreamed)), encoding="utf-8") + return { + "status": "dreamed", + "kb": kb.name, + "input_wiki": str(wiki_root), + "output_store": str(output_root), + "queries": len(dreamed), + "dreamed_at": datetime.now(timezone.utc).isoformat(), + } + + +def load_dream_query_entries(kb: KnowledgeBaseConfig) -> list[Any]: + return load_query_entries(dream_root(kb) / "queries") + + +def dream_root(kb: KnowledgeBaseConfig) -> Path: + return kb.root / ".d_wiki" / "dreams" / "latest" + + +def _dream_page(path: Path, wiki_root: Path) -> DreamedQuery | None: + meta, body = frontmatter(path.read_text(encoding="utf-8", errors="ignore")) + title = str(meta.get("title") or "").strip() + if not title: + return None + curation_body = _curation_body(body) + source = _first(meta.get("sources")) or _wikilink_for_compiled(path, wiki_root) + aliases = _unique([title, *_dream_aliases(title, curation_body)]) + intent = _first(meta.get("intents")) or _dream_intent(title, curation_body) + requirements = _as_list(meta.get("answer_requirements")) or _dream_requirements(title, curation_body, intent) + supporting = _related_sources(body, source) + return DreamedQuery( + slug=path.stem, + title=title, + aliases=tuple(aliases[:18]), + intent=intent, + primary_sources=(source,), + supporting_sources=tuple(supporting[:6]), + answer_requirements=tuple(requirements[:8]), + ) + + +def _dream_aliases(title: str, body: str) -> list[str]: + text = f"{title}\n{body}".lower() + aliases = [ + f"{title}怎么用", + f"如何使用{title}", + ] + if _looks_like_3d_generation(title, text): + aliases.extend( + [ + "3D资产生成怎么用", + "如何使用 3D 资产生成功能", + "我想知道如何使用3d资产生成功能", + "图生3D怎么用", + "如何生成3D模型", + "如何把图片生成3D资产", + ] + ) + if "camera" in text and any(term in text for term in ("点亮", "bring up", "sensor")): + aliases.extend(["如何在X5平台上点亮camera", "X5 camera 点亮", "X5 摄像头 bring up"]) + if any(term in text for term in ("算力", "tops", "bpu")): + aliases.extend(["x5的算力是多少", "X5 BPU 算力"]) + return aliases + + +def _dream_intent(title: str, body: str) -> str: + text = f"{title}\n{body}".lower() + if any(term in text for term in ("如何", "怎么", "步骤", "使用", "创建", "配置", "点亮", "生成", "调试", "bring up")): + return "procedure" + return "fact" + + +def _dream_requirements(title: str, body: str, intent: str) -> list[str]: + text = f"{title}\n{body}".lower() + if _looks_like_3d_generation(title, text): + return [ + "说明 3D 生成是图生 3D 工具,不是镜像资产", + "按准备参考图、生成、保存到 3D 资产的流程回答", + "说明输出默认 glb,生成结果可保存到 3D 资产", + "提醒暂不支持文生 3D,文字描述可先用图像生成产图", + ] + if intent == "procedure": + return ["按步骤回答", "说明关键前提和入口", "引用证据来源"] + return ["直接回答问题", "引用证据来源"] + + +def _looks_like_3d_generation(title: str, text: str) -> bool: + title_text = title.lower() + return ( + "3d 生成" in title_text + or "3d生成" in title_text + or "3d 资产生成" in title_text + or "3d资产生成" in title_text + or "embodygen" in text + or "ai 图生 3d 工具" in text + or "ai 图生3d 工具" in text + ) + + +def _curation_body(body: str) -> str: + lines: list[str] = [] + skip = False + for line in body.splitlines(): + if line.startswith("## "): + skip = line.strip() in {"## Retrieval Hints", "## Related"} + if skip: + continue + if not skip: + lines.append(line) + return "\n".join(lines) + + +def _related_sources(body: str, primary: str) -> list[str]: + links = re.findall(r"\[\[([^\]]+)\]\]", body) + refs = re.findall(r"\^\[([^#\]]+)(?:#[^\]]+)?\]", body) + return _unique([item for item in [*links, *refs] if item and item != primary]) + + +def _query_markdown(item: DreamedQuery) -> str: + lines = [ + "---", + f"title: {_yaml_scalar(item.title)}", + f"aliases: {_yaml_list(item.aliases)}", + f"intent: {item.intent}", + f"primary_sources: {_yaml_list(item.primary_sources)}", + f"supporting_sources: {_yaml_list(item.supporting_sources)}", + f"answer_requirements: {_yaml_list(item.answer_requirements)}", + "provenance: dreamed", + "---", + "", + f"Dreamed query memory for {item.title}.", + "", + ] + return "\n".join(lines) + + +def _readme(kb: KnowledgeBaseConfig, pages: int, queries: int) -> str: + return "\n".join( + [ + "# d-wiki dream output", + "", + "This store is generated from the compiled wiki. It does not modify source documents or compiled concept pages.", + "", + f"- kb: `{kb.name}`", + f"- input_pages: {pages}", + f"- dreamed_queries: {queries}", + "", + ] + ) + + +def _first(value: Any) -> str: + items = _as_list(value) + return items[0] if items else "" + + +def _as_list(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [] + + +def _unique(values: list[str]) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for value in values: + cleaned = re.sub(r"\s+", " ", str(value)).strip() + if not cleaned or cleaned in seen: + continue + seen.add(cleaned) + output.append(cleaned) + return output + + +def _wikilink_for_compiled(path: Path, wiki_root: Path) -> str: + return path.relative_to(wiki_root).with_suffix("").as_posix() + + +def _yaml_scalar(value: str) -> str: + import json + + return json.dumps(value, ensure_ascii=False) + + +def _yaml_list(values: tuple[str, ...] | list[str]) -> str: + return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]" diff --git a/build/lib/d_wiki/llm.py b/build/lib/d_wiki/llm.py new file mode 100644 index 0000000..959e902 --- /dev/null +++ b/build/lib/d_wiki/llm.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class LlmSettings: + base_url: str + model: str + api_key: str + api_style: str = "anthropic" + max_tokens: int = 900 + + +@dataclass(frozen=True) +class LlmPageSynthesis: + summary: str + answer_blueprint: tuple[str, ...] + key_evidence: tuple[str, ...] + confidence: str = "medium" + + +class LlmPageSynthesizer: + def __init__(self, settings: LlmSettings) -> None: + self.settings = settings + + @classmethod + def from_environment(cls) -> "LlmPageSynthesizer | None": + base_url = os.environ.get("D_WIKI_LLM_BASE_URL") or os.environ.get("ROBOGO_LLM_BASE_URL") + model = os.environ.get("D_WIKI_LLM_MODEL") or os.environ.get("ROBOGO_LLM_MODEL") + api_key = os.environ.get("D_WIKI_LLM_API_KEY") or os.environ.get("ROBOGO_LLM_API_KEY") + if not base_url or not model or not api_key: + return None + api_style = os.environ.get("D_WIKI_LLM_API_STYLE") or os.environ.get("ROBOGO_LLM_API_STYLE") or "anthropic" + max_tokens = int(os.environ.get("D_WIKI_LLM_MAX_TOKENS", "900")) + return cls(LlmSettings(base_url=base_url, model=model, api_key=api_key, api_style=api_style, max_tokens=max_tokens)) + + def synthesize(self, prompt: str) -> LlmPageSynthesis | None: + text = self._complete(prompt).strip() + payload = _extract_json(text) + if not isinstance(payload, dict): + return None + summary = str(payload.get("summary") or "").strip() + if not summary: + return None + answer_blueprint = _string_tuple(payload.get("answer_blueprint")) + key_evidence = _string_tuple(payload.get("key_evidence")) + confidence = str(payload.get("confidence") or "medium").strip().lower() + if confidence not in {"low", "medium", "high"}: + confidence = "medium" + return LlmPageSynthesis( + summary=summary, + answer_blueprint=answer_blueprint, + key_evidence=key_evidence, + confidence=confidence, + ) + + def _complete(self, prompt: str) -> str: + if self.settings.api_style == "openai": + return self._complete_openai(prompt) + return self._complete_anthropic(prompt) + + def _complete_anthropic(self, prompt: str) -> str: + payload = { + "model": self.settings.model, + "max_tokens": self.settings.max_tokens, + "temperature": 0, + "system": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.", + "messages": [{"role": "user", "content": prompt}], + } + data = self._post_json(_join_url(self.settings.base_url, "v1/messages"), payload) + content = data.get("content") + if isinstance(content, list): + return "\n".join(str(item.get("text", "")) for item in content if isinstance(item, dict)) + return str(data.get("text") or "") + + def _complete_openai(self, prompt: str) -> str: + payload = { + "model": self.settings.model, + "temperature": 0, + "messages": [ + { + "role": "system", + "content": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.", + }, + {"role": "user", "content": prompt}, + ], + } + data = self._post_json(_join_url(self.settings.base_url, "v1/chat/completions"), payload) + choices = data.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message") + if isinstance(message, dict): + return str(message.get("content") or "") + return "" + + def _post_json(self, url: str, payload: dict[str, Any]) -> dict[str, Any]: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + request = urllib.request.Request( + url, + data=body, + method="POST", + headers={ + "content-type": "application/json", + "accept": "application/json", + "x-api-key": self.settings.api_key, + "authorization": f"Bearer {self.settings.api_key}", + "anthropic-version": "2023-06-01", + }, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="ignore") + raise RuntimeError(f"D_WIKI_LLM_HTTP_ERROR: {exc.code} {detail[:300]}") from exc + + +def _join_url(base_url: str, suffix: str) -> str: + base = base_url.rstrip("/") + if base.endswith("/v1/messages") or base.endswith("/v1/chat/completions"): + return base + if base.endswith("/v1"): + return f"{base}/{suffix.removeprefix('v1/')}" + return f"{base}/{suffix}" + + +def _extract_json(text: str) -> Any: + cleaned = text.strip() + if cleaned.startswith("```"): + cleaned = cleaned.strip("`") + if cleaned.lower().startswith("json"): + cleaned = cleaned[4:].strip() + decoder = json.JSONDecoder() + for index, char in enumerate(cleaned): + if char != "{": + continue + try: + value, _end = decoder.raw_decode(cleaned[index:]) + except json.JSONDecodeError: + continue + return value + return None + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, list): + return () + return tuple(str(item).strip() for item in value if str(item).strip()) diff --git a/build/lib/d_wiki/metadata.py b/build/lib/d_wiki/metadata.py new file mode 100644 index 0000000..dd41dfd --- /dev/null +++ b/build/lib/d_wiki/metadata.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .yaml_compat import safe_load +from .tokenize import tokenize + +QUERY_MATCH_STOP_TOKENS = { + "如何", + "怎么", + "使用", + "创建", + "知道", + "功能", + "资产", +} + + +@dataclass(frozen=True) +class AliasGroup: + name: str + terms: tuple[str, ...] + intent: str | None + related: tuple[str, ...] + + +@dataclass(frozen=True) +class QueryEntry: + path: Path + title: str + aliases: tuple[str, ...] + intent: str | None + primary_sources: tuple[str, ...] + supporting_sources: tuple[str, ...] + answer_requirements: tuple[str, ...] + + +def load_alias_groups(path: Path | None) -> list[AliasGroup]: + if path is None or not path.exists(): + return [] + data = safe_load(path.read_text(encoding="utf-8")) or {} + raw_aliases = data.get("aliases", {}) + if not isinstance(raw_aliases, dict): + return [] + groups: list[AliasGroup] = [] + for name, raw in raw_aliases.items(): + if not isinstance(raw, dict): + continue + groups.append( + AliasGroup( + name=str(name), + terms=tuple(str(term).strip() for term in raw.get("terms", []) if str(term).strip()), + intent=str(raw["intent"]) if raw.get("intent") else None, + related=tuple(str(item).strip() for item in raw.get("related", []) if str(item).strip()), + ) + ) + return groups + + +def load_query_entries(directory: Path | None) -> list[QueryEntry]: + if directory is None or not directory.exists(): + return [] + return [_query_entry(path) for path in sorted(directory.rglob("*.md")) if path.is_file()] + + +def frontmatter(text: str) -> tuple[dict[str, Any], str]: + if not text.startswith("---\n"): + return {}, text + end = text.find("\n---", 4) + if end < 0: + return {}, text + meta = safe_load(text[4:end]) or {} + if not isinstance(meta, dict): + meta = {} + return meta, text[end + 4 :].lstrip("\n") + + +def query_matches_entry(query: str, entry: QueryEntry) -> bool: + normalized = normalize(query) + if any(normalize(alias) in normalized or normalized in normalize(alias) for alias in entry.aliases): + return True + query_terms = _meaningful_query_terms(query) + for candidate in (entry.title, *entry.aliases): + candidate_terms = _meaningful_query_terms(candidate) + if len(query_terms & candidate_terms) >= 2: + return True + return False + + +def _meaningful_query_terms(value: str) -> set[str]: + return {term for term in tokenize(value) if term not in QUERY_MATCH_STOP_TOKENS and len(term.strip()) >= 2} + + +def normalize(value: str) -> str: + return re.sub(r"\s+", " ", value.lower()).strip() + + +def _query_entry(path: Path) -> QueryEntry: + meta, body = frontmatter(path.read_text(encoding="utf-8")) + title = str(meta.get("title") or _first_heading(body) or path.stem) + return QueryEntry( + path=path, + title=title, + aliases=tuple(str(item).strip() for item in meta.get("aliases", []) if str(item).strip()), + intent=str(meta["intent"]) if meta.get("intent") else None, + primary_sources=tuple(str(item).strip() for item in meta.get("primary_sources", []) if str(item).strip()), + supporting_sources=tuple(str(item).strip() for item in meta.get("supporting_sources", []) if str(item).strip()), + answer_requirements=tuple(str(item).strip() for item in meta.get("answer_requirements", []) if str(item).strip()), + ) + + +def _first_heading(text: str) -> str | None: + for line in text.splitlines(): + if match := re.match(r"^#{1,6}\s+(.+)$", line): + return match.group(1).strip() + return None diff --git a/build/lib/d_wiki/search.py b/build/lib/d_wiki/search.py new file mode 100644 index 0000000..07e396f --- /dev/null +++ b/build/lib/d_wiki/search.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import json +import math +import re +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .config import KnowledgeBaseConfig +from .metadata import ( + AliasGroup, + QueryEntry, + frontmatter, + load_alias_groups, + load_query_entries, + normalize, + query_matches_entry, +) +from .dreams import load_dream_query_entries +from .tokenize import tokenize + +SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt") +MAX_BLOCK_CHARS = 520 + + +@dataclass(frozen=True) +class Chunk: + title: str + wikilink: str + source_path: Path + page_type: str + heading: str + text: str + tokens: tuple[str, ...] + index: int + + +def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]: + return { + "status": "ok", + "config": str(config_path), + "knowledge_bases": { + name: { + "root": str(kb.root), + "source_type": kb.source_type, + "exists": kb.root.exists(), + "aliases": str(kb.aliases_path) if kb.aliases_path else None, + "queries": str(kb.queries_dir) if kb.queries_dir else None, + } + for name, kb in bases.items() + }, + } + + +def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]: + chunks = load_chunks(kb) + index_dir = kb.root / ".d_wiki_index" + index_dir.mkdir(exist_ok=True) + payload = { + "kb": kb.name, + "root": str(kb.root), + "source_type": kb.source_type, + "compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None, + "chunks": len(chunks), + "pages": len({chunk.wikilink for chunk in chunks}), + "indexed_at": datetime.now(timezone.utc).isoformat(), + } + (index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + return {"status": "indexed", **payload} + + +def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]: + aliases = load_alias_groups(kb.aliases_path) + query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)] + matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)] + expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries) + chunks = load_chunks(kb) + ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries) + selected = ranked[: max(1, min(max_results, 8))] + context_pack = build_context_pack(query, selected, matched_aliases, matched_entries) + return { + "status": "found" if selected else "no_match", + "query": query, + "knowledge_base": kb.name, + "backend": "lightweight_bm25", + "quality": quality(selected), + "sources": [source_summary(item[1], item[0]) for item in selected], + "context_pack": context_pack, + } + + +def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]: + chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink] + if not chunks: + return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink} + first = chunks[0] + return { + "status": "found", + "knowledge_base": kb.name, + "id": wikilink, + "title": first.title, + "source_path": str(first.source_path), + "blocks": [ + {"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"} + for chunk in chunks + ], + } + + +def expand_query( + query: str, + aliases: list[AliasGroup], + entries: list[QueryEntry], +) -> tuple[list[str], list[AliasGroup]]: + terms = tokenize(query) + normalized_query = normalize(query) + matched: list[AliasGroup] = [] + for group in aliases: + if any(normalize(term) in normalized_query for term in group.terms): + matched.append(group) + for term in group.terms: + terms.extend(tokenize(term)) + for entry in entries: + terms.extend(tokenize(entry.title)) + for alias in entry.aliases: + terms.extend(tokenize(alias)) + return terms, matched + + +def rank_chunks( + chunks: list[Chunk], + terms: list[str], + aliases: list[AliasGroup], + entries: list[QueryEntry], +) -> list[tuple[float, Chunk]]: + if not chunks or not terms: + return [] + term_counts = [Counter(chunk.tokens) for chunk in chunks] + doc_freq: Counter[str] = Counter() + for counts in term_counts: + for term in counts: + doc_freq[term] += 1 + avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks)) + query_counts = Counter(terms) + ranked: list[tuple[float, Chunk]] = [] + related = {item for group in aliases for item in group.related} + primary = {item for entry in entries for item in entry.primary_sources} + supporting = {item for entry in entries for item in entry.supporting_sources} + for chunk, counts in zip(chunks, term_counts): + score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts) + if any(wikilink_matches(chunk.wikilink, item) for item in related): + score += 8.0 + if any(wikilink_matches(chunk.wikilink, item) for item in primary): + score += 16.0 + if any(wikilink_matches(chunk.wikilink, item) for item in supporting): + score += 5.0 + score += heading_score_adjustment(chunk.heading) + if score > 0: + ranked.append((round(score, 3), chunk)) + ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index)) + return ranked + + +def heading_score_adjustment(heading: str) -> float: + if heading == "Retrieval Hints": + return -80.0 + if heading == "Intent Model": + return -40.0 + if heading == "Answer Blueprint": + return -25.0 + if heading == "Evidence Graph": + return 3.0 + if heading == "Key Evidence": + return 5.0 + return 0.0 + + +def bm25_score( + counts: Counter[str], + doc_len: int, + doc_freq: Counter[str], + doc_count: int, + avg_len: float, + query_counts: Counter[str], +) -> float: + k1 = 1.5 + b = 0.75 + score = 0.0 + for term, qf in query_counts.items(): + tf = counts.get(term, 0) + if tf == 0: + continue + idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5)) + denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1))) + score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3) + return score + + +def build_context_pack( + query: str, + selected: list[tuple[float, Chunk]], + aliases: list[AliasGroup], + entries: list[QueryEntry], +) -> dict[str, Any]: + intent = answer_intent(query, aliases, entries) + grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list) + for item in selected: + grouped[item[1].wikilink].append(item) + source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1])) + entry_primary = {item for entry in entries for item in entry.primary_sources} + primary_sources = [ + source_context(wikilink, chunks) + for wikilink, chunks in source_items + if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary) + ] + if not primary_sources: + primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]] + supporting_sources = [ + source_context(wikilink, chunks, blocks_limit=1) + for wikilink, chunks in source_items + if wikilink not in {source["wikilink"] for source in primary_sources} + ][:3] + coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete") + return { + "answer_intent": intent, + "coverage": coverage, + "answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.", + "primary_sources": primary_sources, + "supporting_sources": supporting_sources, + "answer_requirements": answer_requirements(intent, entries), + } + + +def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]: + top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit] + chunk = top[0][1] + return { + "title": chunk.title, + "wikilink": wikilink, + "source_path": str(chunk.source_path), + "reason": "Matched query terms and local metadata.", + "blocks": [ + { + "block_id": f"{block.wikilink}#{block.index}", + "heading": block.heading, + "text": trim_block(block.text), + "score": score, + } + for score, block in top + ], + } + + +def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str: + for entry in entries: + if entry.intent: + return entry.intent + for group in aliases: + if group.intent: + return group.intent + if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")): + return "procedure" + return "fact" + + +def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]: + requirements = unique([req for entry in entries for req in entry.answer_requirements]) + if requirements: + return requirements + if intent == "procedure": + return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"] + return ["直接回答问题", "引用命中的来源"] + + +def unique(values: list[str]) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for value in values: + if value in seen: + continue + seen.add(value) + output.append(value) + return output + + +def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]: + top = selected[0][0] if selected else 0.0 + return { + "confidence": "high" if top >= 8.0 else ("low" if selected else "none"), + "high_confidence": top >= 8.0, + "top_score": top, + "high_confidence_score": 8.0, + } + + +def wikilink_matches(candidate: str, expected: str) -> bool: + return candidate == expected or candidate == f"concepts/{expected}" + + +def source_summary(chunk: Chunk, score: float) -> dict[str, Any]: + return { + "title": chunk.title, + "wikilink": chunk.wikilink, + "source_path": str(chunk.source_path), + "heading": chunk.heading, + "snippet": trim_block(chunk.text, 320), + "score": score, + } + + +def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]: + root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root + paths = source_paths(root) + chunks: list[Chunk] = [] + for path in paths: + chunks.extend(chunks_for_path(path, root)) + return chunks + + +def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]: + chunks: list[Chunk] = [] + for path in source_paths(kb.root): + chunks.extend(chunks_for_path(path, kb.root)) + return chunks + + +def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path: + return kb.root / ".d_wiki" / "wiki" + + +def source_paths(root: Path) -> list[Path]: + if (root / "_sources").is_dir(): + return sorted( + path + for path in (root / "_sources").rglob("*.txt") + if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES) + ) + generated = {"SCHEMA.md", "index.md", "log.md"} + generated_dirs = {".d_wiki", ".d_wiki_index"} + is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki" + return sorted( + path + for path in root.rglob("*.md") + if path.is_file() + and path.name not in generated + and (is_compiled_root or not any(part in generated_dirs for part in path.parts)) + ) + + +def chunks_for_path(path: Path, root: Path) -> list[Chunk]: + raw = path.read_text(encoding="utf-8", errors="ignore") + meta, body = frontmatter(raw) + title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title()) + metadata_text = metadata_terms(meta) + wikilink = wikilink_for(path, root) + sections = split_sections(body, title) + chunks: list[Chunk] = [] + for index, (heading, text) in enumerate(sections): + if heading in {"Source Map", "Related"}: + continue + if not text.strip(): + continue + field_text = " ".join([title, heading, wikilink, metadata_text, text]) + chunks.append( + Chunk( + title=title, + wikilink=wikilink, + source_path=path.resolve(), + page_type="sphinx_source" if "_sources" in path.parts else "markdown", + heading=heading, + text=text.strip(), + tokens=tuple(tokenize(field_text)), + index=index, + ) + ) + return chunks + + +def metadata_terms(meta: dict[str, Any]) -> str: + values: list[str] = [] + for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"): + raw = meta.get(key) + if isinstance(raw, str): + values.append(raw) + elif isinstance(raw, list): + values.extend(str(item) for item in raw if str(item).strip()) + return " ".join(values) + + +def split_sections(body: str, title: str) -> list[tuple[str, str]]: + heading = title + buffer: list[str] = [] + sections: list[tuple[str, str]] = [] + in_fence = False + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("```"): + in_fence = not in_fence + buffer.append(line) + continue + if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)): + if buffer: + sections.extend(paragraphs(heading, "\n".join(buffer))) + buffer = [] + heading = match.group(1).strip() or title + continue + buffer.append(line) + if buffer: + sections.extend(paragraphs(heading, "\n".join(buffer))) + return sections or [(title, body)] + + +def paragraphs(heading: str, text: str) -> list[tuple[str, str]]: + blocks: list[tuple[str, str]] = [] + for part in re.split(r"\n\s*\n", text): + cleaned = clean_block(part) + if cleaned: + blocks.append((heading, cleaned)) + return blocks + + +def clean_block(text: str) -> str: + stripped = text.strip() + if not stripped: + return "" + if re.fullmatch(r"[-*_]{3,}", stripped): + return "" + compact = " ".join(stripped.split()) + if "📷" in compact and len(compact) <= 80: + return "" + if compact.startswith("> 📷") and len(compact) <= 120: + return "" + return stripped + + +def first_heading(body: str) -> str | None: + in_fence = False + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + if match := re.match(r"^#{1,6}\s+(.+)$", line): + return match.group(1).strip() + return None + + +def wikilink_for(path: Path, root: Path) -> str: + try: + relative = path.relative_to(root) + except ValueError: + return path.stem + if relative.parts and relative.parts[0] == "_sources": + relative = Path(*relative.parts[1:]) + value = relative.as_posix() + for suffix in SPHINX_SOURCE_SUFFIXES: + if value.endswith(suffix): + return value[: -len(suffix)] + return value[:-3] if value.endswith(".md") else path.stem + + +def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str: + compact = " ".join(text.split()) + return compact if len(compact) <= limit else compact[:limit].rstrip() diff --git a/build/lib/d_wiki/tokenize.py b/build/lib/d_wiki/tokenize.py new file mode 100644 index 0000000..09823c6 --- /dev/null +++ b/build/lib/d_wiki/tokenize.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import re + +STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上") +STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"} + + +def tokenize(text: str) -> list[str]: + lowered = text.lower() + terms = re.findall(r"[a-z0-9_./+-]+", lowered) + cjk_chars = re.findall(r"[\u4e00-\u9fff]", text) + terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1))) + return [term for term in terms if _useful(term)] + + +def _useful(term: str) -> bool: + if not term or term in STOP_TERMS: + return False + if re.search(r"[\u4e00-\u9fff]", term): + return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term)) + return True diff --git a/build/lib/d_wiki/yaml_compat.py b/build/lib/d_wiki/yaml_compat.py new file mode 100644 index 0000000..61cf7ab --- /dev/null +++ b/build/lib/d_wiki/yaml_compat.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from typing import Any + +try: + import yaml as _yaml # type: ignore +except Exception: # pragma: no cover - exercised in environments without PyYAML + _yaml = None + + +def safe_load(text: str) -> Any: + if _yaml is not None: + return _yaml.safe_load(text) + lines = [ + (len(line) - len(line.lstrip(" ")), line.strip()) + for line in text.splitlines() + if line.strip() and not line.strip().startswith("#") + ] + value, _index = _parse_node(lines, 0, 0) + return value + + +def _parse_node(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]: + if index >= len(lines): + return {}, index + if lines[index][0] < indent: + return {}, index + if lines[index][1].startswith("- "): + values: list[Any] = [] + while index < len(lines) and lines[index][0] == indent and lines[index][1].startswith("- "): + values.append(_scalar(lines[index][1][2:].strip())) + index += 1 + return values, index + mapping: dict[str, Any] = {} + while index < len(lines): + current_indent, stripped = lines[index] + if current_indent < indent: + break + if current_indent > indent: + break + if ":" not in stripped: + index += 1 + continue + key, raw_value = stripped.split(":", 1) + key = key.strip() + raw_value = raw_value.strip() + index += 1 + if raw_value: + mapping[key] = _scalar(raw_value) + continue + child, index = _parse_node(lines, index, indent + 2) + mapping[key] = child + return mapping, index + + +def _scalar(value: str) -> Any: + if value in {"null", "None", "~"}: + return None + if value in {"true", "True"}: + return True + if value in {"false", "False"}: + return False + if value.startswith("[") and value.endswith("]"): + inner = value[1:-1].strip() + if not inner: + return [] + return [_scalar(item.strip()) for item in inner.split(",")] + if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")): + return value[1:-1] + return value diff --git a/d-wiki b/d-wiki new file mode 100755 index 0000000..bb3f6e8 --- /dev/null +++ b/d-wiki @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from d_wiki.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/d-wiki.yml b/d-wiki.yml new file mode 100644 index 0000000..17ebf2b --- /dev/null +++ b/d-wiki.yml @@ -0,0 +1,11 @@ +knowledge_bases: + x5: + root: "/Users/d-robotics/Desktop/evb/sdk_management_system/x5_user_manual/latest" + source_type: "sphinx" + aliases: "knowledge/x5/aliases.yml" + queries: "knowledge/x5/queries" + robogo: + root: "/Users/d-robotics/Documents/New project/read/robogo-user-guide" + source_type: "markdown" + aliases: "knowledge/robogo/aliases.yml" + queries: "knowledge/robogo/queries" diff --git a/d_wiki.egg-info/PKG-INFO b/d_wiki.egg-info/PKG-INFO new file mode 100644 index 0000000..36c79f0 --- /dev/null +++ b/d_wiki.egg-info/PKG-INFO @@ -0,0 +1,5 @@ +Metadata-Version: 2.4 +Name: d-wiki +Version: 0.1.0 +Summary: Lightweight local knowledge retrieval with BM25 and metadata context packs. +Requires-Python: >=3.10 diff --git a/d_wiki.egg-info/SOURCES.txt b/d_wiki.egg-info/SOURCES.txt new file mode 100644 index 0000000..589d687 --- /dev/null +++ b/d_wiki.egg-info/SOURCES.txt @@ -0,0 +1,19 @@ +README.md +pyproject.toml +d_wiki/__init__.py +d_wiki/__main__.py +d_wiki/cli.py +d_wiki/compile.py +d_wiki/config.py +d_wiki/dreams.py +d_wiki/llm.py +d_wiki/metadata.py +d_wiki/search.py +d_wiki/tokenize.py +d_wiki/yaml_compat.py +d_wiki.egg-info/PKG-INFO +d_wiki.egg-info/SOURCES.txt +d_wiki.egg-info/dependency_links.txt +d_wiki.egg-info/entry_points.txt +d_wiki.egg-info/top_level.txt +tests/test_d_wiki.py \ No newline at end of file diff --git a/d_wiki.egg-info/dependency_links.txt b/d_wiki.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/d_wiki.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/d_wiki.egg-info/entry_points.txt b/d_wiki.egg-info/entry_points.txt new file mode 100644 index 0000000..44b01d7 --- /dev/null +++ b/d_wiki.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +d-wiki = d_wiki.cli:main diff --git a/d_wiki.egg-info/top_level.txt b/d_wiki.egg-info/top_level.txt new file mode 100644 index 0000000..c891388 --- /dev/null +++ b/d_wiki.egg-info/top_level.txt @@ -0,0 +1 @@ +d_wiki diff --git a/d_wiki/__init__.py b/d_wiki/__init__.py new file mode 100644 index 0000000..365f5d0 --- /dev/null +++ b/d_wiki/__init__.py @@ -0,0 +1,3 @@ +"""d-wiki lightweight local knowledge retrieval.""" + +__version__ = "0.1.0" diff --git a/d_wiki/__main__.py b/d_wiki/__main__.py new file mode 100644 index 0000000..bfdcd0c --- /dev/null +++ b/d_wiki/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/d_wiki/__pycache__/__init__.cpython-311.pyc b/d_wiki/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..c8e3ec3 Binary files /dev/null and b/d_wiki/__pycache__/__init__.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/__init__.cpython-312.pyc b/d_wiki/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..361d9ba Binary files /dev/null and b/d_wiki/__pycache__/__init__.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/__init__.cpython-314.pyc b/d_wiki/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..5c6a96c Binary files /dev/null and b/d_wiki/__pycache__/__init__.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/__main__.cpython-312.pyc b/d_wiki/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000..fd6e867 Binary files /dev/null and b/d_wiki/__pycache__/__main__.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/__main__.cpython-314.pyc b/d_wiki/__pycache__/__main__.cpython-314.pyc new file mode 100644 index 0000000..142d54a Binary files /dev/null and b/d_wiki/__pycache__/__main__.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/cli.cpython-311.pyc b/d_wiki/__pycache__/cli.cpython-311.pyc new file mode 100644 index 0000000..ed3bfe9 Binary files /dev/null and b/d_wiki/__pycache__/cli.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/cli.cpython-312.pyc b/d_wiki/__pycache__/cli.cpython-312.pyc new file mode 100644 index 0000000..d891e1b Binary files /dev/null and b/d_wiki/__pycache__/cli.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/cli.cpython-314.pyc b/d_wiki/__pycache__/cli.cpython-314.pyc new file mode 100644 index 0000000..795d035 Binary files /dev/null and b/d_wiki/__pycache__/cli.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/compile.cpython-311.pyc b/d_wiki/__pycache__/compile.cpython-311.pyc new file mode 100644 index 0000000..fd4105f Binary files /dev/null and b/d_wiki/__pycache__/compile.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/compile.cpython-312.pyc b/d_wiki/__pycache__/compile.cpython-312.pyc new file mode 100644 index 0000000..bd61aeb Binary files /dev/null and b/d_wiki/__pycache__/compile.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/compile.cpython-314.pyc b/d_wiki/__pycache__/compile.cpython-314.pyc new file mode 100644 index 0000000..ebdd9e1 Binary files /dev/null and b/d_wiki/__pycache__/compile.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/config.cpython-311.pyc b/d_wiki/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000..cfcfaf6 Binary files /dev/null and b/d_wiki/__pycache__/config.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/config.cpython-312.pyc b/d_wiki/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..772d1df Binary files /dev/null and b/d_wiki/__pycache__/config.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/config.cpython-314.pyc b/d_wiki/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000..6c8e8f1 Binary files /dev/null and b/d_wiki/__pycache__/config.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/dreams.cpython-311.pyc b/d_wiki/__pycache__/dreams.cpython-311.pyc new file mode 100644 index 0000000..698d4a2 Binary files /dev/null and b/d_wiki/__pycache__/dreams.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/dreams.cpython-312.pyc b/d_wiki/__pycache__/dreams.cpython-312.pyc new file mode 100644 index 0000000..5ec62ba Binary files /dev/null and b/d_wiki/__pycache__/dreams.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/dreams.cpython-314.pyc b/d_wiki/__pycache__/dreams.cpython-314.pyc new file mode 100644 index 0000000..3458ba5 Binary files /dev/null and b/d_wiki/__pycache__/dreams.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/llm.cpython-311.pyc b/d_wiki/__pycache__/llm.cpython-311.pyc new file mode 100644 index 0000000..3647b88 Binary files /dev/null and b/d_wiki/__pycache__/llm.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/llm.cpython-312.pyc b/d_wiki/__pycache__/llm.cpython-312.pyc new file mode 100644 index 0000000..c1b0884 Binary files /dev/null and b/d_wiki/__pycache__/llm.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/llm.cpython-314.pyc b/d_wiki/__pycache__/llm.cpython-314.pyc new file mode 100644 index 0000000..52d1dc1 Binary files /dev/null and b/d_wiki/__pycache__/llm.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/metadata.cpython-311.pyc b/d_wiki/__pycache__/metadata.cpython-311.pyc new file mode 100644 index 0000000..22c1fb9 Binary files /dev/null and b/d_wiki/__pycache__/metadata.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/metadata.cpython-312.pyc b/d_wiki/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000..86debc0 Binary files /dev/null and b/d_wiki/__pycache__/metadata.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/metadata.cpython-314.pyc b/d_wiki/__pycache__/metadata.cpython-314.pyc new file mode 100644 index 0000000..42e04a2 Binary files /dev/null and b/d_wiki/__pycache__/metadata.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/search.cpython-311.pyc b/d_wiki/__pycache__/search.cpython-311.pyc new file mode 100644 index 0000000..45040c2 Binary files /dev/null and b/d_wiki/__pycache__/search.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/search.cpython-312.pyc b/d_wiki/__pycache__/search.cpython-312.pyc new file mode 100644 index 0000000..ae836f0 Binary files /dev/null and b/d_wiki/__pycache__/search.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/search.cpython-314.pyc b/d_wiki/__pycache__/search.cpython-314.pyc new file mode 100644 index 0000000..3312bf4 Binary files /dev/null and b/d_wiki/__pycache__/search.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/tokenize.cpython-311.pyc b/d_wiki/__pycache__/tokenize.cpython-311.pyc new file mode 100644 index 0000000..055c5e5 Binary files /dev/null and b/d_wiki/__pycache__/tokenize.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/tokenize.cpython-312.pyc b/d_wiki/__pycache__/tokenize.cpython-312.pyc new file mode 100644 index 0000000..5e1bd7b Binary files /dev/null and b/d_wiki/__pycache__/tokenize.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/tokenize.cpython-314.pyc b/d_wiki/__pycache__/tokenize.cpython-314.pyc new file mode 100644 index 0000000..e275cc1 Binary files /dev/null and b/d_wiki/__pycache__/tokenize.cpython-314.pyc differ diff --git a/d_wiki/__pycache__/yaml_compat.cpython-311.pyc b/d_wiki/__pycache__/yaml_compat.cpython-311.pyc new file mode 100644 index 0000000..8425d15 Binary files /dev/null and b/d_wiki/__pycache__/yaml_compat.cpython-311.pyc differ diff --git a/d_wiki/__pycache__/yaml_compat.cpython-312.pyc b/d_wiki/__pycache__/yaml_compat.cpython-312.pyc new file mode 100644 index 0000000..50d28b0 Binary files /dev/null and b/d_wiki/__pycache__/yaml_compat.cpython-312.pyc differ diff --git a/d_wiki/__pycache__/yaml_compat.cpython-314.pyc b/d_wiki/__pycache__/yaml_compat.cpython-314.pyc new file mode 100644 index 0000000..32030cc Binary files /dev/null and b/d_wiki/__pycache__/yaml_compat.cpython-314.pyc differ diff --git a/d_wiki/cli.py b/d_wiki/cli.py new file mode 100644 index 0000000..92d81fc --- /dev/null +++ b/d_wiki/cli.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from .config import DWikiConfig, load_config +from .compile import compile_kb +from .dreams import dream_kb +from .search import get_page, index_payload, search, status_payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="d-wiki") + parser.add_argument("--config", type=Path, default=None) + subparsers = parser.add_subparsers(dest="command", required=True) + + status_parser = subparsers.add_parser("status") + status_parser.add_argument("--format", choices=["json"], default="json") + + index_parser = subparsers.add_parser("index") + index_parser.add_argument("--kb", required=True) + index_parser.add_argument("--format", choices=["json"], default="json") + + compile_parser = subparsers.add_parser("compile") + compile_parser.add_argument("--kb", required=True) + compile_parser.add_argument("--force", action="store_true") + compile_parser.add_argument("--llm", choices=["off", "auto", "require"], default="off") + compile_parser.add_argument("--llm-scope", choices=["queries", "all"], default="queries") + compile_parser.add_argument("--format", choices=["json"], default="json") + + dream_parser = subparsers.add_parser("dream") + dream_parser.add_argument("--kb", required=True) + dream_parser.add_argument("--force", action="store_true") + dream_parser.add_argument("--format", choices=["json"], default="json") + + search_parser = subparsers.add_parser("search") + search_parser.add_argument("--kb", required=True) + search_parser.add_argument("--query", required=True) + search_parser.add_argument("--max-results", type=int, default=5) + search_parser.add_argument("--format", choices=["json"], default="json") + + get_parser = subparsers.add_parser("get") + get_parser.add_argument("--kb", required=True) + get_parser.add_argument("--id", required=True) + get_parser.add_argument("--format", choices=["json"], default="json") + + args = parser.parse_args(argv) + try: + config = load_config(args.config) + payload = dispatch(args, config) + except Exception as exc: + print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 2 + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +def dispatch(args: argparse.Namespace, config: DWikiConfig) -> dict[str, Any]: + if args.command == "status": + return status_payload(config.path, config.knowledge_bases) + kb = config.knowledge_bases.get(args.kb) + if kb is None: + raise ValueError(f"D_WIKI_KB_NOT_FOUND: {args.kb}") + if args.command == "index": + return index_payload(kb) + if args.command == "compile": + return compile_kb(kb, force=args.force, llm_mode=args.llm, llm_scope=args.llm_scope) + if args.command == "dream": + return dream_kb(kb, force=args.force) + if args.command == "search": + return search(kb, args.query, args.max_results) + if args.command == "get": + return get_page(kb, args.id) + raise ValueError(f"D_WIKI_UNKNOWN_COMMAND: {args.command}") diff --git a/d_wiki/compile.py b/d_wiki/compile.py new file mode 100644 index 0000000..bd90cae --- /dev/null +++ b/d_wiki/compile.py @@ -0,0 +1,590 @@ +from __future__ import annotations + +import re +import shutil +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .config import KnowledgeBaseConfig +from .llm import LlmPageSynthesis, LlmPageSynthesizer +from .metadata import load_alias_groups, load_query_entries +from .search import Chunk, compiled_wiki_root, load_source_chunks, trim_block +from .tokenize import tokenize + + +@dataclass(frozen=True) +class CompiledPage: + wikilink: str + title: str + path: Path + + +@dataclass(frozen=True) +class _EvidenceProfile: + tokens: frozenset[str] + phrases: tuple[str, ...] + + +def compile_kb( + kb: KnowledgeBaseConfig, + *, + force: bool = False, + llm_mode: str = "off", + llm_scope: str = "queries", +) -> dict[str, Any]: + chunks = load_source_chunks(kb) + if not chunks: + raise ValueError("D_WIKI_COMPILE_NO_SOURCE_CHUNKS") + synthesizer = LlmPageSynthesizer.from_environment() if llm_mode in {"auto", "require"} else None + if llm_mode == "require" and synthesizer is None: + raise ValueError("D_WIKI_LLM_CONFIG_REQUIRED") + wiki_root = compiled_wiki_root(kb) + concepts_dir = wiki_root / "concepts" + raw_dir = wiki_root / "raw" + if force: + shutil.rmtree(concepts_dir, ignore_errors=True) + concepts_dir.mkdir(parents=True, exist_ok=True) + raw_dir.mkdir(parents=True, exist_ok=True) + + pages: list[CompiledPage] = [] + grouped = _group_chunks(chunks) + query_entries = load_query_entries(kb.queries_dir) + alias_groups = load_alias_groups(kb.aliases_path) + query_by_source = { + source: entry + for entry in query_entries + for source in entry.primary_sources + } + aliases_by_source = { + related: group + for group in alias_groups + for related in group.related + } + + for wikilink, page_chunks in sorted(grouped.items()): + page = _compile_page( + kb, + wikilink, + page_chunks, + query_entry=query_by_source.get(wikilink), + alias_group=aliases_by_source.get(wikilink), + synthesizer=( + synthesizer + if llm_scope == "all" or query_by_source.get(wikilink) is not None + else None + ), + require_llm=llm_mode == "require", + ) + if force or not page.path.exists(): + page.path.write_text(page.content, encoding="utf-8") + pages.append(CompiledPage(wikilink=wikilink, title=page.title, path=page.path)) + + _write_schema(wiki_root, kb) + _write_index(wiki_root, pages) + _append_log(wiki_root, kb, len(chunks), len(pages)) + return { + "status": "compiled", + "kb": kb.name, + "source_root": str(kb.root), + "wiki_root": str(wiki_root), + "source_chunks": len(chunks), + "pages": len(pages), + "llm_mode": llm_mode, + "llm_scope": llm_scope, + "llm_enabled": synthesizer is not None, + "compiled_at": datetime.now(timezone.utc).isoformat(), + } + + +@dataclass(frozen=True) +class _PageDraft: + title: str + path: Path + content: str + + +def _compile_page( + kb: KnowledgeBaseConfig, + wikilink: str, + chunks: list[Chunk], + *, + query_entry: Any | None, + alias_group: Any | None, + synthesizer: LlmPageSynthesizer | None, + require_llm: bool, +) -> _PageDraft: + first = chunks[0] + title = _title_for(first, query_entry) + slug = _slug(wikilink) + path = compiled_wiki_root(kb) / "concepts" / f"{slug}.md" + evidence_profile = _evidence_profile(title, query_entry, alias_group) + tags = _tags(kb, first, alias_group) + aliases = _aliases(title, query_entry, alias_group) + source_ref = first.wikilink + source_path = str(first.source_path) + key_blocks = _key_blocks(chunks, evidence_profile) + intent = _intent(title, query_entry, alias_group) + summary = _summary(chunks, title, evidence_profile, intent=intent) + requirements = _answer_requirements(intent, query_entry) + synthesis = _llm_synthesis( + synthesizer, + title=title, + wikilink=wikilink, + query_entry=query_entry, + alias_group=alias_group, + key_blocks=key_blocks, + require_llm=require_llm, + ) + if synthesis is not None: + summary = synthesis.summary + retrieval_hints = _retrieval_hints(title, aliases, tags, requirements, alias_group) + blueprint = ( + list(synthesis.answer_blueprint) + if synthesis and synthesis.answer_blueprint + else _deterministic_answer_blueprint(intent, requirements, key_blocks) + ) + links = _related_links(query_entry, alias_group) + content = "\n".join( + [ + "---", + f"llmWikiVersion: \"1\"", + f"conceptId: {_yaml_scalar(slug)}", + f"title: {_yaml_scalar(title)}", + f"summary: {_yaml_scalar(summary)}", + f"sources: {_yaml_list([source_ref])}", + f"source_paths: {_yaml_list([source_path])}", + f"kind: concept", + f"tags: {_yaml_list(tags)}", + f"aliases: {_yaml_list(aliases)}", + f"intents: {_yaml_list([intent])}", + f"retrieval_hints: {_yaml_list(retrieval_hints)}", + f"answer_requirements: {_yaml_list(requirements)}", + f"confidence: {synthesis.confidence if synthesis else 'medium'}", + f"provenanceState: {'llm_compiled' if synthesis else 'compiled'}", + "---", + "", + f"# {title}", + "", + summary, + "", + "## Intent Model", + "", + f"- intent: {intent}", + f"- coverage: {'procedure' if intent == 'procedure' else 'fact'}", + f"- answer_requirements: {';'.join(requirements) if requirements else '直接基于证据回答'}", + "", + "## Retrieval Hints", + "", + *[f"- {hint}" for hint in retrieval_hints], + "", + "## Answer Blueprint", + "", + *[f"- {item}" for item in blueprint], + "", + "## Synthesized Evidence" if synthesis and synthesis.key_evidence else "", + "" if synthesis and synthesis.key_evidence else "", + *[f"- {item}" for item in (synthesis.key_evidence if synthesis else ())], + "" if synthesis and synthesis.key_evidence else "", + "## Evidence Graph", + "", + *_evidence_graph_lines(key_blocks), + "", + "## Key Evidence", + "", + *[f"- {trim_block(block.text, 420)} ^[{block.wikilink}#{block.index}]" for block in key_blocks], + "", + "## Source Map", + "", + f"- 原始来源:`{source_ref}`", + f"- 文件路径:`{source_path}`", + "", + "## Related", + "", + *[f"- [[{link}]]" for link in links], + "", + ] + ) + return _PageDraft(title=title, path=path, content=content) + + +def _llm_synthesis( + synthesizer: LlmPageSynthesizer | None, + *, + title: str, + wikilink: str, + query_entry: Any | None, + alias_group: Any | None, + key_blocks: list[Chunk], + require_llm: bool, +) -> LlmPageSynthesis | None: + if synthesizer is None: + return None + prompt = _llm_compile_prompt( + title=title, + wikilink=wikilink, + query_entry=query_entry, + alias_group=alias_group, + key_blocks=key_blocks, + ) + try: + return synthesizer.synthesize(prompt) + except Exception: + if require_llm: + raise + return None + + +def _llm_compile_prompt( + *, + title: str, + wikilink: str, + query_entry: Any | None, + alias_group: Any | None, + key_blocks: list[Chunk], +) -> str: + aliases = list(getattr(query_entry, "aliases", ()) if query_entry is not None else ()) + terms = list(getattr(alias_group, "terms", ()) if alias_group is not None else ()) + requirements = list(getattr(query_entry, "answer_requirements", ()) if query_entry is not None else ()) + intent = getattr(query_entry, "intent", None) if query_entry is not None else getattr(alias_group, "intent", None) + evidence = [ + { + "block_id": f"{block.wikilink}#{block.index}", + "heading": block.heading, + "text": trim_block(block.text, 700), + } + for block in key_blocks[:8] + ] + return "\n".join( + [ + "Compile a d-wiki concept page from the local evidence below.", + "Do not invent facts. Use only the evidence text. Prefer Chinese output when the title or aliases are Chinese.", + "For procedure intent, make answer_blueprint actionable and ordered. For fact intent, put the direct answer first.", + "Return only JSON with this shape:", + '{"summary": "...", "answer_blueprint": ["..."], "key_evidence": ["..."], "confidence": "low|medium|high"}', + "", + json_dumps( + { + "title": title, + "wikilink": wikilink, + "intent": intent, + "aliases": aliases, + "terms": terms, + "answer_requirements": requirements, + "evidence": evidence, + } + ), + ] + ) + + +def _answer_blueprint_lines(synthesis: LlmPageSynthesis | None) -> list[str]: + if synthesis is None or not synthesis.answer_blueprint: + return [] + return ["## Answer Blueprint", "", *[f"- {item}" for item in synthesis.answer_blueprint], ""] + + +def _intent(title: str, query_entry: Any | None, alias_group: Any | None) -> str: + if query_entry is not None and getattr(query_entry, "intent", ""): + return str(query_entry.intent) + if alias_group is not None and getattr(alias_group, "intent", ""): + return str(alias_group.intent) + lowered = title.lower() + if any(term in lowered for term in ("如何", "怎么", "指南", "debug", "调试", "配置", "bring up", "生成")): + return "procedure" + return "fact" + + +def _answer_requirements(intent: str, query_entry: Any | None) -> list[str]: + if query_entry is not None: + requirements = [str(item) for item in getattr(query_entry, "answer_requirements", ()) if str(item).strip()] + if requirements: + return _unique(requirements) + if intent == "procedure": + return ["按步骤回答", "说明关键前提", "说明验证方式", "引用证据来源"] + return ["直接给出结论", "保留关键数值和单位", "引用证据来源"] + + +def _retrieval_hints( + title: str, + aliases: list[str], + tags: list[str], + requirements: list[str], + alias_group: Any | None, +) -> list[str]: + values = [title, *aliases, *tags, *requirements] + if alias_group is not None: + values.extend(str(item) for item in getattr(alias_group, "terms", ())) + hints: list[str] = [] + for value in values: + hints.append(value) + hints.extend(tokenize(value)) + return _unique(hints)[:32] + + +def _deterministic_answer_blueprint(intent: str, requirements: list[str], key_blocks: list[Chunk]) -> list[str]: + if requirements: + return requirements[:6] + if intent == "procedure": + headings = _unique([block.heading for block in key_blocks if block.heading]) + if headings: + return [f"围绕「{heading}」组织步骤和注意事项" for heading in headings[:5]] + return ["按前提、配置、执行、验证的顺序回答"] + return ["先给出直接结论", "补充关键参数、限制和来源"] + + +def _evidence_graph_lines(key_blocks: list[Chunk]) -> list[str]: + lines: list[str] = [] + for block in key_blocks[:8]: + claim = trim_block(block.text, 180) + if not claim: + continue + lines.append(f"- claim: {claim}") + lines.append(f" source: {block.wikilink}#{block.index}") + if block.heading: + lines.append(f" heading: {block.heading}") + return lines or ["- claim: 暂无可用证据", " source: unknown"] + + +def _group_chunks(chunks: list[Chunk]) -> dict[str, list[Chunk]]: + grouped: dict[str, list[Chunk]] = {} + for chunk in chunks: + grouped.setdefault(chunk.wikilink, []).append(chunk) + return grouped + + +def _title_for(chunk: Chunk, query_entry: Any | None) -> str: + if query_entry is not None and getattr(query_entry, "title", ""): + return str(query_entry.title) + return chunk.title + + +def _summary( + chunks: list[Chunk], + title: str, + evidence_profile: _EvidenceProfile | None = None, + *, + intent: str = "fact", +) -> str: + if intent == "procedure": + for chunk in chunks[:12]: + text = trim_block(chunk.text, 220) + if text and not _is_low_signal(text) and re.search(r"(概述|流程|过程|用于|帮助|点亮|调试)", text): + return f"{title}:{text}" + ranked = _rank_evidence(chunks, evidence_profile) + for score, chunk in ranked: + if score <= 0: + continue + text = trim_block(chunk.text, 220) + if text and not _is_low_signal(text): + return f"{title}:{text}" + for chunk in chunks: + text = trim_block(chunk.text, 220) + if text and not _is_low_signal(text) and len(text) >= 20: + return f"{title}:{text}" + return f"{title} 的编译知识页,来源于本地文档。" + + +def _key_blocks(chunks: list[Chunk], evidence_profile: _EvidenceProfile | None = None) -> list[Chunk]: + preferred = [ + chunk + for chunk in chunks + if _is_evidence_candidate(chunk.text) + ] + ranked = _rank_evidence(preferred, evidence_profile) + if not ranked or ranked[0][0] <= 0: + return (preferred or chunks)[:6] + selected: list[Chunk] = [] + seen: set[tuple[str, int]] = set() + for score, chunk in ranked: + if score <= 0: + continue + key = (chunk.wikilink, chunk.index) + if key in seen: + continue + selected.append(chunk) + seen.add(key) + if len(selected) >= 6: + return selected + for chunk in preferred: + key = (chunk.wikilink, chunk.index) + if key in seen: + continue + selected.append(chunk) + seen.add(key) + if len(selected) >= 6: + break + return selected + + +def _evidence_profile(title: str, query_entry: Any | None, alias_group: Any | None) -> _EvidenceProfile: + values = [title] + if query_entry is not None: + values.extend(str(item) for item in getattr(query_entry, "aliases", ())) + values.extend(str(item) for item in getattr(query_entry, "answer_requirements", ())) + if alias_group is not None: + values.extend(str(item) for item in getattr(alias_group, "terms", ())) + phrases = tuple(_unique([_normalize_phrase(value) for value in values if value])) + tokens: set[str] = set() + for value in values: + tokens.update(tokenize(str(value))) + return _EvidenceProfile(tokens=frozenset(tokens), phrases=phrases) + + +def _rank_evidence( + chunks: list[Chunk], + evidence_profile: _EvidenceProfile | None, +) -> list[tuple[float, Chunk]]: + if evidence_profile is None or (not evidence_profile.tokens and not evidence_profile.phrases): + return [] + ranked = [(_evidence_score(chunk, evidence_profile), chunk) for chunk in chunks] + ranked.sort(key=lambda item: (-item[0], item[1].index)) + return ranked + + +def _evidence_score(chunk: Chunk, evidence_profile: _EvidenceProfile) -> float: + text = trim_block(chunk.text, 1000) + normalized = _normalize_phrase(text) + chunk_tokens = set(chunk.tokens) | set(tokenize(text)) + score = 0.0 + score += len(chunk_tokens & evidence_profile.tokens) * 2.0 + for phrase in evidence_profile.phrases: + if len(phrase) >= 2 and phrase in normalized: + score += 5.0 + if "tops" in evidence_profile.tokens and re.search(r"\b\d+(?:\.\d+)?\s*tops\b", normalized): + score += 8.0 + if "算力" in evidence_profile.tokens and "tops" in normalized: + score += 6.0 + if "bpu" in evidence_profile.tokens and "bpu" in normalized: + score += 4.0 + if re.search(r"\b\d+(?:\.\d+)?\s*(tops|ghz|gb|mb|fps|v|a|w)\b", normalized): + score += 2.0 + if _is_low_signal(text): + score -= 8.0 + return score + + +def _is_evidence_candidate(text: str) -> bool: + stripped = text.lstrip() + return not stripped.startswith("|") and not stripped.startswith(">") + + +def _is_low_signal(text: str) -> bool: + stripped = text.strip() + without_marks = re.sub(r"[*_`#\-\s]+", "", stripped) + if len(without_marks) < 8: + return True + if stripped.startswith("```") or "```" in stripped: + return True + if stripped.startswith("|"): + return True + if "![](" in stripped: + return True + if re.search(r"(^|\n)\s*[├│└─]{2,}", stripped): + return True + return False + + +def _tags(kb: KnowledgeBaseConfig, chunk: Chunk, alias_group: Any | None) -> list[str]: + tags = [kb.name] + if alias_group is not None: + tags.append(_slug(str(alias_group.name)).replace("--", "-")) + for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", chunk.title): + lowered = token.lower() + if lowered not in tags: + tags.append(lowered) + return tags[:8] + + +def _aliases(title: str, query_entry: Any | None, alias_group: Any | None) -> list[str]: + values = [title] + if query_entry is not None: + values.extend(str(item) for item in getattr(query_entry, "aliases", ())) + if alias_group is not None: + values.extend(str(item) for item in getattr(alias_group, "terms", ())) + return _unique(values)[:12] + + +def _related_links(query_entry: Any | None, alias_group: Any | None) -> list[str]: + links: list[str] = [] + if query_entry is not None: + links.extend(str(item) for item in getattr(query_entry, "supporting_sources", ())) + if alias_group is not None: + links.extend(str(item) for item in getattr(alias_group, "related", ())) + return _unique(links)[:10] + + +def _unique(values: list[str]) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for value in values: + cleaned = value.strip() + if not cleaned or cleaned in seen: + continue + seen.add(cleaned) + output.append(cleaned) + return output + + +def _normalize_phrase(value: str) -> str: + return re.sub(r"\s+", " ", value.lower()).strip() + + +def _write_schema(wiki_root: Path, kb: KnowledgeBaseConfig) -> None: + (wiki_root / "SCHEMA.md").write_text( + "\n".join( + [ + "# Wiki Schema", + "", + "## Domain", + f"Compiled local knowledge for `{kb.name}`.", + "", + "## Conventions", + "- Pages live under `concepts/`.", + "- Every page has frontmatter with title, summary, sources, tags, aliases, confidence, and provenanceState.", + "- Claims in Key Evidence keep provenance markers pointing back to source block ids.", + "- Raw source files remain immutable; compiled pages are generated artifacts.", + "", + ] + ), + encoding="utf-8", + ) + + +def _write_index(wiki_root: Path, pages: list[CompiledPage]) -> None: + lines = ["# Index", "", "> Content catalog for compiled d-wiki pages.", "", "## Concepts", ""] + for page in sorted(pages, key=lambda item: item.title): + rel = page.path.relative_to(wiki_root).as_posix() + lines.append(f"- [[{rel[:-3]}]] — {page.title}") + lines.append("") + (wiki_root / "index.md").write_text("\n".join(lines), encoding="utf-8") + + +def _append_log(wiki_root: Path, kb: KnowledgeBaseConfig, chunks: int, pages: int) -> None: + line = f"- {datetime.now(timezone.utc).isoformat()} compiled `{kb.name}` from {chunks} chunks into {pages} pages.\n" + path = wiki_root / "log.md" + if not path.exists(): + path.write_text("# Log\n\n", encoding="utf-8") + with path.open("a", encoding="utf-8") as handle: + handle.write(line) + + +def _slug(value: str) -> str: + slug = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "-", value).strip("-").lower() + return slug or "page" + + +def _yaml_scalar(value: str) -> str: + import json + + return json.dumps(value, ensure_ascii=False) + + +def _yaml_list(values: list[str]) -> str: + return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]" + + +def json_dumps(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, indent=2) diff --git a/d_wiki/config.py b/d_wiki/config.py new file mode 100644 index 0000000..b8bee1e --- /dev/null +++ b/d_wiki/config.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .yaml_compat import safe_load + + +@dataclass(frozen=True) +class KnowledgeBaseConfig: + name: str + root: Path + source_type: str + aliases_path: Path | None + queries_dir: Path | None + + +@dataclass(frozen=True) +class DWikiConfig: + path: Path + knowledge_bases: dict[str, KnowledgeBaseConfig] + + +def default_config_path() -> Path: + return Path(os.environ.get("D_WIKI_CONFIG", Path.cwd() / "d-wiki.yml")).expanduser() + + +def load_config(path: Path | None = None) -> DWikiConfig: + config_path = (path or default_config_path()).expanduser().resolve() + data = safe_load(config_path.read_text(encoding="utf-8")) or {} + raw_kbs = data.get("knowledge_bases") + if not isinstance(raw_kbs, dict): + raise ValueError("D_WIKI_CONFIG_MISSING_KNOWLEDGE_BASES") + bases: dict[str, KnowledgeBaseConfig] = {} + for name, raw in raw_kbs.items(): + if not isinstance(raw, dict): + raise ValueError(f"D_WIKI_KB_INVALID: {name}") + root = Path(str(raw.get("root", ""))).expanduser() + if not root.is_absolute(): + root = (config_path.parent / root).resolve() + aliases_path = _optional_path(config_path.parent, raw.get("aliases")) + queries_dir = _optional_path(config_path.parent, raw.get("queries")) + bases[str(name)] = KnowledgeBaseConfig( + name=str(name), + root=root, + source_type=str(raw.get("source_type", "markdown")), + aliases_path=aliases_path, + queries_dir=queries_dir, + ) + return DWikiConfig(path=config_path, knowledge_bases=bases) + + +def _optional_path(base: Path, value: Any) -> Path | None: + if not value: + return None + path = Path(str(value)).expanduser() + return path if path.is_absolute() else (base / path).resolve() diff --git a/d_wiki/dreams.py b/d_wiki/dreams.py new file mode 100644 index 0000000..ffec505 --- /dev/null +++ b/d_wiki/dreams.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import re +import shutil +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .config import KnowledgeBaseConfig +from .metadata import frontmatter, load_query_entries + + +@dataclass(frozen=True) +class DreamedQuery: + slug: str + title: str + aliases: tuple[str, ...] + intent: str + primary_sources: tuple[str, ...] + supporting_sources: tuple[str, ...] + answer_requirements: tuple[str, ...] + + +def dream_kb(kb: KnowledgeBaseConfig, *, force: bool = False) -> dict[str, Any]: + wiki_root = kb.root / ".d_wiki" / "wiki" + if not wiki_root.is_dir(): + raise ValueError("D_WIKI_DREAM_REQUIRES_COMPILED_WIKI") + output_root = dream_root(kb) + queries_dir = output_root / "queries" + if force: + shutil.rmtree(output_root, ignore_errors=True) + queries_dir.mkdir(parents=True, exist_ok=True) + + pages = sorted((wiki_root / "concepts").glob("*.md")) + dreamed = [_dream_page(path, wiki_root) for path in pages] + dreamed = [item for item in dreamed if item is not None] + for item in dreamed: + (queries_dir / f"{item.slug}.md").write_text(_query_markdown(item), encoding="utf-8") + (output_root / "README.md").write_text(_readme(kb, len(pages), len(dreamed)), encoding="utf-8") + return { + "status": "dreamed", + "kb": kb.name, + "input_wiki": str(wiki_root), + "output_store": str(output_root), + "queries": len(dreamed), + "dreamed_at": datetime.now(timezone.utc).isoformat(), + } + + +def load_dream_query_entries(kb: KnowledgeBaseConfig) -> list[Any]: + return load_query_entries(dream_root(kb) / "queries") + + +def dream_root(kb: KnowledgeBaseConfig) -> Path: + return kb.root / ".d_wiki" / "dreams" / "latest" + + +def _dream_page(path: Path, wiki_root: Path) -> DreamedQuery | None: + meta, body = frontmatter(path.read_text(encoding="utf-8", errors="ignore")) + title = str(meta.get("title") or "").strip() + if not title: + return None + curation_body = _curation_body(body) + source = _first(meta.get("sources")) or _wikilink_for_compiled(path, wiki_root) + aliases = _unique([title, *_dream_aliases(title, curation_body)]) + intent = _first(meta.get("intents")) or _dream_intent(title, curation_body) + requirements = _as_list(meta.get("answer_requirements")) or _dream_requirements(title, curation_body, intent) + supporting = _related_sources(body, source) + return DreamedQuery( + slug=path.stem, + title=title, + aliases=tuple(aliases[:18]), + intent=intent, + primary_sources=(source,), + supporting_sources=tuple(supporting[:6]), + answer_requirements=tuple(requirements[:8]), + ) + + +def _dream_aliases(title: str, body: str) -> list[str]: + text = f"{title}\n{body}".lower() + aliases = [ + f"{title}怎么用", + f"如何使用{title}", + ] + if _looks_like_3d_generation(title, text): + aliases.extend( + [ + "3D资产生成怎么用", + "如何使用 3D 资产生成功能", + "我想知道如何使用3d资产生成功能", + "图生3D怎么用", + "如何生成3D模型", + "如何把图片生成3D资产", + ] + ) + if "camera" in text and any(term in text for term in ("点亮", "bring up", "sensor")): + aliases.extend(["如何在X5平台上点亮camera", "X5 camera 点亮", "X5 摄像头 bring up"]) + if any(term in text for term in ("算力", "tops", "bpu")): + aliases.extend(["x5的算力是多少", "X5 BPU 算力"]) + return aliases + + +def _dream_intent(title: str, body: str) -> str: + text = f"{title}\n{body}".lower() + if any(term in text for term in ("如何", "怎么", "步骤", "使用", "创建", "配置", "点亮", "生成", "调试", "bring up")): + return "procedure" + return "fact" + + +def _dream_requirements(title: str, body: str, intent: str) -> list[str]: + text = f"{title}\n{body}".lower() + if _looks_like_3d_generation(title, text): + return [ + "说明 3D 生成是图生 3D 工具,不是镜像资产", + "按准备参考图、生成、保存到 3D 资产的流程回答", + "说明输出默认 glb,生成结果可保存到 3D 资产", + "提醒暂不支持文生 3D,文字描述可先用图像生成产图", + ] + if intent == "procedure": + return ["按步骤回答", "说明关键前提和入口", "引用证据来源"] + return ["直接回答问题", "引用证据来源"] + + +def _looks_like_3d_generation(title: str, text: str) -> bool: + title_text = title.lower() + return ( + "3d 生成" in title_text + or "3d生成" in title_text + or "3d 资产生成" in title_text + or "3d资产生成" in title_text + or "embodygen" in text + or "ai 图生 3d 工具" in text + or "ai 图生3d 工具" in text + ) + + +def _curation_body(body: str) -> str: + lines: list[str] = [] + skip = False + for line in body.splitlines(): + if line.startswith("## "): + skip = line.strip() in {"## Retrieval Hints", "## Related"} + if skip: + continue + if not skip: + lines.append(line) + return "\n".join(lines) + + +def _related_sources(body: str, primary: str) -> list[str]: + links = re.findall(r"\[\[([^\]]+)\]\]", body) + refs = re.findall(r"\^\[([^#\]]+)(?:#[^\]]+)?\]", body) + return _unique([item for item in [*links, *refs] if item and item != primary]) + + +def _query_markdown(item: DreamedQuery) -> str: + lines = [ + "---", + f"title: {_yaml_scalar(item.title)}", + f"aliases: {_yaml_list(item.aliases)}", + f"intent: {item.intent}", + f"primary_sources: {_yaml_list(item.primary_sources)}", + f"supporting_sources: {_yaml_list(item.supporting_sources)}", + f"answer_requirements: {_yaml_list(item.answer_requirements)}", + "provenance: dreamed", + "---", + "", + f"Dreamed query memory for {item.title}.", + "", + ] + return "\n".join(lines) + + +def _readme(kb: KnowledgeBaseConfig, pages: int, queries: int) -> str: + return "\n".join( + [ + "# d-wiki dream output", + "", + "This store is generated from the compiled wiki. It does not modify source documents or compiled concept pages.", + "", + f"- kb: `{kb.name}`", + f"- input_pages: {pages}", + f"- dreamed_queries: {queries}", + "", + ] + ) + + +def _first(value: Any) -> str: + items = _as_list(value) + return items[0] if items else "" + + +def _as_list(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [] + + +def _unique(values: list[str]) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for value in values: + cleaned = re.sub(r"\s+", " ", str(value)).strip() + if not cleaned or cleaned in seen: + continue + seen.add(cleaned) + output.append(cleaned) + return output + + +def _wikilink_for_compiled(path: Path, wiki_root: Path) -> str: + return path.relative_to(wiki_root).with_suffix("").as_posix() + + +def _yaml_scalar(value: str) -> str: + import json + + return json.dumps(value, ensure_ascii=False) + + +def _yaml_list(values: tuple[str, ...] | list[str]) -> str: + return "[" + ", ".join(_yaml_scalar(value) for value in values) + "]" diff --git a/d_wiki/llm.py b/d_wiki/llm.py new file mode 100644 index 0000000..959e902 --- /dev/null +++ b/d_wiki/llm.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class LlmSettings: + base_url: str + model: str + api_key: str + api_style: str = "anthropic" + max_tokens: int = 900 + + +@dataclass(frozen=True) +class LlmPageSynthesis: + summary: str + answer_blueprint: tuple[str, ...] + key_evidence: tuple[str, ...] + confidence: str = "medium" + + +class LlmPageSynthesizer: + def __init__(self, settings: LlmSettings) -> None: + self.settings = settings + + @classmethod + def from_environment(cls) -> "LlmPageSynthesizer | None": + base_url = os.environ.get("D_WIKI_LLM_BASE_URL") or os.environ.get("ROBOGO_LLM_BASE_URL") + model = os.environ.get("D_WIKI_LLM_MODEL") or os.environ.get("ROBOGO_LLM_MODEL") + api_key = os.environ.get("D_WIKI_LLM_API_KEY") or os.environ.get("ROBOGO_LLM_API_KEY") + if not base_url or not model or not api_key: + return None + api_style = os.environ.get("D_WIKI_LLM_API_STYLE") or os.environ.get("ROBOGO_LLM_API_STYLE") or "anthropic" + max_tokens = int(os.environ.get("D_WIKI_LLM_MAX_TOKENS", "900")) + return cls(LlmSettings(base_url=base_url, model=model, api_key=api_key, api_style=api_style, max_tokens=max_tokens)) + + def synthesize(self, prompt: str) -> LlmPageSynthesis | None: + text = self._complete(prompt).strip() + payload = _extract_json(text) + if not isinstance(payload, dict): + return None + summary = str(payload.get("summary") or "").strip() + if not summary: + return None + answer_blueprint = _string_tuple(payload.get("answer_blueprint")) + key_evidence = _string_tuple(payload.get("key_evidence")) + confidence = str(payload.get("confidence") or "medium").strip().lower() + if confidence not in {"low", "medium", "high"}: + confidence = "medium" + return LlmPageSynthesis( + summary=summary, + answer_blueprint=answer_blueprint, + key_evidence=key_evidence, + confidence=confidence, + ) + + def _complete(self, prompt: str) -> str: + if self.settings.api_style == "openai": + return self._complete_openai(prompt) + return self._complete_anthropic(prompt) + + def _complete_anthropic(self, prompt: str) -> str: + payload = { + "model": self.settings.model, + "max_tokens": self.settings.max_tokens, + "temperature": 0, + "system": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.", + "messages": [{"role": "user", "content": prompt}], + } + data = self._post_json(_join_url(self.settings.base_url, "v1/messages"), payload) + content = data.get("content") + if isinstance(content, list): + return "\n".join(str(item.get("text", "")) for item in content if isinstance(item, dict)) + return str(data.get("text") or "") + + def _complete_openai(self, prompt: str) -> str: + payload = { + "model": self.settings.model, + "temperature": 0, + "messages": [ + { + "role": "system", + "content": "You compile local documentation into concise, evidence-grounded wiki pages. Return only JSON.", + }, + {"role": "user", "content": prompt}, + ], + } + data = self._post_json(_join_url(self.settings.base_url, "v1/chat/completions"), payload) + choices = data.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message") + if isinstance(message, dict): + return str(message.get("content") or "") + return "" + + def _post_json(self, url: str, payload: dict[str, Any]) -> dict[str, Any]: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + request = urllib.request.Request( + url, + data=body, + method="POST", + headers={ + "content-type": "application/json", + "accept": "application/json", + "x-api-key": self.settings.api_key, + "authorization": f"Bearer {self.settings.api_key}", + "anthropic-version": "2023-06-01", + }, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="ignore") + raise RuntimeError(f"D_WIKI_LLM_HTTP_ERROR: {exc.code} {detail[:300]}") from exc + + +def _join_url(base_url: str, suffix: str) -> str: + base = base_url.rstrip("/") + if base.endswith("/v1/messages") or base.endswith("/v1/chat/completions"): + return base + if base.endswith("/v1"): + return f"{base}/{suffix.removeprefix('v1/')}" + return f"{base}/{suffix}" + + +def _extract_json(text: str) -> Any: + cleaned = text.strip() + if cleaned.startswith("```"): + cleaned = cleaned.strip("`") + if cleaned.lower().startswith("json"): + cleaned = cleaned[4:].strip() + decoder = json.JSONDecoder() + for index, char in enumerate(cleaned): + if char != "{": + continue + try: + value, _end = decoder.raw_decode(cleaned[index:]) + except json.JSONDecodeError: + continue + return value + return None + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, list): + return () + return tuple(str(item).strip() for item in value if str(item).strip()) diff --git a/d_wiki/metadata.py b/d_wiki/metadata.py new file mode 100644 index 0000000..dd41dfd --- /dev/null +++ b/d_wiki/metadata.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .yaml_compat import safe_load +from .tokenize import tokenize + +QUERY_MATCH_STOP_TOKENS = { + "如何", + "怎么", + "使用", + "创建", + "知道", + "功能", + "资产", +} + + +@dataclass(frozen=True) +class AliasGroup: + name: str + terms: tuple[str, ...] + intent: str | None + related: tuple[str, ...] + + +@dataclass(frozen=True) +class QueryEntry: + path: Path + title: str + aliases: tuple[str, ...] + intent: str | None + primary_sources: tuple[str, ...] + supporting_sources: tuple[str, ...] + answer_requirements: tuple[str, ...] + + +def load_alias_groups(path: Path | None) -> list[AliasGroup]: + if path is None or not path.exists(): + return [] + data = safe_load(path.read_text(encoding="utf-8")) or {} + raw_aliases = data.get("aliases", {}) + if not isinstance(raw_aliases, dict): + return [] + groups: list[AliasGroup] = [] + for name, raw in raw_aliases.items(): + if not isinstance(raw, dict): + continue + groups.append( + AliasGroup( + name=str(name), + terms=tuple(str(term).strip() for term in raw.get("terms", []) if str(term).strip()), + intent=str(raw["intent"]) if raw.get("intent") else None, + related=tuple(str(item).strip() for item in raw.get("related", []) if str(item).strip()), + ) + ) + return groups + + +def load_query_entries(directory: Path | None) -> list[QueryEntry]: + if directory is None or not directory.exists(): + return [] + return [_query_entry(path) for path in sorted(directory.rglob("*.md")) if path.is_file()] + + +def frontmatter(text: str) -> tuple[dict[str, Any], str]: + if not text.startswith("---\n"): + return {}, text + end = text.find("\n---", 4) + if end < 0: + return {}, text + meta = safe_load(text[4:end]) or {} + if not isinstance(meta, dict): + meta = {} + return meta, text[end + 4 :].lstrip("\n") + + +def query_matches_entry(query: str, entry: QueryEntry) -> bool: + normalized = normalize(query) + if any(normalize(alias) in normalized or normalized in normalize(alias) for alias in entry.aliases): + return True + query_terms = _meaningful_query_terms(query) + for candidate in (entry.title, *entry.aliases): + candidate_terms = _meaningful_query_terms(candidate) + if len(query_terms & candidate_terms) >= 2: + return True + return False + + +def _meaningful_query_terms(value: str) -> set[str]: + return {term for term in tokenize(value) if term not in QUERY_MATCH_STOP_TOKENS and len(term.strip()) >= 2} + + +def normalize(value: str) -> str: + return re.sub(r"\s+", " ", value.lower()).strip() + + +def _query_entry(path: Path) -> QueryEntry: + meta, body = frontmatter(path.read_text(encoding="utf-8")) + title = str(meta.get("title") or _first_heading(body) or path.stem) + return QueryEntry( + path=path, + title=title, + aliases=tuple(str(item).strip() for item in meta.get("aliases", []) if str(item).strip()), + intent=str(meta["intent"]) if meta.get("intent") else None, + primary_sources=tuple(str(item).strip() for item in meta.get("primary_sources", []) if str(item).strip()), + supporting_sources=tuple(str(item).strip() for item in meta.get("supporting_sources", []) if str(item).strip()), + answer_requirements=tuple(str(item).strip() for item in meta.get("answer_requirements", []) if str(item).strip()), + ) + + +def _first_heading(text: str) -> str | None: + for line in text.splitlines(): + if match := re.match(r"^#{1,6}\s+(.+)$", line): + return match.group(1).strip() + return None diff --git a/d_wiki/search.py b/d_wiki/search.py new file mode 100644 index 0000000..07e396f --- /dev/null +++ b/d_wiki/search.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import json +import math +import re +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .config import KnowledgeBaseConfig +from .metadata import ( + AliasGroup, + QueryEntry, + frontmatter, + load_alias_groups, + load_query_entries, + normalize, + query_matches_entry, +) +from .dreams import load_dream_query_entries +from .tokenize import tokenize + +SPHINX_SOURCE_SUFFIXES = (".md.txt", ".rst.txt") +MAX_BLOCK_CHARS = 520 + + +@dataclass(frozen=True) +class Chunk: + title: str + wikilink: str + source_path: Path + page_type: str + heading: str + text: str + tokens: tuple[str, ...] + index: int + + +def status_payload(config_path: Path, bases: dict[str, KnowledgeBaseConfig]) -> dict[str, Any]: + return { + "status": "ok", + "config": str(config_path), + "knowledge_bases": { + name: { + "root": str(kb.root), + "source_type": kb.source_type, + "exists": kb.root.exists(), + "aliases": str(kb.aliases_path) if kb.aliases_path else None, + "queries": str(kb.queries_dir) if kb.queries_dir else None, + } + for name, kb in bases.items() + }, + } + + +def index_payload(kb: KnowledgeBaseConfig) -> dict[str, Any]: + chunks = load_chunks(kb) + index_dir = kb.root / ".d_wiki_index" + index_dir.mkdir(exist_ok=True) + payload = { + "kb": kb.name, + "root": str(kb.root), + "source_type": kb.source_type, + "compiled_wiki": str(compiled_wiki_root(kb)) if compiled_wiki_root(kb).exists() else None, + "chunks": len(chunks), + "pages": len({chunk.wikilink for chunk in chunks}), + "indexed_at": datetime.now(timezone.utc).isoformat(), + } + (index_dir / f"{kb.name}.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + return {"status": "indexed", **payload} + + +def search(kb: KnowledgeBaseConfig, query: str, max_results: int = 5) -> dict[str, Any]: + aliases = load_alias_groups(kb.aliases_path) + query_entries = [*load_query_entries(kb.queries_dir), *load_dream_query_entries(kb)] + matched_entries = [entry for entry in query_entries if query_matches_entry(query, entry)] + expanded_terms, matched_aliases = expand_query(query, aliases, matched_entries) + chunks = load_chunks(kb) + ranked = rank_chunks(chunks, expanded_terms, matched_aliases, matched_entries) + selected = ranked[: max(1, min(max_results, 8))] + context_pack = build_context_pack(query, selected, matched_aliases, matched_entries) + return { + "status": "found" if selected else "no_match", + "query": query, + "knowledge_base": kb.name, + "backend": "lightweight_bm25", + "quality": quality(selected), + "sources": [source_summary(item[1], item[0]) for item in selected], + "context_pack": context_pack, + } + + +def get_page(kb: KnowledgeBaseConfig, wikilink: str) -> dict[str, Any]: + chunks = [chunk for chunk in load_chunks(kb) if chunk.wikilink == wikilink] + if not chunks: + return {"status": "not_found", "knowledge_base": kb.name, "id": wikilink} + first = chunks[0] + return { + "status": "found", + "knowledge_base": kb.name, + "id": wikilink, + "title": first.title, + "source_path": str(first.source_path), + "blocks": [ + {"heading": chunk.heading, "text": chunk.text, "block_id": f"{chunk.wikilink}#{chunk.index}"} + for chunk in chunks + ], + } + + +def expand_query( + query: str, + aliases: list[AliasGroup], + entries: list[QueryEntry], +) -> tuple[list[str], list[AliasGroup]]: + terms = tokenize(query) + normalized_query = normalize(query) + matched: list[AliasGroup] = [] + for group in aliases: + if any(normalize(term) in normalized_query for term in group.terms): + matched.append(group) + for term in group.terms: + terms.extend(tokenize(term)) + for entry in entries: + terms.extend(tokenize(entry.title)) + for alias in entry.aliases: + terms.extend(tokenize(alias)) + return terms, matched + + +def rank_chunks( + chunks: list[Chunk], + terms: list[str], + aliases: list[AliasGroup], + entries: list[QueryEntry], +) -> list[tuple[float, Chunk]]: + if not chunks or not terms: + return [] + term_counts = [Counter(chunk.tokens) for chunk in chunks] + doc_freq: Counter[str] = Counter() + for counts in term_counts: + for term in counts: + doc_freq[term] += 1 + avg_len = sum(len(chunk.tokens) for chunk in chunks) / max(1, len(chunks)) + query_counts = Counter(terms) + ranked: list[tuple[float, Chunk]] = [] + related = {item for group in aliases for item in group.related} + primary = {item for entry in entries for item in entry.primary_sources} + supporting = {item for entry in entries for item in entry.supporting_sources} + for chunk, counts in zip(chunks, term_counts): + score = bm25_score(counts, len(chunk.tokens), doc_freq, len(chunks), avg_len, query_counts) + if any(wikilink_matches(chunk.wikilink, item) for item in related): + score += 8.0 + if any(wikilink_matches(chunk.wikilink, item) for item in primary): + score += 16.0 + if any(wikilink_matches(chunk.wikilink, item) for item in supporting): + score += 5.0 + score += heading_score_adjustment(chunk.heading) + if score > 0: + ranked.append((round(score, 3), chunk)) + ranked.sort(key=lambda item: (-item[0], item[1].wikilink, item[1].index)) + return ranked + + +def heading_score_adjustment(heading: str) -> float: + if heading == "Retrieval Hints": + return -80.0 + if heading == "Intent Model": + return -40.0 + if heading == "Answer Blueprint": + return -25.0 + if heading == "Evidence Graph": + return 3.0 + if heading == "Key Evidence": + return 5.0 + return 0.0 + + +def bm25_score( + counts: Counter[str], + doc_len: int, + doc_freq: Counter[str], + doc_count: int, + avg_len: float, + query_counts: Counter[str], +) -> float: + k1 = 1.5 + b = 0.75 + score = 0.0 + for term, qf in query_counts.items(): + tf = counts.get(term, 0) + if tf == 0: + continue + idf = math.log(1 + (doc_count - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5)) + denom = tf + k1 * (1 - b + b * (doc_len / max(avg_len, 1))) + score += idf * ((tf * (k1 + 1)) / denom) * min(qf, 3) + return score + + +def build_context_pack( + query: str, + selected: list[tuple[float, Chunk]], + aliases: list[AliasGroup], + entries: list[QueryEntry], +) -> dict[str, Any]: + intent = answer_intent(query, aliases, entries) + grouped: dict[str, list[tuple[float, Chunk]]] = defaultdict(list) + for item in selected: + grouped[item[1].wikilink].append(item) + source_items = sorted(grouped.items(), key=lambda pair: -max(score for score, _chunk in pair[1])) + entry_primary = {item for entry in entries for item in entry.primary_sources} + primary_sources = [ + source_context(wikilink, chunks) + for wikilink, chunks in source_items + if not entry_primary or any(wikilink_matches(wikilink, item) for item in entry_primary) + ] + if not primary_sources: + primary_sources = [source_context(wikilink, chunks) for wikilink, chunks in source_items[:3]] + supporting_sources = [ + source_context(wikilink, chunks, blocks_limit=1) + for wikilink, chunks in source_items + if wikilink not in {source["wikilink"] for source in primary_sources} + ][:3] + coverage = "insufficient" if not selected else ("partial_procedure" if intent == "procedure" else "complete") + return { + "answer_intent": intent, + "coverage": coverage, + "answer_policy": "Use the evidence blocks to synthesize the answer. Do not answer from a single snippet.", + "primary_sources": primary_sources, + "supporting_sources": supporting_sources, + "answer_requirements": answer_requirements(intent, entries), + } + + +def source_context(wikilink: str, chunks: list[tuple[float, Chunk]], blocks_limit: int = 4) -> dict[str, Any]: + top = sorted(chunks, key=lambda item: (-item[0], item[1].index))[:blocks_limit] + chunk = top[0][1] + return { + "title": chunk.title, + "wikilink": wikilink, + "source_path": str(chunk.source_path), + "reason": "Matched query terms and local metadata.", + "blocks": [ + { + "block_id": f"{block.wikilink}#{block.index}", + "heading": block.heading, + "text": trim_block(block.text), + "score": score, + } + for score, block in top + ], + } + + +def answer_intent(query: str, aliases: list[AliasGroup], entries: list[QueryEntry]) -> str: + for entry in entries: + if entry.intent: + return entry.intent + for group in aliases: + if group.intent: + return group.intent + if any(term in query.lower() for term in ("如何", "怎么", "步骤", "点亮", "bring up", "配置")): + return "procedure" + return "fact" + + +def answer_requirements(intent: str, entries: list[QueryEntry]) -> list[str]: + requirements = unique([req for entry in entries for req in entry.answer_requirements]) + if requirements: + return requirements + if intent == "procedure": + return ["基于 primary_sources 综合步骤化回答", "说明关键前提和验证方式", "引用命中的来源"] + return ["直接回答问题", "引用命中的来源"] + + +def unique(values: list[str]) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for value in values: + if value in seen: + continue + seen.add(value) + output.append(value) + return output + + +def quality(selected: list[tuple[float, Chunk]]) -> dict[str, Any]: + top = selected[0][0] if selected else 0.0 + return { + "confidence": "high" if top >= 8.0 else ("low" if selected else "none"), + "high_confidence": top >= 8.0, + "top_score": top, + "high_confidence_score": 8.0, + } + + +def wikilink_matches(candidate: str, expected: str) -> bool: + return candidate == expected or candidate == f"concepts/{expected}" + + +def source_summary(chunk: Chunk, score: float) -> dict[str, Any]: + return { + "title": chunk.title, + "wikilink": chunk.wikilink, + "source_path": str(chunk.source_path), + "heading": chunk.heading, + "snippet": trim_block(chunk.text, 320), + "score": score, + } + + +def load_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]: + root = compiled_wiki_root(kb) if compiled_wiki_root(kb).is_dir() else kb.root + paths = source_paths(root) + chunks: list[Chunk] = [] + for path in paths: + chunks.extend(chunks_for_path(path, root)) + return chunks + + +def load_source_chunks(kb: KnowledgeBaseConfig) -> list[Chunk]: + chunks: list[Chunk] = [] + for path in source_paths(kb.root): + chunks.extend(chunks_for_path(path, kb.root)) + return chunks + + +def compiled_wiki_root(kb: KnowledgeBaseConfig) -> Path: + return kb.root / ".d_wiki" / "wiki" + + +def source_paths(root: Path) -> list[Path]: + if (root / "_sources").is_dir(): + return sorted( + path + for path in (root / "_sources").rglob("*.txt") + if path.is_file() and path.name.endswith(SPHINX_SOURCE_SUFFIXES) + ) + generated = {"SCHEMA.md", "index.md", "log.md"} + generated_dirs = {".d_wiki", ".d_wiki_index"} + is_compiled_root = root.name == "wiki" and root.parent.name == ".d_wiki" + return sorted( + path + for path in root.rglob("*.md") + if path.is_file() + and path.name not in generated + and (is_compiled_root or not any(part in generated_dirs for part in path.parts)) + ) + + +def chunks_for_path(path: Path, root: Path) -> list[Chunk]: + raw = path.read_text(encoding="utf-8", errors="ignore") + meta, body = frontmatter(raw) + title = str(meta.get("title") or first_heading(body) or path.stem.replace("-", " ").title()) + metadata_text = metadata_terms(meta) + wikilink = wikilink_for(path, root) + sections = split_sections(body, title) + chunks: list[Chunk] = [] + for index, (heading, text) in enumerate(sections): + if heading in {"Source Map", "Related"}: + continue + if not text.strip(): + continue + field_text = " ".join([title, heading, wikilink, metadata_text, text]) + chunks.append( + Chunk( + title=title, + wikilink=wikilink, + source_path=path.resolve(), + page_type="sphinx_source" if "_sources" in path.parts else "markdown", + heading=heading, + text=text.strip(), + tokens=tuple(tokenize(field_text)), + index=index, + ) + ) + return chunks + + +def metadata_terms(meta: dict[str, Any]) -> str: + values: list[str] = [] + for key in ("aliases", "tags", "intents", "related", "retrieval_hints", "answer_requirements"): + raw = meta.get(key) + if isinstance(raw, str): + values.append(raw) + elif isinstance(raw, list): + values.extend(str(item) for item in raw if str(item).strip()) + return " ".join(values) + + +def split_sections(body: str, title: str) -> list[tuple[str, str]]: + heading = title + buffer: list[str] = [] + sections: list[tuple[str, str]] = [] + in_fence = False + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("```"): + in_fence = not in_fence + buffer.append(line) + continue + if not in_fence and (match := re.match(r"^#{1,6}\s+(.+)$", line)): + if buffer: + sections.extend(paragraphs(heading, "\n".join(buffer))) + buffer = [] + heading = match.group(1).strip() or title + continue + buffer.append(line) + if buffer: + sections.extend(paragraphs(heading, "\n".join(buffer))) + return sections or [(title, body)] + + +def paragraphs(heading: str, text: str) -> list[tuple[str, str]]: + blocks: list[tuple[str, str]] = [] + for part in re.split(r"\n\s*\n", text): + cleaned = clean_block(part) + if cleaned: + blocks.append((heading, cleaned)) + return blocks + + +def clean_block(text: str) -> str: + stripped = text.strip() + if not stripped: + return "" + if re.fullmatch(r"[-*_]{3,}", stripped): + return "" + compact = " ".join(stripped.split()) + if "📷" in compact and len(compact) <= 80: + return "" + if compact.startswith("> 📷") and len(compact) <= 120: + return "" + return stripped + + +def first_heading(body: str) -> str | None: + in_fence = False + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + if match := re.match(r"^#{1,6}\s+(.+)$", line): + return match.group(1).strip() + return None + + +def wikilink_for(path: Path, root: Path) -> str: + try: + relative = path.relative_to(root) + except ValueError: + return path.stem + if relative.parts and relative.parts[0] == "_sources": + relative = Path(*relative.parts[1:]) + value = relative.as_posix() + for suffix in SPHINX_SOURCE_SUFFIXES: + if value.endswith(suffix): + return value[: -len(suffix)] + return value[:-3] if value.endswith(".md") else path.stem + + +def trim_block(text: str, limit: int = MAX_BLOCK_CHARS) -> str: + compact = " ".join(text.split()) + return compact if len(compact) <= limit else compact[:limit].rstrip() diff --git a/d_wiki/tokenize.py b/d_wiki/tokenize.py new file mode 100644 index 0000000..09823c6 --- /dev/null +++ b/d_wiki/tokenize.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import re + +STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上") +STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"} + + +def tokenize(text: str) -> list[str]: + lowered = text.lower() + terms = re.findall(r"[a-z0-9_./+-]+", lowered) + cjk_chars = re.findall(r"[\u4e00-\u9fff]", text) + terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1))) + return [term for term in terms if _useful(term)] + + +def _useful(term: str) -> bool: + if not term or term in STOP_TERMS: + return False + if re.search(r"[\u4e00-\u9fff]", term): + return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term)) + return True diff --git a/d_wiki/yaml_compat.py b/d_wiki/yaml_compat.py new file mode 100644 index 0000000..61cf7ab --- /dev/null +++ b/d_wiki/yaml_compat.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from typing import Any + +try: + import yaml as _yaml # type: ignore +except Exception: # pragma: no cover - exercised in environments without PyYAML + _yaml = None + + +def safe_load(text: str) -> Any: + if _yaml is not None: + return _yaml.safe_load(text) + lines = [ + (len(line) - len(line.lstrip(" ")), line.strip()) + for line in text.splitlines() + if line.strip() and not line.strip().startswith("#") + ] + value, _index = _parse_node(lines, 0, 0) + return value + + +def _parse_node(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]: + if index >= len(lines): + return {}, index + if lines[index][0] < indent: + return {}, index + if lines[index][1].startswith("- "): + values: list[Any] = [] + while index < len(lines) and lines[index][0] == indent and lines[index][1].startswith("- "): + values.append(_scalar(lines[index][1][2:].strip())) + index += 1 + return values, index + mapping: dict[str, Any] = {} + while index < len(lines): + current_indent, stripped = lines[index] + if current_indent < indent: + break + if current_indent > indent: + break + if ":" not in stripped: + index += 1 + continue + key, raw_value = stripped.split(":", 1) + key = key.strip() + raw_value = raw_value.strip() + index += 1 + if raw_value: + mapping[key] = _scalar(raw_value) + continue + child, index = _parse_node(lines, index, indent + 2) + mapping[key] = child + return mapping, index + + +def _scalar(value: str) -> Any: + if value in {"null", "None", "~"}: + return None + if value in {"true", "True"}: + return True + if value in {"false", "False"}: + return False + if value.startswith("[") and value.endswith("]"): + inner = value[1:-1].strip() + if not inner: + return [] + return [_scalar(item.strip()) for item in inner.split(",")] + if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")): + return value[1:-1] + return value diff --git a/docs/language-decision.md b/docs/language-decision.md new file mode 100644 index 0000000..aaf929d --- /dev/null +++ b/docs/language-decision.md @@ -0,0 +1,56 @@ +# d-wiki CLI language decision + +## Decision + +Use Python for the first implementation of `d-wiki`. + +The CLI is a local knowledge retrieval component that needs predictable file +system traversal, Markdown/Sphinx source parsing, YAML metadata, BM25 ranking, +JSON output, and straightforward integration with `robogo_assistant_service`. +Python is the lower-risk choice for this first version because the surrounding +service code and tests are already Python, and the implementation intentionally +avoids qmd, embeddings, and Node-specific runtime requirements. + +## Python pros + +- Matches the existing `robogo_assistant_service` stack, test style, and local + developer workflow. +- Strong standard library support for CLI parsing, paths, subprocess-friendly + JSON output, dataclasses, and text processing. +- Easy to keep dependency-light; the current package has no required runtime + dependencies and can optionally use PyYAML when present. +- Good fit for future retrieval upgrades such as ranking experiments, + evaluation scripts, corpus analysis, and possible embedding backends. +- Lower integration friction for service tests that call the CLI from Python. + +## Python cons + +- Single-file executable distribution is less natural than Node/npm CLIs unless + extra packaging tools are introduced. +- Startup can be slower than a bundled JS runtime for very small commands, + though this is not a bottleneck for local document search. +- Type safety depends on discipline and tools rather than the language runtime. + +## TypeScript pros + +- Excellent CLI packaging through npm and strong cross-platform installation + conventions for JS-heavy teams. +- Static types are first-class and make API shape evolution explicit. +- Easier future alignment if `d-wiki` becomes a qmd-native or MCP-first Node + package. + +## TypeScript cons + +- Adds a second runtime stack beside the Python service for the first version. +- The plan explicitly avoids qmd/Node runtime complexity in v1; starting with + TypeScript would pull the project toward the complexity it is trying to defer. +- Requires build tooling, lockfiles, module-format decisions, and generated + output conventions before the retrieval logic itself benefits from them. +- Python service integration would still happen through subprocess JSON, so the + main service does not gain much from a TypeScript implementation today. + +## Revisit triggers + +Reconsider TypeScript if `d-wiki` becomes a distributed npm CLI, adopts qmd as +the default backend, ships an MCP server primarily for Node-based agents, or +needs to share substantial code with a TypeScript UI or SDK. diff --git a/knowledge/robogo/aliases.yml b/knowledge/robogo/aliases.yml new file mode 100644 index 0000000..be0351f --- /dev/null +++ b/knowledge/robogo/aliases.yml @@ -0,0 +1,25 @@ +aliases: + dev_machine: + terms: [开发机, 云主机, SSH, Web 应用, 申请开发机, 创建开发机, 远程开发] + intent: procedure + related: [21-dev-machine] + guagan: + terms: [地瓜干, 计费, 充值, 余额, 消费, 价格, 账户] + intent: fact + related: [03-account-and-guagan] + mirror: + terms: [镜像, 镜像资产, 创建镜像, 保存镜像, fork, 公开镜像, 群组镜像] + intent: procedure + related: [11-mirror-center, 20-cloud-desktop] + three_d_generation: + terms: [3D生成, 3D 生成, 图生3D, 图生 3D, 3D资产生成, 3D 资产生成, 资产生成, 3d资产生成功能, EmbodyGen, glb] + intent: procedure + related: [40-3d-generation, 12-3d-assert, 41-image-generation, 42-urdf-studio] + rdk_agent: + terms: [RDK Agent, VS Code 插件, 插件安装] + intent: procedure + related: [51-rdk-agent, 50-openclaw] + openclaw: + terms: [OpenClaw, MagicBox, openclaw-magicbox, rdkclaw, 智能控制, AI 代理, Agent Platform, 设备配对, Skills 管理, 飞书] + intent: procedure + related: [50-openclaw, 51-rdk-agent] diff --git a/knowledge/robogo/queries/3d-generation.md b/knowledge/robogo/queries/3d-generation.md new file mode 100644 index 0000000..b8438b4 --- /dev/null +++ b/knowledge/robogo/queries/3d-generation.md @@ -0,0 +1,25 @@ +--- +title: 3D 资产生成 +aliases: + - 我想知道如何使用3d资产生成功能 + - 如何使用 3D 资产生成功能 + - 3D资产生成怎么用 + - 3D 生成怎么用 + - 图生3D怎么用 + - 如何生成3D模型 + - 如何把图片生成3D资产 +intent: procedure +primary_sources: + - 40-3d-generation +supporting_sources: + - 12-3d-assert + - 41-image-generation + - 42-urdf-studio +answer_requirements: + - 说明 3D 生成是图生 3D 工具,不是镜像资产 + - 按准备参考图、生成、保存到 3D 资产的流程回答 + - 说明输出默认 glb,生成结果可保存到 3D 资产 + - 提醒暂不支持文生 3D,文字描述可先用图像生成产图 +--- + +RoboGo 3D 资产生成 / 图生 3D 的入口问题。 diff --git a/knowledge/robogo/queries/dev-machine.md b/knowledge/robogo/queries/dev-machine.md new file mode 100644 index 0000000..e69b614 --- /dev/null +++ b/knowledge/robogo/queries/dev-machine.md @@ -0,0 +1,7 @@ +--- +title: 开发机使用 +aliases: [开发机怎么用, 如何使用开发机, 怎么创建开发机, 开发机 SSH, 开发机 Web 应用] +intent: procedure +primary_sources: [21-dev-machine] +answer_requirements: [说明开发机定位, 按创建和登录步骤回答, 说明 Web 应用和 SSH 入口] +--- diff --git a/knowledge/robogo/queries/guagan.md b/knowledge/robogo/queries/guagan.md new file mode 100644 index 0000000..3d2d8e1 --- /dev/null +++ b/knowledge/robogo/queries/guagan.md @@ -0,0 +1,7 @@ +--- +title: 地瓜干计费 +aliases: [地瓜干怎么计费, 地瓜干是什么, 怎么充值, 余额在哪里看] +intent: fact +primary_sources: [03-account-and-guagan] +answer_requirements: [说明地瓜干用途, 说明充值和消费路径, 引用计费口径以平台实际页面为准] +--- diff --git a/knowledge/robogo/queries/mirror.md b/knowledge/robogo/queries/mirror.md new file mode 100644 index 0000000..5416423 --- /dev/null +++ b/knowledge/robogo/queries/mirror.md @@ -0,0 +1,8 @@ +--- +title: 镜像资产使用 +aliases: [镜像怎么使用, 如何创建镜像, 怎么保存镜像, 镜像资产怎么用] +intent: procedure +primary_sources: [11-mirror-center] +supporting_sources: [20-cloud-desktop] +answer_requirements: [区分镜像地址导入和地瓜桌面保存, 说明 Fork 和分享, 引用镜像资产来源] +--- diff --git a/knowledge/robogo/queries/openclaw.md b/knowledge/robogo/queries/openclaw.md new file mode 100644 index 0000000..f7be91f --- /dev/null +++ b/knowledge/robogo/queries/openclaw.md @@ -0,0 +1,8 @@ +--- +title: OpenClaw 使用 +aliases: [如何使用openclaw, OpenClaw 怎么用, 如何创建 OpenClaw, MagicBox 怎么用, OpenClaw 设备配对, OpenClaw Skills] +intent: procedure +primary_sources: [50-openclaw] +supporting_sources: [51-rdk-agent] +answer_requirements: [说明 OpenClaw 定位, 说明实例类型和创建入口, 说明 Skills/飞书/设备配对的使用路径] +--- diff --git a/knowledge/robogo/queries/rdk-agent.md b/knowledge/robogo/queries/rdk-agent.md new file mode 100644 index 0000000..c34cc1e --- /dev/null +++ b/knowledge/robogo/queries/rdk-agent.md @@ -0,0 +1,8 @@ +--- +title: RDK Agent 使用 +aliases: [RDK Agent 怎么用, 如何安装 RDK Agent, RDK Agent 插件, RDK 设备怎么接入] +intent: procedure +primary_sources: [51-rdk-agent] +supporting_sources: [50-openclaw] +answer_requirements: [说明 RDK Agent 是 VS Code 插件入口, 区分设备接入在 OpenClaw MagicBox, 引用页面注意事项] +--- diff --git a/knowledge/x5/aliases.yml b/knowledge/x5/aliases.yml new file mode 100644 index 0000000..742138b --- /dev/null +++ b/knowledge/x5/aliases.yml @@ -0,0 +1,33 @@ +aliases: + camera-bring-up: + terms: + - camera + - 摄像头 + - sensor + - mipi + - csi + - vin + - isp + - camsys + - 点亮 + - bring up + - bring-up + - bringup + - 调试 + - 配置 + - 预览 + intent: procedure + related: + - multimedia_development/7-camsys_debug + - samples/sunrise_camera_user_guide + - samples/sample_pipeline + compute: + terms: + - 算力 + - tops + - bpu + - bayes + - bayers + intent: fact + related: + - preface/overview diff --git a/knowledge/x5/queries/camera-bring-up.md b/knowledge/x5/queries/camera-bring-up.md new file mode 100644 index 0000000..9c1f5d9 --- /dev/null +++ b/knowledge/x5/queries/camera-bring-up.md @@ -0,0 +1,19 @@ +--- +title: X5 camera bring up +aliases: + - 如何在X5平台上点亮camera + - X5 camera 点亮 + - X5 摄像头 bring up +intent: procedure +primary_sources: + - multimedia_development/7-camsys_debug +supporting_sources: + - samples/sunrise_camera_user_guide + - samples/sample_pipeline +answer_requirements: + - 说明这是 X5 camera bring up / 点亮流程 + - 按硬件连接、DTS/GPIO、sensor 驱动、MIPI 配置、运行验证组织 + - 引用 Camera调试指南 和 Sunrise camera 用户使用说明等来源 +--- + +X5 平台点亮 camera 的入口问题。 diff --git a/knowledge/x5/queries/compute.md b/knowledge/x5/queries/compute.md new file mode 100644 index 0000000..2d70ab6 --- /dev/null +++ b/knowledge/x5/queries/compute.md @@ -0,0 +1,14 @@ +--- +title: X5 算力 +aliases: + - x5的算力是多少 + - X5 BPU 算力 +intent: fact +primary_sources: + - preface/overview +answer_requirements: + - 直接给出 X5 BPU 算力 + - 保留 TOPS 和 BPU 信息 +--- + +X5 算力参数入口问题。 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b7d5d1e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "d-wiki" +version = "0.1.0" +description = "Lightweight local knowledge retrieval with BM25 and metadata context packs." +requires-python = ">=3.10" +dependencies = [] + +[project.scripts] +d-wiki = "d_wiki.cli:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["d_wiki*"] diff --git a/tests/__pycache__/test_d_wiki.cpython-311-pytest-9.0.3.pyc b/tests/__pycache__/test_d_wiki.cpython-311-pytest-9.0.3.pyc new file mode 100644 index 0000000..49c77f2 Binary files /dev/null and b/tests/__pycache__/test_d_wiki.cpython-311-pytest-9.0.3.pyc differ diff --git a/tests/__pycache__/test_d_wiki.cpython-312.pyc b/tests/__pycache__/test_d_wiki.cpython-312.pyc new file mode 100644 index 0000000..0a82521 Binary files /dev/null and b/tests/__pycache__/test_d_wiki.cpython-312.pyc differ diff --git a/tests/__pycache__/test_d_wiki.cpython-314.pyc b/tests/__pycache__/test_d_wiki.cpython-314.pyc new file mode 100644 index 0000000..1729824 Binary files /dev/null and b/tests/__pycache__/test_d_wiki.cpython-314.pyc differ diff --git a/tests/test_d_wiki.py b/tests/test_d_wiki.py new file mode 100644 index 0000000..afdeb46 --- /dev/null +++ b/tests/test_d_wiki.py @@ -0,0 +1,509 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from d_wiki.cli import main +from d_wiki.metadata import QueryEntry, query_matches_entry + + +class DWikiTest(unittest.TestCase): + def test_query_entry_matches_rewritten_alias_by_token_overlap(self) -> None: + entry = QueryEntry( + path=Path("camera.md"), + title="camera bring up", + aliases=("如何在X5平台上点亮camera",), + intent="procedure", + primary_sources=("multimedia_development/7-camsys_debug",), + supporting_sources=(), + answer_requirements=(), + ) + + self.assertTrue(query_matches_entry("X5平台 点亮 camera 使能", entry)) + + def test_bm25_metadata_context_pack_for_camera_bring_up(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + config = root / "d-wiki.yml" + output = self._run_json( + [ + "--config", + str(config), + "search", + "--kb", + "x5", + "--query", + "如何在X5平台上点亮camera", + "--format", + "json", + ] + ) + + self.assertEqual(output["status"], "found") + self.assertEqual(output["context_pack"]["answer_intent"], "procedure") + self.assertEqual(output["context_pack"]["coverage"], "partial_procedure") + first = output["context_pack"]["primary_sources"][0] + self.assertEqual(first["wikilink"], "multimedia_development/7-camsys_debug") + self.assertGreaterEqual(len(first["blocks"]), 2) + self.assertIn("点亮 camera", json.dumps(first, ensure_ascii=False)) + + def test_rewritten_camera_query_still_uses_query_entry_primary_source(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + output = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "search", + "--kb", + "x5", + "--query", + "X5平台 点亮 camera 使能", + "--format", + "json", + ] + ) + + self.assertEqual(output["status"], "found") + first = output["context_pack"]["primary_sources"][0] + self.assertEqual(first["wikilink"], "multimedia_development/7-camsys_debug") + + def test_fact_query_for_compute(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + output = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "search", + "--kb", + "x5", + "--query", + "x5的算力是多少", + "--format", + "json", + ] + ) + + self.assertEqual(output["status"], "found") + self.assertEqual(output["context_pack"]["answer_intent"], "fact") + self.assertIn("10 TOPS", json.dumps(output["context_pack"], ensure_ascii=False)) + + def test_unrelated_query_is_insufficient(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + output = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "search", + "--kb", + "x5", + "--query", + "火星咖啡机怎么维修", + "--format", + "json", + ] + ) + + self.assertEqual(output["status"], "no_match") + self.assertEqual(output["context_pack"]["coverage"], "insufficient") + + def test_page_frontmatter_metadata_participates_in_retrieval(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + output = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "search", + "--kb", + "x5", + "--query", + "x5 coredump breadcrumbs", + "--format", + "json", + ] + ) + + self.assertEqual(output["status"], "found") + self.assertEqual(output["sources"][0]["wikilink"], "misc/pstore") + + def test_markdown_cleaning_filters_thematic_breaks_and_screenshot_only_blocks(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + docs = root / "docs" + docs.mkdir() + (docs / "dev-machine.md").write_text( + "# 开发机\n\n" + "> 📷 截图:开发机列表页\n\n" + "---\n\n" + "开发机支持 SSH 登录和 Web 应用访问。\n", + encoding="utf-8", + ) + (root / "d-wiki.yml").write_text( + "knowledge_bases:\n" + " robogo:\n" + f" root: {docs}\n" + " source_type: markdown\n", + encoding="utf-8", + ) + output = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "search", + "--kb", + "robogo", + "--query", + "开发机怎么用", + "--format", + "json", + ] + ) + + snippets = [str(source["snippet"]) for source in output["sources"]] + self.assertNotIn("---", snippets) + self.assertFalse(any("📷" in snippet for snippet in snippets)) + self.assertIn("开发机支持 SSH 登录", snippets[0]) + + def test_compile_builds_wiki_pages_and_search_prefers_compiled_wiki(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + config = root / "d-wiki.yml" + compiled = self._run_json( + [ + "--config", + str(config), + "compile", + "--kb", + "x5", + "--force", + "--format", + "json", + ] + ) + indexed = self._run_json( + [ + "--config", + str(config), + "index", + "--kb", + "x5", + "--format", + "json", + ] + ) + output = self._run_json( + [ + "--config", + str(config), + "search", + "--kb", + "x5", + "--query", + "x5的算力是多少", + "--format", + "json", + ] + ) + wiki_root = Path(str(compiled["wiki_root"])) + self.assertEqual(compiled["status"], "compiled") + self.assertTrue((wiki_root / "SCHEMA.md").exists()) + self.assertTrue((wiki_root / "index.md").exists()) + self.assertTrue((wiki_root / "concepts" / "preface-overview.md").exists()) + self.assertEqual(indexed["compiled_wiki"], str(wiki_root)) + self.assertEqual(output["status"], "found") + self.assertIn(".d_wiki/wiki/concepts", output["sources"][0]["source_path"]) + self.assertIn("10 TOPS", json.dumps(output["context_pack"], ensure_ascii=False)) + + def test_compile_emits_llm_wiki_structure_without_breaking_retrieval(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + config = root / "d-wiki.yml" + compiled = self._run_json( + [ + "--config", + str(config), + "compile", + "--kb", + "x5", + "--force", + "--format", + "json", + ] + ) + self._run_json( + [ + "--config", + str(config), + "index", + "--kb", + "x5", + "--format", + "json", + ] + ) + compute = self._run_json( + [ + "--config", + str(config), + "search", + "--kb", + "x5", + "--query", + "x5的算力是多少", + "--format", + "json", + ] + ) + camera = self._run_json( + [ + "--config", + str(config), + "search", + "--kb", + "x5", + "--query", + "如何在X5平台上点亮camera", + "--format", + "json", + ] + ) + + wiki_root = Path(str(compiled["wiki_root"])) + compute_page = (wiki_root / "concepts" / "preface-overview.md").read_text(encoding="utf-8") + camera_page = ( + wiki_root / "concepts" / "multimedia-development-7-camsys-debug.md" + ).read_text(encoding="utf-8") + + self.assertIn("llmWikiVersion: \"1\"", compute_page) + self.assertIn("## Intent Model", compute_page) + self.assertIn("## Retrieval Hints", compute_page) + self.assertIn("## Answer Blueprint", compute_page) + self.assertIn("## Evidence Graph", compute_page) + self.assertIn("## Key Evidence", compute_page) + self.assertIn("X5 算力", compute_page) + self.assertIn("10 TOPS", compute_page) + self.assertIn("- intent: fact", compute_page) + self.assertIn("- intent: procedure", camera_page) + self.assertIn("按步骤回答", camera_page) + self.assertEqual(compute["status"], "found") + self.assertIn("10 TOPS", json.dumps(compute["context_pack"], ensure_ascii=False)) + self.assertIn( + "Key Evidence", + [block["heading"] for block in compute["context_pack"]["primary_sources"][0]["blocks"]], + ) + self.assertEqual(camera["context_pack"]["answer_intent"], "procedure") + self.assertEqual( + camera["context_pack"]["primary_sources"][0]["wikilink"], + "concepts/multimedia-development-7-camsys-debug", + ) + self.assertNotEqual(camera["sources"][0]["heading"], "Retrieval Hints") + + def test_dream_queries_route_without_manual_query_entry(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + docs = root / "docs" + docs.mkdir() + (docs / "11-mirror-center.md").write_text( + "# 镜像资产使用\n\n" + "镜像资产是 RoboGo 的容器环境仓库,用于保存和复用开发环境。\n", + encoding="utf-8", + ) + (docs / "40-3d-generation.md").write_text( + "# 40 · 3D 生成\n\n" + "3D 生成是平台提供的 AI 图生 3D 工具,基于 EmbodyGen,用一张参考图片生成 3D 模型,默认 glb。\n\n" + "生成结果可以保存到 3D 资产继续使用。\n", + encoding="utf-8", + ) + (root / "knowledge" / "robogo").mkdir(parents=True) + (root / "knowledge" / "robogo" / "aliases.yml").write_text( + "aliases:\n" + " mirror:\n" + " terms: [镜像, 镜像资产, 创建镜像]\n" + " intent: procedure\n" + " related: [11-mirror-center]\n", + encoding="utf-8", + ) + (root / "d-wiki.yml").write_text( + "knowledge_bases:\n" + " robogo:\n" + f" root: {docs}\n" + " source_type: markdown\n" + " aliases: knowledge/robogo/aliases.yml\n", + encoding="utf-8", + ) + compiled = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "compile", + "--kb", + "robogo", + "--force", + "--format", + "json", + ] + ) + dreamed = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "dream", + "--kb", + "robogo", + "--force", + "--format", + "json", + ] + ) + self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "index", + "--kb", + "robogo", + "--format", + "json", + ] + ) + output = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "search", + "--kb", + "robogo", + "--query", + "我想知道如何使用3d资产生成功能", + "--format", + "json", + ] + ) + page = Path(str(compiled["wiki_root"])) / "concepts" / "40-3d-generation.md" + page_content = page.read_text(encoding="utf-8") + dreamed_query = Path(str(dreamed["output_store"])) / "queries" / "40-3d-generation.md" + dreamed_content = dreamed_query.read_text(encoding="utf-8") + + self.assertNotIn("dream_queries:", page_content) + self.assertIn("3D资产生成怎么用", dreamed_content) + self.assertIn("provenance: dreamed", dreamed_content) + self.assertEqual(output["status"], "found") + self.assertEqual(output["context_pack"]["primary_sources"][0]["wikilink"], "concepts/40-3d-generation") + self.assertNotIn("镜像资产使用", json.dumps(output["context_pack"]["answer_requirements"], ensure_ascii=False)) + + def test_compile_llm_auto_falls_back_without_llm_config(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = self._fixture(Path(tmp)) + compiled = self._run_json( + [ + "--config", + str(root / "d-wiki.yml"), + "compile", + "--kb", + "x5", + "--force", + "--llm", + "auto", + "--format", + "json", + ] + ) + page = Path(str(compiled["wiki_root"])) / "concepts" / "preface-overview.md" + page_content = page.read_text(encoding="utf-8") + + self.assertEqual(compiled["status"], "compiled") + self.assertEqual(compiled["llm_mode"], "auto") + self.assertEqual(compiled["llm_scope"], "queries") + self.assertFalse(compiled["llm_enabled"]) + self.assertIn("provenanceState: compiled", page_content) + + def _run_json(self, argv: list[str]) -> dict[str, object]: + from contextlib import redirect_stdout + from io import StringIO + + out = StringIO() + with redirect_stdout(out): + code = main(argv) + self.assertEqual(code, 0, out.getvalue()) + return json.loads(out.getvalue()) + + def _fixture(self, root: Path) -> Path: + sources = root / "manual" / "_sources" + (sources / "multimedia_development").mkdir(parents=True) + (sources / "preface").mkdir(parents=True) + (sources / "misc").mkdir(parents=True) + (sources / "multimedia_development" / "7-camsys_debug.md.txt").write_text( + "# Camera调试指南\n\n" + "## 一 范围\n\n" + "本章节概述了 camera bring up 的过程,用于帮助读者快速了解并掌握 X5 camera 框架," + "如何快速的新增camera 配置,并点亮 camera。\n\n" + "## 1.1 sensor gpio 配置\n\n" + "vcon 是 X5 camera 用于管理 sensor 硬件相关的 dts 节点。\n\n" + "## 2 sensor 驱动文件添加\n\n" + "需要将原厂 sensor 驱动转换成 X5 camera 驱动代码,并编译生成 so 库。\n", + encoding="utf-8", + ) + (sources / "preface" / "overview.md.txt").write_text( + "# 前言\n\n## 关键特性\n\n- 贝叶斯架构BPU,提供最高 10 TOPS 算力。\n", + encoding="utf-8", + ) + (sources / "misc" / "pstore.md.txt").write_text( + "---\n" + "title: pstore\n" + "aliases: [coredump breadcrumbs]\n" + "tags: [debug, reboot]\n" + "---\n" + "# pstore\n\n在X5平台上,当系统异常重启时会保存日志。\n", + encoding="utf-8", + ) + (root / "knowledge" / "x5" / "queries").mkdir(parents=True) + (root / "knowledge" / "x5" / "aliases.yml").write_text( + "aliases:\n" + " camera:\n" + " terms: [camera, 摄像头, sensor, mipi, csi, vin, isp, 点亮, bring up, 调试, 配置]\n" + " intent: procedure\n" + " related: [multimedia_development/7-camsys_debug]\n" + " compute:\n" + " terms: [算力, tops, bpu]\n" + " intent: fact\n" + " related: [preface/overview]\n", + encoding="utf-8", + ) + (root / "knowledge" / "x5" / "queries" / "camera.md").write_text( + "---\n" + "title: camera bring up\n" + "aliases: [如何在X5平台上点亮camera]\n" + "intent: procedure\n" + "primary_sources: [multimedia_development/7-camsys_debug]\n" + "answer_requirements: [按步骤回答]\n" + "---\n", + encoding="utf-8", + ) + (root / "knowledge" / "x5" / "queries" / "compute.md").write_text( + "---\n" + "title: X5 算力\n" + "aliases: [x5的算力是多少]\n" + "intent: fact\n" + "primary_sources: [preface/overview]\n" + "---\n", + encoding="utf-8", + ) + (root / "d-wiki.yml").write_text( + "knowledge_bases:\n" + " x5:\n" + f" root: {root / 'manual'}\n" + " source_type: sphinx\n" + " aliases: knowledge/x5/aliases.yml\n" + " queries: knowledge/x5/queries\n", + encoding="utf-8", + ) + return root + + +if __name__ == "__main__": + unittest.main()