wdb/d_wiki/tokenize.py
2026-05-28 19:02:16 +08:00

23 lines
767 B
Python

from __future__ import annotations
import re
STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上")
STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"}
def tokenize(text: str) -> list[str]:
lowered = text.lower()
terms = re.findall(r"[a-z0-9_./+-]+", lowered)
cjk_chars = re.findall(r"[\u4e00-\u9fff]", text)
terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1)))
return [term for term in terms if _useful(term)]
def _useful(term: str) -> bool:
if not term or term in STOP_TERMS:
return False
if re.search(r"[\u4e00-\u9fff]", term):
return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term))
return True