forked from d-cloud/d-wiki
23 lines
767 B
Python
23 lines
767 B
Python
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
STOP_CJK = set("的是多少什么怎么如何吗呢啊了在上")
|
|
STOP_TERMS = {"x5", "如何", "怎么", "什么", "多少", "平台"}
|
|
|
|
|
|
def tokenize(text: str) -> list[str]:
|
|
lowered = text.lower()
|
|
terms = re.findall(r"[a-z0-9_./+-]+", lowered)
|
|
cjk_chars = re.findall(r"[\u4e00-\u9fff]", text)
|
|
terms.extend("".join(cjk_chars[index : index + 2]) for index in range(max(0, len(cjk_chars) - 1)))
|
|
return [term for term in terms if _useful(term)]
|
|
|
|
|
|
def _useful(term: str) -> bool:
|
|
if not term or term in STOP_TERMS:
|
|
return False
|
|
if re.search(r"[\u4e00-\u9fff]", term):
|
|
return not (all(char in STOP_CJK for char in term) or any(char in STOP_CJK for char in term))
|
|
return True
|