diff --git a/serve.py b/serve.py new file mode 100644 index 0000000..11262d2 --- /dev/null +++ b/serve.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""serve.py — jev r41 sidecar 推理服务 (Laya DecisionModel, mmBERT-base) + +打分协议 (与训练/评测严格一致): + 1. 文档按 win_tokens=800 切窗 (≤4 窗, 开头起) + 2. 每窗 build_sequence(max_len=1024, head_len=160) → marker logit + 3. p_win = sigmoid(z / T_WIN) # per-window 温度 (ID 校准) + 4. p_doc_mean = mean(p_win) # r41 证明 mean 优于 max + 5. p_final = sigmoid(logit(p_doc_mean) / T_AGG) # 聚合后二次校准 + +POST /score {query, doc_text} → {p_win[], p_mean, p_max, p_final} +POST /score_batch [{query, doc_text}...] → [...] +GET /health → {status, windows_protocol, T_WIN, T_AGG} +""" +import json +import math +import os +import sys + +sys.path.insert(0, "/srv/laya") # vendored laya.common (build_sequence/DecisionModel) + +import torch +from fastapi import FastAPI +from pydantic import BaseModel +from transformers import AutoConfig, AutoModel, AutoTokenizer +from peft import LoraConfig, get_peft_model +from laya.common import DecisionModel, build_sequence + +MODEL_DIR = os.environ.get("MODEL_DIR", "/model") +T_WIN = float(os.environ.get("T_WIN", "1.35")) +T_AGG = float(os.environ.get("T_AGG", "1.3")) +MAX_LEN = int(os.environ.get("MAX_LEN", "1024")) +HEAD_LEN = int(os.environ.get("HEAD_LEN", "160")) +WIN_TOKENS = int(os.environ.get("WIN_TOKENS", "800")) +MAX_WIN = int(os.environ.get("MAX_WIN", "4")) + +INSTR = "Does this document contain the information needed to answer the user's question? Answer based only on the document content." +Q = {"t": "noul", "ins": INSTR, "crit": None} + +app = FastAPI(title="jev-r41-sidecar") +state = {} + +def load_model(): + tok = AutoTokenizer.from_pretrained(os.path.join(MODEL_DIR, "mmbert")) + cfg = AutoConfig.from_pretrained(os.path.join(MODEL_DIR, "mmbert")) + enc = AutoModel.from_config(cfg) + sd = torch.load(os.path.join(MODEL_DIR, "mmbert", "pytorch_model.bin"), + map_location="cpu", weights_only=False) + sd2 = {k[len("model."):] if k.startswith("model.") else k: v for k, v in sd.items()} + enc.load_state_dict(sd2, strict=False) + dm = DecisionModel(enc, head_layers=2, n_act=2) + lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, + target_modules=["Wqkv", "Wo"], task_type=None) + dm.encoder = get_peft_model(dm.encoder, lora) + ad = torch.load(os.path.join(MODEL_DIR, "adapter.pt"), map_location="cpu", weights_only=False) + missing, unexpected = dm.load_state_dict(ad, strict=False) + assert len(missing) == 134 and not unexpected, f"adapter mismatch missing={len(missing)} unexpected={len(unexpected)}" + dm.eval() + state["tok"] = tok + state["dm"] = dm + state["ready"] = True + +def doc_windows(text): + ids = state["tok"](text[:12000], add_special_tokens=False)["input_ids"] + if len(ids) <= WIN_TOKENS: + return [text] + wins = [] + for s in range(0, min(len(ids), WIN_TOKENS * MAX_WIN), WIN_TOKENS): + wins.append(state["tok"].decode(ids[s:s + WIN_TOKENS])) + if len(wins) >= MAX_WIN: + break + return wins or [text[:6000]] + +@torch.no_grad() +def score_windows(query, wins): + tok = state["tok"] + dm = state["dm"] + ids_l, mm_l, mp_l, lens = [], [], [], [] + for w in wins: + st = f"用户问题: {query}\n\n文档内容:\n{w}" + ids, markers = build_sequence(tok, st, Q, MAX_LEN, HEAD_LEN) + lens.append(len(ids)) + ids_l.append(ids + [0] * (MAX_LEN - len(ids))) + mm_l.append([True] * len(markers) + [False] * (8 - len(markers))) + mp_l.append((markers + [0] * (8 - len(markers)))[:8]) + b = { + "input_ids": torch.tensor(ids_l), + "attention_mask": torch.tensor( + [[1] * n + [0] * (MAX_LEN - n) for n in lens]), + "marker_pos": torch.tensor(mp_l), + "marker_mask": torch.tensor(mm_l).bool(), + "qtype": torch.tensor([2] * len(ids_l)), + } + logits, _ = dm(**b) + z = logits[..., 1].float().tolist() + p_win = [1 / (1 + math.exp(-v / T_WIN)) for v in z] + return p_win + +def score_one(query, doc_text): + wins = doc_windows(doc_text) + p_win = score_windows(query, wins) + p_mean = sum(p_win) / len(p_win) + p_max = max(p_win) + lg_mean = math.log(max(p_mean, 1e-6) / max(1 - p_mean, 1e-6)) + p_final = 1 / (1 + math.exp(-lg_mean / T_AGG)) + return {"p_win": [round(p, 5) for p in p_win], + "p_mean": round(p_mean, 5), + "p_max": round(p_max, 5), + "p_final": round(p_final, 5), + "n_win": len(wins)} + +class ScoreReq(BaseModel): + query: str + doc_text: str + +class BatchReq(BaseModel): + items: list + +@app.get("/health") +def health(): + return {"status": "ok" if state.get("ready") else "loading", + "protocol": {"max_len": MAX_LEN, "head_len": HEAD_LEN, + "win_tokens": WIN_TOKENS, "max_win": MAX_WIN}, + "T_WIN": T_WIN, "T_AGG": T_AGG} + +@app.post("/score") +def score(req: ScoreReq): + return score_one(req.query, req.doc_text) + +@app.post("/score_batch") +def score_batch(req: BatchReq): + return [score_one(it["query"], it["doc_text"]) for it in req.items] + +if __name__ == "__main__": + import uvicorn + load_model() + uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8000")), log_level="warning")