laya repo modules + v0.5 decision-format data (4K balanced)

This commit is contained in:
dingbin.wang 2026-10-21 14:42:42 +08:00
parent 05967cce9e
commit 9927a3859d
10 changed files with 12792 additions and 0 deletions

168
laya_repo/__init__.py Normal file
View File

@ -0,0 +1,168 @@
"""Laya: Fast, non-autoregressive System 1 decision engine with calibrated probabilities."""
import os as _os
# Tell transformers not to import TensorFlow, before anything can import transformers.
#
# Laya's model path is torch-only, but on transformers 4.x `build_model` reaches
# `transformers.modeling_utils` for `no_init_weights`, and importing that module pulls a chain
# (`loss_utils` -> `loss_d_fine` -> `loss_for_object_detection` -> `image_transforms`) that ends in
# `import tensorflow`. In an environment where TensorFlow is installed but cannot load, that is a
# native crash rather than an exception -- `Fatal Python error: Bus error` -- from a library Laya
# never uses (#915).
#
# `setdefault`, not assignment: a caller who set `USE_TF` to something deliberate keeps it, and one
# who wants TensorFlow in the same process can still have it by setting the variable themselves. CI,
# the Dockerfile and the test suite already set `USE_TF=0` for exactly this reason; this makes the
# package behave the same way without the caller having to know.
#
# It has to be here, above the imports below, because transformers decides TensorFlow's
# availability when it is first imported -- and `import laya` deliberately imports no transformers
# (the torch-backed names are lazy, see `_LAZY_ATTRS`), so this runs first in the normal case.
_os.environ.setdefault("USE_TF", "0")
from .email import clean_email_body, email_state
from .hooks import AsyncHook, BaseHook, Hook, OptionStabilityHook, PredictContext, PredictHook
from .lang import analyse as detect_language
from .lang import detect_script, is_english
from .presets import (
email_questions,
guard_questions,
moderation_questions,
router_questions,
triage_questions,
)
from .router import DEFAULT_MODELS, RouteDecision, Router
from .structured import DecisionResult, decide, decide_batch
__version__ = "0.4.12"
# Routing, language detection and email cleaning are pure Python. The torch-backed names are
# resolved lazily so that `import laya` -- and therefore `from laya import Router` or
# `from laya.lang import detect_script` -- does not pay torch's import time and memory.
_LAZY_ATTRS = {
# stdlib-only module, but keep it lazy so `import laya` does not pull in hashlib
"PINNED_REVISIONS": (".revisions", "PINNED_REVISIONS"),
"Agent": (".agent", "Agent"),
"RLAgent": (".agent", "RLAgent"),
"load": (".agent", "load"),
"fit_temperatures": (".calibrate", "fit_temperatures"),
"fit_one_temperature": (".calibrate", "fit_one_temperature"),
"fit_temperature_map": (".calibrate", "fit_temperature_map"),
"fit_abstention_thresholds": (".calibrate", "fit_abstention_thresholds"),
"fit_binning_map": (".calibrate", "fit_binning_map"),
"apply_binning_map": (".calibrate", "apply_binning_map"),
"proper_reward": (".common", "proper_reward"),
"td_lambda_targets": (".common", "td_lambda_targets"),
"ece_score": (".common", "ece_score"),
"answer_confidence": (".common", "answer_confidence"),
"confidence_from_probs": (".common", "confidence_from_probs"),
"check_min_confidence": (".confidence", "check_min_confidence"),
"check_min_confidence_map": (".confidence", "check_min_confidence_map"),
"resolve_min_confidence": (".confidence", "resolve_min_confidence"),
"flag_low_confidence": (".confidence", "flag_low_confidence"),
"apply_confidence_gate": (".confidence", "apply_confidence_gate"),
"GATE_STATES": (".confidence", "GATE_STATES"),
"jev_confidence": (".confidence", "jev_confidence"),
"render_options": (".common", "render_options"),
"QTYPES": (".common", "QTYPES"),
"QTYPE_NAMES": (".common", "QTYPE_NAMES"),
"shortlist_choice": (".shortlist", "shortlist_choice"),
"predict_shortlist": (".shortlist", "predict_shortlist"),
"predict_tournament": (".shortlist", "predict_tournament"),
"embed_fn_from_agent": (".shortlist", "embed_fn_from_agent"),
"cached_embed_fn": (".shortlist", "cached_embed_fn"),
"LayaRouter": (".integrations", "LayaRouter"),
"LayaGuardrail": (".integrations", "LayaGuardrail"),
"LayaGuardrailError": (".integrations", "LayaGuardrailError"),
"LayaTriage": (".integrations", "LayaTriage"),
"LayaEvaluator": (".integrations", "LayaEvaluator"),
"LayaDecision": (".integrations", "LayaDecision"),
}
def __getattr__(name):
try:
module_name, attr = _LAZY_ATTRS[name]
except KeyError:
raise AttributeError("module %r has no attribute %r" % (__name__, name)) from None
import importlib
value = getattr(importlib.import_module(module_name, __name__), attr)
globals()[name] = value # cache: __getattr__ runs at most once per name
return value
def __dir__():
return sorted(list(globals()) + list(_LAZY_ATTRS))
__all__ = [
"Agent",
"RLAgent",
"load",
"fit_temperatures",
"fit_one_temperature",
"fit_temperature_map",
"fit_abstention_thresholds",
"fit_binning_map",
"apply_binning_map",
"Router",
"RouteDecision",
"DEFAULT_MODELS",
"shortlist_choice",
"predict_shortlist",
"predict_tournament",
"embed_fn_from_agent",
"cached_embed_fn",
"detect_language",
"detect_script",
"is_english",
"clean_email_body",
"email_questions",
"email_state",
"guard_questions",
"moderation_questions",
"router_questions",
"triage_questions",
"proper_reward",
"td_lambda_targets",
"ece_score",
"answer_confidence",
"confidence_from_probs",
"check_min_confidence",
"check_min_confidence_map",
"resolve_min_confidence",
"flag_low_confidence",
"apply_confidence_gate",
"GATE_STATES",
"jev_confidence",
"render_options",
"QTYPES",
"QTYPE_NAMES",
"LayaRouter",
"LayaGuardrail",
"LayaGuardrailError",
"LayaTriage",
"LayaEvaluator",
"LayaDecision",
"PredictContext",
"PredictHook",
"Hook",
"BaseHook",
"AsyncHook",
"OptionStabilityHook",
"decide",
"decide_batch",
"DecisionResult",
"PINNED_REVISIONS",
"__version__",
# Submodules for direct import paths
"confidence",
"email",
"hooks",
"lang",
"presets",
"router",
"structured",
]

2156
laya_repo/agent.py Normal file

File diff suppressed because it is too large Load Diff

822
laya_repo/calibrate.py Normal file
View File

@ -0,0 +1,822 @@
"""Fit and persist per-bucket temperatures for Laya logits.
Ports the notebook LBFGS fitter (NLL on softmax(z/T)) and groups records by
`temp_bucket(qtype, k)` so the runtime's `temperature_by_options` lookup has a writer.
Fitting is CPU-side: records are `(qtype, logits, target, k)` and do not load Hub weights.
Two sample floors, on purpose:
- `MIN_BUCKET_N` is the per-bucket floor. Buckets with fewer examples are omitted from
`temperature_by_options`; the type-level scalar covers them at lookup time.
- `MIN_TYPE_N` is the lower floor used only for that type-level scalar. It stays at the
previous floor of 10 so a realistic dataset that fills no single bucket still gets a
temperature instead of remaining at 1.0. Do not raise it to `MIN_BUCKET_N`.
Fitted temperatures go through `clamp_temperature` and therefore `[TEMP_MIN, TEMP_MAX]`
from `laya.common`, the same guard checkpoint load uses. There is no second pair of
bounds in this module. A clamp that moved a fit is reported in `fit_temperature_map`'s
`temperature_clamped`, since the clamped value alone cannot say whether the optimum was
the bound or merely beyond it.
"""
import warnings
from numbers import Integral
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
import numpy as np
import torch
from .common import (
QTYPE_NAMES,
QTYPES,
TEMP_MAX,
TEMP_MIN,
clamp_temperature,
collate_items,
ece_score,
temp_bucket,
temperature_rejected,
)
# Per-bucket floor. Below this, `temperature_by_options` omits the bucket.
MIN_BUCKET_N = 2000
# Type-level floor only. Lower than MIN_BUCKET_N; see the module docstring.
MIN_TYPE_N = 10
# Fraction of each bucket held out when `compute_ece=True`. Unused otherwise.
ECE_HOLDOUT_FRAC = 0.2
# Log-space search margin, in e-folds, outside the `[TEMP_MIN, TEMP_MAX]` answer band.
# The optimiser runs on log T with no bound of its own; any scale this far below
# `TEMP_MIN` already clamps to `TEMP_MIN`, so the band costs no real minimum, while an
# unbounded walk reaches `exp()` underflow (~-88 in float32), divides the objective by
# zero, and hands `clamp_temperature` a NaN -- which it reports as the neutral 1.0.
LOG_T_MARGIN = 10.0
# Payload schema. Files written before identity was recorded have no version key;
# readers treat that absence as version 1.
CALIBRATION_VERSION = 2
N_QTYPES = len(QTYPES)
# (qtype:int, logits:1d, target:1d, k:int)
Record = Tuple[int, Any, Any, int]
_CONFIG_IDENTITY_SKIP = ("temperature", "temperature_by_options")
def _vec(x) -> np.ndarray:
if isinstance(x, torch.Tensor):
x = x.detach().cpu().numpy()
return np.asarray(x, dtype=np.float32).reshape(-1)
def _validated_pair(logits, target, k=None):
vectors = []
for name, value in (("logits", logits), ("target", target)):
if isinstance(value, torch.Tensor):
value = value.detach().cpu().numpy()
try:
vector = np.asarray(value)
if vector.ndim != 1 or vector.dtype.kind not in "iuf":
raise ValueError("%s must be a one-dimensional numeric vector" % name)
with np.errstate(over="ignore", invalid="ignore"):
vector = vector.astype(np.float32)
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError("%s must be a one-dimensional numeric vector" % name) from exc
if not np.isfinite(vector).all():
raise ValueError("%s must contain only finite values" % name)
vectors.append(vector)
logits, target = vectors
if len(logits) != len(target):
raise ValueError("logits and target must have the same length")
if k is None:
k = len(logits)
if isinstance(k, bool) or not isinstance(k, Integral) or k < 1 or k > len(logits):
raise ValueError("k must be an integer between 1 and the vector length")
logits, target = logits[:k], target[:k]
if (target < 0).any() or not np.isclose(target.sum(), 1.0, rtol=1e-5, atol=1e-6):
raise ValueError("target must be a nonnegative probability vector summing to 1")
return logits, target, int(k)
def _positive_int(name: str, value: int) -> int:
"""A count parameter: an integer >= 1, normalised to a Python `int`, or a `ValueError` naming it.
This is the rule the module already applies everywhere a count is checked -- `min_n` in
:func:`fit_one_temperature`, `k` in :func:`_validated_pair` (which returns `int(k)` for the same
reason), and the `"bins"` of every `binning_map` entry :func:`apply_calibration_payload` reads --
applied to the two fitting parameters that were checked with less. The asymmetry was not
cosmetic: a `bool` is an `Integral`, so `bins` computed as a comparison (`len(bucket) > 15`)
passed the old `bins < 1` guard and emitted `{"bins": True, ...}`, a map the loader refuses on
its boolean arm. And a `numpy` integer is an `Integral` but not a Python `int`, so a `bins`
derived from an array shape emitted a map `Agent.save_calibration`'s own `json.dump` cannot write
-- a fit that produced an artifact the file format rejects. `int()` keeps that second case
working. A float, a string or `None` is a mistake, and now names itself instead of raising
`TypeError` from `range()` (or, for a `min_bucket_n` that compares `nan` to a length, silently
disabling the sample floor the parameter exists to enforce).
"""
if isinstance(value, bool) or not isinstance(value, Integral) or value < 1:
raise ValueError("%s must be a positive integer, got %r" % (name, value))
return int(value)
def _pairs_by_width(pairs: Sequence) -> List[Tuple[torch.Tensor, torch.Tensor]]:
"""One (logits, target) matrix per distinct option count, narrowest first.
A ragged pool -- a question type holding 2-option and 12-option decisions -- cannot be
one matrix. Any value used to square off the short rows is a real column in the single
softmax every row then goes through, so it competes for the probability mass the fit is
trying to allocate; at `-1e4` it wins the row it pads once the row's own logits drop
below it, and it overflows to `inf`/`NaN` in float32 when the scale gets small. Grouping
keeps each row inside its own option count, which is what the objective means.
"""
groups: Dict[int, List[Tuple[np.ndarray, np.ndarray]]] = {}
for z, t in pairs:
groups.setdefault(len(_vec(z)), []).append((_vec(z), _vec(t)))
return [(torch.from_numpy(np.stack([z for z, _ in groups[w]])),
torch.from_numpy(np.stack([t for _t, t in groups[w]]))) for w in sorted(groups)]
def _fit_unclamped_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
"""The optimised scale before `clamp_temperature`, for callers that report the clamp.
`fit_one_temperature` is this value clamped, and that is the value anything installs.
Only the clamp reporting in `_fit_groups` reads the raw scale, so that a bucket whose
optimum sits outside `[TEMP_MIN, TEMP_MAX]` can be told apart from one whose optimum
really is a bound. The search band is `LOG_T_MARGIN` e-folds wider than the answer
band, so a raw value can be far outside it; a NaN from the underflow case described in
`fit_one_temperature` reaches the caller as a NaN rather than as 1.0.
"""
if min_n is None:
min_n = MIN_BUCKET_N
if isinstance(min_n, bool) or not isinstance(min_n, Integral) or min_n < 1:
raise ValueError("min_n must be a positive integer")
sel = []
for pair in pairs:
try:
z, t = pair
except (TypeError, ValueError) as exc:
raise ValueError("pair must be (logits, target)") from exc
z, t, _k = _validated_pair(z, t)
sel.append((z, t))
if len(sel) < min_n:
return 1.0
groups = _pairs_by_width(sel)
n_rows = float(len(sel))
log_lo = float(np.log(TEMP_MIN)) - LOG_T_MARGIN
log_hi = float(np.log(TEMP_MAX)) + LOG_T_MARGIN
with torch.enable_grad():
log_t = torch.zeros(1, requires_grad=True)
opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)
def closure():
opt.zero_grad()
scale = log_t.clamp(min=log_lo, max=log_hi).exp()
nll = sum(-(t * torch.log_softmax(z / scale, -1)).sum(-1).sum()
for z, t in groups)
loss = nll / n_rows
loss.backward()
return loss
opt.step(closure)
return float(log_t.detach().clamp(min=log_lo, max=log_hi).exp().item())
def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
"""Fit one scalar T by NLL + LBFGS on log T.
The result is `clamp_temperature` of the optimised scale, so it lies in
`[TEMP_MIN, TEMP_MAX]` (or is the neutral 1.0 when the value is not a number).
The search is bounded to `LOG_T_MARGIN` e-folds outside that band, which covers any
minimum the answer can use; past it the float32 scale drifts toward `exp()` underflow,
where the objective divides by zero and every gradient becomes NaN, and
`clamp_temperature` reads that NaN back as the neutral 1.0.
Returns 1.0 when fewer than `min_n` pairs are given. `min_n` defaults to
`MIN_BUCKET_N` (the per-bucket floor). Type-level fits pass `MIN_TYPE_N`, which
is lower, so a dataset that fills no bucket still gets a scalar instead of
staying at 1.0.
A clamped result is indistinguishable from an optimum that really is a bound, because
both are the bound. `fit_temperature_map`'s `temperature_clamped` is where the
difference is reported; it carries the unclamped scale beside the value installed.
"""
return clamp_temperature(_fit_unclamped_temperature(pairs, min_n), TEMP_MIN, TEMP_MAX)
def _iter_records(records: Iterable) -> List[Tuple[int, np.ndarray, np.ndarray, int]]:
out = []
for rec in records:
if not isinstance(rec, (list, tuple)) or len(rec) not in (3, 4):
raise ValueError("record must be (qtype, logits, target[, k])")
if len(rec) == 4:
qtype, logits, target, k = rec
if k is None:
raise ValueError("k must be an integer between 1 and the vector length")
else:
qtype, logits, target = rec
k = None
if isinstance(qtype, bool) or not isinstance(qtype, Integral) or not 0 <= qtype < N_QTYPES:
raise ValueError("qtype must be an integer between 0 and %d" % (N_QTYPES - 1))
logits, target, k = _validated_pair(logits, target, k)
out.append((int(qtype), logits, target, k))
return out
def _softmax(z, t_scale: float) -> np.ndarray:
z = np.asarray(z, dtype=np.float64) / max(1e-3, float(t_scale))
z = z - z.max()
p = np.exp(z)
return p / p.sum()
def _ece_report(recs, temperature, temperature_by_options) -> Dict[str, Any]:
conf_b, cor_b, conf_a, cor_a = [], [], [], []
for qt, z, t, k in recs:
y = int(np.argmax(t[:k]))
p0 = _softmax(z[:k], 1.0)
conf_b.append(float(p0.max()))
cor_b.append(float(int(p0.argmax()) == y))
t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
p1 = _softmax(z[:k], t_scale)
conf_a.append(float(p1.max()))
cor_a.append(float(int(p1.argmax()) == y))
return {
"ece_before": ece_score(np.asarray(conf_b), np.asarray(cor_b)),
"ece_after": ece_score(np.asarray(conf_a), np.asarray(cor_a)),
"n": float(len(recs)),
}
def _group_records(recs):
by_type = {qt: [] for qt in range(N_QTYPES)}
by_bucket = {}
for qt, z, t, k in recs:
if qt in by_type:
by_type[qt].append((z, t))
key = temp_bucket(qt, k)
by_bucket.setdefault(key, []).append((z, t))
return by_type, by_bucket
def _ece_split(recs, seed):
"""Split records into fit and eval, stratified by `temp_bucket`.
Returns `(fit_recs, eval_recs, excluded_buckets)`. The split is a deterministic
partition (`numpy.random.RandomState(seed)`): every record is in exactly one side.
A bucket whose fit side would fall below `MIN_BUCKET_N` is not split; all of its
records stay in `fit_recs` and its key is listed in `excluded_buckets`.
"""
by_bucket = {}
for i, rec in enumerate(recs):
qt, _z, _t, k = rec
by_bucket.setdefault(temp_bucket(qt, k), []).append(i)
rng = np.random.RandomState(seed)
fit_idx = []
eval_idx = []
excluded = []
for key in sorted(by_bucket):
idxs = by_bucket[key]
n = len(idxs)
n_eval = int(round(n * ECE_HOLDOUT_FRAC))
if n - n_eval < MIN_BUCKET_N:
fit_idx.extend(idxs)
excluded.append(key)
continue
perm = rng.permutation(n)
for j in perm[:n_eval]:
eval_idx.append(idxs[int(j)])
for j in perm[n_eval:]:
fit_idx.append(idxs[int(j)])
fit_idx.sort()
eval_idx.sort()
fit_recs = [recs[i] for i in fit_idx]
eval_recs = [recs[i] for i in eval_idx]
return fit_recs, eval_recs, excluded
def _clamp_entry(raw: float, applied: float) -> Optional[Dict[str, Any]]:
"""Describe a fit the clamp moved, or None when it left the fit alone.
A NaN raw scale is the underflow case, not a clamp: `clamp_temperature` answers the
neutral 1.0 for it, which is a different fact and is not reported here.
"""
if not np.isfinite(raw) or TEMP_MIN <= raw <= TEMP_MAX:
return None
return {"fitted": raw, "applied": applied, "bound": "min" if raw < TEMP_MIN else "max"}
def _fit_groups(by_type, by_bucket):
temperature = [1.0] * N_QTYPES
clamped: Dict[str, Dict[str, Any]] = {}
for qt in range(N_QTYPES):
pairs = by_type[qt]
# MIN_TYPE_N, not MIN_BUCKET_N: the scalar has to exist when every bucket is small.
if len(pairs) >= MIN_TYPE_N:
raw = _fit_unclamped_temperature(pairs, min_n=MIN_TYPE_N)
temperature[qt] = clamp_temperature(raw, TEMP_MIN, TEMP_MAX)
entry = _clamp_entry(raw, temperature[qt])
if entry is not None:
clamped[QTYPE_NAMES[qt]] = entry
temperature_by_options = {}
for key, pairs in by_bucket.items():
if len(pairs) < MIN_BUCKET_N:
continue
raw = _fit_unclamped_temperature(pairs)
temperature_by_options[key] = clamp_temperature(raw, TEMP_MIN, TEMP_MAX)
entry = _clamp_entry(raw, temperature_by_options[key])
if entry is not None:
clamped[key] = entry
return temperature, temperature_by_options, clamped
def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: int = 0) -> Dict[str, Any]:
"""Fit type-level scalars and per-bucket temperatures.
`MIN_BUCKET_N` is the per-bucket floor: smaller buckets are omitted from
`temperature_by_options` and the type-level scalar covers them. `MIN_TYPE_N` is
the separate, lower floor for that scalar only.
`compute_ece=False` (the default, and the path `Agent.fit_temperatures` stores)
fits on every record and returns no `report` key. `seed` is ignored on this path.
`compute_ece=True` holds out `ECE_HOLDOUT_FRAC` of each bucket, stratified by
`temp_bucket`, using `seed` so the same records always split the same way. Temperatures
are fit on the remainder only and ECE is scored only on the held-out records.
`seed` is this module's own count rule -- an integer in `[0, 2**32-1]`, which is what
`numpy.random.RandomState` can take -- so a `10.0`, a `"7"`, a `True` or an out-of-range
value is a `ValueError` naming `seed` on either path, not numpy's cast error on one path
and silence on the other. `seed=None` stays numpy's entropy seed: accepted, and the split
is then not reproducible.
`temperature_clamped` names every fit the clamp moved, on either path: a bucket key as
`temperature_by_options` spells it, or a question-type name for one of the `temperature`
scalars. Each entry carries `fitted` (the unclamped optimum), `applied` (the value in
this result) and `bound`, `"min"` or `"max"`. It is empty when nothing was clamped, and
it reports a fact a caller cannot otherwise recover: a value of exactly `TEMP_MIN` or
`TEMP_MAX` is what both a clamped fit and a genuine fit on the bound look like, so
without this an optimum of 0.387 installs as 0.5 and reads as a finished fit. The
clamped value is still the one to install, because `load_calibration` applies the same
bounds at checkpoint load; what the entry adds is that the data wanted to go further.
`report["n"]` is the number of records passed in; `report["n_eval"]` is the held-out
count the ECE rests on. A bucket that would drop below `MIN_BUCKET_N` after the
holdout is fit on all of its records, left out of the eval set, and named in
`report["buckets_excluded_from_eval"]` instead of being dropped. `n_by_bucket`
always counts the full input, including when the fit itself used a subset.
"""
if seed is not None and (isinstance(seed, bool) or not isinstance(seed, Integral)
or not 0 <= seed < 2 ** 32):
raise ValueError("seed must be None or an integer in [0, 2**32-1], got %r" % (seed,))
recs = _iter_records(records)
all_by_type, all_buckets = _group_records(recs)
n_by_bucket = {key: len(pairs) for key, pairs in all_buckets.items()}
if compute_ece:
_fit_recs, eval_recs, excluded = _ece_split(recs, seed)
by_type, by_bucket = _group_records(_fit_recs)
else:
eval_recs = None
excluded = []
by_type, by_bucket = all_by_type, all_buckets
temperature, temperature_by_options, clamped = _fit_groups(by_type, by_bucket)
out: Dict[str, Any] = {
"temperature": temperature,
"temperature_by_options": temperature_by_options,
"n_by_bucket": n_by_bucket,
"temperature_clamped": clamped,
}
if compute_ece:
report = _ece_report(eval_recs, temperature, temperature_by_options)
# `n` stays the size of the input. `n_eval` is the held-out count ECE used.
report["n"] = float(len(recs))
report["n_eval"] = float(len(eval_recs))
report["buckets_excluded_from_eval"] = excluded
out["report"] = report
return out
fit_temperatures = fit_temperature_map
# Per-bucket floor for abstention thresholds. Lower than `MIN_BUCKET_N` (which governs temperature
# fitting): a threshold is a single order statistic of the calibrated confidences, so it stabilises
# on far fewer examples than an LBFGS temperature does. Buckets below this are omitted, and only the
# map's own "default" entry covers them afterwards -- `resolve_min_confidence` falls from "default"
# to 0.0, which gates nothing. There is no scalar behind that fall-through, unlike a temperature
# bucket omitted from `temperature_by_options`, whose type-level scalar really is the fallback: one
# call's `min_confidence` is either a float or a map, never both, so no caller's float is in scope
# here. A map produced with this floor should name a "default" if any bucket under it must abstain.
MIN_ABSTAIN_BUCKET_N = 100
def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_error: float,
conservative: bool) -> float:
"""Smallest confidence `tau` whose accepted set (`conf >= tau`) keeps error <= `target_error`.
`pairs` are `(confidence, correct)`. Sweeping from the most confident down maximises coverage
at the target risk (the selective-classification / split-conformal cut). `conservative` adds one
pseudo-error so a bucket does not clear the gate on a lucky short run. Returns 1.0 when no cut
holds the risk -- the bucket is too unreliable to accept anything short of a reported certainty.
"""
ordered = sorted(pairs, key=lambda cc: cc[0], reverse=True)
levels = sorted({c for c, _ in ordered}, reverse=True)
n = len(ordered)
n_acc = n_err = idx = 0
best = None
# Evaluate the error over the WHOLE accepted set {conf >= tau} at each distinct level, not
# incrementally within a tie: a threshold accepts every answer at its own confidence, so a tie
# group's errors must all be counted before the level is judged (else the cut sinks into a bad
# cohort on its first few correct members).
for tau in levels:
while idx < n and ordered[idx][0] >= tau:
n_acc += 1
n_err += 0 if ordered[idx][1] else 1
idx += 1
rate = (n_err + 1.0) / (n_acc + 1.0) if conservative else (n_err / n_acc)
if n_acc > 0 and rate <= target_error:
best = tau
if best is None:
return 1.0
return float(min(1.0, max(0.0, best)))
def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
temperature_by_options: Dict[str, float], *,
binning_map: Optional[Dict[str, Dict[str, Any]]] = None,
target_error: float = 0.10,
min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,
conservative: bool = True) -> Dict[str, float]:
"""Fit a per-`temp_bucket` abstention threshold so a gate keeps a target error in each fitted bucket.
A single `min_confidence` does not transfer across option counts (#394): the calibrated
confidence of a 2-option and a 12-option answer live on different scales, so one cut over- or
under-abstains depending on the question. This fits one cut per bucket instead, keyed exactly
like `temperature_by_options` (`common.temp_bucket`, e.g. ``"choice:3-5"``), and the result is a
`min_confidence` map that :func:`laya.confidence.check_min_confidence` /
:func:`laya.confidence.apply_confidence_gate` accept directly -- with one exception to keep:
every bucket below `min_bucket_n` is omitted, so a calibration set too small to fill a single
bucket returns ``{}``, and the validator refuses an empty map ("must be a non-empty dict"). Pass
no gate in that case rather than passing what came back; `laya.train` writes the map into a
checkpoint only `if abstention_thresholds:`.
`records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes
(`records_from_labeled` builds them). Confidence is the **calibrated** `max(p)` -- the logits are
scaled by the fitted `temperature` / `temperature_by_options` first, so thresholds and the
numbers the runtime reports are on the same scale. `target_error` is the tolerated error among
accepted answers; `min_bucket_n` omits buckets too small to fit, and `conservative` adds a
one-sample margin. An omitted bucket is not caught by a fallback: the map lookup falls to its own
``"default"`` entry and then to 0.0, which gates nothing, so a `min_bucket_n` that excludes a
bucket leaves that bucket ungated unless the map names a ``"default"``. The thresholds are
empirical cuts on the calibration set, not a formal coverage guarantee -- validate on held-out
data (`fit_temperature_map(..., compute_ece=True)` gives a held-out split) for a production gate.
Pass `binning_map` when the agent that will serve these thresholds has one installed -- by
`Agent.fit_binning`, or by a calibration payload that carries `binning_map` -- because the
runtime recalibrates `answer_confidence` through that map before anything reads it, so a cut
fitted without it is a cut on a scale the gate never sees. The thresholds are then on the binned
scale, and the order the two were fitted in stops mattering. Measured on 1,200 synthetic
12-option records at `target_error=0.10`: the cut fitted without a map holds 9.8% error over 50%
coverage on un-binned confidences, and admits 94.5% of answers at 25.6% error once the same
number is compared against binned ones.
"""
# A bool is the one value a range test cannot see: `0.0 <= True <= 1.0` holds, so `False` would
# be read as 0.0 -- the cut that abstains on every answer -- and `True` as 1.0, which gates
# nothing. Both fit a map, so nothing downstream would mention it. `TrainConfig` refuses the
# same bool on the way to this call (`train.py`), as does `_iter_records` for a `qtype`.
if isinstance(target_error, bool) or not 0.0 <= target_error <= 1.0:
raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))
min_bucket_n = _positive_int("min_bucket_n", min_bucket_n)
recs = _iter_records(records)
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
for qt, z, t, k in recs:
y = int(np.argmax(t[:k]))
bucket = temp_bucket(qt, k)
t_scale = temperature_by_options.get(bucket, temperature[qt])
p = _softmax(z[:k], t_scale)
conf = float(p.max())
if binning_map:
# The runtime bins before anyone reads `answer_confidence` (`Agent._decode_answers`),
# so the cut has to be chosen on the binned scale or it gates a different quantity.
conf = apply_binning_map(conf, bucket, binning_map)
# `_decode_answers` reports `answer_confidence` rounded to 4 decimals, after binning, and
# the gate compares that. A cut picked at full precision can sit above the rounded value of
# its own cohort (bin 5/6 is reported as 0.8333 < 0.83333...), which abstains all of it.
conf = round(conf, 4)
by_bucket.setdefault(bucket, []).append((conf, int(int(p.argmax()) == y)))
out: Dict[str, float] = {}
for key, pairs in by_bucket.items():
if len(pairs) < min_bucket_n:
continue
out[key] = _select_abstention_threshold(pairs, target_error, conservative)
return out
# Per-bucket floor for histogram-binning recalibration. Higher than the abstention floor because a
# binning map splits each bucket's examples across `bins`, so each bin needs its own sample; lower
# than the temperature floor because binning is a count per bin, not an optimisation.
MIN_BINNING_BUCKET_N = 200
def fit_binning_map(records: Iterable, temperature: Sequence[float],
temperature_by_options: Dict[str, float], *, bins: int = 15,
min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Dict[str, Any]]:
"""Fit a per-`temp_bucket` histogram-binning recalibration map for `answer_confidence`.
Temperature scaling applies one scalar per bucket; it cannot fix a bucket whose reliability
curve is not a simple sharpening/softening (the pathological `choice:11+` the shipped English
checkpoint carries is one). Histogram binning is the non-parametric alternative: split the
calibrated confidences of a bucket into `bins` equal-width bins over [0, 1], and map every
confidence that lands in a bin to that bin's empirical accuracy. It needs no monotonicity
assumption and no extra dependency (NumPy only; isotonic regression would pull in scikit-learn).
`records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes;
confidence is the calibrated `max(p)` (logits scaled by the fitted `temperature` /
`temperature_by_options` first), so a binning map composes on top of a temperature map rather
than replacing it. Returns `{bucket: {"bins": N, "values": [recalibrated confidence per bin]}}`;
buckets below `min_bucket_n` are omitted. Apply it with :func:`apply_binning_map`. An empty bin
(a confidence range the calibration set never produced) maps to that bin's centre, a point
inside the region the value came from: the value is still rewritten, and moves by at most half a
bin width. What the centre buys is that an unseen confidence is never recalibrated to a
fabricated 0, not that it passes through -- only a bucket the map has no entry for does that
(:func:`apply_binning_map`).
"""
bins = _positive_int("bins", bins)
min_bucket_n = _positive_int("min_bucket_n", min_bucket_n)
recs = _iter_records(records)
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
for qt, z, t, k in recs:
y = int(np.argmax(t[:k]))
bucket = temp_bucket(qt, k)
t_scale = temperature_by_options.get(bucket, temperature[qt])
p = _softmax(z[:k], t_scale)
by_bucket.setdefault(bucket, []).append((float(p.max()), int(int(p.argmax()) == y)))
out: Dict[str, Dict[str, Any]] = {}
for key, pairs in by_bucket.items():
if len(pairs) < min_bucket_n:
continue
conf = np.asarray([c for c, _ in pairs], dtype=float)
corr = np.asarray([c for _, c in pairs], dtype=float)
idx = np.clip((conf * bins).astype(int), 0, bins - 1)
values = []
for b in range(bins):
mask = idx == b
values.append(float(corr[mask].mean()) if mask.any() else (b + 0.5) / bins)
out[key] = {"bins": bins, "values": values}
return out
def apply_binning_map(confidence: float, bucket: str,
binning_map: Dict[str, Dict[str, Any]]) -> float:
"""Recalibrate one `answer_confidence` for its option-count `bucket` (`common.temp_bucket`).
Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map
was not fit for passes through rather than being forced to a wrong value.
"""
entry = binning_map.get(bucket)
if not entry:
return float(confidence)
bins = int(entry["bins"])
b = min(bins - 1, max(0, int(float(confidence) * bins)))
return float(entry["values"][b])
@torch.no_grad()
def records_from_labeled(agent, pairs: Sequence) -> List[Record]:
"""Collect CPU records from `(state, questions, targets)`.
Logits are the raw rows `_decode_answers` scales, not the probabilities it returns.
`_encode_state` builds one state's items, `_forward` runs them, and `_option_logits`
slices each question to the same marker width `_decode_answers` divides by temperature.
No Hub download and no `model.safetensors` write. Tests stay weight-free by passing
synthetic records to `fit_temperature_map` instead of calling this.
"""
# Imported lazily: `agent` imports this module at load, and `_option_logits` is defined
# on the way through that import.
from .agent import _option_logits
records: List[Record] = []
for state, questions, targets in pairs:
ids = list(questions.keys())
if not ids:
continue
for qid in ids:
agent._check_question(qid, questions[qid])
internal = {qid: agent._to_internal(questions[qid]) for qid in ids}
items = agent._encode_state(state, ids, internal)
batch = collate_items([items], agent.tok.pad_token_id)
if batch is None:
continue
logits, _act = agent._forward(batch)
rows = _option_logits(logits, items, 0)
for j, qid in enumerate(ids):
k = len(items[j]["markers"])
qt = int(items[j]["qtype"])
records.append((qt, _vec(rows[j]), _vec(targets[qid])[:k], k))
return records
def _config_identity(cfg):
"""Checkpoint config without the temperature fields this file itself stores."""
if cfg is None:
return None
return {str(k): cfg[k] for k in cfg if k not in _CONFIG_IDENTITY_SKIP}
def calibration_payload(
temperature,
temperature_by_options,
model_id_or_path=None,
subfolder=None,
config=None,
binning_map=None,
) -> Dict[str, Any]:
"""JSON body written by `Agent.save_calibration` (no weights).
`version` is `CALIBRATION_VERSION`. Payloads written before identity was recorded have
no version key; `apply_calibration_payload` treats that as version 1 and still loads.
`model_id_or_path`, `subfolder`, and `config` say which checkpoint the map was fitted
against. `config` is `rl_agent_config.json` without `temperature` /
`temperature_by_options` (those live at the top of this payload).
`binning_map` is the optional histogram-binning recalibration map fitted by
`fit_binning_map`; the key is omitted when no map is installed, which is `None`, not an empty
map. A map that fitted nothing -- `{}`, the return value when no bucket reached the sample
floor -- is still written, so the round trip through :func:`apply_calibration_payload` gives
`{}` back rather than `None`.
"""
payload = {
"version": CALIBRATION_VERSION,
"temperature": [float(x) for x in temperature],
"temperature_by_options": {
str(k): float(v) for k, v in dict(temperature_by_options).items()
},
"model_id_or_path": model_id_or_path,
"subfolder": subfolder,
"config": _config_identity(config),
}
if binning_map is not None:
payload["binning_map"] = binning_map
return payload
def _warn_if_identity_mismatch(obj, payload) -> None:
recorded_model = payload.get("model_id_or_path")
recorded_sub = payload.get("subfolder")
recorded_cfg = payload.get("config")
current_model = getattr(obj, "model_id_or_path", None)
current_sub = getattr(obj, "subfolder", None)
current_cfg = _config_identity(getattr(obj, "cfg", None))
if (recorded_model, recorded_sub, recorded_cfg) == (current_model, current_sub, current_cfg):
return
cfg_note = ""
if recorded_cfg != current_cfg:
cfg_note = " Checkpoint config identity differs."
warnings.warn(
"calibration was fitted for model_id_or_path=%r subfolder=%r but this agent is "
"model_id_or_path=%r subfolder=%r.%s Loading the temperatures anyway."
% (recorded_model, recorded_sub, current_model, current_sub, cfg_note),
UserWarning,
stacklevel=3,
)
def _rejected_temperatures(entries):
"""Names whose raw value is not the number `clamp_temperature` kept, matching checkpoint load."""
rejected = []
for name, raw, applied in entries:
if not temperature_rejected(raw, applied):
continue
rejected.append("%s=%r -> %g" % (name, raw, applied))
return rejected
def _install_temperatures(obj, temperature, temperature_by_options, warn: bool = True) -> None:
"""Store a map the way checkpoint load does: raw values, and the clamped ones inference reads.
`warn=False` is the fitter path: those values were already passed through `clamp_temperature`,
so a second pass must not report them as a bad file.
"""
raw_t = list(temperature)
applied_t = [clamp_temperature(t) for t in raw_t]
raw_b = {str(k): v for k, v in dict(temperature_by_options).items()}
applied_b = {k: clamp_temperature(v) for k, v in raw_b.items()}
obj.temperature_raw = raw_t
obj.temperature = applied_t
obj.temperature_by_options_raw = raw_b
obj.temperature_by_options = applied_b
if not warn:
return
entries = [("temperature[%d]" % i, raw, applied_t[i]) for i, raw in enumerate(raw_t)]
entries += [(key, raw_b[key], applied_b[key]) for key in raw_b]
rejected = _rejected_temperatures(entries)
if rejected:
warnings.warn(
"laya: calibration temperatures are invalid or outside [%g, %g]; "
"using %s. Treat confidence from the affected entries as uncalibrated."
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
RuntimeWarning,
stacklevel=3,
)
def apply_calibration_payload(obj, payload: Dict[str, Any]) -> None:
"""Copy `temperature`, `temperature_by_options` and `binning_map` onto `obj`.
Those are the three fields this reads and the three it writes. `binning_map` is installed
whatever the payload holds, including nothing: a payload with no ``binning_map`` key -- every
file written before histogram binning existed, and every agent that saved before
:meth:`Agent.fit_binning` ran -- installs `None`, which clears a map this object already
fitted. The file is the whole calibration state, not a patch onto the current one.
A missing `version` is version 1 (temperatures only, no checkpoint identity). Version
`CALIBRATION_VERSION` records the checkpoint the map was fitted for; a mismatch warns
and still loads, so an older file never becomes a hard failure. Each temperature is passed
through `clamp_temperature`, so a non-numeric or out-of-range entry cannot crash a later
forward the way an unclamped zero used to.
`temperature` and `binning_map` take opposite value policies, and that is the sentence to read
before writing a file by hand: a bad *temperature* is forgiven and clamped into
`[TEMP_MIN, TEMP_MAX]`, while a bad *binning value* is refused. Nothing recalibrates a
confidence the way an out-of-range binning value would, so there is no defensible fallback
for it.
The payload's *shape* is checked before any of that, and refused with a `ValueError` naming
the field: the values may be junk the clamp forgives, but the payload itself not being an
object, a `version` that is not an integer, a `temperature` that is not a list of three, a
`temperature_by_options` that is not an object, a `binning_map` that is not an object, or a
`binning_map` entry that is not an object with an integer `bins` >= 1 and `values` of exactly
that length holding finite numbers in [0, 1] is a file this code cannot read -- JSON gives
`int`, `str`, `list` and `dict` for each of those mistakes just as happily as it gives the
right shapes, and each used to fail with a raw `TypeError`/`AttributeError` from
`len()`/`dict()`, or worse, to load: a `{"a": 1, "b": 2, "c": 3}` or `"abc"` has `len` 3 and
used to pass the length check and install its *keys* as temperatures.
"""
if not isinstance(payload, dict):
raise ValueError("calibration JSON must be an object, got %s" % type(payload).__name__)
version = payload.get("version", 1)
if version is None:
version = 1
if isinstance(version, bool) or not isinstance(version, (int, float, str)):
raise ValueError("calibration JSON version must be an integer, got %r" % (version,))
if isinstance(version, float) and not version.is_integer():
# `int()` truncates, and the truncation is not cosmetic: a file whose `version` is 1.9
# became 1, which is below CALIBRATION_VERSION, so the identity check at the foot of this
# function never ran and a map fitted for another checkpoint loaded silently. The string
# form of the same mistake already raises below; a whole number written as `2.0` is an
# integer and still loads.
raise ValueError("calibration JSON version must be an integer, got %r" % (version,))
try:
version = int(version)
except (TypeError, ValueError, OverflowError) as exc: # Overflow: JSON's 1e999 is inf
raise ValueError(
"calibration JSON version must be an integer, got %r" % (version,)) from exc
temps = payload.get("temperature")
if not isinstance(temps, (list, tuple)) or len(temps) != N_QTYPES:
raise ValueError("calibration JSON must contain temperature: [3 floats]")
by_options = payload.get("temperature_by_options")
if by_options is None: # absent, or an explicit null: a file from before bucket temperatures
by_options = {}
if not isinstance(by_options, dict):
raise ValueError(
"calibration JSON temperature_by_options must be an object of bucket -> float, "
"got %s" % type(by_options).__name__)
binning = payload.get("binning_map")
if binning is not None:
if not isinstance(binning, dict):
raise ValueError(
"calibration JSON binning_map must be an object of bucket -> {bins, values}, "
"got %s" % type(binning).__name__)
for name, entry in binning.items():
if not isinstance(entry, dict):
raise ValueError(
"calibration JSON binning_map[%r] must be an object with \"bins\" and \"values\", "
"got %s" % (name, type(entry).__name__))
n_bins = entry.get("bins")
values = entry.get("values")
if isinstance(n_bins, bool) or not isinstance(n_bins, int) or n_bins < 1:
raise ValueError(
"calibration JSON binning_map[%r] must have an integer \"bins\" >= 1, got %r" % (name, n_bins))
if not isinstance(values, (list, tuple)) or len(values) != n_bins:
raise ValueError(
"calibration JSON binning_map[%r] must have \"values\" of length \"bins\" (%d)"
% (name, n_bins))
try:
parsed_values = [float(v) for v in values]
except (TypeError, ValueError) as exc:
raise ValueError(
"calibration JSON binning_map[%r] values must be numbers, got %r" % (name, values)) from exc
for v in values:
if isinstance(v, bool) or not isinstance(v, (int, float)):
raise ValueError(
"calibration JSON binning_map[%r] values must be numbers, got %r" % (name, values))
for v in parsed_values:
if not np.isfinite(v):
raise ValueError(
"calibration JSON binning_map[%r] values must be finite, got %r" % (name, values))
if not 0.0 <= v <= 1.0:
raise ValueError(
"calibration JSON binning_map[%r] values must be in [0, 1], got %r" % (name, values))
obj.binning_map = binning
if version >= CALIBRATION_VERSION:
_warn_if_identity_mismatch(obj, payload)
_install_temperatures(obj, temps, by_options)

1018
laya_repo/common.py Normal file

File diff suppressed because it is too large Load Diff

315
laya_repo/email.py Normal file
View File

@ -0,0 +1,315 @@
"""Email utilities for cleaning and structuring email inputs in laya.
The markers below cover English, Portuguese, Spanish and French mail clients. The Router already sends
Portuguese and Spanish states to the multilingual checkpoint, but with English-only markers their
cleaning was a no-op: Gmail's `Em ... escreveu:`, Outlook's `-----Mensagem original-----`, the
`Atenciosamente` sign-off and the confidentiality footer all reached the model, and the quoted
history (often a *different* request) weighed on the answer as much as the new message did.
French mail reads the same way through an English-only cleaner: Gmail's `Le ... a écrit :`,
Outlook's `-----Message d'origine-----`, `Cordialement` and `Envoyé depuis mon iPhone` all survived.
"""
import re
import unicodedata
from typing import Dict, List, Optional
# One definition, in the module that holds the other presets. Re-exported here because
# `from laya.email import email_questions` is a path callers already have.
from .presets import email_questions # noqa: F401
_QUOTE_HEADERS = [
re.compile(r"^\s*On .{0,300}wrote:\s*$", re.I),
# "Em resposta ao que você escreveu:" is body text; a client's attribution always carries a date
re.compile(r"^\s*Em (?=.*\d).{0,300}escreveu:\s*$", re.I),
re.compile(r"^\s*El (?=.*\d).{0,300}escribi[óo]:\s*$", re.I),
# Gmail's French attribution always carries a date too ("Le lun. ... a écrit :"); a bare
# "Le rapport que vous avez écrit :" is body text, the same rule as above. French typography
# puts a space before the colon, so the verb allows one (`a écrit :`, never `a écrit:`).
re.compile(r"^\s*Le (?=.*\d).{0,300}a [eé]crit\s*:\s*$", re.I),
re.compile(r"^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}", re.I),
re.compile(r"^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado)|Message d'origine)\s*-{2,}", re.I),
re.compile(r"^\s*_{8,}\s*$"),
# `From:` opens ordinary prose too ("From: my side the integration works, but please
# refund..."), and a reply header always carries the sender, so the header is only
# recognised when an address follows -- the same rule as `De:` below. A bare
# `From: Name` header is caught by _HEADER_FROM_NAME/_HEADER_NEXT instead, which need the
# header's own `Sent:`/`Date:` line to tell it apart from a sentence.
re.compile(r"^\s*From:\s.*[@<]", re.I),
# `De:` also opens ordinary Portuguese/Spanish lines ("De: 10/09 a 15/09"), so the Outlook
# header is only recognised when it carries an address. French Outlook writes `De :` with a
# space, so the marker allows one; the address rule is unchanged
re.compile(r"^\s*De\s*:\s.*[@<]", re.I),
]
# Gmail wraps a long attribution line, leaving `fulano@x.com> escreveu:` alone on the next line.
# That tail cuts too, and takes the `On/Em/El/Le ...` head it belongs to with it. The French tail
# keeps its spaced colon (`support@x.com> a écrit :`).
_ATTRIBUTION_TAIL = re.compile(r"^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]|a [eé]crit)\s*:\s*$", re.I)
_ATTRIBUTION_HEAD = re.compile(r"^\s*(On|Em|El|Le) (?=.*\d)", re.I)
# Exchange often leaves the address out of Outlook's reply header ("De: Maria Souza"), so a bare `De:`
# only cuts when the header's own `Enviado:` line, or a dated `Data:`/`Fecha:` line, follows it.
# `Para:` is not enough: "De: 10/09 / Para: 15/09" is how a leave request reads.
#
# The same is true of a bare English `From: Maria Souza`, which is why the marker above needs
# this rule: the English client lines are the translations of the two `De:` neighbours. A line
# that only looks like prose still has to be told apart from a header by its neighbours, so the
# English pair is "From: <name>" followed by "Sent:"/"Date:". French Outlook writes the same
# header as `De : Marie Dupont` with `Envoyé :` underneath, so the pair gains that translation too.
# Both allow the French spaced colon (`De :`, `Envoyé :`); the neighbours still do the telling apart.
_HEADER_FROM_NAME = re.compile(r"^\s*(De|From)\s*:\s+\S", re.I)
_HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Envoy[ée]( le)?\s*:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})", re.I)
# A closing's name starts with a letter that is not lowercase: capitalised in any script
# (`Łukasz`, `Дмитрий`) or caseless (`山田`). `re` cannot say "not a lowercase letter in any
# script" -- a class has to list ranges, and `[^\W\d_a-zß-öø-ÿ]` stops at Latin-1, so
# `Thanks, żaneta` read as a name and the line counted as a sign-off. The tail is matched
# structurally instead, and each token's first letter is judged by category below -- the same
# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before
# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port. `re` has no `\p{M}` either, and a
# class cannot list those ranges any more than it could list the lowercase ones, so a mark that
# rides on a character is dropped before the tail is matched and the letter it rides on answers
# the case question. A mark with no base -- opening the tail, or following a space -- is left
# where it is, so it still breaks the token as the port's leading `\p{Lu}\p{Lt}\p{Lo}` does.
# `\p{M}` is marks only: ZWJ and ZWNJ are `Cf`, so `Thanks, क्‌ष` is kept here and in the port alike.
_SIGNOFF_HEAD = re.compile(
r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"
r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"
)
_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w'-]*[\s,.]*){0,3}$")
_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w'-]*")
def _drop_marks(text: str) -> str:
"""Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`.
Only a mark that rides on a preceding character goes; one that opens the string or
follows a space has no base to ride on and stays, so it still separates tokens.
"""
kept = []
for ch in text:
if unicodedata.category(ch).startswith("M") and kept and not kept[-1].isspace():
continue
kept.append(ch)
return "".join(kept)
def _is_english_signoff(line: str) -> bool:
"""True when a closing word is followed by nothing but punctuation and a short name."""
m = _SIGNOFF_HEAD.match(line)
if m is None:
return False
tail = _drop_marks(line[m.end():])
if _SIGNOFF_TAIL.match(tail) is None:
return False
return all(unicodedata.category(token[0]) in ("Lu", "Lt", "Lo")
for token in _SIGNOFF_TOKEN.findall(tail))
def _marker_matches(marker, line: str) -> bool:
"""One `_SIGNATURE_MARKERS` entry: a compiled pattern, or a callable for a rule a pattern
cannot express (the English sign-off)."""
return bool(marker(line)) if callable(marker) else bool(marker.match(line))
_SIGNATURE_MARKERS = [
re.compile(r"^\s*--\s*$"),
# A closing line is the closing word plus punctuation and at most a name. Anything else on
# the line is a sentence, and the case of the next word is what separates the two: a name is
# capitalised, "for" in "Thanks for the quick reply." is not. The closing words are matched
# case-insensitively, the name is not, so the flag is scoped instead of global.
# `warmest` and `and/& regards` are closings the alternation did not reach; the name that may
# follow is judged in `_is_english_signoff` above, a callable because `re` cannot express its
# rule. `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not.
_is_english_signoff,
# Portuguese/Spanish sign-offs match only on their own: "Obrigado pelo retorno, mas ..." is a
# request, not a signature, so unlike the English marker no trailing words are allowed.
# French closings keep the same rule: "Merci pour votre aide, mais ..." stays, while a bare
# "Cordialement," or "Merci," goes with the name underneath it.
re.compile(
r"^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?"
r"( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|"
r"(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?|"
r"(bien )?cordialement|salutations( distinguées)?|bien à vous|merci( d'avance)?|"
r"bonne journée)[\s,!.]*$",
re.I,
),
]
# Mobile and mail-app footers. Only a line that is nothing *but* the footer matches -- "Enviado do meu
# celular o comprovante ontem." is a request -- and such a line may run to 60 characters, since
# Samsung's default ("Enviado do meu smartphone Samsung Galaxy.") is longer than a sign-off's 40.
# Apple's French default ("Envoyé depuis mon iPhone") reads the same way.
_DEVICE = (r"iphone|ipad|android|ios|mobile|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|"
r"outlook|yahoo|mail|e-?mail|gmail|windows")
_DEVICE_FOOTER = re.compile(
r"^\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?|"
r"envoy[ée] (depuis|de)( mon| ma| mes)?)"
r" (%s)( (%s|para|for|no|na|\d+|phone|device|pro|max|mini|plus|using [a-z][a-z0-9_.+-]*))*"
r"|(obter o|get) outlook (para|for) (ios|android))[\s.!]*$"
% (_DEVICE, _DEVICE),
re.I,
)
_DISCLAIMER = re.compile(
# English: tied to a disclaimer noun and a disclaimer tail, the way the Portuguese
# branches below are. The bare word matched any sentence that merely mentioned it,
# so "Is this confidential?" and "Confidential: I need a refund." were deleted whole.
# `[^.]` rather than `[^.\n]`: a footer wraps, so "are\nconfidential" must still match.
r"(\b(e-?mail|message|information|communication|transmission|contents?)\b[^.]{0,60}"
r"\bconfidential\b[^.]{0,60}\b(intended|solely|addressee|recipient|privileged|"
r"disclos|unauthori[sz]ed)|"
r"\bconfidential\b[^.]{0,60}\b(and (may|is) (also )?privileged)|"
r"if you (have )?received this (e-?mail|message) in error|"
# Portuguese/Spanish: tied to "this message/e-mail" rather than the bare word `confidencial`,
# which a sender's own request ("preciso do contrato confidencial") uses just as often
r"\b(esta|este) (mensagem|e-?mail|mensaje|correo)\b[^.]{0,80}(confidencia|sigilos|privilegiad)|"
r"\b(uso exclusivo|exclusivamente|únicamente|unicamente)\b[^.]{0,30}"
r"(destinatári|destinatari|pessoa|persona|entidade|entidad)|"
r"\b(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)\b[^.]{0,20} por (engano|erro)|"
r"\b(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)\b[^.]{0,20} por error|"
# the "think before printing" footer, tied to its environmental ending rather than to
# `antes de imprimir`, which a request uses too ("antes de imprimir o boleto, confira o valor")
r"\bantes de imprimir\b[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)|"
r"\b(meio|medio) ambiente\b[^.]{0,30}antes de imprimir|"
# French: tied to "ce message/cet e-mail" rather than the bare word `confidentiel`,
# which a sender's own request ("le contrat confidentiel") uses just as often
r"\b(ce|cet|cette) (message|e-?mail|mail|courriel)\b[^.]{0,80}(confidentiel|privil[eé]gi)|"
r"\bavez re[çc]u (ce|cet|cette) (message|e-?mail|mail)\b[^.]{0,20} par erreur|"
r"\b(usage exclusif|exclusivement|uniquement)\b[^.]{0,30}destinataire)",
re.I,
)
_SENTENCE = re.compile(r"(?<=[.!?])\s+")
def _starts_new_sentence(line: str) -> bool:
"""First letter is uppercase: a fresh sentence, not a wrapped line.
Lines in uncased scripts (CJK, Devanagari, ...) never start a new piece,
so wrapped boilerplate in those scripts still drops whole.
"""
for ch in line:
if ch.isalpha():
return ch.isupper()
return False
def _split_fused_lines(sentence: str) -> List[str]:
"""Split a fused boilerplate-positive sentence at sentence-starting newlines.
An unpunctuated request line glued to a disclaimer line ("locked\\nThis ...")
splits at the newline because the next line starts uppercase; a lowercase
continuation ("are\\nconfidential") belongs to the same sentence, so a wrapped
boilerplate footer still drops whole.
"""
if "\n" not in sentence:
return [sentence]
pieces, buf = [], ""
for line in (ln.strip() for ln in sentence.split("\n")):
if not line:
continue
if buf and _starts_new_sentence(line):
pieces.append(buf)
buf = line
else:
buf = (buf + " " + line) if buf else line
if buf:
pieces.append(buf)
return pieces
def _strip_disclaimer(paragraph: str) -> str:
"""Drop boilerplate disclaimer text from one paragraph.
A paragraph is dropped whole only when *every* sentence in it is boilerplate; otherwise only
the boilerplate sentences go. A footer that runs on without a blank line used to take the
sender's actual request with it, which is worse than leaving one boilerplate line behind.
"""
if not _DISCLAIMER.search(paragraph):
return paragraph # nothing to do: keep the original line structure
parts = [p.strip() for p in _SENTENCE.split(paragraph) if p.strip()]
pieces = []
for p in parts:
pieces.extend(_split_fused_lines(p) if _DISCLAIMER.search(p) else [p])
return " ".join(p for p in pieces if not _DISCLAIMER.search(p))
def clean_email_body(body: str, max_chars: int = 3000) -> str:
"""Remove quoted email history, signatures and disclaimers to keep input focused.
`max_chars` is the length the result is cut to, 3000 characters unless raised -- see
`email_state`, which takes the same budget and passes it through.
It is a slice budget, not a floor, and it is applied twice: as the `max_chars * 4` pre-cut
that bounds the regex work below, and as the `text[:max_chars]` that returns the result. The
pre-cut runs first, so the quote, signature and disclaimer rules see only the first
`4 * max_chars` characters -- a footer that begins past that window is never recognized, and
the paragraph holding it survives to the final cut. So `0` answers with the empty string, and a
negative value trims from the end: its length test is against a negative number, so the pre-cut
always fires and removes `4 * |max_chars|`, and the final slice removes `|max_chars|` more,
which is `5 * |max_chars|` off the body and empty once that reaches the body's length. That is
specified behaviour, recorded by the `budget-zero` and `budget-negative-*` cases of
`laya-java/fixtures/email.json` that the Java and TypeScript ports are tested against, so "no
limit" means a budget larger than the body -- never `0` and never negative.
"""
text = (body or "").replace("\r\n", "\n").replace("\r", "\n").replace("\\n", "\n")
# Bound regex work before the expensive patterns below: _DISCLAIMER uses
# [^.]{0,60/80/100} alternations whose cost grows with input length, and only
# max_chars are ever returned. Truncate lines too so one MB-long line cannot
# dominate matching.
if len(text) > max_chars * 4:
text = text[:max_chars * 4]
lines = []
src = text.split("\n")
for i, line in enumerate(src):
if any(p.match(line) for p in _QUOTE_HEADERS) and lines:
break
if (lines and _HEADER_FROM_NAME.match(line) and i + 1 < len(src)
and _HEADER_NEXT.match(src[i + 1])):
break
if _ATTRIBUTION_TAIL.match(line) and lines:
if _ATTRIBUTION_HEAD.match(lines[-1]):
lines.pop()
break
if line.lstrip().startswith(">"):
continue
lines.append(line.rstrip())
cut = len(lines)
for i in range(max(1, min(int(len(lines) * 0.6), len(lines) - 8)), len(lines)):
n = len(lines[i].strip())
if (n <= 40 and any(_marker_matches(p, lines[i]) for p in _SIGNATURE_MARKERS)) or (
n <= 60 and _DEVICE_FOOTER.match(lines[i])):
cut = i
break
lines = lines[:cut]
paragraphs = [_strip_disclaimer(p) for p in re.split(r"\n\s*\n", "\n".join(lines))]
text = re.sub(r"[ \t]+", " ", "\n\n".join(p.strip() for p in paragraphs if p.strip()))
return text[:max_chars]
def email_state(subject: str, body: str, sender: Optional[str] = None, clean: bool = True,
max_chars: int = 3000, **extra) -> Dict:
"""Construct a clean state dictionary for email classification.
`max_chars` is the budget `clean_email_body` cuts the body to, and it is worth raising for a
long message: at the default the body stops after 3000 characters, so a request that arrives in
the last paragraphs never reaches the model -- including through `predict_long`, which scans a
state in windows precisely so it can read past one window's worth. Lowering it does the same
thing more surely: it is a slice budget, so `0` gives the model a subject and an empty body, and
a negative value cuts from the end, which is where that late request lives. `clean_email_body`
states the arithmetic, and the `budget-zero` and `budget-negative-*` cases of
`laya-java/fixtures/email.json` pin it for every port, so it is behaviour rather than an error --
pass a budget larger than the body when the message is long, never `0` or a negative to mean "no
limit". Ignored when `clean=False`, which passes the body through whole.
Any other keyword becomes a field of the state, so it is read by the model; a typo here is an
input mutation, not an error. The exception is a keyword whose value is exactly ``None``: it is
dropped rather than added, so ``email_state(..., thread_id=None)`` gives the same state as
leaving the keyword out. The key is absent, not null -- and the distinction reaches the model,
because `laya.common.serialize_state` turns the state into JSON before it is tokenized, so a
retained ``None`` would put ``"thread_id": null`` in the prompt. ``sender`` is dropped on the
same call by a different rule, falsiness rather than ``None``, so ``sender=""`` leaves no
``from`` field either.
"""
state = {
"subject": (subject or "").strip(),
"body": clean_email_body(body, max_chars=max_chars) if clean else (body or ""),
}
if sender:
state["from"] = sender
state.update({k: v for k, v in extra.items() if v is not None})
return state

867
laya_repo/lang.py Normal file
View File

@ -0,0 +1,867 @@
"""Dependency-free language/script detection used to route between Laya checkpoints.
Routing only needs one decision: *is this English Latin text, or is it something the English
checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint
collapsing to near-random on non-Latin scripts (Hindi 0.100, Korean 0.103, Swahili 0.103,
Tamil 0.113 at 20 options, where random is 0.050), while holding up far better on Latin-script
languages (French 0.487, Spanish 0.480). So the signal that matters most is *script*, and the
secondary signal is whether Latin text is English.
Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic and
is explicitly best-effort: pass an explicit model or `lang=` when you already know the language.
"""
import re
import unicodedata
from collections import Counter
from collections.abc import Mapping
from typing import Dict, List, Optional, Union
# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
_SCRIPT_RANGES = [
("greek", ((0x0370, 0x03FF), (0x1F00, 0x1FFF))),
("cyrillic", ((0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F))),
("armenian", ((0x0530, 0x058F),)),
("hebrew", ((0x0590, 0x05FF),)),
("arabic", ((0x0600, 0x06FF), (0x0750, 0x077F), (0x08A0, 0x08FF), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF))),
("devanagari", ((0x0900, 0x097F), (0xA8E0, 0xA8FF))),
("bengali", ((0x0980, 0x09FF),)),
("gurmukhi", ((0x0A00, 0x0A7F),)),
("gujarati", ((0x0A80, 0x0AFF),)),
("oriya", ((0x0B00, 0x0B7F),)),
("tamil", ((0x0B80, 0x0BFF),)),
("telugu", ((0x0C00, 0x0C7F),)),
("kannada", ((0x0C80, 0x0CFF),)),
("malayalam", ((0x0D00, 0x0D7F),)),
("sinhala", ((0x0D80, 0x0DFF),)),
("thai", ((0x0E00, 0x0E7F),)),
("lao", ((0x0E80, 0x0EFF),)),
("tibetan", ((0x0F00, 0x0FFF),)),
("myanmar", ((0x1000, 0x109F),)),
("georgian", ((0x10A0, 0x10FF),)),
("ethiopic", ((0x1200, 0x137F),)),
("khmer", ((0x1780, 0x17FF),)),
("hangul", ((0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF))),
("kana", ((0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF))),
("han", ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF))),
]
# Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
# weighted and a margin is required before calling something non-English.
#
# The Romance lists (fr/es/pt/it) deliberately carry the *unaccented* function words as well as the
# accented ones. A state that lost its accents -- mail clients, ticket systems and any pipeline that
# normalises to ASCII strip them -- keeps no diacritic rate for the non-English signal to read, so
# `la`, `un`, `y`, `e`, `et`, `deux` and friends are the only evidence left. With a list of mostly
# accented words such a state produced one hit or none, fell under the two-hit margin below, and was
# handed to the English checkpoint as undecided-but-not-non-English text (see #172 and #54).
_STOP = {
"en": {"the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
"this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
"will", "can", "would", "there", "their", "what", "which", "please", "we", "i"},
"fr": {"le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
"pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
"et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
"ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois", "très", "bien",
"tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut", "dois", "doit",
"merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
"alors", "donc"},
"de": {"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
"auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
"ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine", "meinen",
"meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen", "einem", "einer",
"wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei", "nach", "noch", "bitte",
"heute", "jetzt", "kann", "kannst", "habe", "gibt", "wird",
# shared with English on purpose: counted for English alone, they outvoted short German
"in", "was"},
"es": {"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
"pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
# `de`/`en` are Spanish too, but they are common English tokens as well (`de facto`,
# `en-US`, `en route`, `Rio de Janeiro`), and a state of those alone already carries
# no English function word for the margin below to weigh them against, so they stay out.
"la", "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos",
"ni", "dos", "tres", "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede",
"pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde",
"porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo",
"aquí", "hoy", "gracias"},
"pt": {"os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
"dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela",
# `no` is Portuguese too, and among the most frequent words it has; it is also one of the
# most frequent English words, so it stays out and short Portuguese states that lean on it
# alone are left to the diacritic rate, as before.
"o", "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou",
"tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu",
"sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais",
"já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
"olá",
# Brazilian support text: `você` and the unaccented `nao`/`voce`/`sao`/`ja` that a stripped
# state keeps (#172), and the chat abbreviations `vc`/`pra`. Without them "Voce pode me
# mandar a nota fiscal?" matched one word and went to the English checkpoint, which on
# `pt` reports 0.97 mean confidence at 0.47 accuracy. `ate`, `bom`, `sim` and `cade` stay
# out: each is an English token too (ate, BOM, SIM, Cade).
"você", "vocês", "voce", "voces", "vc", "vcs", "nao", "sao", "ja", "até", "tá", "pra",
"gostaria", "obrigada", "também", "tambem", "estou", "estamos", "meus", "minhas",
"nosso", "nossa", "consigo", "cadê", "boa", "tarde", "noite",
# the words a ticket keeps once the jargon is English ("Deu erro 500 no endpoint de login
# depois do update"): time and person words plus the past tenses a bug report is told in
"depois", "antes", "então", "entao", "ninguém", "ninguem", "alguém", "alguem", "nenhum",
"nenhuma", "estava", "ficou", "fiz", "deu"},
"it": {"il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
"questo", "questa", "anche", "come", "più", "sono", "nella", "alla",
"la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",
"ci", "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
"devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
"dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso", "oggi",
"ieri", "grazie", "ciao", "scusa",
# the articulated prepositions: Italian-only words, which is what lets a state made of
# shared articles (`la fattura`) still name the language rather than stay undecided
"nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei",
"delle", "dello", "degli", "agli", "alle", "col"},
"nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
"door", "maar", "ook", "worden", "deze", "naar", "wordt"},
# Swedish function words and common auxiliaries. Several overlap with English or German
# (`i`, `kan`, `har`), so the distinctive words below are what lets Swedish text survive an
# ASCII-normalising ticket pipeline without treating one stray Nordic letter as the only hint.
"sv": {"jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",
"vill", "måste", "också", "dessa", "detta", "säger", "upp", "utan", "mitt", "min", "om",
"kommer", "här", "två", "vi", "nästa", "gör", "göra",
"hjälp", "hjälpa", "mig", "återbetalning", "återbetala", "faktura", "gång", "gånger",
"hittar", "inställningen", "inställningarna", "lösenord", "när", "öppnar", "spårningen",
# Common spellings from ticket systems that strip Swedish diacritics.
"aterbetalning", "aterbetala", "behover", "fel", "ganger", "hjalp", "hjalpa",
"installningen", "installningarna", "kraschar", "kvittot", "losenord", "nar",
"oppnar", "paket", "skicka", "sparningen", "tva", "uppdaterats", "blivit", "debiterade",
"appen"},
# Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a
# French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately
# left out for that reason, and the diacritic signal below carries the rest.
"ro": {"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
"ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
"acesta", "aceasta", "mi", "ți", "vă", "nu"},
# Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali
# keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English
# and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence. Spelling
# is not standardised, so the common variants are listed (`bhalo`/`valo`, `korchi`/`korsi`).
# Left out on purpose: frequent Bangla words that are also English words -- `ache` (is),
# `are` (is there), `to` (so), `take` (to him), `age` (before), `pore` (later), `mane`
# (meaning), `din` (give), `sob` (all), `tar` (his), `dao` (give), `eta` (this; ETA) --
# ordinary words of a neighbouring language (`ora`, `nei`, `vai`), and words another list
# already claims (`na`, `o`, `e`, `je`, `ta`, `por`, `hoy`), so adding `bn` cannot move a
# state of any other language.
"bn": {"ami", "amar", "amake", "amra", "amader", "apni", "apnar", "apnake", "apnara",
"tumi", "tomar", "tomake", "tomra", "tader", "ota", "eita", "oita",
"ekta", "ei", "oi", "ki", "keno", "kivabe", "kibhabe", "kothay", "kokhon", "kobe",
"koto", "kintu", "jodi", "tahole", "ar", "theke", "jonno", "sathe", "shathe", "diye",
"niye", "moddhe", "kore", "korte", "korchi", "korsi", "korbo", "korechi", "koreche",
"korun", "koren", "korlam", "hobe", "hoyeche", "hoise", "hocche", "hoyni",
"chai", "chaina", "lagbe", "parchi", "parbo", "parchina", "peyechi", "paini",
"dite", "dilam", "diyechi", "nai", "khub", "onek", "ekhon", "akhon", "ekhono",
"abar", "ekbar", "duibar", "ajke", "kalke", "taka", "bhalo", "valo", "kharap",
"shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu", "kichu", "bolte", "bolun",
"parben", "asbe", "jabe", "pabo", "ferot", "dorkar", "hoye", "geche", "gese"},
# "her", "ne", "men", "de" collide with English, French and Romanian, and "ki" with the
# romanized Bangla list, so they are left out.
"az": {"və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",
"var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
"hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
"ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"},
}
# Short support fragments often consist of only two or three words, so they do not reach the
# four-token minimum used by the general language guess. These spellings are specific enough to
# identify Swedish in that narrow case; generic words such as `fel`, `hjälp`, `paket` and `appen`
# are deliberately not sufficient on their own. Keep ASCII-normalised variants beside the forms
# users commonly type without Swedish characters.
_SHORT_SWEDISH_WORDS = {
"åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",
"glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto", "kvittot",
"spårningen", "sparningen", "inställningen", "installningen", "felmeddelande",
"abonnemanget",
}
# Letters that ordinary English does not use. This is the signal that catches a Latin-script
# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),
# which is the difference between routing it to the multilingual checkpoint and silently
# handing it to the English one.
_NON_EN_DIACRITICS = set(
"àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" # Western European
"ăâîșțşţ" # Romanian
"ąćęłńśźż" # Polish
"čďěňřšťůž" # Czech / Slovak
"őű" # Hungarian
"ğı" # Turkish (text is lowercased before matching)
"āēģīķļņūž" # Baltic
"đ" # Serbo-Croatian / Vietnamese
"ə" # Azerbaijani
)
# Words that more than one list claims. `la`, `un`, `e`, `que`, `una` and friends are function words
# of several of these languages at once, so matching one says "not English" without saying *which*
# language: a shared word may not name a winner by itself (Romanian text was reported as French that
# way), though it still counts toward the total of a language that also matched a word of its own.
_SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}
if sum(w in words for words in _STOP.values()) > 1}
# Danish is not in `_STOP`, but several of its common words also occur in the Swedish list.
# Do not let these words alone name a Danish sentence as Swedish; they remain useful score hits
# when another, more distinctive Swedish word is present.
_NORDIC_OVERLAP_WORDS = {
"hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi",
}
_SHARED_WORDS.update(_NORDIC_OVERLAP_WORDS)
# English function words no other list holds (`in`, `is`, `as`, `was` are shared with German,
# Dutch and Portuguese). They alone carry the English rescue of `latin_profile`.
_EN_ONLY_WORDS = _STOP["en"] - _SHARED_WORDS
# A few of these function words are also ordinary English words: `come` (Italian), `son` (Spanish),
# `do` (Portuguese), `care` (Romanian), `todo` (a to-do list), `per`, `plus`, and `im` -- German
# `im`, and also how `I'm` is written without the apostrophe. For those a repeat is evidence of the
# foreign language exactly once: "do more, do less" repeats an English word, and counting both hits
# let one such word clear the `best >= 2` bar below and send plain English to the multilingual
# checkpoint. Every other word keeps counting occurrences: `der`, `los` and `sa` are nobody's
# English, so a request whose only evidence is `der` twice ("reduzieren der helligkeit der lichter")
# is still German, and deduping every word instead sent 791 of 148,700 non-English MASSIVE test rows
# to the English checkpoint. Dutch `van` is the word this list deliberately leaves out: "liedje van
# ... van" is ordinary Dutch, and deduping it moved 14 of those rows while rescuing no English one.
_EN_COLLISION_WORDS = {"come", "son", "do", "care", "todo", "im", "per", "plus"} & {
w for lg, sw in _STOP.items() if lg != "en" for w in sw}
_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,
# `user@acme.com`, `v1.2.3`, `U.S.A.`. `_WORD` splits them into pieces that collide with real
# function words -- `com` is Portuguese for "with", `o` is its article, `e` is Italian "e" -- so a
# state that was mostly links scored a language it does not contain, and two domains were enough to
# cross the margin below. A sentence-final period (`arrivato.`) keeps its word: the pattern needs
# word characters on both sides of the dot.
# The lookbehind is what keeps this linear. Without it the greedy `[\w-]*` is retried
# at every offset inside a run of word characters, and each attempt rescans the run
# before failing on the absent `[.@]` -- quadratic in the run's length, so 4000
# characters of one token cost 205 ms against 0.45 ms for ordinary prose. It removes
# no match: `[\w-]` and `[.@]` are disjoint, so an attempt from inside a run consumes
# to exactly the same separator as an attempt from the run's start and the two always
# succeed or fail together -- a leftmost match can only ever begin at a run start.
_IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0) -> List[str]:
"""Collect the string leaves of a state, so detection sees real content.
Keys are ignored: they are usually English field names. Any mapping counts, not only
`dict` -- a UserDict or mappingproxy has the same string values and used to contribute
nothing, which made `analyse` report no letters and the router fall through to English.
"""
if _depth > 6 or state is None:
return []
if isinstance(state, str):
return [state]
if isinstance(state, (bytes, bytearray)):
try:
return [bytes(state).decode("utf-8")]
except UnicodeDecodeError:
return []
if isinstance(state, Mapping):
out = []
for v in state.values():
out.extend(_iter_text(v, _depth + 1))
return out
if isinstance(state, (list, tuple)):
out = []
for v in state:
out.extend(_iter_text(v, _depth + 1))
return out
return []
def state_text(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000) -> str:
"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""
parts: List[str] = []
budget = max_chars
for leaf in _iter_text(state):
if budget <= 0:
break
if len(leaf) > budget:
parts.append(leaf[:budget])
break
parts.append(leaf)
# Account for the joining space without materializing the full text first.
budget -= len(leaf) + 1
return " ".join(parts)[:max_chars]
def _script_counts(text: str) -> Dict[str, int]:
"""Count the alphabetic characters of `text` by script, in one pass.
Latin is inserted last so `_script_from_counts` keeps `detect_script`'s tie-break: a named
script wins a tie against Latin, because `max` returns the first of equal values and Latin
is the last key. `analyse` needs both the dominant script and the per-script fractions, and
used to walk the text twice (once per function) to get them; one pass serves both.
"""
counts: Dict[str, int] = {}
latin = 0
for ch in text:
if not ch.isalpha():
continue
cp = ord(ch)
if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
latin += 1 # Latin, IPA Extensions, Ext-Additional, fullwidth
continue
for name, ranges in _SCRIPT_RANGES:
if any(lo <= cp <= hi for lo, hi in ranges):
counts[name] = counts.get(name, 0) + 1
break
else:
# An alphabetic character no range claims used to be counted nowhere, so text
# written only in an unlisted script produced a total of 0 and was reported as
# "unknown" -- and `analyse` treats "unknown" as English, sending it to the
# checkpoint that has no tokens for it. 68% of Unicode's alphabetic codepoints
# are outside _SCRIPT_RANGES (the CJK extensions, kana supplements, bopomofo,
# halfwidth katakana, and dozens of smaller scripts), so enumerating them all is
# not maintainable. Counting the remainder under "other" keeps them visible and
# non-Latin, which is the safe direction: an unreadable script must not be
# handed to the English checkpoint.
counts["other"] = counts.get("other", 0) + 1
counts["latin"] = latin
return counts
def _script_from_counts(counts: Dict[str, int]) -> str:
if not any(counts.values()):
return "unknown"
return max(counts.items(), key=lambda kv: kv[1])[0]
def _profile_from_counts(counts: Dict[str, int]) -> Dict[str, float]:
total = sum(counts.values())
if not total:
return {}
# Keep Latin first in the public mapping, the order callers saw before this was refactored.
ordered: Dict[str, int] = {}
if counts.get("latin"):
ordered["latin"] = counts["latin"]
for name, value in counts.items():
if name != "latin":
ordered[name] = value
return {k: v / total for k, v in ordered.items() if v}
def detect_script(text: str) -> str:
"""Dominant script of `text`: 'latin', 'han', 'devanagari', ... or 'unknown' if there are no letters."""
return _script_from_counts(_script_counts(text))
def script_profile(text: str) -> Dict[str, float]:
"""Fraction of alphabetic characters belonging to each detected script."""
return _profile_from_counts(_script_counts(text))
# A diacritic rate above this is taken as evidence the text is not English, even when no
# stopword list matches it.
NON_EN_DIACRITIC_RATE = 0.02
# One accented loanword or proper noun (`café`, `résumé`, `José`) must not alone pull otherwise
# plain English off the English checkpoint: the rate is measured over every character, so a
# single `é` in a short sentence clears the floor above. English function words keep their say
# only through the rescue below -- two distinct words no other list holds, at most one
# non-English letter word, and a rate still well above the floor vetoes regardless.
ENGLISH_RESCUE_DIACRITIC_RATE = 0.06
# Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a
# brand name or order code outvotes the CJK request around it letter for letter, though one CJK
# character carries far more than a letter. A short message needs a large share to count; a long
# payload (ticket fields, English agent turns) dilutes the share, so there a sentence's worth of
# letters counts too.
NON_LATIN_FRACTION = 0.2
NON_LATIN_MIN_FRACTION = 0.1
NON_LATIN_MIN_LETTERS = 10
def _non_latin_carries(non_latin: float, n_non_latin: int) -> bool:
"""Whether the non-Latin letters of a Latin-plurality text are enough to carry the state."""
return (non_latin >= NON_LATIN_FRACTION
or (non_latin >= NON_LATIN_MIN_FRACTION and n_non_latin >= NON_LATIN_MIN_LETTERS))
def _script_of(ch: str) -> Optional[str]:
"""The named non-Latin script of one letter, or None for Latin and for unclaimed letters."""
cp = ord(ch)
if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
return None
for name, ranges in _SCRIPT_RANGES:
if any(lo <= cp <= hi for lo, hi in ranges):
return name
return None
def _non_latin_words(text: str) -> List[str]:
"""Non-Latin runs that read as words rather than as annotation inside English prose.
English prose carries three kinds of non-Latin letters that are not a request written in
another script, and each is excluded here: a symbol (`Set α to 0.05`, one letter), a proper
name (`Дмитрий Петрович Савицкий`, capitalised), and a pronunciation (`[vlɐˈdʲimʲɪr]`, which
no script range claims). A combining mark belongs to the letter before it and never splits a
word, so `Влади́мир` stays one capitalised name rather than becoming `Влади` + `мир`.
"""
runs, cur, script = [], "", None
for ch in text:
if unicodedata.combining(ch):
continue
s = _script_of(ch)
if s is not None and s == script:
cur += ch
continue
if cur:
runs.append(cur)
cur, script = (ch, s) if s is not None else ("", None)
if cur:
runs.append(cur)
return [w for w in runs if len(w) >= 2 and not w[0].isupper()]
def _english_rescued_by_words(words: List[str], diac_rate: float) -> bool:
"""Whether plain-English function words outvote a marginal diacritic rate (#337).
The rate is measured over every character, so one accented loanword or proper noun in a
short sentence clears `NON_EN_DIACRITIC_RATE` outright. English still wins when it shows at
least two distinct function words no other list holds and at most one word carrying a
non-English letter: one loanword is not a non-English vocabulary, while the odd word a
Danish or Swedish sentence picks up (`i`, `at`, `for`, `have`) is not an English sentence
either. A rate well above `ENGLISH_RESCUE_DIACRITIC_RATE` vetoes regardless.
"""
if diac_rate >= ENGLISH_RESCUE_DIACRITIC_RATE:
return False
if len(set(words) & _EN_ONLY_WORDS) < 2:
return False
return sum(1 for w in set(words) if any(ch in _NON_EN_DIACRITICS for ch in w)) <= 1
def latin_profile(text: str) -> Dict[str, object]:
"""Evidence behind the Latin-script language guess.
Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and
`looks_non_english`. The latter can be true for non-English diacritics or an overlapping
Swedish-Danish marker even when this heuristic cannot name the language. `analyse` needs the
evidence and not just the verdict, because "undecided" and "English" are different answers
and only one of them is safe to send to the English checkpoint.
A non-English language is only named when it matched at least one word that no other list
claims: shared function words alone (`la`, `e`, `o`) identify no particular language.
"""
# 'İ'.lower() is 'i' + a combining dot, which matches no word list
words = _WORD.findall(_IDENTIFIER.sub(" ", text).replace("İ", "i").lower())
lowered = text.lower()
diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)
diac_rate = diac / max(1, len(lowered))
non_english = diac_rate >= NON_EN_DIACRITIC_RATE
nordic_overlap = bool(set(words) & _NORDIC_OVERLAP_WORDS) and not bool(set(words) & _EN_ONLY_WORDS)
if 1 < len(words) < 4 and set(words) & _SHORT_SWEDISH_WORDS:
return {"language": "sv", "english_hits": 0, "diacritic_rate": diac_rate,
"looks_non_english": non_english}
if len(words) < 4:
return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,
"looks_non_english": non_english or nordic_overlap}
# A collision word counts once however often it repeats; every other word counts its hits.
counts = Counter(words)
scores = {lg: sum(1 if w in _EN_COLLISION_WORDS else n for w, n in counts.items() if w in sw)
for lg, sw in _STOP.items()}
en = scores.get("en", 0)
# Only a language that matched at least one word no other list claims may be named. Without
# that condition the top score can be pure overlap -- `la` and `e` in Romanian text made
# Italian the winner -- which is a guess dressed as a detection. Such a language is dropped
# from the running rather than merely losing the tie, so a lesser score with real evidence
# still gets named, and the text stays undecided when no list has any.
evidenced = {lg: s for lg, s in scores.items()
if lg != "en" and any(w not in _SHARED_WORDS for w in set(words) & _STOP[lg])}
best_lg, best = max(evidenced.items(), key=lambda kv: kv[1], default=(None, 0))
lang = None
if best_lg and best >= max(2, en + 2):
# a non-English language needs a clear margin over English function words
lang = best_lg
elif (best_lg == "sv" and "inte" in words and "kan" in words
and words[0] in {"kan", "jag", "vi"} and en <= 1):
# Short login requests such as "kan inte logga in" carry a distinctive Swedish phrase
# but also one English-shaped token (`in`). Do not treat `kan` alone as Swedish: it is
# common in Danish and Norwegian too.
lang = best_lg
elif best_lg and non_english and best >= max(2, en):
# Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish
# on the strength of the diacritics alone, which is a guess dressed as a detection.
lang = best_lg
elif en and (not non_english or _english_rescued_by_words(words, diac_rate)):
lang = "en"
return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,
"looks_non_english": non_english or (lang is None and nordic_overlap)}
def guess_latin_language(text: str) -> Optional[str]:
"""Best-effort language code for Latin-script text, or None when undecided.
Scores function-word hits per language and requires the winner to beat English by a margin and
to have matched at least one word of its own, so ordinary English is never misrouted and a
word of several languages at once names none of them. Short inputs usually return None on
purpose.
"""
return latin_profile(text)["language"]
# Code is not prose in any language, but split into words it reads as one: `os.path` is Portuguese
# (`os`), `round(el, 2)` Spanish (`el`), `non_english` Italian (`non`). A line pasted from a program
# into an English request must not count as a foreign segment, so a line carrying code syntax --
# `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers
# are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.
_CODE_LINE = re.compile(r"[=;{}\[\]]|\w\(")
# Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese
# (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`). A whitespace token
# holding a letter or digit, a joiner (`.`, `_`, `/`, `\`) and another letter or digit is an
# identifier or a compound and is dropped whole. The pattern has a fixed length on purpose: an
# open-ended `\w+(?:[._]\w+)+` backtracks quadratically on a long run of letters with no joiner,
# and a state is user input.
_JOINED = re.compile(r"[^\W_][._/\\][^\W_]")
# An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`
# are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written
# entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.
_LETTER_RUN = re.compile(r"[^\W\d_]{2,}")
# ...but a line of nothing but acronyms has no lowercase either, so it skips the blanking above
# and is read as prose. `MON DES EST LA` -- a hockey team, a state, a time zone and an airport --
# was named French, and `STORES LOS ANGELES LAS VEGAS EL PASO CLOSED` Spanish. An all-caps line
# must therefore show evidence that is more than acronym-shaped before it is believed.
#
# The bar is on the tokens that actually scored, not on the line. Requiring one long word
# anywhere does almost nothing by itself: `ANGELES`, `VEGAS` and `SEGUNDO` are long and are not
# what named Spanish -- `LOS`, `LAS` and `EL` did.
#
# Three corpora decide these two numbers, and `research/evals/shouted_bar_corpora.py` builds all
# three and prints the table below: upper-cased MASSIVE test rows that the rule names as their
# own locale with no bar at all (genuine shouted prose, must survive); 500,000 lines of 4-8
# acronyms; 50,000 all-caps US address and signage lines. Pinned to the MASSIVE test split at
# revision 940fd47a and `random.Random(0)`, with the acronym and place-name pools committed in
# that script, so every row here can be rechecked when someone wants to move a constant. Each
# row applies one bar, named in full, so the rows are comparable:
#
# rule prose kept acronym FP address FP
# none (no bar at all) 10,960 100.00% 4,219 141
# diacritics only 10,960 100.00% 4,219 141
# word >= 5 only 10,854 99.03% 431 141 <- addresses stand
# stopword >= 3 only 10,729 97.89% 3,751 109 <- keeps LOS/LAS/EL
# stopword >= 4 only 8,841 80.67% 0 0
# stopword >= 5 only 6,963 63.53% 0 0
# stopword >= 3 AND word >= 5 10,625 96.94% 386 109
# stopword >= 4 AND word >= 5 8,762 79.95% 0 0 <- chosen
# stopword >= 5 AND word >= 5 6,902 62.97% 0 0
# stopword >= 4 AND word >= 6 8,418 76.81% 0 0
# stopword >= 4 AND word >= 7 7,459 68.06% 0 0
# stopword >= 4 AND word >= 5, no dia 6,749 61.58% 0 0
#
# `LOS`, `LAS` and `EL` are three, three and two letters, so any bar of 4 or more rejects them;
# what the table has to decide is the other side of that, and it is 3 that it rules out -- at 3
# the reported address line still reads as Spanish, and 3,751 of 500,000 acronym runs and 109 of
# 50,000 address lines still read as prose. 4 closes both columns outright. Going on to 5 buys
# nothing measurable and costs 1,860 more real lines. Non-English diacritics are accepted in
# place of a long stopword, because the module already treats them as non-English evidence
# (NON_EN_DIACRITIC_RATE) and neither an acronym nor a US place name carries any: that disjunct
# is free on both false-positive corpora (0 and 0 either way) and lifts prose kept from 61.58%
# to 79.95%.
_SHOUTED_MIN_STOPWORD = 4
# A length bar as well: a token of this many letters is a word rather than an acronym. Used
# twice -- here, on an all-caps line, and in `_analyse_text` to decide whether an all-caps run
# inside mixed-case text is an acronym worth blanking.
#
# Stated plainly, this bar is not justified by the table above: at a stopword bar of 4 both
# false-positive columns are already 0, so the 79 real prose lines it costs (8,841 -> 8,762) buy
# nothing those corpora can see. It is powerful on its own -- row three takes acronym false
# positives from 4,219 to 431 -- and it is kept because the corpora it was first chosen against
# were larger than the ones committed here (5,670 acronyms and 18,718 place names, neither
# committed, so their counts are not reproducible and are not quoted) and did report false
# positives surviving a stopword bar of 4. Absence on this corpus is not absence. Anyone who
# wants those 79 lines back should widen the acronym pool in that script first and show the
# column stays at 0. Swept with the stopword bar at 4: no bar 8,841 prose, >= 5 8,762,
# >= 6 8,418, >= 7 7,459.
#
# What the pair costs is not small and is not only short lines: 2,198 of the 10,960
# shouted-prose lines are given up, 1,816 of them (82.62%) six tokens or longer, the longest a
# 34-token Dutch sentence. On MASSIVE as written none of this is visible -- not one of the
# 151,674 rows holds a single uppercase character -- so it has to be measured on rows that do:
# over the 77,324 rows of the 26 non-English Latin locales, upper-cased, the multilingual share
# falls 0.5912 -> 0.5430, which is 3,727 rows. The casualties are sentences.
_SHOUTED_MIN_WORD = 5
def _shouted(text: str) -> bool:
"""True for a *cased* segment written entirely in capitals.
The `isupper` half is not redundant. A caseless script -- Devanagari, Bengali, CJK -- has no
lowercase either, so testing only `islower` would call every such segment shouted and hold it
to a bar it cannot clear: `_WORD` splits at every combining mark, so the tokens are short by
construction. Nothing is routed on that path today (`_STOP["bn"]` is romanised, so a
Bengali-script line is named by script and never reaches here), but the bar belongs to cased
text and saying so here keeps it that way.
That argument covers *pure* caseless text, which never reaches a caps bar at all. Text that
merely contains some has no lowercase either and does reach one; `_analyse_text` keeps both
bars off it, because a bar that works by discarding Latin evidence has nothing to say about
the half of the state it cannot read.
"""
return any(ch.isupper() for ch in text) and not any(ch.islower() for ch in text)
def _shouted_evidence(tokens, lang: str, diacritic_rate: float) -> bool:
"""Whether an all-caps line's evidence for `lang` is more than acronym-shaped tokens."""
if not any(len(t) >= _SHOUTED_MIN_WORD for t in tokens):
return False
if diacritic_rate >= NON_EN_DIACRITIC_RATE:
return True
matched = {t.lower() for t in tokens} & _STOP.get(lang, set())
return any(len(w) >= _SHOUTED_MIN_STOPWORD for w in matched)
def _named_prose_language(segment: str):
"""Language code for one non-code line, or None when it does not name a foreign language.
Same evidence bar as `_non_english_segment`: four words, a language `latin_profile` will name,
and two *different* words of that language. Acronyms and slash compounds are not words. A
segment in all capitals keeps its acronym-shaped tokens -- shouting is not a foreign language
-- so it must also clear `_shouted_evidence`.
"""
if not segment.strip() or _CODE_LINE.search(segment):
return None
prose = " ".join(tok for tok in segment.split() if not _JOINED.search(tok))
shouted = _shouted(prose)
if not shouted:
prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)
tokens = _WORD.findall(prose)
if len(tokens) < 4:
return None
prof = latin_profile(prose)
lang = prof["language"]
if lang in (None, "en"):
return None
if shouted and not _shouted_evidence(tokens, lang, float(prof["diacritic_rate"])):
return None
if len({w.lower() for w in tokens} & _STOP.get(lang, set())) < 2:
return None
return lang
def _non_english_segment(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000):
"""First line or field that, read on its own, is named a non-English language, else None.
Returns (language, segment). A segment needs the evidence a whole state needs -- at least four
words, and a language named by `latin_profile` -- and, because one line carries far less text
than a state, two things more: the words that name the language must be two *different* ones
(`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash
compounds are not words. This adds no new way to call English text foreign; it only stops a
longer English part from outvoting a foreign one. Reads at most `max_chars` characters in all.
"""
seen = 0
for leaf in _iter_text(state):
for seg in leaf.split("\n"):
if seen >= max_chars:
return None
seg = seg[:max_chars - seen]
seen += len(seg)
lang = _named_prose_language(seg)
if lang:
return lang, seg.strip()
return None
def _analyse_text(text: str) -> Dict[str, object]:
"""Detection result for one already-flattened string.
Does not look for a foreign line inside mostly-English text. `analyse` does that, because it
needs the original state and not only the joined window.
"""
counts = _script_counts(text)
prof = _profile_from_counts(counts)
script = _script_from_counts(counts)
non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0
n_non_latin = round(non_latin * sum(ch.isalpha() for ch in text))
if (script == "latin" and _non_latin_words(text)
and _non_latin_carries(non_latin, n_non_latin)):
script = max((s for s in prof if s != "latin"), key=prof.get)
if script == "unknown":
return {"script": "unknown", "script_profile": prof, "language": None,
"is_english": True, "language_undecided": True, "diacritic_rate": 0.0,
"non_latin_fraction": 0.0, "mixed_segment": None}
if script != "latin":
return {"script": script, "script_profile": prof, "language": None,
"is_english": False, "language_undecided": True, "diacritic_rate": 0.0,
"non_latin_fraction": non_latin, "mixed_segment": None}
prof_lat = latin_profile(text)
lang = prof_lat["language"]
# The acronym bar belongs here too, not only in the segment scan. The scan runs only once a
# state already reads English overall, so a state that is *nothing but* an acronym line --
# or one diluted by fewer English lines than it takes to tip this verdict -- never reached it
# and was named foreign outright: `MON DES EST LA` routed multilingual on its own, and so did
# the same line under one or two lines of English. Vetoing here leaves the text undecided, so
# `looks_non_english` still decides it: a shouted line with non-English diacritics is kept.
# Neither bar applies when the state is carried by a caseless script. Both of them answer
# the verdict by taking evidence *away* from the Latin letters, and `looks_non_english`,
# which decides what is left, reads Latin diacritics only -- it cannot see the caseless half.
# Nothing else catches the fall, either: the script promotion above needs
# `_non_latin_words`, which drops any run starting with a capital, so a shouted mixed-script
# line has no promotable word by construction. `ПРИВЕТ MON DES EST LA` is 35% Cyrillic
# letters and was being sent to the English checkpoint on the strength of disbelieving `MON
# DES EST LA`. Whatever the Latin part is worth, text this far from Latin is not English, so
# there is nothing for either bar to buy here and a wrong language name costs nothing: the
# checkpoint is chosen on `is_english`.
if lang not in (None, "en") and not _non_latin_carries(non_latin, n_non_latin):
if _shouted(text):
if not _shouted_evidence(_WORD.findall(text), lang,
float(prof_lat["diacritic_rate"])):
lang = None
else:
# Mixed-case text: the segment scan has always held that an acronym is not a word,
# but the whole-state verdict never applied that rule, so the acronyms voted in it.
# That is what let a short English state be outvoted -- one or two lines of English
# above `MON DES EST LA` still read as French overall, and only at three did English
# win the margin. Re-take the verdict without the all-caps runs. `looks_non_english`
# and `diacritic_rate` stay measured on the original text, so a foreign word that
# happens to be shouted cannot be blanked out of the diacritic safety net.
#
# Only *acronym-shaped* runs are blanked. Blanking every all-caps run deleted
# emphasis capitals, which are ordinary in real prose and are usually on the words
# that carry the sentence, so when the shouted words were the ones that named the
# language the evidence simply went: `sag mir das HEUTIGE DATUM` and `quiero
# cancelar mi PEDIDO POR FAVOR` both lost their verdict and routed english. It also
# broke monotonicity -- the fully upper-cased form held the shouted bar, but adding
# one lowercase English token moved the text to this branch, where it had no bar at
# all. So a run is blanked only while every one of them is short enough to be an
# acronym, by `_shouted_evidence`'s own first test: a run of `_SHOUTED_MIN_WORD`
# letters is a word, and a word is not blanked out of its own sentence. Measured
# over the 20,708 Latin-locale MASSIVE test rows, emphasis-capitalised two ways
# (see the commit message): regressions 1,753 -> 293 and 3,835 -> 752.
caps = [m.group() for m in _LETTER_RUN.finditer(text) if m.group().isupper()]
if caps and not any(len(c) >= _SHOUTED_MIN_WORD for c in caps):
blanked = _LETTER_RUN.sub(
lambda m: " " if m.group().isupper() else m.group(), text)
if blanked != text:
lang = latin_profile(blanked)["language"]
# Undecided is not English. Treating it as English sent every Latin-script language we hold no
# stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the
# language, non-English letters or a shared Swedish-Danish marker can still prefer the
# multilingual checkpoint; text with neither signal still goes to the English one.
undecided = lang is None
english = lang == "en" or (undecided and not prof_lat["looks_non_english"])
return {"script": "latin", "script_profile": prof, "language": lang,
"is_english": english, "language_undecided": undecided,
"diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),
"non_latin_fraction": non_latin, "mixed_segment": None}
def _leaf_non_english(leaf: str) -> Optional[Dict[str, object]]:
"""A string value that is itself not safe for the English checkpoint, else None.
A one-word name ("José") and a capitalised non-Latin name stay out: the same rules
`latin_profile` and `_non_latin_words` already use, so a name field cannot pull an
English ticket onto the multilingual checkpoint. Code lines, acronyms and slash compounds
stay out too, matching `_non_english_segment`, so a pasted traceback is not a message.
Each line is capped at 4000 characters; unlike the segment scan, a long earlier field
does not consume the budget of the next one (#384).
"""
best_n = -1
best: Optional[Dict[str, object]] = None
for line in leaf.split("\n"):
# A line this short cannot be selected, so skip it before the slice, `strip`,
# `_CODE_LINE` and a full `_analyse_text` pass. Each branch below needs four `_WORD`
# tokens or NON_LATIN_MIN_LETTERS letters. `_WORD` matches maximal runs of letters and
# letter-like numerals, so four tokens need three separators between them: seven
# characters, and ten letters need ten. What makes seven safe rather than six is that
# every one of those counts is taken on the raw line -- `İ` lowers to two code points, so
# counting on `sample.lower()` would let a four-character line reach four tokens, which is
# why `latin_profile` replaces `İ` before lowering.
if len(line) < 7:
continue
sample = line[:4000]
if not sample.strip() or _CODE_LINE.search(sample):
continue
det = _analyse_text(sample)
if det["is_english"]:
continue
# A named language still has to survive `_named_prose_language`: acronyms and slash
# compounds are not words. But a veto is not a verdict of English. Dropping the line here
# skipped the diacritic branch below, which exists for exactly this -- text that carries
# non-English letters and that no stopword list can name. `WIE SPÄT IST ES IN KÖLN`
# was vetoed for having no long stopword and then thrown away, so a German field routed
# english; now it falls through and its umlauts carry it. That is also why
# `language_undecided` is no longer required below: a language named and then disbelieved
# is in the same evidential position as one never named.
#
# The ASCII spelling `WIE SPAET IST ES IN KOELN` is a different case and is not fixed
# here. It has no diacritics to fall through to, and its longest matched German stopword
# is `wie`, three letters, so `_SHOUTED_MIN_STOPWORD` vetoes it and nothing recovers it:
# genuine German on the English checkpoint, one of the 3,727 upper-cased casualties
# counted above and not an example of what this branch buys.
named = (det["language"] not in (None, "en")
and _named_prose_language(sample) is not None)
if not named:
if det["script"] not in ("latin", "unknown"):
if not (_non_latin_words(sample) and sum(ch.isalpha() for ch in sample) >= NON_LATIN_MIN_LETTERS):
continue
elif not (float(det["diacritic_rate"]) >= NON_EN_DIACRITIC_RATE
and len(_WORD.findall(sample)) >= 4):
continue
n_alpha = sum(ch.isalpha() for ch in sample)
if n_alpha > best_n:
best_n = n_alpha
best = det
return best
def analyse(state: Union[str, bytes, Mapping, list, None]) -> Dict[str, object]:
"""Full detection result for a state.
Returns `script`, `script_profile`, `language` (best effort, may be None),
`is_english`, `non_latin_fraction` and `mixed_segment`. `mixed_segment` names the line or field
the segment scan itself singled out, else None: a state flipped non-English by the per-value
read below is reported with `mixed_segment` still None, because that read keeps the offending
value in its own local and never writes it back.
String values are what get read. When a state has several of them, one non-English value is
enough: joining every value into one window let a long English note fill the 4000 characters,
or outvote a short German message, and that message was then sent to the English checkpoint
(#384). The segment scan still stops at 4000 characters, which is what keeps a huge field
cheap; a value it did not reach is read on its own afterwards.
"""
result = _analyse_text(state_text(state))
if result["script"] == "latin" and result["is_english"]:
# A Portuguese ticket with an English stack trace, error payload or form template reads as
# English as a whole, because the English part is longer -- yet the part a question is about
# is the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47
# accuracy on `pt`). The cost is lopsided: English sent to multilingual loses a few points,
# the reverse loses calibration. So a state that would go to English is checked line by line
# and field by field.
leaves = _iter_text(state)
# a single line has no other part to be outvoted by, and was just read whole
if len(leaves) > 1 or any("\n" in leaf for leaf in leaves):
found = _non_english_segment(state)
if found:
lang, mixed = found
result = dict(result)
result["language"] = lang
result["is_english"] = False
result["language_undecided"] = False
result["mixed_segment"] = mixed
# A plain string was just read whole. A structured state can still hide a message past the
# segment cap, or in a script `latin_profile` does not name.
if isinstance(state, (str, bytes, bytearray)) or state is None or not result["is_english"]:
return result
best_n = -1
best: Optional[Dict[str, object]] = None
for leaf in _iter_text(state):
det = _leaf_non_english(leaf)
if det is None:
continue
n_alpha = sum(ch.isalpha() for ch in leaf[:4000])
if n_alpha > best_n:
best_n = n_alpha
best = det
if best is None:
return result
updated = dict(result)
updated["language"] = best["language"]
updated["is_english"] = False
updated["language_undecided"] = best["language_undecided"]
return updated
def is_english(state: Union[str, bytes, Mapping, list, None]) -> bool:
"""True when the English checkpoint can be expected to read this state."""
return bool(analyse(state)["is_english"])

210
laya_repo/presets.py Normal file
View File

@ -0,0 +1,210 @@
"""Ready-to-use question presets for common production decision workflows."""
import re
from typing import Dict, Optional
# A preset says which part of the state it reads by naming it in backticks: "What does the
# customer want in `message`?". That convention is what `state_field` below depends on, and it is
# the reason no table of preset -> key has to be kept in step by hand.
_STATE_FIELD_RE = re.compile(r"`(\w+)`")
def state_field(questions: Dict) -> Optional[str]:
"""The state key ``questions`` reads, or ``None`` when that is not exactly one key.
Every built-in preset names a single field -- `message`, `body`, `prompt`, `post`, `request` --
and a caller that puts its text under a different key is asking the model about a field that is
not in the state. Surfacing the name lets a caller place the text correctly instead of guessing.
``None`` covers both "names nothing" and "names several", because then only the caller knows
which field the request belongs in.
"""
named = {match
for spec in questions.values()
for match in _STATE_FIELD_RE.findall(spec.get("instructions") or "")}
if len(named) != 1:
return None
return next(iter(named))
def triage_questions() -> Dict:
"""Preset questions for customer support ticket triage."""
return {
"intent": {
"type": "choice",
"instructions": "What does the customer want in `message`?",
"criteria": {
"refund": "money returned or a duplicate charge reversed",
"technical_help": "a bug, outage or integration problem",
"billing_question": "a question about an invoice, plan or payment method",
"information": "general information, pricing or how-to",
"cancellation": "wants to cancel or downgrade",
"other": "none of the other options fits",
},
},
"is_urgent": {
"type": "noul",
"instructions": "Does `message` communicate time pressure or a deadline?",
},
"frustration": {
"type": "score",
"instructions": "How frustrated does the customer sound in `message`?",
"criteria": [
"calm and neutral",
"concerned but civil",
"clearly annoyed",
"very angry or using strong language",
],
},
"refund_requested": {
"type": "noul",
"instructions": "Does the customer ask for money back?",
},
"churn_risk": {
"type": "noul",
"instructions": "Does `message` suggest the customer may leave for a competitor or cancel?",
},
}
def email_questions(categories: Optional[Dict[str, str]] = None) -> Dict:
"""Preset questions for inbound email triage and threat filtering."""
categories = categories or {
"billing": "invoices, payments, refunds",
"technical": "bugs, outages, integrations",
"sales": "pricing, demos, new purchases",
"security": "phishing, scams, account compromise",
"hr": "hiring, leave, payroll",
"other": "none of the above",
}
return {
"category": {
"type": "choice",
"instructions": "Which team should handle the email in `body`?",
"criteria": categories,
},
"is_spam": {
"type": "noul",
"instructions": "Is this email unsolicited spam or bulk marketing?",
},
"is_phishing": {
"type": "noul",
"instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?",
"criteria": {"true": "phishing, scam, or fraud", "false": "a legitimate email"},
},
"urgency": {
"type": "score",
"instructions": "How urgent is the request in `body`?",
"criteria": ["no time pressure", "needs attention soon", "blocking issue or hard deadline"],
},
"needs_reply": {
"type": "noul",
"instructions": "Does the sender expect a reply?",
},
}
def guard_questions() -> Dict:
"""Preset questions for real-time LLM input guardrails."""
return {
"jailbreak": {
"type": "noul",
"instructions": "Does `prompt` try to make an AI assistant ignore its rules, policies or system instructions?",
},
"prompt_injection": {
"type": "noul",
"instructions": "Does `prompt` contain instructions aimed at the AI system rather than a genuine user request?",
},
"sensitive_data": {
"type": "noul",
"instructions": "Does `prompt` contain credentials, personal data or other sensitive information?",
},
"harm_severity": {
"type": "score",
"instructions": "How much harm would complying with `prompt` cause?",
"criteria": [
"none: ordinary request",
"minor: mildly inappropriate",
"serious: unsafe advice or abuse",
"severe: dangerous or illegal",
],
},
"topic": {
"type": "choice",
"instructions": "What is `prompt` about?",
"criteria": {
"product_support": None,
"coding": None,
"general_knowledge": None,
"personal_advice": None,
"security_testing": None,
"other": None,
},
},
}
def moderation_questions() -> Dict:
"""Preset questions for content safety and moderation."""
return {
"toxic": {
"type": "noul",
"instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?",
},
"harassment": {
"type": "noul",
"instructions": "Does `post` target or harass a specific person?",
},
"threat": {
"type": "noul",
"instructions": "Does `post` threaten violence, harm or intimidation?",
},
"spam": {
"type": "noul",
"instructions": "Is `post` spam or advertising?",
},
"severity": {
"type": "score",
"instructions": "How severe is any rule-breaking in `post`?",
"criteria": [
"no rule-breaking: ordinary on-topic post",
"mild: rude tone or off-topic, no target",
"clear violation: insults, harassment or spam aimed at someone",
"severe: threats, hate speech or calls for violence",
],
},
}
def router_questions() -> Dict:
"""Preset questions for intelligent model routing."""
return {
"difficulty": {
"type": "score",
"instructions": "How hard is `request` for a language model?",
"criteria": [
"trivial: a lookup or one-liner",
"easy: short answer, no reasoning",
"moderate: several steps",
"hard: long multi-step reasoning or specialist knowledge",
],
},
"domain": {
"type": "choice",
"instructions": "What domain does `request` belong to?",
"criteria": {
"code": "software engineering, programming, refactoring, architecture, debugging",
"math_or_logic": "mathematics, logic puzzles, proofs, complex calculation",
"writing": "creative writing, essays, emails, blog posts, copywriting",
"factual_lookup": "facts, definitions, trivia, history",
"data_analysis": "statistics, SQL, data manipulation, metrics",
"chitchat": "casual conversation, greetings, small talk",
},
},
"needs_tools": {
"type": "noul",
"instructions": "Does answering `request` require external tools, search or private data?",
},
"is_sensitive": {
"type": "noul",
"instructions": "Does `request` involve money, legal, medical or safety consequences?",
},
}

1905
laya_repo/router.py Normal file

File diff suppressed because it is too large Load Diff

1331
laya_repo/train.py Normal file

File diff suppressed because it is too large Load Diff

4000
laya_v05_decisions.jsonl Normal file

File diff suppressed because one or more lines are too long