laya repo modules + v0.5 decision-format data (4K balanced)
This commit is contained in:
parent
05967cce9e
commit
9927a3859d
168
laya_repo/__init__.py
Normal file
168
laya_repo/__init__.py
Normal file
@ -0,0 +1,168 @@
|
||||
"""Laya: Fast, non-autoregressive System 1 decision engine with calibrated probabilities."""
|
||||
|
||||
import os as _os
|
||||
|
||||
# Tell transformers not to import TensorFlow, before anything can import transformers.
|
||||
#
|
||||
# Laya's model path is torch-only, but on transformers 4.x `build_model` reaches
|
||||
# `transformers.modeling_utils` for `no_init_weights`, and importing that module pulls a chain
|
||||
# (`loss_utils` -> `loss_d_fine` -> `loss_for_object_detection` -> `image_transforms`) that ends in
|
||||
# `import tensorflow`. In an environment where TensorFlow is installed but cannot load, that is a
|
||||
# native crash rather than an exception -- `Fatal Python error: Bus error` -- from a library Laya
|
||||
# never uses (#915).
|
||||
#
|
||||
# `setdefault`, not assignment: a caller who set `USE_TF` to something deliberate keeps it, and one
|
||||
# who wants TensorFlow in the same process can still have it by setting the variable themselves. CI,
|
||||
# the Dockerfile and the test suite already set `USE_TF=0` for exactly this reason; this makes the
|
||||
# package behave the same way without the caller having to know.
|
||||
#
|
||||
# It has to be here, above the imports below, because transformers decides TensorFlow's
|
||||
# availability when it is first imported -- and `import laya` deliberately imports no transformers
|
||||
# (the torch-backed names are lazy, see `_LAZY_ATTRS`), so this runs first in the normal case.
|
||||
_os.environ.setdefault("USE_TF", "0")
|
||||
|
||||
from .email import clean_email_body, email_state
|
||||
from .hooks import AsyncHook, BaseHook, Hook, OptionStabilityHook, PredictContext, PredictHook
|
||||
from .lang import analyse as detect_language
|
||||
from .lang import detect_script, is_english
|
||||
from .presets import (
|
||||
email_questions,
|
||||
guard_questions,
|
||||
moderation_questions,
|
||||
router_questions,
|
||||
triage_questions,
|
||||
)
|
||||
from .router import DEFAULT_MODELS, RouteDecision, Router
|
||||
from .structured import DecisionResult, decide, decide_batch
|
||||
|
||||
__version__ = "0.4.12"
|
||||
|
||||
# Routing, language detection and email cleaning are pure Python. The torch-backed names are
|
||||
# resolved lazily so that `import laya` -- and therefore `from laya import Router` or
|
||||
# `from laya.lang import detect_script` -- does not pay torch's import time and memory.
|
||||
_LAZY_ATTRS = {
|
||||
# stdlib-only module, but keep it lazy so `import laya` does not pull in hashlib
|
||||
"PINNED_REVISIONS": (".revisions", "PINNED_REVISIONS"),
|
||||
"Agent": (".agent", "Agent"),
|
||||
"RLAgent": (".agent", "RLAgent"),
|
||||
"load": (".agent", "load"),
|
||||
"fit_temperatures": (".calibrate", "fit_temperatures"),
|
||||
"fit_one_temperature": (".calibrate", "fit_one_temperature"),
|
||||
"fit_temperature_map": (".calibrate", "fit_temperature_map"),
|
||||
"fit_abstention_thresholds": (".calibrate", "fit_abstention_thresholds"),
|
||||
"fit_binning_map": (".calibrate", "fit_binning_map"),
|
||||
"apply_binning_map": (".calibrate", "apply_binning_map"),
|
||||
"proper_reward": (".common", "proper_reward"),
|
||||
"td_lambda_targets": (".common", "td_lambda_targets"),
|
||||
"ece_score": (".common", "ece_score"),
|
||||
"answer_confidence": (".common", "answer_confidence"),
|
||||
"confidence_from_probs": (".common", "confidence_from_probs"),
|
||||
"check_min_confidence": (".confidence", "check_min_confidence"),
|
||||
"check_min_confidence_map": (".confidence", "check_min_confidence_map"),
|
||||
"resolve_min_confidence": (".confidence", "resolve_min_confidence"),
|
||||
"flag_low_confidence": (".confidence", "flag_low_confidence"),
|
||||
"apply_confidence_gate": (".confidence", "apply_confidence_gate"),
|
||||
"GATE_STATES": (".confidence", "GATE_STATES"),
|
||||
"jev_confidence": (".confidence", "jev_confidence"),
|
||||
"render_options": (".common", "render_options"),
|
||||
"QTYPES": (".common", "QTYPES"),
|
||||
"QTYPE_NAMES": (".common", "QTYPE_NAMES"),
|
||||
"shortlist_choice": (".shortlist", "shortlist_choice"),
|
||||
"predict_shortlist": (".shortlist", "predict_shortlist"),
|
||||
"predict_tournament": (".shortlist", "predict_tournament"),
|
||||
"embed_fn_from_agent": (".shortlist", "embed_fn_from_agent"),
|
||||
"cached_embed_fn": (".shortlist", "cached_embed_fn"),
|
||||
"LayaRouter": (".integrations", "LayaRouter"),
|
||||
"LayaGuardrail": (".integrations", "LayaGuardrail"),
|
||||
"LayaGuardrailError": (".integrations", "LayaGuardrailError"),
|
||||
"LayaTriage": (".integrations", "LayaTriage"),
|
||||
"LayaEvaluator": (".integrations", "LayaEvaluator"),
|
||||
"LayaDecision": (".integrations", "LayaDecision"),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
try:
|
||||
module_name, attr = _LAZY_ATTRS[name]
|
||||
except KeyError:
|
||||
raise AttributeError("module %r has no attribute %r" % (__name__, name)) from None
|
||||
import importlib
|
||||
|
||||
value = getattr(importlib.import_module(module_name, __name__), attr)
|
||||
globals()[name] = value # cache: __getattr__ runs at most once per name
|
||||
return value
|
||||
|
||||
|
||||
def __dir__():
|
||||
return sorted(list(globals()) + list(_LAZY_ATTRS))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Agent",
|
||||
"RLAgent",
|
||||
"load",
|
||||
"fit_temperatures",
|
||||
"fit_one_temperature",
|
||||
"fit_temperature_map",
|
||||
"fit_abstention_thresholds",
|
||||
"fit_binning_map",
|
||||
"apply_binning_map",
|
||||
"Router",
|
||||
"RouteDecision",
|
||||
"DEFAULT_MODELS",
|
||||
"shortlist_choice",
|
||||
"predict_shortlist",
|
||||
"predict_tournament",
|
||||
"embed_fn_from_agent",
|
||||
"cached_embed_fn",
|
||||
"detect_language",
|
||||
"detect_script",
|
||||
"is_english",
|
||||
"clean_email_body",
|
||||
"email_questions",
|
||||
"email_state",
|
||||
"guard_questions",
|
||||
"moderation_questions",
|
||||
"router_questions",
|
||||
"triage_questions",
|
||||
"proper_reward",
|
||||
"td_lambda_targets",
|
||||
"ece_score",
|
||||
"answer_confidence",
|
||||
"confidence_from_probs",
|
||||
"check_min_confidence",
|
||||
"check_min_confidence_map",
|
||||
"resolve_min_confidence",
|
||||
"flag_low_confidence",
|
||||
"apply_confidence_gate",
|
||||
"GATE_STATES",
|
||||
"jev_confidence",
|
||||
"render_options",
|
||||
"QTYPES",
|
||||
"QTYPE_NAMES",
|
||||
"LayaRouter",
|
||||
"LayaGuardrail",
|
||||
"LayaGuardrailError",
|
||||
"LayaTriage",
|
||||
"LayaEvaluator",
|
||||
"LayaDecision",
|
||||
"PredictContext",
|
||||
"PredictHook",
|
||||
"Hook",
|
||||
"BaseHook",
|
||||
"AsyncHook",
|
||||
"OptionStabilityHook",
|
||||
"decide",
|
||||
"decide_batch",
|
||||
"DecisionResult",
|
||||
"PINNED_REVISIONS",
|
||||
"__version__",
|
||||
# Submodules for direct import paths
|
||||
"confidence",
|
||||
"email",
|
||||
"hooks",
|
||||
"lang",
|
||||
"presets",
|
||||
"router",
|
||||
"structured",
|
||||
]
|
||||
2156
laya_repo/agent.py
Normal file
2156
laya_repo/agent.py
Normal file
File diff suppressed because it is too large
Load Diff
822
laya_repo/calibrate.py
Normal file
822
laya_repo/calibrate.py
Normal file
@ -0,0 +1,822 @@
|
||||
"""Fit and persist per-bucket temperatures for Laya logits.
|
||||
|
||||
Ports the notebook LBFGS fitter (NLL on softmax(z/T)) and groups records by
|
||||
`temp_bucket(qtype, k)` so the runtime's `temperature_by_options` lookup has a writer.
|
||||
Fitting is CPU-side: records are `(qtype, logits, target, k)` and do not load Hub weights.
|
||||
|
||||
Two sample floors, on purpose:
|
||||
|
||||
- `MIN_BUCKET_N` is the per-bucket floor. Buckets with fewer examples are omitted from
|
||||
`temperature_by_options`; the type-level scalar covers them at lookup time.
|
||||
- `MIN_TYPE_N` is the lower floor used only for that type-level scalar. It stays at the
|
||||
previous floor of 10 so a realistic dataset that fills no single bucket still gets a
|
||||
temperature instead of remaining at 1.0. Do not raise it to `MIN_BUCKET_N`.
|
||||
|
||||
Fitted temperatures go through `clamp_temperature` and therefore `[TEMP_MIN, TEMP_MAX]`
|
||||
from `laya.common`, the same guard checkpoint load uses. There is no second pair of
|
||||
bounds in this module. A clamp that moved a fit is reported in `fit_temperature_map`'s
|
||||
`temperature_clamped`, since the clamped value alone cannot say whether the optimum was
|
||||
the bound or merely beyond it.
|
||||
"""
|
||||
import warnings
|
||||
from numbers import Integral
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from .common import (
|
||||
QTYPE_NAMES,
|
||||
QTYPES,
|
||||
TEMP_MAX,
|
||||
TEMP_MIN,
|
||||
clamp_temperature,
|
||||
collate_items,
|
||||
ece_score,
|
||||
temp_bucket,
|
||||
temperature_rejected,
|
||||
)
|
||||
|
||||
# Per-bucket floor. Below this, `temperature_by_options` omits the bucket.
|
||||
MIN_BUCKET_N = 2000
|
||||
# Type-level floor only. Lower than MIN_BUCKET_N; see the module docstring.
|
||||
MIN_TYPE_N = 10
|
||||
# Fraction of each bucket held out when `compute_ece=True`. Unused otherwise.
|
||||
ECE_HOLDOUT_FRAC = 0.2
|
||||
# Log-space search margin, in e-folds, outside the `[TEMP_MIN, TEMP_MAX]` answer band.
|
||||
# The optimiser runs on log T with no bound of its own; any scale this far below
|
||||
# `TEMP_MIN` already clamps to `TEMP_MIN`, so the band costs no real minimum, while an
|
||||
# unbounded walk reaches `exp()` underflow (~-88 in float32), divides the objective by
|
||||
# zero, and hands `clamp_temperature` a NaN -- which it reports as the neutral 1.0.
|
||||
LOG_T_MARGIN = 10.0
|
||||
# Payload schema. Files written before identity was recorded have no version key;
|
||||
# readers treat that absence as version 1.
|
||||
CALIBRATION_VERSION = 2
|
||||
N_QTYPES = len(QTYPES)
|
||||
|
||||
# (qtype:int, logits:1d, target:1d, k:int)
|
||||
Record = Tuple[int, Any, Any, int]
|
||||
|
||||
_CONFIG_IDENTITY_SKIP = ("temperature", "temperature_by_options")
|
||||
|
||||
|
||||
def _vec(x) -> np.ndarray:
|
||||
if isinstance(x, torch.Tensor):
|
||||
x = x.detach().cpu().numpy()
|
||||
return np.asarray(x, dtype=np.float32).reshape(-1)
|
||||
|
||||
|
||||
def _validated_pair(logits, target, k=None):
|
||||
vectors = []
|
||||
for name, value in (("logits", logits), ("target", target)):
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = value.detach().cpu().numpy()
|
||||
try:
|
||||
vector = np.asarray(value)
|
||||
if vector.ndim != 1 or vector.dtype.kind not in "iuf":
|
||||
raise ValueError("%s must be a one-dimensional numeric vector" % name)
|
||||
with np.errstate(over="ignore", invalid="ignore"):
|
||||
vector = vector.astype(np.float32)
|
||||
except (TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError("%s must be a one-dimensional numeric vector" % name) from exc
|
||||
if not np.isfinite(vector).all():
|
||||
raise ValueError("%s must contain only finite values" % name)
|
||||
vectors.append(vector)
|
||||
logits, target = vectors
|
||||
if len(logits) != len(target):
|
||||
raise ValueError("logits and target must have the same length")
|
||||
if k is None:
|
||||
k = len(logits)
|
||||
if isinstance(k, bool) or not isinstance(k, Integral) or k < 1 or k > len(logits):
|
||||
raise ValueError("k must be an integer between 1 and the vector length")
|
||||
logits, target = logits[:k], target[:k]
|
||||
if (target < 0).any() or not np.isclose(target.sum(), 1.0, rtol=1e-5, atol=1e-6):
|
||||
raise ValueError("target must be a nonnegative probability vector summing to 1")
|
||||
return logits, target, int(k)
|
||||
|
||||
|
||||
def _positive_int(name: str, value: int) -> int:
|
||||
"""A count parameter: an integer >= 1, normalised to a Python `int`, or a `ValueError` naming it.
|
||||
|
||||
This is the rule the module already applies everywhere a count is checked -- `min_n` in
|
||||
:func:`fit_one_temperature`, `k` in :func:`_validated_pair` (which returns `int(k)` for the same
|
||||
reason), and the `"bins"` of every `binning_map` entry :func:`apply_calibration_payload` reads --
|
||||
applied to the two fitting parameters that were checked with less. The asymmetry was not
|
||||
cosmetic: a `bool` is an `Integral`, so `bins` computed as a comparison (`len(bucket) > 15`)
|
||||
passed the old `bins < 1` guard and emitted `{"bins": True, ...}`, a map the loader refuses on
|
||||
its boolean arm. And a `numpy` integer is an `Integral` but not a Python `int`, so a `bins`
|
||||
derived from an array shape emitted a map `Agent.save_calibration`'s own `json.dump` cannot write
|
||||
-- a fit that produced an artifact the file format rejects. `int()` keeps that second case
|
||||
working. A float, a string or `None` is a mistake, and now names itself instead of raising
|
||||
`TypeError` from `range()` (or, for a `min_bucket_n` that compares `nan` to a length, silently
|
||||
disabling the sample floor the parameter exists to enforce).
|
||||
"""
|
||||
if isinstance(value, bool) or not isinstance(value, Integral) or value < 1:
|
||||
raise ValueError("%s must be a positive integer, got %r" % (name, value))
|
||||
return int(value)
|
||||
|
||||
|
||||
def _pairs_by_width(pairs: Sequence) -> List[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""One (logits, target) matrix per distinct option count, narrowest first.
|
||||
|
||||
A ragged pool -- a question type holding 2-option and 12-option decisions -- cannot be
|
||||
one matrix. Any value used to square off the short rows is a real column in the single
|
||||
softmax every row then goes through, so it competes for the probability mass the fit is
|
||||
trying to allocate; at `-1e4` it wins the row it pads once the row's own logits drop
|
||||
below it, and it overflows to `inf`/`NaN` in float32 when the scale gets small. Grouping
|
||||
keeps each row inside its own option count, which is what the objective means.
|
||||
"""
|
||||
groups: Dict[int, List[Tuple[np.ndarray, np.ndarray]]] = {}
|
||||
for z, t in pairs:
|
||||
groups.setdefault(len(_vec(z)), []).append((_vec(z), _vec(t)))
|
||||
return [(torch.from_numpy(np.stack([z for z, _ in groups[w]])),
|
||||
torch.from_numpy(np.stack([t for _t, t in groups[w]]))) for w in sorted(groups)]
|
||||
|
||||
|
||||
def _fit_unclamped_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
|
||||
"""The optimised scale before `clamp_temperature`, for callers that report the clamp.
|
||||
|
||||
`fit_one_temperature` is this value clamped, and that is the value anything installs.
|
||||
Only the clamp reporting in `_fit_groups` reads the raw scale, so that a bucket whose
|
||||
optimum sits outside `[TEMP_MIN, TEMP_MAX]` can be told apart from one whose optimum
|
||||
really is a bound. The search band is `LOG_T_MARGIN` e-folds wider than the answer
|
||||
band, so a raw value can be far outside it; a NaN from the underflow case described in
|
||||
`fit_one_temperature` reaches the caller as a NaN rather than as 1.0.
|
||||
"""
|
||||
if min_n is None:
|
||||
min_n = MIN_BUCKET_N
|
||||
if isinstance(min_n, bool) or not isinstance(min_n, Integral) or min_n < 1:
|
||||
raise ValueError("min_n must be a positive integer")
|
||||
sel = []
|
||||
for pair in pairs:
|
||||
try:
|
||||
z, t = pair
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError("pair must be (logits, target)") from exc
|
||||
z, t, _k = _validated_pair(z, t)
|
||||
sel.append((z, t))
|
||||
if len(sel) < min_n:
|
||||
return 1.0
|
||||
groups = _pairs_by_width(sel)
|
||||
n_rows = float(len(sel))
|
||||
log_lo = float(np.log(TEMP_MIN)) - LOG_T_MARGIN
|
||||
log_hi = float(np.log(TEMP_MAX)) + LOG_T_MARGIN
|
||||
with torch.enable_grad():
|
||||
log_t = torch.zeros(1, requires_grad=True)
|
||||
opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)
|
||||
|
||||
def closure():
|
||||
opt.zero_grad()
|
||||
scale = log_t.clamp(min=log_lo, max=log_hi).exp()
|
||||
nll = sum(-(t * torch.log_softmax(z / scale, -1)).sum(-1).sum()
|
||||
for z, t in groups)
|
||||
loss = nll / n_rows
|
||||
loss.backward()
|
||||
return loss
|
||||
|
||||
opt.step(closure)
|
||||
return float(log_t.detach().clamp(min=log_lo, max=log_hi).exp().item())
|
||||
|
||||
|
||||
def fit_one_temperature(pairs: Sequence, min_n: Optional[int] = None) -> float:
|
||||
"""Fit one scalar T by NLL + LBFGS on log T.
|
||||
|
||||
The result is `clamp_temperature` of the optimised scale, so it lies in
|
||||
`[TEMP_MIN, TEMP_MAX]` (or is the neutral 1.0 when the value is not a number).
|
||||
The search is bounded to `LOG_T_MARGIN` e-folds outside that band, which covers any
|
||||
minimum the answer can use; past it the float32 scale drifts toward `exp()` underflow,
|
||||
where the objective divides by zero and every gradient becomes NaN, and
|
||||
`clamp_temperature` reads that NaN back as the neutral 1.0.
|
||||
Returns 1.0 when fewer than `min_n` pairs are given. `min_n` defaults to
|
||||
`MIN_BUCKET_N` (the per-bucket floor). Type-level fits pass `MIN_TYPE_N`, which
|
||||
is lower, so a dataset that fills no bucket still gets a scalar instead of
|
||||
staying at 1.0.
|
||||
|
||||
A clamped result is indistinguishable from an optimum that really is a bound, because
|
||||
both are the bound. `fit_temperature_map`'s `temperature_clamped` is where the
|
||||
difference is reported; it carries the unclamped scale beside the value installed.
|
||||
"""
|
||||
return clamp_temperature(_fit_unclamped_temperature(pairs, min_n), TEMP_MIN, TEMP_MAX)
|
||||
|
||||
|
||||
def _iter_records(records: Iterable) -> List[Tuple[int, np.ndarray, np.ndarray, int]]:
|
||||
out = []
|
||||
for rec in records:
|
||||
if not isinstance(rec, (list, tuple)) or len(rec) not in (3, 4):
|
||||
raise ValueError("record must be (qtype, logits, target[, k])")
|
||||
if len(rec) == 4:
|
||||
qtype, logits, target, k = rec
|
||||
if k is None:
|
||||
raise ValueError("k must be an integer between 1 and the vector length")
|
||||
else:
|
||||
qtype, logits, target = rec
|
||||
k = None
|
||||
if isinstance(qtype, bool) or not isinstance(qtype, Integral) or not 0 <= qtype < N_QTYPES:
|
||||
raise ValueError("qtype must be an integer between 0 and %d" % (N_QTYPES - 1))
|
||||
logits, target, k = _validated_pair(logits, target, k)
|
||||
out.append((int(qtype), logits, target, k))
|
||||
return out
|
||||
|
||||
|
||||
def _softmax(z, t_scale: float) -> np.ndarray:
|
||||
z = np.asarray(z, dtype=np.float64) / max(1e-3, float(t_scale))
|
||||
z = z - z.max()
|
||||
p = np.exp(z)
|
||||
return p / p.sum()
|
||||
|
||||
|
||||
def _ece_report(recs, temperature, temperature_by_options) -> Dict[str, Any]:
|
||||
conf_b, cor_b, conf_a, cor_a = [], [], [], []
|
||||
for qt, z, t, k in recs:
|
||||
y = int(np.argmax(t[:k]))
|
||||
p0 = _softmax(z[:k], 1.0)
|
||||
conf_b.append(float(p0.max()))
|
||||
cor_b.append(float(int(p0.argmax()) == y))
|
||||
t_scale = temperature_by_options.get(temp_bucket(qt, k), temperature[qt])
|
||||
p1 = _softmax(z[:k], t_scale)
|
||||
conf_a.append(float(p1.max()))
|
||||
cor_a.append(float(int(p1.argmax()) == y))
|
||||
return {
|
||||
"ece_before": ece_score(np.asarray(conf_b), np.asarray(cor_b)),
|
||||
"ece_after": ece_score(np.asarray(conf_a), np.asarray(cor_a)),
|
||||
"n": float(len(recs)),
|
||||
}
|
||||
|
||||
|
||||
def _group_records(recs):
|
||||
by_type = {qt: [] for qt in range(N_QTYPES)}
|
||||
by_bucket = {}
|
||||
for qt, z, t, k in recs:
|
||||
if qt in by_type:
|
||||
by_type[qt].append((z, t))
|
||||
key = temp_bucket(qt, k)
|
||||
by_bucket.setdefault(key, []).append((z, t))
|
||||
return by_type, by_bucket
|
||||
|
||||
|
||||
def _ece_split(recs, seed):
|
||||
"""Split records into fit and eval, stratified by `temp_bucket`.
|
||||
|
||||
Returns `(fit_recs, eval_recs, excluded_buckets)`. The split is a deterministic
|
||||
partition (`numpy.random.RandomState(seed)`): every record is in exactly one side.
|
||||
A bucket whose fit side would fall below `MIN_BUCKET_N` is not split; all of its
|
||||
records stay in `fit_recs` and its key is listed in `excluded_buckets`.
|
||||
"""
|
||||
by_bucket = {}
|
||||
for i, rec in enumerate(recs):
|
||||
qt, _z, _t, k = rec
|
||||
by_bucket.setdefault(temp_bucket(qt, k), []).append(i)
|
||||
|
||||
rng = np.random.RandomState(seed)
|
||||
fit_idx = []
|
||||
eval_idx = []
|
||||
excluded = []
|
||||
for key in sorted(by_bucket):
|
||||
idxs = by_bucket[key]
|
||||
n = len(idxs)
|
||||
n_eval = int(round(n * ECE_HOLDOUT_FRAC))
|
||||
if n - n_eval < MIN_BUCKET_N:
|
||||
fit_idx.extend(idxs)
|
||||
excluded.append(key)
|
||||
continue
|
||||
perm = rng.permutation(n)
|
||||
for j in perm[:n_eval]:
|
||||
eval_idx.append(idxs[int(j)])
|
||||
for j in perm[n_eval:]:
|
||||
fit_idx.append(idxs[int(j)])
|
||||
|
||||
fit_idx.sort()
|
||||
eval_idx.sort()
|
||||
fit_recs = [recs[i] for i in fit_idx]
|
||||
eval_recs = [recs[i] for i in eval_idx]
|
||||
return fit_recs, eval_recs, excluded
|
||||
|
||||
|
||||
def _clamp_entry(raw: float, applied: float) -> Optional[Dict[str, Any]]:
|
||||
"""Describe a fit the clamp moved, or None when it left the fit alone.
|
||||
|
||||
A NaN raw scale is the underflow case, not a clamp: `clamp_temperature` answers the
|
||||
neutral 1.0 for it, which is a different fact and is not reported here.
|
||||
"""
|
||||
if not np.isfinite(raw) or TEMP_MIN <= raw <= TEMP_MAX:
|
||||
return None
|
||||
return {"fitted": raw, "applied": applied, "bound": "min" if raw < TEMP_MIN else "max"}
|
||||
|
||||
|
||||
def _fit_groups(by_type, by_bucket):
|
||||
temperature = [1.0] * N_QTYPES
|
||||
clamped: Dict[str, Dict[str, Any]] = {}
|
||||
for qt in range(N_QTYPES):
|
||||
pairs = by_type[qt]
|
||||
# MIN_TYPE_N, not MIN_BUCKET_N: the scalar has to exist when every bucket is small.
|
||||
if len(pairs) >= MIN_TYPE_N:
|
||||
raw = _fit_unclamped_temperature(pairs, min_n=MIN_TYPE_N)
|
||||
temperature[qt] = clamp_temperature(raw, TEMP_MIN, TEMP_MAX)
|
||||
entry = _clamp_entry(raw, temperature[qt])
|
||||
if entry is not None:
|
||||
clamped[QTYPE_NAMES[qt]] = entry
|
||||
|
||||
temperature_by_options = {}
|
||||
for key, pairs in by_bucket.items():
|
||||
if len(pairs) < MIN_BUCKET_N:
|
||||
continue
|
||||
raw = _fit_unclamped_temperature(pairs)
|
||||
temperature_by_options[key] = clamp_temperature(raw, TEMP_MIN, TEMP_MAX)
|
||||
entry = _clamp_entry(raw, temperature_by_options[key])
|
||||
if entry is not None:
|
||||
clamped[key] = entry
|
||||
return temperature, temperature_by_options, clamped
|
||||
|
||||
|
||||
def fit_temperature_map(records: Iterable, compute_ece: bool = False, seed: int = 0) -> Dict[str, Any]:
|
||||
"""Fit type-level scalars and per-bucket temperatures.
|
||||
|
||||
`MIN_BUCKET_N` is the per-bucket floor: smaller buckets are omitted from
|
||||
`temperature_by_options` and the type-level scalar covers them. `MIN_TYPE_N` is
|
||||
the separate, lower floor for that scalar only.
|
||||
|
||||
`compute_ece=False` (the default, and the path `Agent.fit_temperatures` stores)
|
||||
fits on every record and returns no `report` key. `seed` is ignored on this path.
|
||||
|
||||
`compute_ece=True` holds out `ECE_HOLDOUT_FRAC` of each bucket, stratified by
|
||||
`temp_bucket`, using `seed` so the same records always split the same way. Temperatures
|
||||
are fit on the remainder only and ECE is scored only on the held-out records.
|
||||
`seed` is this module's own count rule -- an integer in `[0, 2**32-1]`, which is what
|
||||
`numpy.random.RandomState` can take -- so a `10.0`, a `"7"`, a `True` or an out-of-range
|
||||
value is a `ValueError` naming `seed` on either path, not numpy's cast error on one path
|
||||
and silence on the other. `seed=None` stays numpy's entropy seed: accepted, and the split
|
||||
is then not reproducible.
|
||||
`temperature_clamped` names every fit the clamp moved, on either path: a bucket key as
|
||||
`temperature_by_options` spells it, or a question-type name for one of the `temperature`
|
||||
scalars. Each entry carries `fitted` (the unclamped optimum), `applied` (the value in
|
||||
this result) and `bound`, `"min"` or `"max"`. It is empty when nothing was clamped, and
|
||||
it reports a fact a caller cannot otherwise recover: a value of exactly `TEMP_MIN` or
|
||||
`TEMP_MAX` is what both a clamped fit and a genuine fit on the bound look like, so
|
||||
without this an optimum of 0.387 installs as 0.5 and reads as a finished fit. The
|
||||
clamped value is still the one to install, because `load_calibration` applies the same
|
||||
bounds at checkpoint load; what the entry adds is that the data wanted to go further.
|
||||
|
||||
`report["n"]` is the number of records passed in; `report["n_eval"]` is the held-out
|
||||
count the ECE rests on. A bucket that would drop below `MIN_BUCKET_N` after the
|
||||
holdout is fit on all of its records, left out of the eval set, and named in
|
||||
`report["buckets_excluded_from_eval"]` instead of being dropped. `n_by_bucket`
|
||||
always counts the full input, including when the fit itself used a subset.
|
||||
"""
|
||||
if seed is not None and (isinstance(seed, bool) or not isinstance(seed, Integral)
|
||||
or not 0 <= seed < 2 ** 32):
|
||||
raise ValueError("seed must be None or an integer in [0, 2**32-1], got %r" % (seed,))
|
||||
recs = _iter_records(records)
|
||||
all_by_type, all_buckets = _group_records(recs)
|
||||
n_by_bucket = {key: len(pairs) for key, pairs in all_buckets.items()}
|
||||
|
||||
if compute_ece:
|
||||
_fit_recs, eval_recs, excluded = _ece_split(recs, seed)
|
||||
by_type, by_bucket = _group_records(_fit_recs)
|
||||
else:
|
||||
eval_recs = None
|
||||
excluded = []
|
||||
by_type, by_bucket = all_by_type, all_buckets
|
||||
|
||||
temperature, temperature_by_options, clamped = _fit_groups(by_type, by_bucket)
|
||||
out: Dict[str, Any] = {
|
||||
"temperature": temperature,
|
||||
"temperature_by_options": temperature_by_options,
|
||||
"n_by_bucket": n_by_bucket,
|
||||
"temperature_clamped": clamped,
|
||||
}
|
||||
if compute_ece:
|
||||
report = _ece_report(eval_recs, temperature, temperature_by_options)
|
||||
# `n` stays the size of the input. `n_eval` is the held-out count ECE used.
|
||||
report["n"] = float(len(recs))
|
||||
report["n_eval"] = float(len(eval_recs))
|
||||
report["buckets_excluded_from_eval"] = excluded
|
||||
out["report"] = report
|
||||
return out
|
||||
|
||||
|
||||
fit_temperatures = fit_temperature_map
|
||||
|
||||
|
||||
# Per-bucket floor for abstention thresholds. Lower than `MIN_BUCKET_N` (which governs temperature
|
||||
# fitting): a threshold is a single order statistic of the calibrated confidences, so it stabilises
|
||||
# on far fewer examples than an LBFGS temperature does. Buckets below this are omitted, and only the
|
||||
# map's own "default" entry covers them afterwards -- `resolve_min_confidence` falls from "default"
|
||||
# to 0.0, which gates nothing. There is no scalar behind that fall-through, unlike a temperature
|
||||
# bucket omitted from `temperature_by_options`, whose type-level scalar really is the fallback: one
|
||||
# call's `min_confidence` is either a float or a map, never both, so no caller's float is in scope
|
||||
# here. A map produced with this floor should name a "default" if any bucket under it must abstain.
|
||||
MIN_ABSTAIN_BUCKET_N = 100
|
||||
|
||||
|
||||
def _select_abstention_threshold(pairs: Sequence[Tuple[float, int]], target_error: float,
|
||||
conservative: bool) -> float:
|
||||
"""Smallest confidence `tau` whose accepted set (`conf >= tau`) keeps error <= `target_error`.
|
||||
|
||||
`pairs` are `(confidence, correct)`. Sweeping from the most confident down maximises coverage
|
||||
at the target risk (the selective-classification / split-conformal cut). `conservative` adds one
|
||||
pseudo-error so a bucket does not clear the gate on a lucky short run. Returns 1.0 when no cut
|
||||
holds the risk -- the bucket is too unreliable to accept anything short of a reported certainty.
|
||||
"""
|
||||
ordered = sorted(pairs, key=lambda cc: cc[0], reverse=True)
|
||||
levels = sorted({c for c, _ in ordered}, reverse=True)
|
||||
n = len(ordered)
|
||||
n_acc = n_err = idx = 0
|
||||
best = None
|
||||
# Evaluate the error over the WHOLE accepted set {conf >= tau} at each distinct level, not
|
||||
# incrementally within a tie: a threshold accepts every answer at its own confidence, so a tie
|
||||
# group's errors must all be counted before the level is judged (else the cut sinks into a bad
|
||||
# cohort on its first few correct members).
|
||||
for tau in levels:
|
||||
while idx < n and ordered[idx][0] >= tau:
|
||||
n_acc += 1
|
||||
n_err += 0 if ordered[idx][1] else 1
|
||||
idx += 1
|
||||
rate = (n_err + 1.0) / (n_acc + 1.0) if conservative else (n_err / n_acc)
|
||||
if n_acc > 0 and rate <= target_error:
|
||||
best = tau
|
||||
if best is None:
|
||||
return 1.0
|
||||
return float(min(1.0, max(0.0, best)))
|
||||
|
||||
|
||||
def fit_abstention_thresholds(records: Iterable, temperature: Sequence[float],
|
||||
temperature_by_options: Dict[str, float], *,
|
||||
binning_map: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
target_error: float = 0.10,
|
||||
min_bucket_n: int = MIN_ABSTAIN_BUCKET_N,
|
||||
conservative: bool = True) -> Dict[str, float]:
|
||||
"""Fit a per-`temp_bucket` abstention threshold so a gate keeps a target error in each fitted bucket.
|
||||
|
||||
A single `min_confidence` does not transfer across option counts (#394): the calibrated
|
||||
confidence of a 2-option and a 12-option answer live on different scales, so one cut over- or
|
||||
under-abstains depending on the question. This fits one cut per bucket instead, keyed exactly
|
||||
like `temperature_by_options` (`common.temp_bucket`, e.g. ``"choice:3-5"``), and the result is a
|
||||
`min_confidence` map that :func:`laya.confidence.check_min_confidence` /
|
||||
:func:`laya.confidence.apply_confidence_gate` accept directly -- with one exception to keep:
|
||||
every bucket below `min_bucket_n` is omitted, so a calibration set too small to fill a single
|
||||
bucket returns ``{}``, and the validator refuses an empty map ("must be a non-empty dict"). Pass
|
||||
no gate in that case rather than passing what came back; `laya.train` writes the map into a
|
||||
checkpoint only `if abstention_thresholds:`.
|
||||
|
||||
`records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes
|
||||
(`records_from_labeled` builds them). Confidence is the **calibrated** `max(p)` -- the logits are
|
||||
scaled by the fitted `temperature` / `temperature_by_options` first, so thresholds and the
|
||||
numbers the runtime reports are on the same scale. `target_error` is the tolerated error among
|
||||
accepted answers; `min_bucket_n` omits buckets too small to fit, and `conservative` adds a
|
||||
one-sample margin. An omitted bucket is not caught by a fallback: the map lookup falls to its own
|
||||
``"default"`` entry and then to 0.0, which gates nothing, so a `min_bucket_n` that excludes a
|
||||
bucket leaves that bucket ungated unless the map names a ``"default"``. The thresholds are
|
||||
empirical cuts on the calibration set, not a formal coverage guarantee -- validate on held-out
|
||||
data (`fit_temperature_map(..., compute_ece=True)` gives a held-out split) for a production gate.
|
||||
|
||||
Pass `binning_map` when the agent that will serve these thresholds has one installed -- by
|
||||
`Agent.fit_binning`, or by a calibration payload that carries `binning_map` -- because the
|
||||
runtime recalibrates `answer_confidence` through that map before anything reads it, so a cut
|
||||
fitted without it is a cut on a scale the gate never sees. The thresholds are then on the binned
|
||||
scale, and the order the two were fitted in stops mattering. Measured on 1,200 synthetic
|
||||
12-option records at `target_error=0.10`: the cut fitted without a map holds 9.8% error over 50%
|
||||
coverage on un-binned confidences, and admits 94.5% of answers at 25.6% error once the same
|
||||
number is compared against binned ones.
|
||||
"""
|
||||
# A bool is the one value a range test cannot see: `0.0 <= True <= 1.0` holds, so `False` would
|
||||
# be read as 0.0 -- the cut that abstains on every answer -- and `True` as 1.0, which gates
|
||||
# nothing. Both fit a map, so nothing downstream would mention it. `TrainConfig` refuses the
|
||||
# same bool on the way to this call (`train.py`), as does `_iter_records` for a `qtype`.
|
||||
if isinstance(target_error, bool) or not 0.0 <= target_error <= 1.0:
|
||||
raise ValueError("target_error must be in [0.0, 1.0], got %r" % (target_error,))
|
||||
min_bucket_n = _positive_int("min_bucket_n", min_bucket_n)
|
||||
recs = _iter_records(records)
|
||||
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
|
||||
for qt, z, t, k in recs:
|
||||
y = int(np.argmax(t[:k]))
|
||||
bucket = temp_bucket(qt, k)
|
||||
t_scale = temperature_by_options.get(bucket, temperature[qt])
|
||||
p = _softmax(z[:k], t_scale)
|
||||
conf = float(p.max())
|
||||
if binning_map:
|
||||
# The runtime bins before anyone reads `answer_confidence` (`Agent._decode_answers`),
|
||||
# so the cut has to be chosen on the binned scale or it gates a different quantity.
|
||||
conf = apply_binning_map(conf, bucket, binning_map)
|
||||
# `_decode_answers` reports `answer_confidence` rounded to 4 decimals, after binning, and
|
||||
# the gate compares that. A cut picked at full precision can sit above the rounded value of
|
||||
# its own cohort (bin 5/6 is reported as 0.8333 < 0.83333...), which abstains all of it.
|
||||
conf = round(conf, 4)
|
||||
by_bucket.setdefault(bucket, []).append((conf, int(int(p.argmax()) == y)))
|
||||
out: Dict[str, float] = {}
|
||||
for key, pairs in by_bucket.items():
|
||||
if len(pairs) < min_bucket_n:
|
||||
continue
|
||||
out[key] = _select_abstention_threshold(pairs, target_error, conservative)
|
||||
return out
|
||||
|
||||
|
||||
# Per-bucket floor for histogram-binning recalibration. Higher than the abstention floor because a
|
||||
# binning map splits each bucket's examples across `bins`, so each bin needs its own sample; lower
|
||||
# than the temperature floor because binning is a count per bin, not an optimisation.
|
||||
MIN_BINNING_BUCKET_N = 200
|
||||
|
||||
|
||||
def fit_binning_map(records: Iterable, temperature: Sequence[float],
|
||||
temperature_by_options: Dict[str, float], *, bins: int = 15,
|
||||
min_bucket_n: int = MIN_BINNING_BUCKET_N) -> Dict[str, Dict[str, Any]]:
|
||||
"""Fit a per-`temp_bucket` histogram-binning recalibration map for `answer_confidence`.
|
||||
|
||||
Temperature scaling applies one scalar per bucket; it cannot fix a bucket whose reliability
|
||||
curve is not a simple sharpening/softening (the pathological `choice:11+` the shipped English
|
||||
checkpoint carries is one). Histogram binning is the non-parametric alternative: split the
|
||||
calibrated confidences of a bucket into `bins` equal-width bins over [0, 1], and map every
|
||||
confidence that lands in a bin to that bin's empirical accuracy. It needs no monotonicity
|
||||
assumption and no extra dependency (NumPy only; isotonic regression would pull in scikit-learn).
|
||||
|
||||
`records` are the same `(qtype, logits, target[, k])` tuples `fit_temperature_map` consumes;
|
||||
confidence is the calibrated `max(p)` (logits scaled by the fitted `temperature` /
|
||||
`temperature_by_options` first), so a binning map composes on top of a temperature map rather
|
||||
than replacing it. Returns `{bucket: {"bins": N, "values": [recalibrated confidence per bin]}}`;
|
||||
buckets below `min_bucket_n` are omitted. Apply it with :func:`apply_binning_map`. An empty bin
|
||||
(a confidence range the calibration set never produced) maps to that bin's centre, a point
|
||||
inside the region the value came from: the value is still rewritten, and moves by at most half a
|
||||
bin width. What the centre buys is that an unseen confidence is never recalibrated to a
|
||||
fabricated 0, not that it passes through -- only a bucket the map has no entry for does that
|
||||
(:func:`apply_binning_map`).
|
||||
"""
|
||||
bins = _positive_int("bins", bins)
|
||||
min_bucket_n = _positive_int("min_bucket_n", min_bucket_n)
|
||||
recs = _iter_records(records)
|
||||
by_bucket: Dict[str, List[Tuple[float, int]]] = {}
|
||||
for qt, z, t, k in recs:
|
||||
y = int(np.argmax(t[:k]))
|
||||
bucket = temp_bucket(qt, k)
|
||||
t_scale = temperature_by_options.get(bucket, temperature[qt])
|
||||
p = _softmax(z[:k], t_scale)
|
||||
by_bucket.setdefault(bucket, []).append((float(p.max()), int(int(p.argmax()) == y)))
|
||||
out: Dict[str, Dict[str, Any]] = {}
|
||||
for key, pairs in by_bucket.items():
|
||||
if len(pairs) < min_bucket_n:
|
||||
continue
|
||||
conf = np.asarray([c for c, _ in pairs], dtype=float)
|
||||
corr = np.asarray([c for _, c in pairs], dtype=float)
|
||||
idx = np.clip((conf * bins).astype(int), 0, bins - 1)
|
||||
values = []
|
||||
for b in range(bins):
|
||||
mask = idx == b
|
||||
values.append(float(corr[mask].mean()) if mask.any() else (b + 0.5) / bins)
|
||||
out[key] = {"bins": bins, "values": values}
|
||||
return out
|
||||
|
||||
|
||||
def apply_binning_map(confidence: float, bucket: str,
|
||||
binning_map: Dict[str, Dict[str, Any]]) -> float:
|
||||
"""Recalibrate one `answer_confidence` for its option-count `bucket` (`common.temp_bucket`).
|
||||
|
||||
Returns the confidence unchanged when the map has no entry for the bucket, so a bucket the map
|
||||
was not fit for passes through rather than being forced to a wrong value.
|
||||
"""
|
||||
entry = binning_map.get(bucket)
|
||||
if not entry:
|
||||
return float(confidence)
|
||||
bins = int(entry["bins"])
|
||||
b = min(bins - 1, max(0, int(float(confidence) * bins)))
|
||||
return float(entry["values"][b])
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def records_from_labeled(agent, pairs: Sequence) -> List[Record]:
|
||||
"""Collect CPU records from `(state, questions, targets)`.
|
||||
|
||||
Logits are the raw rows `_decode_answers` scales, not the probabilities it returns.
|
||||
`_encode_state` builds one state's items, `_forward` runs them, and `_option_logits`
|
||||
slices each question to the same marker width `_decode_answers` divides by temperature.
|
||||
No Hub download and no `model.safetensors` write. Tests stay weight-free by passing
|
||||
synthetic records to `fit_temperature_map` instead of calling this.
|
||||
"""
|
||||
# Imported lazily: `agent` imports this module at load, and `_option_logits` is defined
|
||||
# on the way through that import.
|
||||
from .agent import _option_logits
|
||||
|
||||
records: List[Record] = []
|
||||
for state, questions, targets in pairs:
|
||||
ids = list(questions.keys())
|
||||
if not ids:
|
||||
continue
|
||||
for qid in ids:
|
||||
agent._check_question(qid, questions[qid])
|
||||
internal = {qid: agent._to_internal(questions[qid]) for qid in ids}
|
||||
items = agent._encode_state(state, ids, internal)
|
||||
batch = collate_items([items], agent.tok.pad_token_id)
|
||||
if batch is None:
|
||||
continue
|
||||
logits, _act = agent._forward(batch)
|
||||
rows = _option_logits(logits, items, 0)
|
||||
for j, qid in enumerate(ids):
|
||||
k = len(items[j]["markers"])
|
||||
qt = int(items[j]["qtype"])
|
||||
records.append((qt, _vec(rows[j]), _vec(targets[qid])[:k], k))
|
||||
return records
|
||||
|
||||
|
||||
def _config_identity(cfg):
|
||||
"""Checkpoint config without the temperature fields this file itself stores."""
|
||||
if cfg is None:
|
||||
return None
|
||||
return {str(k): cfg[k] for k in cfg if k not in _CONFIG_IDENTITY_SKIP}
|
||||
|
||||
|
||||
def calibration_payload(
|
||||
temperature,
|
||||
temperature_by_options,
|
||||
model_id_or_path=None,
|
||||
subfolder=None,
|
||||
config=None,
|
||||
binning_map=None,
|
||||
) -> Dict[str, Any]:
|
||||
"""JSON body written by `Agent.save_calibration` (no weights).
|
||||
|
||||
`version` is `CALIBRATION_VERSION`. Payloads written before identity was recorded have
|
||||
no version key; `apply_calibration_payload` treats that as version 1 and still loads.
|
||||
|
||||
`model_id_or_path`, `subfolder`, and `config` say which checkpoint the map was fitted
|
||||
against. `config` is `rl_agent_config.json` without `temperature` /
|
||||
`temperature_by_options` (those live at the top of this payload).
|
||||
|
||||
`binning_map` is the optional histogram-binning recalibration map fitted by
|
||||
`fit_binning_map`; the key is omitted when no map is installed, which is `None`, not an empty
|
||||
map. A map that fitted nothing -- `{}`, the return value when no bucket reached the sample
|
||||
floor -- is still written, so the round trip through :func:`apply_calibration_payload` gives
|
||||
`{}` back rather than `None`.
|
||||
"""
|
||||
payload = {
|
||||
"version": CALIBRATION_VERSION,
|
||||
"temperature": [float(x) for x in temperature],
|
||||
"temperature_by_options": {
|
||||
str(k): float(v) for k, v in dict(temperature_by_options).items()
|
||||
},
|
||||
"model_id_or_path": model_id_or_path,
|
||||
"subfolder": subfolder,
|
||||
"config": _config_identity(config),
|
||||
}
|
||||
if binning_map is not None:
|
||||
payload["binning_map"] = binning_map
|
||||
return payload
|
||||
|
||||
|
||||
def _warn_if_identity_mismatch(obj, payload) -> None:
|
||||
recorded_model = payload.get("model_id_or_path")
|
||||
recorded_sub = payload.get("subfolder")
|
||||
recorded_cfg = payload.get("config")
|
||||
current_model = getattr(obj, "model_id_or_path", None)
|
||||
current_sub = getattr(obj, "subfolder", None)
|
||||
current_cfg = _config_identity(getattr(obj, "cfg", None))
|
||||
if (recorded_model, recorded_sub, recorded_cfg) == (current_model, current_sub, current_cfg):
|
||||
return
|
||||
cfg_note = ""
|
||||
if recorded_cfg != current_cfg:
|
||||
cfg_note = " Checkpoint config identity differs."
|
||||
warnings.warn(
|
||||
"calibration was fitted for model_id_or_path=%r subfolder=%r but this agent is "
|
||||
"model_id_or_path=%r subfolder=%r.%s Loading the temperatures anyway."
|
||||
% (recorded_model, recorded_sub, current_model, current_sub, cfg_note),
|
||||
UserWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
|
||||
|
||||
def _rejected_temperatures(entries):
|
||||
"""Names whose raw value is not the number `clamp_temperature` kept, matching checkpoint load."""
|
||||
rejected = []
|
||||
for name, raw, applied in entries:
|
||||
if not temperature_rejected(raw, applied):
|
||||
continue
|
||||
rejected.append("%s=%r -> %g" % (name, raw, applied))
|
||||
return rejected
|
||||
|
||||
|
||||
def _install_temperatures(obj, temperature, temperature_by_options, warn: bool = True) -> None:
|
||||
"""Store a map the way checkpoint load does: raw values, and the clamped ones inference reads.
|
||||
|
||||
`warn=False` is the fitter path: those values were already passed through `clamp_temperature`,
|
||||
so a second pass must not report them as a bad file.
|
||||
"""
|
||||
raw_t = list(temperature)
|
||||
applied_t = [clamp_temperature(t) for t in raw_t]
|
||||
raw_b = {str(k): v for k, v in dict(temperature_by_options).items()}
|
||||
applied_b = {k: clamp_temperature(v) for k, v in raw_b.items()}
|
||||
obj.temperature_raw = raw_t
|
||||
obj.temperature = applied_t
|
||||
obj.temperature_by_options_raw = raw_b
|
||||
obj.temperature_by_options = applied_b
|
||||
if not warn:
|
||||
return
|
||||
entries = [("temperature[%d]" % i, raw, applied_t[i]) for i, raw in enumerate(raw_t)]
|
||||
entries += [(key, raw_b[key], applied_b[key]) for key in raw_b]
|
||||
rejected = _rejected_temperatures(entries)
|
||||
if rejected:
|
||||
warnings.warn(
|
||||
"laya: calibration temperatures are invalid or outside [%g, %g]; "
|
||||
"using %s. Treat confidence from the affected entries as uncalibrated."
|
||||
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
|
||||
RuntimeWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
|
||||
|
||||
def apply_calibration_payload(obj, payload: Dict[str, Any]) -> None:
|
||||
"""Copy `temperature`, `temperature_by_options` and `binning_map` onto `obj`.
|
||||
|
||||
Those are the three fields this reads and the three it writes. `binning_map` is installed
|
||||
whatever the payload holds, including nothing: a payload with no ``binning_map`` key -- every
|
||||
file written before histogram binning existed, and every agent that saved before
|
||||
:meth:`Agent.fit_binning` ran -- installs `None`, which clears a map this object already
|
||||
fitted. The file is the whole calibration state, not a patch onto the current one.
|
||||
|
||||
A missing `version` is version 1 (temperatures only, no checkpoint identity). Version
|
||||
`CALIBRATION_VERSION` records the checkpoint the map was fitted for; a mismatch warns
|
||||
and still loads, so an older file never becomes a hard failure. Each temperature is passed
|
||||
through `clamp_temperature`, so a non-numeric or out-of-range entry cannot crash a later
|
||||
forward the way an unclamped zero used to.
|
||||
|
||||
`temperature` and `binning_map` take opposite value policies, and that is the sentence to read
|
||||
before writing a file by hand: a bad *temperature* is forgiven and clamped into
|
||||
`[TEMP_MIN, TEMP_MAX]`, while a bad *binning value* is refused. Nothing recalibrates a
|
||||
confidence the way an out-of-range binning value would, so there is no defensible fallback
|
||||
for it.
|
||||
|
||||
The payload's *shape* is checked before any of that, and refused with a `ValueError` naming
|
||||
the field: the values may be junk the clamp forgives, but the payload itself not being an
|
||||
object, a `version` that is not an integer, a `temperature` that is not a list of three, a
|
||||
`temperature_by_options` that is not an object, a `binning_map` that is not an object, or a
|
||||
`binning_map` entry that is not an object with an integer `bins` >= 1 and `values` of exactly
|
||||
that length holding finite numbers in [0, 1] is a file this code cannot read -- JSON gives
|
||||
`int`, `str`, `list` and `dict` for each of those mistakes just as happily as it gives the
|
||||
right shapes, and each used to fail with a raw `TypeError`/`AttributeError` from
|
||||
`len()`/`dict()`, or worse, to load: a `{"a": 1, "b": 2, "c": 3}` or `"abc"` has `len` 3 and
|
||||
used to pass the length check and install its *keys* as temperatures.
|
||||
"""
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("calibration JSON must be an object, got %s" % type(payload).__name__)
|
||||
version = payload.get("version", 1)
|
||||
if version is None:
|
||||
version = 1
|
||||
if isinstance(version, bool) or not isinstance(version, (int, float, str)):
|
||||
raise ValueError("calibration JSON version must be an integer, got %r" % (version,))
|
||||
if isinstance(version, float) and not version.is_integer():
|
||||
# `int()` truncates, and the truncation is not cosmetic: a file whose `version` is 1.9
|
||||
# became 1, which is below CALIBRATION_VERSION, so the identity check at the foot of this
|
||||
# function never ran and a map fitted for another checkpoint loaded silently. The string
|
||||
# form of the same mistake already raises below; a whole number written as `2.0` is an
|
||||
# integer and still loads.
|
||||
raise ValueError("calibration JSON version must be an integer, got %r" % (version,))
|
||||
try:
|
||||
version = int(version)
|
||||
except (TypeError, ValueError, OverflowError) as exc: # Overflow: JSON's 1e999 is inf
|
||||
raise ValueError(
|
||||
"calibration JSON version must be an integer, got %r" % (version,)) from exc
|
||||
temps = payload.get("temperature")
|
||||
if not isinstance(temps, (list, tuple)) or len(temps) != N_QTYPES:
|
||||
raise ValueError("calibration JSON must contain temperature: [3 floats]")
|
||||
by_options = payload.get("temperature_by_options")
|
||||
if by_options is None: # absent, or an explicit null: a file from before bucket temperatures
|
||||
by_options = {}
|
||||
if not isinstance(by_options, dict):
|
||||
raise ValueError(
|
||||
"calibration JSON temperature_by_options must be an object of bucket -> float, "
|
||||
"got %s" % type(by_options).__name__)
|
||||
binning = payload.get("binning_map")
|
||||
if binning is not None:
|
||||
if not isinstance(binning, dict):
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map must be an object of bucket -> {bins, values}, "
|
||||
"got %s" % type(binning).__name__)
|
||||
for name, entry in binning.items():
|
||||
if not isinstance(entry, dict):
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map[%r] must be an object with \"bins\" and \"values\", "
|
||||
"got %s" % (name, type(entry).__name__))
|
||||
n_bins = entry.get("bins")
|
||||
values = entry.get("values")
|
||||
if isinstance(n_bins, bool) or not isinstance(n_bins, int) or n_bins < 1:
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map[%r] must have an integer \"bins\" >= 1, got %r" % (name, n_bins))
|
||||
if not isinstance(values, (list, tuple)) or len(values) != n_bins:
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map[%r] must have \"values\" of length \"bins\" (%d)"
|
||||
% (name, n_bins))
|
||||
try:
|
||||
parsed_values = [float(v) for v in values]
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map[%r] values must be numbers, got %r" % (name, values)) from exc
|
||||
for v in values:
|
||||
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map[%r] values must be numbers, got %r" % (name, values))
|
||||
for v in parsed_values:
|
||||
if not np.isfinite(v):
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map[%r] values must be finite, got %r" % (name, values))
|
||||
if not 0.0 <= v <= 1.0:
|
||||
raise ValueError(
|
||||
"calibration JSON binning_map[%r] values must be in [0, 1], got %r" % (name, values))
|
||||
obj.binning_map = binning
|
||||
if version >= CALIBRATION_VERSION:
|
||||
_warn_if_identity_mismatch(obj, payload)
|
||||
_install_temperatures(obj, temps, by_options)
|
||||
1018
laya_repo/common.py
Normal file
1018
laya_repo/common.py
Normal file
File diff suppressed because it is too large
Load Diff
315
laya_repo/email.py
Normal file
315
laya_repo/email.py
Normal file
@ -0,0 +1,315 @@
|
||||
"""Email utilities for cleaning and structuring email inputs in laya.
|
||||
|
||||
The markers below cover English, Portuguese, Spanish and French mail clients. The Router already sends
|
||||
Portuguese and Spanish states to the multilingual checkpoint, but with English-only markers their
|
||||
cleaning was a no-op: Gmail's `Em ... escreveu:`, Outlook's `-----Mensagem original-----`, the
|
||||
`Atenciosamente` sign-off and the confidentiality footer all reached the model, and the quoted
|
||||
history (often a *different* request) weighed on the answer as much as the new message did.
|
||||
French mail reads the same way through an English-only cleaner: Gmail's `Le ... a écrit :`,
|
||||
Outlook's `-----Message d'origine-----`, `Cordialement` and `Envoyé depuis mon iPhone` all survived.
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# One definition, in the module that holds the other presets. Re-exported here because
|
||||
# `from laya.email import email_questions` is a path callers already have.
|
||||
from .presets import email_questions # noqa: F401
|
||||
|
||||
_QUOTE_HEADERS = [
|
||||
re.compile(r"^\s*On .{0,300}wrote:\s*$", re.I),
|
||||
# "Em resposta ao que você escreveu:" is body text; a client's attribution always carries a date
|
||||
re.compile(r"^\s*Em (?=.*\d).{0,300}escreveu:\s*$", re.I),
|
||||
re.compile(r"^\s*El (?=.*\d).{0,300}escribi[óo]:\s*$", re.I),
|
||||
# Gmail's French attribution always carries a date too ("Le lun. ... a écrit :"); a bare
|
||||
# "Le rapport que vous avez écrit :" is body text, the same rule as above. French typography
|
||||
# puts a space before the colon, so the verb allows one (`a écrit :`, never `a écrit:`).
|
||||
re.compile(r"^\s*Le (?=.*\d).{0,300}a [eé]crit\s*:\s*$", re.I),
|
||||
re.compile(r"^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}", re.I),
|
||||
re.compile(r"^\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado)|Message d'origine)\s*-{2,}", re.I),
|
||||
re.compile(r"^\s*_{8,}\s*$"),
|
||||
# `From:` opens ordinary prose too ("From: my side the integration works, but please
|
||||
# refund..."), and a reply header always carries the sender, so the header is only
|
||||
# recognised when an address follows -- the same rule as `De:` below. A bare
|
||||
# `From: Name` header is caught by _HEADER_FROM_NAME/_HEADER_NEXT instead, which need the
|
||||
# header's own `Sent:`/`Date:` line to tell it apart from a sentence.
|
||||
re.compile(r"^\s*From:\s.*[@<]", re.I),
|
||||
# `De:` also opens ordinary Portuguese/Spanish lines ("De: 10/09 a 15/09"), so the Outlook
|
||||
# header is only recognised when it carries an address. French Outlook writes `De :` with a
|
||||
# space, so the marker allows one; the address rule is unchanged
|
||||
re.compile(r"^\s*De\s*:\s.*[@<]", re.I),
|
||||
]
|
||||
# Gmail wraps a long attribution line, leaving `fulano@x.com> escreveu:` alone on the next line.
|
||||
# That tail cuts too, and takes the `On/Em/El/Le ...` head it belongs to with it. The French tail
|
||||
# keeps its spaced colon (`support@x.com> a écrit :`).
|
||||
_ATTRIBUTION_TAIL = re.compile(r"^.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]|a [eé]crit)\s*:\s*$", re.I)
|
||||
_ATTRIBUTION_HEAD = re.compile(r"^\s*(On|Em|El|Le) (?=.*\d)", re.I)
|
||||
# Exchange often leaves the address out of Outlook's reply header ("De: Maria Souza"), so a bare `De:`
|
||||
# only cuts when the header's own `Enviado:` line, or a dated `Data:`/`Fecha:` line, follows it.
|
||||
# `Para:` is not enough: "De: 10/09 / Para: 15/09" is how a leave request reads.
|
||||
#
|
||||
# The same is true of a bare English `From: Maria Souza`, which is why the marker above needs
|
||||
# this rule: the English client lines are the translations of the two `De:` neighbours. A line
|
||||
# that only looks like prose still has to be told apart from a header by its neighbours, so the
|
||||
# English pair is "From: <name>" followed by "Sent:"/"Date:". French Outlook writes the same
|
||||
# header as `De : Marie Dupont` with `Envoyé :` underneath, so the pair gains that translation too.
|
||||
# Both allow the French spaced colon (`De :`, `Envoyé :`); the neighbours still do the telling apart.
|
||||
_HEADER_FROM_NAME = re.compile(r"^\s*(De|From)\s*:\s+\S", re.I)
|
||||
_HEADER_NEXT = re.compile(r"^\s*(Enviad[oa]( em| el)?:\s|Envoy[ée]( le)?\s*:\s|Sent:\s|(Data|Fecha|Date):\s.*\d{4})", re.I)
|
||||
# A closing's name starts with a letter that is not lowercase: capitalised in any script
|
||||
# (`Łukasz`, `Дмитрий`) or caseless (`山田`). `re` cannot say "not a lowercase letter in any
|
||||
# script" -- a class has to list ranges, and `[^\W\d_a-zß-öø-ÿ]` stops at Latin-1, so
|
||||
# `Thanks, żaneta` read as a name and the line counted as a sign-off. The tail is matched
|
||||
# structurally instead, and each token's first letter is judged by category below -- the same
|
||||
# rule as the TS port's `\p{Lu}\p{Lt}\p{Lo}`. Combining marks ride along with the letter before
|
||||
# them (`Jose\u0301` is `José`), as `\p{M}` allows in the port. `re` has no `\p{M}` either, and a
|
||||
# class cannot list those ranges any more than it could list the lowercase ones, so a mark that
|
||||
# rides on a character is dropped before the tail is matched and the letter it rides on answers
|
||||
# the case question. A mark with no base -- opening the tail, or following a space -- is left
|
||||
# where it is, so it still breaks the token as the port's leading `\p{Lu}\p{Lt}\p{Lo}` does.
|
||||
# `\p{M}` is marks only: ZWJ and ZWNJ are `Cf`, so `Thanks, क्ष` is kept here and in the port alike.
|
||||
_SIGNOFF_HEAD = re.compile(
|
||||
r"^\s*(?i:best|kind|warmest|warm|many thanks|thanks|thank you|regards|cheers|sincerely)"
|
||||
r"(?i:\s+(?:and|&)\s+regards|\s+(?:regards|wishes|again|in advance|a lot|so much|very much))?"
|
||||
)
|
||||
_SIGNOFF_TAIL = re.compile(r"^[\s,;:!.]*(?:[^\W\d_][\w'-]*[\s,.]*){0,3}$")
|
||||
_SIGNOFF_TOKEN = re.compile(r"[^\W\d_][\w'-]*")
|
||||
|
||||
|
||||
def _drop_marks(text: str) -> str:
|
||||
"""Remove combining marks, the `Mn`/`Mc`/`Me` categories the port spells `\\p{M}`.
|
||||
|
||||
Only a mark that rides on a preceding character goes; one that opens the string or
|
||||
follows a space has no base to ride on and stays, so it still separates tokens.
|
||||
"""
|
||||
kept = []
|
||||
for ch in text:
|
||||
if unicodedata.category(ch).startswith("M") and kept and not kept[-1].isspace():
|
||||
continue
|
||||
kept.append(ch)
|
||||
return "".join(kept)
|
||||
|
||||
|
||||
def _is_english_signoff(line: str) -> bool:
|
||||
"""True when a closing word is followed by nothing but punctuation and a short name."""
|
||||
m = _SIGNOFF_HEAD.match(line)
|
||||
if m is None:
|
||||
return False
|
||||
tail = _drop_marks(line[m.end():])
|
||||
if _SIGNOFF_TAIL.match(tail) is None:
|
||||
return False
|
||||
return all(unicodedata.category(token[0]) in ("Lu", "Lt", "Lo")
|
||||
for token in _SIGNOFF_TOKEN.findall(tail))
|
||||
|
||||
|
||||
def _marker_matches(marker, line: str) -> bool:
|
||||
"""One `_SIGNATURE_MARKERS` entry: a compiled pattern, or a callable for a rule a pattern
|
||||
cannot express (the English sign-off)."""
|
||||
return bool(marker(line)) if callable(marker) else bool(marker.match(line))
|
||||
|
||||
|
||||
_SIGNATURE_MARKERS = [
|
||||
re.compile(r"^\s*--\s*$"),
|
||||
# A closing line is the closing word plus punctuation and at most a name. Anything else on
|
||||
# the line is a sentence, and the case of the next word is what separates the two: a name is
|
||||
# capitalised, "for" in "Thanks for the quick reply." is not. The closing words are matched
|
||||
# case-insensitively, the name is not, so the flag is scoped instead of global.
|
||||
# `warmest` and `and/& regards` are closings the alternation did not reach; the name that may
|
||||
# follow is judged in `_is_english_signoff` above, a callable because `re` cannot express its
|
||||
# rule. `Regards, Łukasz` is a sign-off, `Thanks for the reply` is not.
|
||||
_is_english_signoff,
|
||||
# Portuguese/Spanish sign-offs match only on their own: "Obrigado pelo retorno, mas ..." is a
|
||||
# request, not a signature, so unlike the English marker no trailing words are allowed.
|
||||
# French closings keep the same rule: "Merci pour votre aide, mais ..." stays, while a bare
|
||||
# "Cordialement," or "Merci," goes with the name underneath it.
|
||||
re.compile(
|
||||
r"^\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?"
|
||||
r"( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|"
|
||||
r"(un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?|"
|
||||
r"(bien )?cordialement|salutations( distinguées)?|bien à vous|merci( d'avance)?|"
|
||||
r"bonne journée)[\s,!.]*$",
|
||||
re.I,
|
||||
),
|
||||
]
|
||||
# Mobile and mail-app footers. Only a line that is nothing *but* the footer matches -- "Enviado do meu
|
||||
# celular o comprovante ontem." is a request -- and such a line may run to 60 characters, since
|
||||
# Samsung's default ("Enviado do meu smartphone Samsung Galaxy.") is longer than a sign-off's 40.
|
||||
# Apple's French default ("Envoyé depuis mon iPhone") reads the same way.
|
||||
_DEVICE = (r"iphone|ipad|android|ios|mobile|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|"
|
||||
r"outlook|yahoo|mail|e-?mail|gmail|windows")
|
||||
_DEVICE_FOOTER = re.compile(
|
||||
r"^\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?|"
|
||||
r"envoy[ée] (depuis|de)( mon| ma| mes)?)"
|
||||
r" (%s)( (%s|para|for|no|na|\d+|phone|device|pro|max|mini|plus|using [a-z][a-z0-9_.+-]*))*"
|
||||
r"|(obter o|get) outlook (para|for) (ios|android))[\s.!]*$"
|
||||
% (_DEVICE, _DEVICE),
|
||||
re.I,
|
||||
)
|
||||
_DISCLAIMER = re.compile(
|
||||
# English: tied to a disclaimer noun and a disclaimer tail, the way the Portuguese
|
||||
# branches below are. The bare word matched any sentence that merely mentioned it,
|
||||
# so "Is this confidential?" and "Confidential: I need a refund." were deleted whole.
|
||||
# `[^.]` rather than `[^.\n]`: a footer wraps, so "are\nconfidential" must still match.
|
||||
r"(\b(e-?mail|message|information|communication|transmission|contents?)\b[^.]{0,60}"
|
||||
r"\bconfidential\b[^.]{0,60}\b(intended|solely|addressee|recipient|privileged|"
|
||||
r"disclos|unauthori[sz]ed)|"
|
||||
r"\bconfidential\b[^.]{0,60}\b(and (may|is) (also )?privileged)|"
|
||||
r"if you (have )?received this (e-?mail|message) in error|"
|
||||
# Portuguese/Spanish: tied to "this message/e-mail" rather than the bare word `confidencial`,
|
||||
# which a sender's own request ("preciso do contrato confidencial") uses just as often
|
||||
r"\b(esta|este) (mensagem|e-?mail|mensaje|correo)\b[^.]{0,80}(confidencia|sigilos|privilegiad)|"
|
||||
r"\b(uso exclusivo|exclusivamente|únicamente|unicamente)\b[^.]{0,30}"
|
||||
r"(destinatári|destinatari|pessoa|persona|entidade|entidad)|"
|
||||
r"\b(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)\b[^.]{0,20} por (engano|erro)|"
|
||||
r"\b(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)\b[^.]{0,20} por error|"
|
||||
# the "think before printing" footer, tied to its environmental ending rather than to
|
||||
# `antes de imprimir`, which a request uses too ("antes de imprimir o boleto, confira o valor")
|
||||
r"\bantes de imprimir\b[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)|"
|
||||
r"\b(meio|medio) ambiente\b[^.]{0,30}antes de imprimir|"
|
||||
# French: tied to "ce message/cet e-mail" rather than the bare word `confidentiel`,
|
||||
# which a sender's own request ("le contrat confidentiel") uses just as often
|
||||
r"\b(ce|cet|cette) (message|e-?mail|mail|courriel)\b[^.]{0,80}(confidentiel|privil[eé]gi)|"
|
||||
r"\bavez re[çc]u (ce|cet|cette) (message|e-?mail|mail)\b[^.]{0,20} par erreur|"
|
||||
r"\b(usage exclusif|exclusivement|uniquement)\b[^.]{0,30}destinataire)",
|
||||
re.I,
|
||||
)
|
||||
_SENTENCE = re.compile(r"(?<=[.!?])\s+")
|
||||
|
||||
|
||||
def _starts_new_sentence(line: str) -> bool:
|
||||
"""First letter is uppercase: a fresh sentence, not a wrapped line.
|
||||
|
||||
Lines in uncased scripts (CJK, Devanagari, ...) never start a new piece,
|
||||
so wrapped boilerplate in those scripts still drops whole.
|
||||
"""
|
||||
for ch in line:
|
||||
if ch.isalpha():
|
||||
return ch.isupper()
|
||||
return False
|
||||
|
||||
|
||||
def _split_fused_lines(sentence: str) -> List[str]:
|
||||
"""Split a fused boilerplate-positive sentence at sentence-starting newlines.
|
||||
|
||||
An unpunctuated request line glued to a disclaimer line ("locked\\nThis ...")
|
||||
splits at the newline because the next line starts uppercase; a lowercase
|
||||
continuation ("are\\nconfidential") belongs to the same sentence, so a wrapped
|
||||
boilerplate footer still drops whole.
|
||||
"""
|
||||
if "\n" not in sentence:
|
||||
return [sentence]
|
||||
pieces, buf = [], ""
|
||||
for line in (ln.strip() for ln in sentence.split("\n")):
|
||||
if not line:
|
||||
continue
|
||||
if buf and _starts_new_sentence(line):
|
||||
pieces.append(buf)
|
||||
buf = line
|
||||
else:
|
||||
buf = (buf + " " + line) if buf else line
|
||||
if buf:
|
||||
pieces.append(buf)
|
||||
return pieces
|
||||
|
||||
|
||||
def _strip_disclaimer(paragraph: str) -> str:
|
||||
"""Drop boilerplate disclaimer text from one paragraph.
|
||||
|
||||
A paragraph is dropped whole only when *every* sentence in it is boilerplate; otherwise only
|
||||
the boilerplate sentences go. A footer that runs on without a blank line used to take the
|
||||
sender's actual request with it, which is worse than leaving one boilerplate line behind.
|
||||
"""
|
||||
if not _DISCLAIMER.search(paragraph):
|
||||
return paragraph # nothing to do: keep the original line structure
|
||||
parts = [p.strip() for p in _SENTENCE.split(paragraph) if p.strip()]
|
||||
pieces = []
|
||||
for p in parts:
|
||||
pieces.extend(_split_fused_lines(p) if _DISCLAIMER.search(p) else [p])
|
||||
return " ".join(p for p in pieces if not _DISCLAIMER.search(p))
|
||||
|
||||
|
||||
def clean_email_body(body: str, max_chars: int = 3000) -> str:
|
||||
"""Remove quoted email history, signatures and disclaimers to keep input focused.
|
||||
|
||||
`max_chars` is the length the result is cut to, 3000 characters unless raised -- see
|
||||
`email_state`, which takes the same budget and passes it through.
|
||||
|
||||
It is a slice budget, not a floor, and it is applied twice: as the `max_chars * 4` pre-cut
|
||||
that bounds the regex work below, and as the `text[:max_chars]` that returns the result. The
|
||||
pre-cut runs first, so the quote, signature and disclaimer rules see only the first
|
||||
`4 * max_chars` characters -- a footer that begins past that window is never recognized, and
|
||||
the paragraph holding it survives to the final cut. So `0` answers with the empty string, and a
|
||||
negative value trims from the end: its length test is against a negative number, so the pre-cut
|
||||
always fires and removes `4 * |max_chars|`, and the final slice removes `|max_chars|` more,
|
||||
which is `5 * |max_chars|` off the body and empty once that reaches the body's length. That is
|
||||
specified behaviour, recorded by the `budget-zero` and `budget-negative-*` cases of
|
||||
`laya-java/fixtures/email.json` that the Java and TypeScript ports are tested against, so "no
|
||||
limit" means a budget larger than the body -- never `0` and never negative.
|
||||
"""
|
||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n").replace("\\n", "\n")
|
||||
# Bound regex work before the expensive patterns below: _DISCLAIMER uses
|
||||
# [^.]{0,60/80/100} alternations whose cost grows with input length, and only
|
||||
# max_chars are ever returned. Truncate lines too so one MB-long line cannot
|
||||
# dominate matching.
|
||||
if len(text) > max_chars * 4:
|
||||
text = text[:max_chars * 4]
|
||||
lines = []
|
||||
src = text.split("\n")
|
||||
for i, line in enumerate(src):
|
||||
if any(p.match(line) for p in _QUOTE_HEADERS) and lines:
|
||||
break
|
||||
if (lines and _HEADER_FROM_NAME.match(line) and i + 1 < len(src)
|
||||
and _HEADER_NEXT.match(src[i + 1])):
|
||||
break
|
||||
if _ATTRIBUTION_TAIL.match(line) and lines:
|
||||
if _ATTRIBUTION_HEAD.match(lines[-1]):
|
||||
lines.pop()
|
||||
break
|
||||
if line.lstrip().startswith(">"):
|
||||
continue
|
||||
lines.append(line.rstrip())
|
||||
cut = len(lines)
|
||||
for i in range(max(1, min(int(len(lines) * 0.6), len(lines) - 8)), len(lines)):
|
||||
n = len(lines[i].strip())
|
||||
if (n <= 40 and any(_marker_matches(p, lines[i]) for p in _SIGNATURE_MARKERS)) or (
|
||||
n <= 60 and _DEVICE_FOOTER.match(lines[i])):
|
||||
cut = i
|
||||
break
|
||||
lines = lines[:cut]
|
||||
paragraphs = [_strip_disclaimer(p) for p in re.split(r"\n\s*\n", "\n".join(lines))]
|
||||
text = re.sub(r"[ \t]+", " ", "\n\n".join(p.strip() for p in paragraphs if p.strip()))
|
||||
return text[:max_chars]
|
||||
|
||||
|
||||
def email_state(subject: str, body: str, sender: Optional[str] = None, clean: bool = True,
|
||||
max_chars: int = 3000, **extra) -> Dict:
|
||||
"""Construct a clean state dictionary for email classification.
|
||||
|
||||
`max_chars` is the budget `clean_email_body` cuts the body to, and it is worth raising for a
|
||||
long message: at the default the body stops after 3000 characters, so a request that arrives in
|
||||
the last paragraphs never reaches the model -- including through `predict_long`, which scans a
|
||||
state in windows precisely so it can read past one window's worth. Lowering it does the same
|
||||
thing more surely: it is a slice budget, so `0` gives the model a subject and an empty body, and
|
||||
a negative value cuts from the end, which is where that late request lives. `clean_email_body`
|
||||
states the arithmetic, and the `budget-zero` and `budget-negative-*` cases of
|
||||
`laya-java/fixtures/email.json` pin it for every port, so it is behaviour rather than an error --
|
||||
pass a budget larger than the body when the message is long, never `0` or a negative to mean "no
|
||||
limit". Ignored when `clean=False`, which passes the body through whole.
|
||||
|
||||
Any other keyword becomes a field of the state, so it is read by the model; a typo here is an
|
||||
input mutation, not an error. The exception is a keyword whose value is exactly ``None``: it is
|
||||
dropped rather than added, so ``email_state(..., thread_id=None)`` gives the same state as
|
||||
leaving the keyword out. The key is absent, not null -- and the distinction reaches the model,
|
||||
because `laya.common.serialize_state` turns the state into JSON before it is tokenized, so a
|
||||
retained ``None`` would put ``"thread_id": null`` in the prompt. ``sender`` is dropped on the
|
||||
same call by a different rule, falsiness rather than ``None``, so ``sender=""`` leaves no
|
||||
``from`` field either.
|
||||
"""
|
||||
state = {
|
||||
"subject": (subject or "").strip(),
|
||||
"body": clean_email_body(body, max_chars=max_chars) if clean else (body or ""),
|
||||
}
|
||||
if sender:
|
||||
state["from"] = sender
|
||||
state.update({k: v for k, v in extra.items() if v is not None})
|
||||
return state
|
||||
867
laya_repo/lang.py
Normal file
867
laya_repo/lang.py
Normal file
@ -0,0 +1,867 @@
|
||||
"""Dependency-free language/script detection used to route between Laya checkpoints.
|
||||
|
||||
Routing only needs one decision: *is this English Latin text, or is it something the English
|
||||
checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint
|
||||
collapsing to near-random on non-Latin scripts (Hindi 0.100, Korean 0.103, Swahili 0.103,
|
||||
Tamil 0.113 at 20 options, where random is 0.050), while holding up far better on Latin-script
|
||||
languages (French 0.487, Spanish 0.480). So the signal that matters most is *script*, and the
|
||||
secondary signal is whether Latin text is English.
|
||||
|
||||
Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic and
|
||||
is explicitly best-effort: pass an explicit model or `lang=` when you already know the language.
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from collections.abc import Mapping
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
|
||||
_SCRIPT_RANGES = [
|
||||
("greek", ((0x0370, 0x03FF), (0x1F00, 0x1FFF))),
|
||||
("cyrillic", ((0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F))),
|
||||
("armenian", ((0x0530, 0x058F),)),
|
||||
("hebrew", ((0x0590, 0x05FF),)),
|
||||
("arabic", ((0x0600, 0x06FF), (0x0750, 0x077F), (0x08A0, 0x08FF), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF))),
|
||||
("devanagari", ((0x0900, 0x097F), (0xA8E0, 0xA8FF))),
|
||||
("bengali", ((0x0980, 0x09FF),)),
|
||||
("gurmukhi", ((0x0A00, 0x0A7F),)),
|
||||
("gujarati", ((0x0A80, 0x0AFF),)),
|
||||
("oriya", ((0x0B00, 0x0B7F),)),
|
||||
("tamil", ((0x0B80, 0x0BFF),)),
|
||||
("telugu", ((0x0C00, 0x0C7F),)),
|
||||
("kannada", ((0x0C80, 0x0CFF),)),
|
||||
("malayalam", ((0x0D00, 0x0D7F),)),
|
||||
("sinhala", ((0x0D80, 0x0DFF),)),
|
||||
("thai", ((0x0E00, 0x0E7F),)),
|
||||
("lao", ((0x0E80, 0x0EFF),)),
|
||||
("tibetan", ((0x0F00, 0x0FFF),)),
|
||||
("myanmar", ((0x1000, 0x109F),)),
|
||||
("georgian", ((0x10A0, 0x10FF),)),
|
||||
("ethiopic", ((0x1200, 0x137F),)),
|
||||
("khmer", ((0x1780, 0x17FF),)),
|
||||
("hangul", ((0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF))),
|
||||
("kana", ((0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF))),
|
||||
("han", ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF))),
|
||||
]
|
||||
|
||||
# Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
|
||||
# weighted and a margin is required before calling something non-English.
|
||||
#
|
||||
# The Romance lists (fr/es/pt/it) deliberately carry the *unaccented* function words as well as the
|
||||
# accented ones. A state that lost its accents -- mail clients, ticket systems and any pipeline that
|
||||
# normalises to ASCII strip them -- keeps no diacritic rate for the non-English signal to read, so
|
||||
# `la`, `un`, `y`, `e`, `et`, `deux` and friends are the only evidence left. With a list of mostly
|
||||
# accented words such a state produced one hit or none, fell under the two-hit margin below, and was
|
||||
# handed to the English checkpoint as undecided-but-not-non-English text (see #172 and #54).
|
||||
_STOP = {
|
||||
"en": {"the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
|
||||
"this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
|
||||
"will", "can", "would", "there", "their", "what", "which", "please", "we", "i"},
|
||||
"fr": {"le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
|
||||
"pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce",
|
||||
"et", "du", "au", "ou", "je", "tu", "il", "elle", "ils", "elles", "mon", "ton",
|
||||
"ma", "ta", "sa", "mes", "tes", "ses", "ces", "deux", "trois", "très", "bien",
|
||||
"tout", "tous", "toute", "fait", "veux", "veut", "peux", "peut", "dois", "doit",
|
||||
"merci", "bonjour", "jour", "jours", "mois", "fois", "quand", "comment", "pourquoi",
|
||||
"alors", "donc"},
|
||||
"de": {"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
|
||||
"auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber",
|
||||
"ich", "wir", "mir", "mich", "dir", "dich", "uns", "mein", "meine", "meinen",
|
||||
"meinem", "meiner", "diese", "dieser", "diesen", "dieses", "einen", "einem", "einer",
|
||||
"wie", "wo", "wann", "welche", "im", "zum", "zur", "aus", "bei", "nach", "noch", "bitte",
|
||||
"heute", "jetzt", "kann", "kannst", "habe", "gibt", "wird",
|
||||
# shared with English on purpose: counted for English alone, they outvoted short German
|
||||
"in", "was"},
|
||||
"es": {"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
|
||||
"pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus",
|
||||
# `de`/`en` are Spanish too, but they are common English tokens as well (`de facto`,
|
||||
# `en-US`, `en route`, `Rio de Janeiro`), and a state of those alone already carries
|
||||
# no English function word for the margin below to weigh them against, so they stay out.
|
||||
"la", "un", "y", "al", "lo", "le", "les", "su", "mi", "tu", "nos",
|
||||
"ni", "dos", "tres", "fue", "fueron", "ser", "tiene", "tienen", "tengo", "puede",
|
||||
"pueden", "quiero", "necesito", "hemos", "han", "sobre", "entre", "cuando", "donde",
|
||||
"porque", "aunque", "también", "ya", "eso", "esto", "esa", "ese", "nada", "algo",
|
||||
"aquí", "hoy", "gracias"},
|
||||
"pt": {"os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
|
||||
"dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela",
|
||||
# `no` is Portuguese too, and among the most frequent words it has; it is also one of the
|
||||
# most frequent English words, so it stays out and short Portuguese states that lean on it
|
||||
# alone are left to the diacritic rate, as before.
|
||||
"o", "e", "na", "nas", "nos", "ao", "aos", "por", "foi", "era", "ser", "sou",
|
||||
"tem", "tenho", "pode", "podem", "quero", "preciso", "eu", "meu", "minha", "seu",
|
||||
"sua", "isso", "isto", "aqui", "ali", "como", "quando", "onde", "porque", "mais",
|
||||
"já", "ainda", "agora", "hoje", "ontem", "dois", "três", "tudo", "nada", "obrigado",
|
||||
"olá",
|
||||
# Brazilian support text: `você` and the unaccented `nao`/`voce`/`sao`/`ja` that a stripped
|
||||
# state keeps (#172), and the chat abbreviations `vc`/`pra`. Without them "Voce pode me
|
||||
# mandar a nota fiscal?" matched one word and went to the English checkpoint, which on
|
||||
# `pt` reports 0.97 mean confidence at 0.47 accuracy. `ate`, `bom`, `sim` and `cade` stay
|
||||
# out: each is an English token too (ate, BOM, SIM, Cade).
|
||||
"você", "vocês", "voce", "voces", "vc", "vcs", "nao", "sao", "ja", "até", "tá", "pra",
|
||||
"gostaria", "obrigada", "também", "tambem", "estou", "estamos", "meus", "minhas",
|
||||
"nosso", "nossa", "consigo", "cadê", "boa", "tarde", "noite",
|
||||
# the words a ticket keeps once the jargon is English ("Deu erro 500 no endpoint de login
|
||||
# depois do update"): time and person words plus the past tenses a bug report is told in
|
||||
"depois", "antes", "então", "entao", "ninguém", "ninguem", "alguém", "alguem", "nenhum",
|
||||
"nenhuma", "estava", "ficou", "fiz", "deu"},
|
||||
"it": {"il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
|
||||
"questo", "questa", "anche", "come", "più", "sono", "nella", "alla",
|
||||
"la", "le", "un", "uno", "una", "e", "ed", "o", "da", "su", "tra", "fra", "mi",
|
||||
"ci", "ne", "ho", "hai", "ha", "abbiamo", "avete", "hanno", "era", "stato", "stata",
|
||||
"devo", "deve", "devono", "voglio", "vorrei", "mio", "mia", "tuo", "sua", "quando",
|
||||
"dove", "perche", "molto", "poco", "sempre", "mai", "già", "ancora", "adesso", "oggi",
|
||||
"ieri", "grazie", "ciao", "scusa",
|
||||
# the articulated prepositions: Italian-only words, which is what lets a state made of
|
||||
# shared articles (`la fattura`) still name the language rather than stay undecided
|
||||
"nel", "nell", "negli", "sul", "sulla", "sulle", "dal", "dalla", "dallo", "dagli", "dei",
|
||||
"delle", "dello", "degli", "agli", "alle", "col"},
|
||||
"nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
|
||||
"door", "maar", "ook", "worden", "deze", "naar", "wordt"},
|
||||
# Swedish function words and common auxiliaries. Several overlap with English or German
|
||||
# (`i`, `kan`, `har`), so the distinctive words below are what lets Swedish text survive an
|
||||
# ASCII-normalising ticket pipeline without treating one stray Nordic letter as the only hint.
|
||||
"sv": {"jag", "är", "och", "inte", "att", "från", "till", "behöver", "får", "skulle", "ska",
|
||||
"vill", "måste", "också", "dessa", "detta", "säger", "upp", "utan", "mitt", "min", "om",
|
||||
"kommer", "här", "två", "vi", "nästa", "gör", "göra",
|
||||
"hjälp", "hjälpa", "mig", "återbetalning", "återbetala", "faktura", "gång", "gånger",
|
||||
"hittar", "inställningen", "inställningarna", "lösenord", "när", "öppnar", "spårningen",
|
||||
# Common spellings from ticket systems that strip Swedish diacritics.
|
||||
"aterbetalning", "aterbetala", "behover", "fel", "ganger", "hjalp", "hjalpa",
|
||||
"installningen", "installningarna", "kraschar", "kvittot", "losenord", "nar",
|
||||
"oppnar", "paket", "skicka", "sparningen", "tva", "uppdaterats", "blivit", "debiterade",
|
||||
"appen"},
|
||||
# Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a
|
||||
# French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately
|
||||
# left out for that reason, and the diacritic signal below carries the rest.
|
||||
"ro": {"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
|
||||
"ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
|
||||
"acesta", "aceasta", "mi", "ți", "vă", "nu"},
|
||||
# Romanized Bangla ("Banglish"): how Bangla is typed in chats, tickets and email when no Bengali
|
||||
# keyboard is at hand. It has no diacritics, so without a list it read as undecided-but-English
|
||||
# and went to the English checkpoint, which scores 0.08 on Bangla at 0.94 confidence. Spelling
|
||||
# is not standardised, so the common variants are listed (`bhalo`/`valo`, `korchi`/`korsi`).
|
||||
# Left out on purpose: frequent Bangla words that are also English words -- `ache` (is),
|
||||
# `are` (is there), `to` (so), `take` (to him), `age` (before), `pore` (later), `mane`
|
||||
# (meaning), `din` (give), `sob` (all), `tar` (his), `dao` (give), `eta` (this; ETA) --
|
||||
# ordinary words of a neighbouring language (`ora`, `nei`, `vai`), and words another list
|
||||
# already claims (`na`, `o`, `e`, `je`, `ta`, `por`, `hoy`), so adding `bn` cannot move a
|
||||
# state of any other language.
|
||||
"bn": {"ami", "amar", "amake", "amra", "amader", "apni", "apnar", "apnake", "apnara",
|
||||
"tumi", "tomar", "tomake", "tomra", "tader", "ota", "eita", "oita",
|
||||
"ekta", "ei", "oi", "ki", "keno", "kivabe", "kibhabe", "kothay", "kokhon", "kobe",
|
||||
"koto", "kintu", "jodi", "tahole", "ar", "theke", "jonno", "sathe", "shathe", "diye",
|
||||
"niye", "moddhe", "kore", "korte", "korchi", "korsi", "korbo", "korechi", "koreche",
|
||||
"korun", "koren", "korlam", "hobe", "hoyeche", "hoise", "hocche", "hoyni",
|
||||
"chai", "chaina", "lagbe", "parchi", "parbo", "parchina", "peyechi", "paini",
|
||||
"dite", "dilam", "diyechi", "nai", "khub", "onek", "ekhon", "akhon", "ekhono",
|
||||
"abar", "ekbar", "duibar", "ajke", "kalke", "taka", "bhalo", "valo", "kharap",
|
||||
"shomossa", "somossa", "dhonnobad", "bhai", "shob", "keu", "kichu", "bolte", "bolun",
|
||||
"parben", "asbe", "jabe", "pabo", "ferot", "dorkar", "hoye", "geche", "gese"},
|
||||
|
||||
# "her", "ne", "men", "de" collide with English, French and Romanian, and "ki" with the
|
||||
# romanized Bangla list, so they are left out.
|
||||
"az": {"və", "ve", "bir", "bu", "üçün", "ucun", "ilə", "ile", "olan", "olub", "olmasa",
|
||||
"var", "yox", "yoxdur", "mən", "sən", "biz", "siz", "onlar", "daha", "çox", "cox",
|
||||
"hər", "nə", "kimi", "görə", "sonra", "əgər", "eger", "deyil", "lakin", "amma",
|
||||
"ancaq", "artıq", "artiq", "də", "isə", "həm", "yalnız", "yalniz"},
|
||||
}
|
||||
|
||||
# Short support fragments often consist of only two or three words, so they do not reach the
|
||||
# four-token minimum used by the general language guess. These spellings are specific enough to
|
||||
# identify Swedish in that narrow case; generic words such as `fel`, `hjälp`, `paket` and `appen`
|
||||
# are deliberately not sufficient on their own. Keep ASCII-normalised variants beside the forms
|
||||
# users commonly type without Swedish characters.
|
||||
_SHORT_SWEDISH_WORDS = {
|
||||
"åtkomst", "atkomst", "lösenord", "losenord", "fakturan", "betalningen", "inloggningen",
|
||||
"glömt", "glomt", "behöver", "behover", "återbetalning", "aterbetalning", "kvitto", "kvittot",
|
||||
"spårningen", "sparningen", "inställningen", "installningen", "felmeddelande",
|
||||
"abonnemanget",
|
||||
}
|
||||
# Letters that ordinary English does not use. This is the signal that catches a Latin-script
|
||||
# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),
|
||||
# which is the difference between routing it to the multilingual checkpoint and silently
|
||||
# handing it to the English one.
|
||||
_NON_EN_DIACRITICS = set(
|
||||
"àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" # Western European
|
||||
"ăâîșțşţ" # Romanian
|
||||
"ąćęłńśźż" # Polish
|
||||
"čďěňřšťůž" # Czech / Slovak
|
||||
"őű" # Hungarian
|
||||
"ğı" # Turkish (text is lowercased before matching)
|
||||
"āēģīķļņūž" # Baltic
|
||||
"đ" # Serbo-Croatian / Vietnamese
|
||||
"ə" # Azerbaijani
|
||||
)
|
||||
# Words that more than one list claims. `la`, `un`, `e`, `que`, `una` and friends are function words
|
||||
# of several of these languages at once, so matching one says "not English" without saying *which*
|
||||
# language: a shared word may not name a winner by itself (Romanian text was reported as French that
|
||||
# way), though it still counts toward the total of a language that also matched a word of its own.
|
||||
_SHARED_WORDS = {w for w in {word for words in _STOP.values() for word in words}
|
||||
if sum(w in words for words in _STOP.values()) > 1}
|
||||
# Danish is not in `_STOP`, but several of its common words also occur in the Swedish list.
|
||||
# Do not let these words alone name a Danish sentence as Swedish; they remain useful score hits
|
||||
# when another, more distinctive Swedish word is present.
|
||||
_NORDIC_OVERLAP_WORDS = {
|
||||
"hej", "ja", "nej", "jo", "tack", "mig", "min", "om", "kommer", "får", "skulle", "vi",
|
||||
}
|
||||
_SHARED_WORDS.update(_NORDIC_OVERLAP_WORDS)
|
||||
# English function words no other list holds (`in`, `is`, `as`, `was` are shared with German,
|
||||
# Dutch and Portuguese). They alone carry the English rescue of `latin_profile`.
|
||||
_EN_ONLY_WORDS = _STOP["en"] - _SHARED_WORDS
|
||||
# A few of these function words are also ordinary English words: `come` (Italian), `son` (Spanish),
|
||||
# `do` (Portuguese), `care` (Romanian), `todo` (a to-do list), `per`, `plus`, and `im` -- German
|
||||
# `im`, and also how `I'm` is written without the apostrophe. For those a repeat is evidence of the
|
||||
# foreign language exactly once: "do more, do less" repeats an English word, and counting both hits
|
||||
# let one such word clear the `best >= 2` bar below and send plain English to the multilingual
|
||||
# checkpoint. Every other word keeps counting occurrences: `der`, `los` and `sa` are nobody's
|
||||
# English, so a request whose only evidence is `der` twice ("reduzieren der helligkeit der lichter")
|
||||
# is still German, and deduping every word instead sent 791 of 148,700 non-English MASSIVE test rows
|
||||
# to the English checkpoint. Dutch `van` is the word this list deliberately leaves out: "liedje van
|
||||
# ... van" is ordinary Dutch, and deduping it moved 14 of those rows while rescuing no English one.
|
||||
_EN_COLLISION_WORDS = {"come", "son", "do", "care", "todo", "im", "per", "plus"} & {
|
||||
w for lg, sw in _STOP.items() if lg != "en" for w in sw}
|
||||
|
||||
_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
|
||||
# A token whose dot or @ joins word characters is an identifier, not prose: `github.com`,
|
||||
# `user@acme.com`, `v1.2.3`, `U.S.A.`. `_WORD` splits them into pieces that collide with real
|
||||
# function words -- `com` is Portuguese for "with", `o` is its article, `e` is Italian "e" -- so a
|
||||
# state that was mostly links scored a language it does not contain, and two domains were enough to
|
||||
# cross the margin below. A sentence-final period (`arrivato.`) keeps its word: the pattern needs
|
||||
# word characters on both sides of the dot.
|
||||
# The lookbehind is what keeps this linear. Without it the greedy `[\w-]*` is retried
|
||||
# at every offset inside a run of word characters, and each attempt rescans the run
|
||||
# before failing on the absent `[.@]` -- quadratic in the run's length, so 4000
|
||||
# characters of one token cost 205 ms against 0.45 ms for ordinary prose. It removes
|
||||
# no match: `[\w-]` and `[.@]` are disjoint, so an attempt from inside a run consumes
|
||||
# to exactly the same separator as an attempt from the run's start and the two always
|
||||
# succeed or fail together -- a leftmost match can only ever begin at a run start.
|
||||
_IDENTIFIER = re.compile(r"(?<![\w-])[\w-]*(?:[.@][\w-]+)+", re.UNICODE)
|
||||
|
||||
|
||||
def _iter_text(state: Union[str, bytes, Mapping, list, None], _depth: int = 0) -> List[str]:
|
||||
"""Collect the string leaves of a state, so detection sees real content.
|
||||
|
||||
Keys are ignored: they are usually English field names. Any mapping counts, not only
|
||||
`dict` -- a UserDict or mappingproxy has the same string values and used to contribute
|
||||
nothing, which made `analyse` report no letters and the router fall through to English.
|
||||
"""
|
||||
if _depth > 6 or state is None:
|
||||
return []
|
||||
if isinstance(state, str):
|
||||
return [state]
|
||||
if isinstance(state, (bytes, bytearray)):
|
||||
try:
|
||||
return [bytes(state).decode("utf-8")]
|
||||
except UnicodeDecodeError:
|
||||
return []
|
||||
if isinstance(state, Mapping):
|
||||
out = []
|
||||
for v in state.values():
|
||||
out.extend(_iter_text(v, _depth + 1))
|
||||
return out
|
||||
if isinstance(state, (list, tuple)):
|
||||
out = []
|
||||
for v in state:
|
||||
out.extend(_iter_text(v, _depth + 1))
|
||||
return out
|
||||
return []
|
||||
|
||||
|
||||
def state_text(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000) -> str:
|
||||
"""Flatten a state into the text used for detection (keys are ignored: they are usually English)."""
|
||||
parts: List[str] = []
|
||||
budget = max_chars
|
||||
for leaf in _iter_text(state):
|
||||
if budget <= 0:
|
||||
break
|
||||
if len(leaf) > budget:
|
||||
parts.append(leaf[:budget])
|
||||
break
|
||||
parts.append(leaf)
|
||||
# Account for the joining space without materializing the full text first.
|
||||
budget -= len(leaf) + 1
|
||||
return " ".join(parts)[:max_chars]
|
||||
|
||||
|
||||
def _script_counts(text: str) -> Dict[str, int]:
|
||||
"""Count the alphabetic characters of `text` by script, in one pass.
|
||||
|
||||
Latin is inserted last so `_script_from_counts` keeps `detect_script`'s tie-break: a named
|
||||
script wins a tie against Latin, because `max` returns the first of equal values and Latin
|
||||
is the last key. `analyse` needs both the dominant script and the per-script fractions, and
|
||||
used to walk the text twice (once per function) to get them; one pass serves both.
|
||||
"""
|
||||
counts: Dict[str, int] = {}
|
||||
latin = 0
|
||||
for ch in text:
|
||||
if not ch.isalpha():
|
||||
continue
|
||||
cp = ord(ch)
|
||||
if cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
|
||||
latin += 1 # Latin, IPA Extensions, Ext-Additional, fullwidth
|
||||
continue
|
||||
for name, ranges in _SCRIPT_RANGES:
|
||||
if any(lo <= cp <= hi for lo, hi in ranges):
|
||||
counts[name] = counts.get(name, 0) + 1
|
||||
break
|
||||
else:
|
||||
# An alphabetic character no range claims used to be counted nowhere, so text
|
||||
# written only in an unlisted script produced a total of 0 and was reported as
|
||||
# "unknown" -- and `analyse` treats "unknown" as English, sending it to the
|
||||
# checkpoint that has no tokens for it. 68% of Unicode's alphabetic codepoints
|
||||
# are outside _SCRIPT_RANGES (the CJK extensions, kana supplements, bopomofo,
|
||||
# halfwidth katakana, and dozens of smaller scripts), so enumerating them all is
|
||||
# not maintainable. Counting the remainder under "other" keeps them visible and
|
||||
# non-Latin, which is the safe direction: an unreadable script must not be
|
||||
# handed to the English checkpoint.
|
||||
counts["other"] = counts.get("other", 0) + 1
|
||||
counts["latin"] = latin
|
||||
return counts
|
||||
|
||||
|
||||
def _script_from_counts(counts: Dict[str, int]) -> str:
|
||||
if not any(counts.values()):
|
||||
return "unknown"
|
||||
return max(counts.items(), key=lambda kv: kv[1])[0]
|
||||
|
||||
|
||||
def _profile_from_counts(counts: Dict[str, int]) -> Dict[str, float]:
|
||||
total = sum(counts.values())
|
||||
if not total:
|
||||
return {}
|
||||
# Keep Latin first in the public mapping, the order callers saw before this was refactored.
|
||||
ordered: Dict[str, int] = {}
|
||||
if counts.get("latin"):
|
||||
ordered["latin"] = counts["latin"]
|
||||
for name, value in counts.items():
|
||||
if name != "latin":
|
||||
ordered[name] = value
|
||||
return {k: v / total for k, v in ordered.items() if v}
|
||||
|
||||
|
||||
def detect_script(text: str) -> str:
|
||||
"""Dominant script of `text`: 'latin', 'han', 'devanagari', ... or 'unknown' if there are no letters."""
|
||||
return _script_from_counts(_script_counts(text))
|
||||
|
||||
|
||||
def script_profile(text: str) -> Dict[str, float]:
|
||||
"""Fraction of alphabetic characters belonging to each detected script."""
|
||||
return _profile_from_counts(_script_counts(text))
|
||||
|
||||
|
||||
# A diacritic rate above this is taken as evidence the text is not English, even when no
|
||||
# stopword list matches it.
|
||||
NON_EN_DIACRITIC_RATE = 0.02
|
||||
|
||||
# One accented loanword or proper noun (`café`, `résumé`, `José`) must not alone pull otherwise
|
||||
# plain English off the English checkpoint: the rate is measured over every character, so a
|
||||
# single `é` in a short sentence clears the floor above. English function words keep their say
|
||||
# only through the rescue below -- two distinct words no other list holds, at most one
|
||||
# non-English letter word, and a rate still well above the floor vetoes regardless.
|
||||
ENGLISH_RESCUE_DIACRITIC_RATE = 0.06
|
||||
|
||||
# Non-Latin text is not for the English checkpoint even when Latin letters are the plurality: a
|
||||
# brand name or order code outvotes the CJK request around it letter for letter, though one CJK
|
||||
# character carries far more than a letter. A short message needs a large share to count; a long
|
||||
# payload (ticket fields, English agent turns) dilutes the share, so there a sentence's worth of
|
||||
# letters counts too.
|
||||
NON_LATIN_FRACTION = 0.2
|
||||
NON_LATIN_MIN_FRACTION = 0.1
|
||||
NON_LATIN_MIN_LETTERS = 10
|
||||
|
||||
|
||||
def _non_latin_carries(non_latin: float, n_non_latin: int) -> bool:
|
||||
"""Whether the non-Latin letters of a Latin-plurality text are enough to carry the state."""
|
||||
return (non_latin >= NON_LATIN_FRACTION
|
||||
or (non_latin >= NON_LATIN_MIN_FRACTION and n_non_latin >= NON_LATIN_MIN_LETTERS))
|
||||
|
||||
|
||||
def _script_of(ch: str) -> Optional[str]:
|
||||
"""The named non-Latin script of one letter, or None for Latin and for unclaimed letters."""
|
||||
cp = ord(ch)
|
||||
if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A:
|
||||
return None
|
||||
for name, ranges in _SCRIPT_RANGES:
|
||||
if any(lo <= cp <= hi for lo, hi in ranges):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def _non_latin_words(text: str) -> List[str]:
|
||||
"""Non-Latin runs that read as words rather than as annotation inside English prose.
|
||||
|
||||
English prose carries three kinds of non-Latin letters that are not a request written in
|
||||
another script, and each is excluded here: a symbol (`Set α to 0.05`, one letter), a proper
|
||||
name (`Дмитрий Петрович Савицкий`, capitalised), and a pronunciation (`[vlɐˈdʲimʲɪr]`, which
|
||||
no script range claims). A combining mark belongs to the letter before it and never splits a
|
||||
word, so `Влади́мир` stays one capitalised name rather than becoming `Влади` + `мир`.
|
||||
"""
|
||||
runs, cur, script = [], "", None
|
||||
for ch in text:
|
||||
if unicodedata.combining(ch):
|
||||
continue
|
||||
s = _script_of(ch)
|
||||
if s is not None and s == script:
|
||||
cur += ch
|
||||
continue
|
||||
if cur:
|
||||
runs.append(cur)
|
||||
cur, script = (ch, s) if s is not None else ("", None)
|
||||
if cur:
|
||||
runs.append(cur)
|
||||
return [w for w in runs if len(w) >= 2 and not w[0].isupper()]
|
||||
|
||||
|
||||
def _english_rescued_by_words(words: List[str], diac_rate: float) -> bool:
|
||||
"""Whether plain-English function words outvote a marginal diacritic rate (#337).
|
||||
|
||||
The rate is measured over every character, so one accented loanword or proper noun in a
|
||||
short sentence clears `NON_EN_DIACRITIC_RATE` outright. English still wins when it shows at
|
||||
least two distinct function words no other list holds and at most one word carrying a
|
||||
non-English letter: one loanword is not a non-English vocabulary, while the odd word a
|
||||
Danish or Swedish sentence picks up (`i`, `at`, `for`, `have`) is not an English sentence
|
||||
either. A rate well above `ENGLISH_RESCUE_DIACRITIC_RATE` vetoes regardless.
|
||||
"""
|
||||
if diac_rate >= ENGLISH_RESCUE_DIACRITIC_RATE:
|
||||
return False
|
||||
if len(set(words) & _EN_ONLY_WORDS) < 2:
|
||||
return False
|
||||
return sum(1 for w in set(words) if any(ch in _NON_EN_DIACRITICS for ch in w)) <= 1
|
||||
|
||||
|
||||
def latin_profile(text: str) -> Dict[str, object]:
|
||||
"""Evidence behind the Latin-script language guess.
|
||||
|
||||
Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and
|
||||
`looks_non_english`. The latter can be true for non-English diacritics or an overlapping
|
||||
Swedish-Danish marker even when this heuristic cannot name the language. `analyse` needs the
|
||||
evidence and not just the verdict, because "undecided" and "English" are different answers
|
||||
and only one of them is safe to send to the English checkpoint.
|
||||
|
||||
A non-English language is only named when it matched at least one word that no other list
|
||||
claims: shared function words alone (`la`, `e`, `o`) identify no particular language.
|
||||
"""
|
||||
# 'İ'.lower() is 'i' + a combining dot, which matches no word list
|
||||
words = _WORD.findall(_IDENTIFIER.sub(" ", text).replace("İ", "i").lower())
|
||||
lowered = text.lower()
|
||||
diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)
|
||||
diac_rate = diac / max(1, len(lowered))
|
||||
non_english = diac_rate >= NON_EN_DIACRITIC_RATE
|
||||
nordic_overlap = bool(set(words) & _NORDIC_OVERLAP_WORDS) and not bool(set(words) & _EN_ONLY_WORDS)
|
||||
if 1 < len(words) < 4 and set(words) & _SHORT_SWEDISH_WORDS:
|
||||
return {"language": "sv", "english_hits": 0, "diacritic_rate": diac_rate,
|
||||
"looks_non_english": non_english}
|
||||
if len(words) < 4:
|
||||
return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,
|
||||
"looks_non_english": non_english or nordic_overlap}
|
||||
|
||||
# A collision word counts once however often it repeats; every other word counts its hits.
|
||||
counts = Counter(words)
|
||||
scores = {lg: sum(1 if w in _EN_COLLISION_WORDS else n for w, n in counts.items() if w in sw)
|
||||
for lg, sw in _STOP.items()}
|
||||
en = scores.get("en", 0)
|
||||
# Only a language that matched at least one word no other list claims may be named. Without
|
||||
# that condition the top score can be pure overlap -- `la` and `e` in Romanian text made
|
||||
# Italian the winner -- which is a guess dressed as a detection. Such a language is dropped
|
||||
# from the running rather than merely losing the tie, so a lesser score with real evidence
|
||||
# still gets named, and the text stays undecided when no list has any.
|
||||
evidenced = {lg: s for lg, s in scores.items()
|
||||
if lg != "en" and any(w not in _SHARED_WORDS for w in set(words) & _STOP[lg])}
|
||||
best_lg, best = max(evidenced.items(), key=lambda kv: kv[1], default=(None, 0))
|
||||
|
||||
lang = None
|
||||
if best_lg and best >= max(2, en + 2):
|
||||
# a non-English language needs a clear margin over English function words
|
||||
lang = best_lg
|
||||
elif (best_lg == "sv" and "inte" in words and "kan" in words
|
||||
and words[0] in {"kan", "jag", "vi"} and en <= 1):
|
||||
# Short login requests such as "kan inte logga in" carry a distinctive Swedish phrase
|
||||
# but also one English-shaped token (`in`). Do not treat `kan` alone as Swedish: it is
|
||||
# common in Danish and Norwegian too.
|
||||
lang = best_lg
|
||||
elif best_lg and non_english and best >= max(2, en):
|
||||
# Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish
|
||||
# on the strength of the diacritics alone, which is a guess dressed as a detection.
|
||||
lang = best_lg
|
||||
elif en and (not non_english or _english_rescued_by_words(words, diac_rate)):
|
||||
lang = "en"
|
||||
return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,
|
||||
"looks_non_english": non_english or (lang is None and nordic_overlap)}
|
||||
|
||||
|
||||
def guess_latin_language(text: str) -> Optional[str]:
|
||||
"""Best-effort language code for Latin-script text, or None when undecided.
|
||||
|
||||
Scores function-word hits per language and requires the winner to beat English by a margin and
|
||||
to have matched at least one word of its own, so ordinary English is never misrouted and a
|
||||
word of several languages at once names none of them. Short inputs usually return None on
|
||||
purpose.
|
||||
"""
|
||||
return latin_profile(text)["language"]
|
||||
|
||||
|
||||
# Code is not prose in any language, but split into words it reads as one: `os.path` is Portuguese
|
||||
# (`os`), `round(el, 2)` Spanish (`el`), `non_english` Italian (`non`). A line pasted from a program
|
||||
# into an English request must not count as a foreign segment, so a line carrying code syntax --
|
||||
# `=`, `;`, braces, brackets or a call `name(` -- is skipped, and dotted or underscored identifiers
|
||||
# are dropped from the rest. Prose keeps "Deu erro (500)": the parenthesis follows a space.
|
||||
_CODE_LINE = re.compile(r"[=;{}\[\]]|\w\(")
|
||||
# Slash and backslash compounds are names, not sentences: `Nav/Com` and `OS/2` read as Portuguese
|
||||
# (`com`, `os`), `C:\DOS\mode` as Portuguese (`dos`), `ESA/UN` as Spanish (`un`). A whitespace token
|
||||
# holding a letter or digit, a joiner (`.`, `_`, `/`, `\`) and another letter or digit is an
|
||||
# identifier or a compound and is dropped whole. The pattern has a fixed length on purpose: an
|
||||
# open-ended `\w+(?:[._]\w+)+` backtracks quadratically on a long run of letters with no joiner,
|
||||
# and a state is user input.
|
||||
_JOINED = re.compile(r"[^\W_][._/\\][^\W_]")
|
||||
# An all-caps token inside mixed-case text is an acronym or a code: `MON`, `LA`, `EST`, `COM`, `DES`
|
||||
# are hockey teams, states, time zones and radio bands, not French or Portuguese. A segment written
|
||||
# entirely in capitals keeps its words -- a customer shouting in Portuguese is still Portuguese.
|
||||
_LETTER_RUN = re.compile(r"[^\W\d_]{2,}")
|
||||
# ...but a line of nothing but acronyms has no lowercase either, so it skips the blanking above
|
||||
# and is read as prose. `MON DES EST LA` -- a hockey team, a state, a time zone and an airport --
|
||||
# was named French, and `STORES LOS ANGELES LAS VEGAS EL PASO CLOSED` Spanish. An all-caps line
|
||||
# must therefore show evidence that is more than acronym-shaped before it is believed.
|
||||
#
|
||||
# The bar is on the tokens that actually scored, not on the line. Requiring one long word
|
||||
# anywhere does almost nothing by itself: `ANGELES`, `VEGAS` and `SEGUNDO` are long and are not
|
||||
# what named Spanish -- `LOS`, `LAS` and `EL` did.
|
||||
#
|
||||
# Three corpora decide these two numbers, and `research/evals/shouted_bar_corpora.py` builds all
|
||||
# three and prints the table below: upper-cased MASSIVE test rows that the rule names as their
|
||||
# own locale with no bar at all (genuine shouted prose, must survive); 500,000 lines of 4-8
|
||||
# acronyms; 50,000 all-caps US address and signage lines. Pinned to the MASSIVE test split at
|
||||
# revision 940fd47a and `random.Random(0)`, with the acronym and place-name pools committed in
|
||||
# that script, so every row here can be rechecked when someone wants to move a constant. Each
|
||||
# row applies one bar, named in full, so the rows are comparable:
|
||||
#
|
||||
# rule prose kept acronym FP address FP
|
||||
# none (no bar at all) 10,960 100.00% 4,219 141
|
||||
# diacritics only 10,960 100.00% 4,219 141
|
||||
# word >= 5 only 10,854 99.03% 431 141 <- addresses stand
|
||||
# stopword >= 3 only 10,729 97.89% 3,751 109 <- keeps LOS/LAS/EL
|
||||
# stopword >= 4 only 8,841 80.67% 0 0
|
||||
# stopword >= 5 only 6,963 63.53% 0 0
|
||||
# stopword >= 3 AND word >= 5 10,625 96.94% 386 109
|
||||
# stopword >= 4 AND word >= 5 8,762 79.95% 0 0 <- chosen
|
||||
# stopword >= 5 AND word >= 5 6,902 62.97% 0 0
|
||||
# stopword >= 4 AND word >= 6 8,418 76.81% 0 0
|
||||
# stopword >= 4 AND word >= 7 7,459 68.06% 0 0
|
||||
# stopword >= 4 AND word >= 5, no dia 6,749 61.58% 0 0
|
||||
#
|
||||
# `LOS`, `LAS` and `EL` are three, three and two letters, so any bar of 4 or more rejects them;
|
||||
# what the table has to decide is the other side of that, and it is 3 that it rules out -- at 3
|
||||
# the reported address line still reads as Spanish, and 3,751 of 500,000 acronym runs and 109 of
|
||||
# 50,000 address lines still read as prose. 4 closes both columns outright. Going on to 5 buys
|
||||
# nothing measurable and costs 1,860 more real lines. Non-English diacritics are accepted in
|
||||
# place of a long stopword, because the module already treats them as non-English evidence
|
||||
# (NON_EN_DIACRITIC_RATE) and neither an acronym nor a US place name carries any: that disjunct
|
||||
# is free on both false-positive corpora (0 and 0 either way) and lifts prose kept from 61.58%
|
||||
# to 79.95%.
|
||||
_SHOUTED_MIN_STOPWORD = 4
|
||||
# A length bar as well: a token of this many letters is a word rather than an acronym. Used
|
||||
# twice -- here, on an all-caps line, and in `_analyse_text` to decide whether an all-caps run
|
||||
# inside mixed-case text is an acronym worth blanking.
|
||||
#
|
||||
# Stated plainly, this bar is not justified by the table above: at a stopword bar of 4 both
|
||||
# false-positive columns are already 0, so the 79 real prose lines it costs (8,841 -> 8,762) buy
|
||||
# nothing those corpora can see. It is powerful on its own -- row three takes acronym false
|
||||
# positives from 4,219 to 431 -- and it is kept because the corpora it was first chosen against
|
||||
# were larger than the ones committed here (5,670 acronyms and 18,718 place names, neither
|
||||
# committed, so their counts are not reproducible and are not quoted) and did report false
|
||||
# positives surviving a stopword bar of 4. Absence on this corpus is not absence. Anyone who
|
||||
# wants those 79 lines back should widen the acronym pool in that script first and show the
|
||||
# column stays at 0. Swept with the stopword bar at 4: no bar 8,841 prose, >= 5 8,762,
|
||||
# >= 6 8,418, >= 7 7,459.
|
||||
#
|
||||
# What the pair costs is not small and is not only short lines: 2,198 of the 10,960
|
||||
# shouted-prose lines are given up, 1,816 of them (82.62%) six tokens or longer, the longest a
|
||||
# 34-token Dutch sentence. On MASSIVE as written none of this is visible -- not one of the
|
||||
# 151,674 rows holds a single uppercase character -- so it has to be measured on rows that do:
|
||||
# over the 77,324 rows of the 26 non-English Latin locales, upper-cased, the multilingual share
|
||||
# falls 0.5912 -> 0.5430, which is 3,727 rows. The casualties are sentences.
|
||||
_SHOUTED_MIN_WORD = 5
|
||||
|
||||
|
||||
def _shouted(text: str) -> bool:
|
||||
"""True for a *cased* segment written entirely in capitals.
|
||||
|
||||
The `isupper` half is not redundant. A caseless script -- Devanagari, Bengali, CJK -- has no
|
||||
lowercase either, so testing only `islower` would call every such segment shouted and hold it
|
||||
to a bar it cannot clear: `_WORD` splits at every combining mark, so the tokens are short by
|
||||
construction. Nothing is routed on that path today (`_STOP["bn"]` is romanised, so a
|
||||
Bengali-script line is named by script and never reaches here), but the bar belongs to cased
|
||||
text and saying so here keeps it that way.
|
||||
|
||||
That argument covers *pure* caseless text, which never reaches a caps bar at all. Text that
|
||||
merely contains some has no lowercase either and does reach one; `_analyse_text` keeps both
|
||||
bars off it, because a bar that works by discarding Latin evidence has nothing to say about
|
||||
the half of the state it cannot read.
|
||||
"""
|
||||
return any(ch.isupper() for ch in text) and not any(ch.islower() for ch in text)
|
||||
|
||||
|
||||
def _shouted_evidence(tokens, lang: str, diacritic_rate: float) -> bool:
|
||||
"""Whether an all-caps line's evidence for `lang` is more than acronym-shaped tokens."""
|
||||
if not any(len(t) >= _SHOUTED_MIN_WORD for t in tokens):
|
||||
return False
|
||||
if diacritic_rate >= NON_EN_DIACRITIC_RATE:
|
||||
return True
|
||||
matched = {t.lower() for t in tokens} & _STOP.get(lang, set())
|
||||
return any(len(w) >= _SHOUTED_MIN_STOPWORD for w in matched)
|
||||
|
||||
|
||||
def _named_prose_language(segment: str):
|
||||
"""Language code for one non-code line, or None when it does not name a foreign language.
|
||||
|
||||
Same evidence bar as `_non_english_segment`: four words, a language `latin_profile` will name,
|
||||
and two *different* words of that language. Acronyms and slash compounds are not words. A
|
||||
segment in all capitals keeps its acronym-shaped tokens -- shouting is not a foreign language
|
||||
-- so it must also clear `_shouted_evidence`.
|
||||
"""
|
||||
if not segment.strip() or _CODE_LINE.search(segment):
|
||||
return None
|
||||
prose = " ".join(tok for tok in segment.split() if not _JOINED.search(tok))
|
||||
shouted = _shouted(prose)
|
||||
if not shouted:
|
||||
prose = _LETTER_RUN.sub(lambda m: " " if m.group().isupper() else m.group(), prose)
|
||||
tokens = _WORD.findall(prose)
|
||||
if len(tokens) < 4:
|
||||
return None
|
||||
prof = latin_profile(prose)
|
||||
lang = prof["language"]
|
||||
if lang in (None, "en"):
|
||||
return None
|
||||
if shouted and not _shouted_evidence(tokens, lang, float(prof["diacritic_rate"])):
|
||||
return None
|
||||
if len({w.lower() for w in tokens} & _STOP.get(lang, set())) < 2:
|
||||
return None
|
||||
return lang
|
||||
|
||||
|
||||
def _non_english_segment(state: Union[str, bytes, Mapping, list, None], max_chars: int = 4000):
|
||||
"""First line or field that, read on its own, is named a non-English language, else None.
|
||||
|
||||
Returns (language, segment). A segment needs the evidence a whole state needs -- at least four
|
||||
words, and a language named by `latin_profile` -- and, because one line carries far less text
|
||||
than a state, two things more: the words that name the language must be two *different* ones
|
||||
(`COM ... COM` in an English radio listing is one word seen twice), and acronyms and slash
|
||||
compounds are not words. This adds no new way to call English text foreign; it only stops a
|
||||
longer English part from outvoting a foreign one. Reads at most `max_chars` characters in all.
|
||||
"""
|
||||
seen = 0
|
||||
for leaf in _iter_text(state):
|
||||
for seg in leaf.split("\n"):
|
||||
if seen >= max_chars:
|
||||
return None
|
||||
seg = seg[:max_chars - seen]
|
||||
seen += len(seg)
|
||||
lang = _named_prose_language(seg)
|
||||
if lang:
|
||||
return lang, seg.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _analyse_text(text: str) -> Dict[str, object]:
|
||||
"""Detection result for one already-flattened string.
|
||||
|
||||
Does not look for a foreign line inside mostly-English text. `analyse` does that, because it
|
||||
needs the original state and not only the joined window.
|
||||
"""
|
||||
counts = _script_counts(text)
|
||||
prof = _profile_from_counts(counts)
|
||||
script = _script_from_counts(counts)
|
||||
non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0
|
||||
n_non_latin = round(non_latin * sum(ch.isalpha() for ch in text))
|
||||
if (script == "latin" and _non_latin_words(text)
|
||||
and _non_latin_carries(non_latin, n_non_latin)):
|
||||
script = max((s for s in prof if s != "latin"), key=prof.get)
|
||||
if script == "unknown":
|
||||
return {"script": "unknown", "script_profile": prof, "language": None,
|
||||
"is_english": True, "language_undecided": True, "diacritic_rate": 0.0,
|
||||
"non_latin_fraction": 0.0, "mixed_segment": None}
|
||||
if script != "latin":
|
||||
return {"script": script, "script_profile": prof, "language": None,
|
||||
"is_english": False, "language_undecided": True, "diacritic_rate": 0.0,
|
||||
"non_latin_fraction": non_latin, "mixed_segment": None}
|
||||
prof_lat = latin_profile(text)
|
||||
lang = prof_lat["language"]
|
||||
# The acronym bar belongs here too, not only in the segment scan. The scan runs only once a
|
||||
# state already reads English overall, so a state that is *nothing but* an acronym line --
|
||||
# or one diluted by fewer English lines than it takes to tip this verdict -- never reached it
|
||||
# and was named foreign outright: `MON DES EST LA` routed multilingual on its own, and so did
|
||||
# the same line under one or two lines of English. Vetoing here leaves the text undecided, so
|
||||
# `looks_non_english` still decides it: a shouted line with non-English diacritics is kept.
|
||||
# Neither bar applies when the state is carried by a caseless script. Both of them answer
|
||||
# the verdict by taking evidence *away* from the Latin letters, and `looks_non_english`,
|
||||
# which decides what is left, reads Latin diacritics only -- it cannot see the caseless half.
|
||||
# Nothing else catches the fall, either: the script promotion above needs
|
||||
# `_non_latin_words`, which drops any run starting with a capital, so a shouted mixed-script
|
||||
# line has no promotable word by construction. `ПРИВЕТ MON DES EST LA` is 35% Cyrillic
|
||||
# letters and was being sent to the English checkpoint on the strength of disbelieving `MON
|
||||
# DES EST LA`. Whatever the Latin part is worth, text this far from Latin is not English, so
|
||||
# there is nothing for either bar to buy here and a wrong language name costs nothing: the
|
||||
# checkpoint is chosen on `is_english`.
|
||||
if lang not in (None, "en") and not _non_latin_carries(non_latin, n_non_latin):
|
||||
if _shouted(text):
|
||||
if not _shouted_evidence(_WORD.findall(text), lang,
|
||||
float(prof_lat["diacritic_rate"])):
|
||||
lang = None
|
||||
else:
|
||||
# Mixed-case text: the segment scan has always held that an acronym is not a word,
|
||||
# but the whole-state verdict never applied that rule, so the acronyms voted in it.
|
||||
# That is what let a short English state be outvoted -- one or two lines of English
|
||||
# above `MON DES EST LA` still read as French overall, and only at three did English
|
||||
# win the margin. Re-take the verdict without the all-caps runs. `looks_non_english`
|
||||
# and `diacritic_rate` stay measured on the original text, so a foreign word that
|
||||
# happens to be shouted cannot be blanked out of the diacritic safety net.
|
||||
#
|
||||
# Only *acronym-shaped* runs are blanked. Blanking every all-caps run deleted
|
||||
# emphasis capitals, which are ordinary in real prose and are usually on the words
|
||||
# that carry the sentence, so when the shouted words were the ones that named the
|
||||
# language the evidence simply went: `sag mir das HEUTIGE DATUM` and `quiero
|
||||
# cancelar mi PEDIDO POR FAVOR` both lost their verdict and routed english. It also
|
||||
# broke monotonicity -- the fully upper-cased form held the shouted bar, but adding
|
||||
# one lowercase English token moved the text to this branch, where it had no bar at
|
||||
# all. So a run is blanked only while every one of them is short enough to be an
|
||||
# acronym, by `_shouted_evidence`'s own first test: a run of `_SHOUTED_MIN_WORD`
|
||||
# letters is a word, and a word is not blanked out of its own sentence. Measured
|
||||
# over the 20,708 Latin-locale MASSIVE test rows, emphasis-capitalised two ways
|
||||
# (see the commit message): regressions 1,753 -> 293 and 3,835 -> 752.
|
||||
caps = [m.group() for m in _LETTER_RUN.finditer(text) if m.group().isupper()]
|
||||
if caps and not any(len(c) >= _SHOUTED_MIN_WORD for c in caps):
|
||||
blanked = _LETTER_RUN.sub(
|
||||
lambda m: " " if m.group().isupper() else m.group(), text)
|
||||
if blanked != text:
|
||||
lang = latin_profile(blanked)["language"]
|
||||
# Undecided is not English. Treating it as English sent every Latin-script language we hold no
|
||||
# stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the
|
||||
# language, non-English letters or a shared Swedish-Danish marker can still prefer the
|
||||
# multilingual checkpoint; text with neither signal still goes to the English one.
|
||||
undecided = lang is None
|
||||
english = lang == "en" or (undecided and not prof_lat["looks_non_english"])
|
||||
return {"script": "latin", "script_profile": prof, "language": lang,
|
||||
"is_english": english, "language_undecided": undecided,
|
||||
"diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),
|
||||
"non_latin_fraction": non_latin, "mixed_segment": None}
|
||||
|
||||
|
||||
def _leaf_non_english(leaf: str) -> Optional[Dict[str, object]]:
|
||||
"""A string value that is itself not safe for the English checkpoint, else None.
|
||||
|
||||
A one-word name ("José") and a capitalised non-Latin name stay out: the same rules
|
||||
`latin_profile` and `_non_latin_words` already use, so a name field cannot pull an
|
||||
English ticket onto the multilingual checkpoint. Code lines, acronyms and slash compounds
|
||||
stay out too, matching `_non_english_segment`, so a pasted traceback is not a message.
|
||||
Each line is capped at 4000 characters; unlike the segment scan, a long earlier field
|
||||
does not consume the budget of the next one (#384).
|
||||
"""
|
||||
best_n = -1
|
||||
best: Optional[Dict[str, object]] = None
|
||||
for line in leaf.split("\n"):
|
||||
# A line this short cannot be selected, so skip it before the slice, `strip`,
|
||||
# `_CODE_LINE` and a full `_analyse_text` pass. Each branch below needs four `_WORD`
|
||||
# tokens or NON_LATIN_MIN_LETTERS letters. `_WORD` matches maximal runs of letters and
|
||||
# letter-like numerals, so four tokens need three separators between them: seven
|
||||
# characters, and ten letters need ten. What makes seven safe rather than six is that
|
||||
# every one of those counts is taken on the raw line -- `İ` lowers to two code points, so
|
||||
# counting on `sample.lower()` would let a four-character line reach four tokens, which is
|
||||
# why `latin_profile` replaces `İ` before lowering.
|
||||
if len(line) < 7:
|
||||
continue
|
||||
sample = line[:4000]
|
||||
if not sample.strip() or _CODE_LINE.search(sample):
|
||||
continue
|
||||
det = _analyse_text(sample)
|
||||
if det["is_english"]:
|
||||
continue
|
||||
# A named language still has to survive `_named_prose_language`: acronyms and slash
|
||||
# compounds are not words. But a veto is not a verdict of English. Dropping the line here
|
||||
# skipped the diacritic branch below, which exists for exactly this -- text that carries
|
||||
# non-English letters and that no stopword list can name. `WIE SPÄT IST ES IN KÖLN`
|
||||
# was vetoed for having no long stopword and then thrown away, so a German field routed
|
||||
# english; now it falls through and its umlauts carry it. That is also why
|
||||
# `language_undecided` is no longer required below: a language named and then disbelieved
|
||||
# is in the same evidential position as one never named.
|
||||
#
|
||||
# The ASCII spelling `WIE SPAET IST ES IN KOELN` is a different case and is not fixed
|
||||
# here. It has no diacritics to fall through to, and its longest matched German stopword
|
||||
# is `wie`, three letters, so `_SHOUTED_MIN_STOPWORD` vetoes it and nothing recovers it:
|
||||
# genuine German on the English checkpoint, one of the 3,727 upper-cased casualties
|
||||
# counted above and not an example of what this branch buys.
|
||||
named = (det["language"] not in (None, "en")
|
||||
and _named_prose_language(sample) is not None)
|
||||
if not named:
|
||||
if det["script"] not in ("latin", "unknown"):
|
||||
if not (_non_latin_words(sample) and sum(ch.isalpha() for ch in sample) >= NON_LATIN_MIN_LETTERS):
|
||||
continue
|
||||
elif not (float(det["diacritic_rate"]) >= NON_EN_DIACRITIC_RATE
|
||||
and len(_WORD.findall(sample)) >= 4):
|
||||
continue
|
||||
n_alpha = sum(ch.isalpha() for ch in sample)
|
||||
if n_alpha > best_n:
|
||||
best_n = n_alpha
|
||||
best = det
|
||||
return best
|
||||
|
||||
|
||||
def analyse(state: Union[str, bytes, Mapping, list, None]) -> Dict[str, object]:
|
||||
"""Full detection result for a state.
|
||||
|
||||
Returns `script`, `script_profile`, `language` (best effort, may be None),
|
||||
`is_english`, `non_latin_fraction` and `mixed_segment`. `mixed_segment` names the line or field
|
||||
the segment scan itself singled out, else None: a state flipped non-English by the per-value
|
||||
read below is reported with `mixed_segment` still None, because that read keeps the offending
|
||||
value in its own local and never writes it back.
|
||||
|
||||
String values are what get read. When a state has several of them, one non-English value is
|
||||
enough: joining every value into one window let a long English note fill the 4000 characters,
|
||||
or outvote a short German message, and that message was then sent to the English checkpoint
|
||||
(#384). The segment scan still stops at 4000 characters, which is what keeps a huge field
|
||||
cheap; a value it did not reach is read on its own afterwards.
|
||||
"""
|
||||
result = _analyse_text(state_text(state))
|
||||
if result["script"] == "latin" and result["is_english"]:
|
||||
# A Portuguese ticket with an English stack trace, error payload or form template reads as
|
||||
# English as a whole, because the English part is longer -- yet the part a question is about
|
||||
# is the customer's, and the English checkpoint cannot read it (0.97 confidence at 0.47
|
||||
# accuracy on `pt`). The cost is lopsided: English sent to multilingual loses a few points,
|
||||
# the reverse loses calibration. So a state that would go to English is checked line by line
|
||||
# and field by field.
|
||||
leaves = _iter_text(state)
|
||||
# a single line has no other part to be outvoted by, and was just read whole
|
||||
if len(leaves) > 1 or any("\n" in leaf for leaf in leaves):
|
||||
found = _non_english_segment(state)
|
||||
if found:
|
||||
lang, mixed = found
|
||||
result = dict(result)
|
||||
result["language"] = lang
|
||||
result["is_english"] = False
|
||||
result["language_undecided"] = False
|
||||
result["mixed_segment"] = mixed
|
||||
# A plain string was just read whole. A structured state can still hide a message past the
|
||||
# segment cap, or in a script `latin_profile` does not name.
|
||||
if isinstance(state, (str, bytes, bytearray)) or state is None or not result["is_english"]:
|
||||
return result
|
||||
best_n = -1
|
||||
best: Optional[Dict[str, object]] = None
|
||||
for leaf in _iter_text(state):
|
||||
det = _leaf_non_english(leaf)
|
||||
if det is None:
|
||||
continue
|
||||
n_alpha = sum(ch.isalpha() for ch in leaf[:4000])
|
||||
if n_alpha > best_n:
|
||||
best_n = n_alpha
|
||||
best = det
|
||||
if best is None:
|
||||
return result
|
||||
updated = dict(result)
|
||||
updated["language"] = best["language"]
|
||||
updated["is_english"] = False
|
||||
updated["language_undecided"] = best["language_undecided"]
|
||||
return updated
|
||||
|
||||
|
||||
def is_english(state: Union[str, bytes, Mapping, list, None]) -> bool:
|
||||
"""True when the English checkpoint can be expected to read this state."""
|
||||
return bool(analyse(state)["is_english"])
|
||||
210
laya_repo/presets.py
Normal file
210
laya_repo/presets.py
Normal file
@ -0,0 +1,210 @@
|
||||
"""Ready-to-use question presets for common production decision workflows."""
|
||||
import re
|
||||
from typing import Dict, Optional
|
||||
|
||||
# A preset says which part of the state it reads by naming it in backticks: "What does the
|
||||
# customer want in `message`?". That convention is what `state_field` below depends on, and it is
|
||||
# the reason no table of preset -> key has to be kept in step by hand.
|
||||
_STATE_FIELD_RE = re.compile(r"`(\w+)`")
|
||||
|
||||
|
||||
def state_field(questions: Dict) -> Optional[str]:
|
||||
"""The state key ``questions`` reads, or ``None`` when that is not exactly one key.
|
||||
|
||||
Every built-in preset names a single field -- `message`, `body`, `prompt`, `post`, `request` --
|
||||
and a caller that puts its text under a different key is asking the model about a field that is
|
||||
not in the state. Surfacing the name lets a caller place the text correctly instead of guessing.
|
||||
``None`` covers both "names nothing" and "names several", because then only the caller knows
|
||||
which field the request belongs in.
|
||||
"""
|
||||
named = {match
|
||||
for spec in questions.values()
|
||||
for match in _STATE_FIELD_RE.findall(spec.get("instructions") or "")}
|
||||
if len(named) != 1:
|
||||
return None
|
||||
return next(iter(named))
|
||||
|
||||
|
||||
def triage_questions() -> Dict:
|
||||
"""Preset questions for customer support ticket triage."""
|
||||
return {
|
||||
"intent": {
|
||||
"type": "choice",
|
||||
"instructions": "What does the customer want in `message`?",
|
||||
"criteria": {
|
||||
"refund": "money returned or a duplicate charge reversed",
|
||||
"technical_help": "a bug, outage or integration problem",
|
||||
"billing_question": "a question about an invoice, plan or payment method",
|
||||
"information": "general information, pricing or how-to",
|
||||
"cancellation": "wants to cancel or downgrade",
|
||||
"other": "none of the other options fits",
|
||||
},
|
||||
},
|
||||
"is_urgent": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `message` communicate time pressure or a deadline?",
|
||||
},
|
||||
"frustration": {
|
||||
"type": "score",
|
||||
"instructions": "How frustrated does the customer sound in `message`?",
|
||||
"criteria": [
|
||||
"calm and neutral",
|
||||
"concerned but civil",
|
||||
"clearly annoyed",
|
||||
"very angry or using strong language",
|
||||
],
|
||||
},
|
||||
"refund_requested": {
|
||||
"type": "noul",
|
||||
"instructions": "Does the customer ask for money back?",
|
||||
},
|
||||
"churn_risk": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `message` suggest the customer may leave for a competitor or cancel?",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def email_questions(categories: Optional[Dict[str, str]] = None) -> Dict:
|
||||
"""Preset questions for inbound email triage and threat filtering."""
|
||||
categories = categories or {
|
||||
"billing": "invoices, payments, refunds",
|
||||
"technical": "bugs, outages, integrations",
|
||||
"sales": "pricing, demos, new purchases",
|
||||
"security": "phishing, scams, account compromise",
|
||||
"hr": "hiring, leave, payroll",
|
||||
"other": "none of the above",
|
||||
}
|
||||
return {
|
||||
"category": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team should handle the email in `body`?",
|
||||
"criteria": categories,
|
||||
},
|
||||
"is_spam": {
|
||||
"type": "noul",
|
||||
"instructions": "Is this email unsolicited spam or bulk marketing?",
|
||||
},
|
||||
"is_phishing": {
|
||||
"type": "noul",
|
||||
"instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?",
|
||||
"criteria": {"true": "phishing, scam, or fraud", "false": "a legitimate email"},
|
||||
},
|
||||
"urgency": {
|
||||
"type": "score",
|
||||
"instructions": "How urgent is the request in `body`?",
|
||||
"criteria": ["no time pressure", "needs attention soon", "blocking issue or hard deadline"],
|
||||
},
|
||||
"needs_reply": {
|
||||
"type": "noul",
|
||||
"instructions": "Does the sender expect a reply?",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def guard_questions() -> Dict:
|
||||
"""Preset questions for real-time LLM input guardrails."""
|
||||
return {
|
||||
"jailbreak": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `prompt` try to make an AI assistant ignore its rules, policies or system instructions?",
|
||||
},
|
||||
"prompt_injection": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `prompt` contain instructions aimed at the AI system rather than a genuine user request?",
|
||||
},
|
||||
"sensitive_data": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `prompt` contain credentials, personal data or other sensitive information?",
|
||||
},
|
||||
"harm_severity": {
|
||||
"type": "score",
|
||||
"instructions": "How much harm would complying with `prompt` cause?",
|
||||
"criteria": [
|
||||
"none: ordinary request",
|
||||
"minor: mildly inappropriate",
|
||||
"serious: unsafe advice or abuse",
|
||||
"severe: dangerous or illegal",
|
||||
],
|
||||
},
|
||||
"topic": {
|
||||
"type": "choice",
|
||||
"instructions": "What is `prompt` about?",
|
||||
"criteria": {
|
||||
"product_support": None,
|
||||
"coding": None,
|
||||
"general_knowledge": None,
|
||||
"personal_advice": None,
|
||||
"security_testing": None,
|
||||
"other": None,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def moderation_questions() -> Dict:
|
||||
"""Preset questions for content safety and moderation."""
|
||||
return {
|
||||
"toxic": {
|
||||
"type": "noul",
|
||||
"instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?",
|
||||
},
|
||||
"harassment": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `post` target or harass a specific person?",
|
||||
},
|
||||
"threat": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `post` threaten violence, harm or intimidation?",
|
||||
},
|
||||
"spam": {
|
||||
"type": "noul",
|
||||
"instructions": "Is `post` spam or advertising?",
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"instructions": "How severe is any rule-breaking in `post`?",
|
||||
"criteria": [
|
||||
"no rule-breaking: ordinary on-topic post",
|
||||
"mild: rude tone or off-topic, no target",
|
||||
"clear violation: insults, harassment or spam aimed at someone",
|
||||
"severe: threats, hate speech or calls for violence",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def router_questions() -> Dict:
|
||||
"""Preset questions for intelligent model routing."""
|
||||
return {
|
||||
"difficulty": {
|
||||
"type": "score",
|
||||
"instructions": "How hard is `request` for a language model?",
|
||||
"criteria": [
|
||||
"trivial: a lookup or one-liner",
|
||||
"easy: short answer, no reasoning",
|
||||
"moderate: several steps",
|
||||
"hard: long multi-step reasoning or specialist knowledge",
|
||||
],
|
||||
},
|
||||
"domain": {
|
||||
"type": "choice",
|
||||
"instructions": "What domain does `request` belong to?",
|
||||
"criteria": {
|
||||
"code": "software engineering, programming, refactoring, architecture, debugging",
|
||||
"math_or_logic": "mathematics, logic puzzles, proofs, complex calculation",
|
||||
"writing": "creative writing, essays, emails, blog posts, copywriting",
|
||||
"factual_lookup": "facts, definitions, trivia, history",
|
||||
"data_analysis": "statistics, SQL, data manipulation, metrics",
|
||||
"chitchat": "casual conversation, greetings, small talk",
|
||||
},
|
||||
},
|
||||
"needs_tools": {
|
||||
"type": "noul",
|
||||
"instructions": "Does answering `request` require external tools, search or private data?",
|
||||
},
|
||||
"is_sensitive": {
|
||||
"type": "noul",
|
||||
"instructions": "Does `request` involve money, legal, medical or safety consequences?",
|
||||
},
|
||||
}
|
||||
1905
laya_repo/router.py
Normal file
1905
laya_repo/router.py
Normal file
File diff suppressed because it is too large
Load Diff
1331
laya_repo/train.py
Normal file
1331
laya_repo/train.py
Normal file
File diff suppressed because it is too large
Load Diff
4000
laya_v05_decisions.jsonl
Normal file
4000
laya_v05_decisions.jsonl
Normal file
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user