From eeacb069344523a15d2344dd7e142c9cc6ff7cce Mon Sep 17 00:00:00 2001 From: eust-w Date: Thu, 25 Jun 2026 13:04:09 +0800 Subject: [PATCH] Compress backend JSON responses --- docs/product-readiness.md | 27 ++++++++++----- scripts/verify-real-backend-editing.mjs | 46 +++++++++++++++++++++++++ server/headless_motrix_bridge.py | 26 ++++++++++++-- 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/docs/product-readiness.md b/docs/product-readiness.md index 95bc6f2..a4b3c89 100644 --- a/docs/product-readiness.md +++ b/docs/product-readiness.md @@ -60,6 +60,10 @@ Important properties: sends `If-None-Match`, the backend can return `304 Not Modified` with no JSON body, and the older semantic frame comparison remains as a fallback after parsing. +- Large JSON responses are gzip-compressed by the bridge when the client + advertises support. `/compose` inline manifests and `/manifest` expose + `X-Motrix-Json-Bytes`, `X-Motrix-Wire-Bytes`, and `X-Motrix-Encoding`, making + tunnel transfer cost measurable in real backend tests. - Large-scene diagnostics use adaptive stats refresh: user-visible events still force an immediate panel update, while background periodic refreshes drop to a lower cadence for large or paused scenes so long diagnostic strings do not @@ -269,13 +273,16 @@ Representative product gate baseline for the current thresholds: - Real backend doctor: `14 passed / 0 warned / 0 failed` - Real backend startup: synced and restarted the Wuying bridge, checked the local tunnel, and passed the editing verifier before browser acceptance. -- Real backend compose/editing: `43 passed / 0 warned / 0 failed` +- Real backend compose/editing: `46 passed / 0 warned / 0 failed` - Physical compose: `/compose` returned `objectCount=636`, `bodyCount=573`, and `4/4` product-gate primitive bodies in inline manifest, fetched manifest, - and live state. The latest representative gate measured `371ms`. + and live state. The latest representative gate measured `451ms`. +- JSON transport compression: `/compose` inline manifest compressed from + `257041` bytes to `11205` bytes (`ratio=0.044`), cached `/compose` from + `257040` to `11199`, and `/manifest` from `256695` to `11042`. - Repeated physical compose cache: a second identical `/compose` request returns `cached=true` with the same compose key; the current real-backend verification - measured `156ms`. + measured `222ms`. - Static shelf stability: selected shelf anchors had `max drift 0.00000m` after a 2500 ms resume window, preventing the shelf-toppling regression. - Fixed inventory protection: a pre-stocked product body returned HTTP `400` @@ -638,15 +645,17 @@ Bundle evidence: Control evidence: -- Cold compose endpoint: `264ms`, `4/4` composed object bodies synced. -- Repeated compose cache: `72ms`, `cached=true`, same compose key. +- Cold compose endpoint: `451ms`, `4/4` composed object bodies synced. +- Repeated compose cache: `222ms`, `cached=true`, same compose key. +- JSON transport compression: `/compose` and `/manifest` wire payloads are + about `11 KiB` after gzip instead of about `257 KiB` uncompressed. - Active state endpoint: full `/state` returned `602` bodies; active - `/state?bodies=active` returned `34` bodies in `59ms`, filtered `568` fixed + `/state?bodies=active` returned `34` bodies in `62ms`, filtered `568` fixed bodies, and retained `4/4` composed dynamic bodies. -- Scene Builder UI compose: representative run `3559ms` end to end with real +- Scene Builder UI compose: representative run `3756ms` end to end with real backend diagnostics. -- Share-link restore: representative full browser restore `6403ms`. -- Config import restore: representative full browser restore `6894ms`. +- Share-link restore: representative full browser restore `13118ms`. +- Config import restore: representative full browser restore `14236ms`. - Composed object body edit: `47ms`. - Fixed inventory edit rejection: `59ms`. - Dynamic composed object drop: `0.48012m`. diff --git a/scripts/verify-real-backend-editing.mjs b/scripts/verify-real-backend-editing.mjs index 944f310..ba111c6 100644 --- a/scripts/verify-real-backend-editing.mjs +++ b/scripts/verify-real-backend-editing.mjs @@ -109,6 +109,28 @@ function stateHashFromHeaders(headers) { return String(raw).trim().replace(/^W\//, "").replace(/^"|"$/g, "") || undefined; } +function numberHeader(headers, name) { + const value = Number(headers?.[name]); + return Number.isFinite(value) ? value : undefined; +} + +function compressedJsonDetail(response) { + const jsonBytes = numberHeader(response?.headers, "x-motrix-json-bytes"); + const wireBytes = numberHeader(response?.headers, "x-motrix-wire-bytes"); + const encoding = response?.headers?.["x-motrix-encoding"] ?? response?.headers?.["content-encoding"] ?? "missing"; + return { + encoding, + jsonBytes, + wireBytes, + ratio: jsonBytes && wireBytes ? wireBytes / jsonBytes : undefined, + ok: encoding === "gzip" + && Number.isFinite(jsonBytes) + && Number.isFinite(wireBytes) + && wireBytes > 0 + && wireBytes < jsonBytes, + }; +} + function bodySet(items) { return new Set(items.filter((item) => typeof item === "string" && item.length > 0)); } @@ -145,6 +167,9 @@ let composeUrl; let health; let composeResult; let composeCacheResult; +let composeCompression; +let composeCacheCompression; +let manifestCompression; let initialState; let initialStateHash; let repeatInitialStateHash; @@ -197,6 +222,12 @@ try { )); checks.push(check("physical-compose-latency", composeResult.elapsedMs <= maxComposeLatencyMs, `${composeResult.elapsedMs}ms <= ${maxComposeLatencyMs}ms`, "warn")); checks.push(check("physical-compose-manifest", inlineComposeObjects.length > 0, `${inlineComposeObjects.length} inline objects`)); + composeCompression = compressedJsonDetail(composeResult); + checks.push(check( + "physical-compose-json-gzip", + composeCompression.ok, + `${composeCompression.encoding} ${composeCompression.wireBytes ?? "-"} / ${composeCompression.jsonBytes ?? "-"} bytes ratio=${composeCompression.ratio?.toFixed(3) ?? "-"}`, + )); checks.push(check( "physical-compose-objects", composeObjects.every((item) => inlineComposeBodies.has(item.id)), @@ -218,10 +249,22 @@ try { `${composeCacheResult.elapsedMs}ms <= ${maxControlLatencyMs}ms`, "warn", )); + composeCacheCompression = compressedJsonDetail(composeCacheResult); + checks.push(check( + "physical-compose-cache-json-gzip", + composeCacheCompression.ok, + `${composeCacheCompression.encoding} ${composeCacheCompression.wireBytes ?? "-"} / ${composeCacheCompression.jsonBytes ?? "-"} bytes ratio=${composeCacheCompression.ratio?.toFixed(3) ?? "-"}`, + )); const manifestResponse = await fetchJson(manifestUrl.href, {}, fetchTimeoutMs); manifest = manifestResponse.json; checks.push(check("manifest", manifestResponse.ok && Array.isArray(manifest?.objects), `HTTP ${manifestResponse.status} in ${manifestResponse.elapsedMs}ms`)); + manifestCompression = compressedJsonDetail(manifestResponse); + checks.push(check( + "manifest-json-gzip", + manifestCompression.ok, + `${manifestCompression.encoding} ${manifestCompression.wireBytes ?? "-"} / ${manifestCompression.jsonBytes ?? "-"} bytes ratio=${manifestCompression.ratio?.toFixed(3) ?? "-"}`, + )); stateUrl = manifest?.state?.url ? new URL(manifest.state.url, manifestUrl).href : undefined; controlUrl = manifest?.state?.controlUrl @@ -447,6 +490,7 @@ const report = { jointCount: composeResult.json?.jointCount, actuatorCount: composeResult.json?.actuatorCount, elapsedMs: composeResult.elapsedMs, + compression: composeCompression, objectIds: composeObjects.map((item) => item.id), } : undefined, @@ -456,6 +500,7 @@ const report = { cached: composeCacheResult.json?.cached, composeKey: composeCacheResult.json?.composeKey, objectCount: composeCacheResult.json?.objectCount, + compression: composeCacheCompression, } : undefined, scene: manifest @@ -471,6 +516,7 @@ const report = { acc[role] = (acc[role] ?? 0) + 1; return acc; }, {}), + compression: manifestCompression, } : undefined, selectedGroup: selectedGroup diff --git a/server/headless_motrix_bridge.py b/server/headless_motrix_bridge.py index 69aecf7..9394db2 100644 --- a/server/headless_motrix_bridge.py +++ b/server/headless_motrix_bridge.py @@ -14,6 +14,7 @@ not used here; the browser does the visual rendering. from __future__ import annotations import argparse +import gzip import hashlib import json import mimetypes @@ -79,6 +80,11 @@ PRODUCT_BODY_PREFIXES = ( "Box_", "Drink_", ) +JSON_COMPRESSION_MIN_BYTES = 1024 +JSON_EXPOSED_HEADERS = ( + "X-Motrix-State-Hash, ETag, X-Motrix-Json-Bytes, " + "X-Motrix-Wire-Bytes, X-Motrix-Encoding" +) def safe_name(raw: Any, fallback: str) -> str: @@ -1364,12 +1370,26 @@ def main() -> int: return def send_json(self, status: int, payload: dict[str, Any], headers: dict[str, str] | None = None) -> None: - body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + raw_body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + body = raw_body + encoding = "identity" + accepts_gzip = "gzip" in (self.headers.get("Accept-Encoding") or "").lower() + if accepts_gzip and len(raw_body) >= JSON_COMPRESSION_MIN_BYTES: + compressed = gzip.compress(raw_body, compresslevel=5) + if len(compressed) < len(raw_body): + body = compressed + encoding = "gzip" self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Access-Control-Allow-Origin", "*") - self.send_header("Access-Control-Expose-Headers", "X-Motrix-State-Hash, ETag") + self.send_header("Access-Control-Expose-Headers", JSON_EXPOSED_HEADERS) self.send_header("Cache-Control", "no-store") + self.send_header("Vary", "Accept-Encoding") + self.send_header("X-Motrix-Json-Bytes", str(len(raw_body))) + self.send_header("X-Motrix-Wire-Bytes", str(len(body))) + self.send_header("X-Motrix-Encoding", encoding) + if encoding == "gzip": + self.send_header("Content-Encoding", "gzip") for name, value in (headers or {}).items(): self.send_header(name, value) self.send_header("Content-Length", str(len(body))) @@ -1379,7 +1399,7 @@ def main() -> int: def send_not_modified(self, state_hash: str) -> None: self.send_response(304) self.send_header("Access-Control-Allow-Origin", "*") - self.send_header("Access-Control-Expose-Headers", "X-Motrix-State-Hash, ETag") + self.send_header("Access-Control-Expose-Headers", JSON_EXPOSED_HEADERS) self.send_header("Cache-Control", "no-store") self.send_header("X-Motrix-State-Hash", state_hash) self.send_header("ETag", f'"{state_hash}"')