🔎 Add bounded read-only diagnostics and isolated permission verification

This commit is contained in:
Codex 2026-11-10 08:46:26 -08:00
parent e150a8a24c
commit a98127c42d
17 changed files with 899 additions and 12 deletions

View File

@ -1,8 +1,11 @@
FROM golang:1.24 AS build
WORKDIR /build
COPY cmd/diagnostic-gateway/main.go .
RUN CGO_ENABLED=0 go build -trimpath -o /gateway main.go
COPY go.mod go.sum ./
RUN go mod download
COPY cmd/diagnostic-gateway ./cmd/diagnostic-gateway
RUN CGO_ENABLED=0 go build -trimpath -o /gateway ./cmd/diagnostic-gateway
FROM scratch
COPY --from=build /gateway /gateway
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
USER 65532:65532
ENTRYPOINT ["/gateway"]

View File

@ -0,0 +1,55 @@
package main
import (
"context"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"log"
"net/http"
"os"
"strings"
"time"
)
func configureDiagnostics(g *gateway, client *http.Client, ns, tokenFile string) {
d := newDiagnostics()
g.diagnostics = d
if os.Getenv("DIAGNOSTIC_LOGS_ENABLED") == "true" {
d.logs = logLoader(client, "https://kubernetes.default.svc", ns, func() (string, error) { b, e := os.ReadFile(tokenFile); return strings.TrimSpace(string(b)), e })
}
if file := os.Getenv("DIAGNOSTIC_MONGO_URI_FILE"); file != "" {
raw, e := os.ReadFile(file)
if e != nil {
log.Fatal("diagnostic Mongo credential unavailable")
}
org := os.Getenv("DIAGNOSTIC_ORGANIZATION")
if org == "" {
log.Fatal("diagnostic organization required")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
c, e := mongo.Connect(ctx, options.Client().ApplyURI(strings.TrimSpace(string(raw))).SetMaxPoolSize(2).SetRetryReads(false).SetRetryWrites(false).SetServerSelectionTimeout(2*time.Second))
if e != nil {
log.Fatal("diagnostic Mongo connection failed")
}
if checkPrivileges(ctx, c, "drobodata-fanyu") != nil {
log.Fatal("diagnostic Mongo identity must have find-only access to the four views")
}
d.task = mongoTaskLoader(c, "drobodata-fanyu", org)
}
if file := os.Getenv("DIAGNOSTIC_BCM_CONFIG_FILE"); file != "" {
collect, e := newBCMCollector(file)
if e != nil {
log.Fatal("invalid BCM configuration")
}
go func() {
for {
v := collect()
d.mu.Lock()
d.egress = v
d.mu.Unlock()
time.Sleep(time.Minute)
}
}()
}
}

View File

@ -0,0 +1,206 @@
package main
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
"time"
)
type diagnosticResult struct {
Available bool `json:"available"`
ObservedAt string `json:"observedAt"`
Evidence any `json:"evidence,omitempty"`
Limitations []string `json:"limitations,omitempty"`
Error string `json:"error,omitempty"`
}
type diagnosticLoader func(context.Context, string) (diagnosticResult, error)
type cachedDiagnostic struct {
at time.Time
result diagnosticResult
}
type diagnostics struct {
task, logs diagnosticLoader
mu sync.Mutex
cache map[string]cachedDiagnostic
busy chan struct{}
egress diagnosticResult
}
var taskIDPattern = regexp.MustCompile(`^annotation-task-[A-Za-z0-9_-]{1,100}$`)
var requestIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
func newDiagnostics() *diagnostics {
return &diagnostics{cache: map[string]cachedDiagnostic{}, busy: make(chan struct{}, 2), egress: diagnosticResult{Error: "bcm_not_configured"}}
}
func (d *diagnostics) serve(w http.ResponseWriter, r *http.Request) bool {
path := strings.TrimPrefix(r.URL.Path, "/diagnostics/v1/")
var loader diagnosticLoader
arg := ""
switch {
case path == "egress":
d.mu.Lock()
v := d.egress
d.mu.Unlock()
if stamp, err := time.Parse(time.RFC3339, v.ObservedAt); v.Available && (err != nil || time.Since(stamp) > 2*time.Minute) {
v = diagnosticResult{Error: "bcm_snapshot_stale"}
}
writeDiagnostic(w, v)
return true
case strings.HasPrefix(path, "tasks/"):
arg = strings.TrimPrefix(path, "tasks/")
if !taskIDPattern.MatchString(arg) {
http.Error(w, `{"error":"invalid_task_id"}`, 400)
return true
}
loader = d.task
case strings.HasPrefix(path, "logs/api/"):
arg = strings.TrimPrefix(path, "logs/api/")
if !requestIDPattern.MatchString(arg) {
http.Error(w, `{"error":"invalid_request_id"}`, 400)
return true
}
loader = d.logs
default:
return false
}
if loader == nil {
writeDiagnostic(w, diagnosticResult{Error: "adapter_not_configured"})
return true
}
d.mu.Lock()
cached, ok := d.cache[path]
d.mu.Unlock()
if ok && time.Since(cached.at) < 15*time.Second {
writeDiagnostic(w, cached.result)
return true
}
select {
case d.busy <- struct{}{}:
defer func() { <-d.busy }()
default:
http.Error(w, `{"error":"diagnostic_busy"}`, 429)
return true
}
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
defer cancel()
v, e := loader(ctx, arg)
if e != nil {
v = diagnosticResult{Error: "upstream_unavailable_or_timed_out"}
}
v.ObservedAt = time.Now().UTC().Format(time.RFC3339)
d.mu.Lock()
if len(d.cache) >= 128 {
d.cache = map[string]cachedDiagnostic{}
}
d.cache[path] = cachedDiagnostic{time.Now(), v}
d.mu.Unlock()
writeDiagnostic(w, v)
return true
}
func writeDiagnostic(w http.ResponseWriter, v diagnosticResult) {
if !v.Available {
w.WriteHeader(503)
}
json.NewEncoder(w).Encode(v)
}
// Only already structured request traces are eligible. Unknown text never leaves the gateway.
var tracePattern = regexp.MustCompile(`request_trace id=([a-f0-9]{32}) kind=(login|diagnostic|slow)(?: method=(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) path=\S+)? status=([0-9]{3}) elapsed_ms=([0-9]{1,9})(?:\s|$)`)
func parseTraces(raw, requestID string) []map[string]string {
out := []map[string]string{}
for _, line := range strings.Split(raw, "\n") {
m := tracePattern.FindStringSubmatch(line)
if len(m) == 0 || m[1] != requestID {
continue
}
stamp := strings.Fields(line)[0]
if _, e := time.Parse(time.RFC3339Nano, stamp); e != nil {
continue
}
out = append(out, map[string]string{"timestamp": stamp, "requestId": m[1], "kind": m[2], "method": m[3], "status": m[4], "elapsedMs": m[5]})
}
return out
}
func getBounded(ctx context.Context, client *http.Client, base, path, token string, limit int) ([]byte, error) {
req, e := http.NewRequestWithContext(ctx, "GET", base+path, nil)
if e != nil {
return nil, e
}
req.Header.Set("Authorization", "Bearer "+token)
resp, e := client.Do(req)
if e != nil {
return nil, e
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, errors.New("upstream unavailable")
}
b, e := io.ReadAll(io.LimitReader(resp.Body, int64(limit+1)))
if e != nil || len(b) > limit {
return nil, errors.New("response limit")
}
return b, nil
}
func logLoader(client *http.Client, base, ns string, token func() (string, error)) diagnosticLoader {
return func(ctx context.Context, id string) (diagnosticResult, error) {
t, e := token()
if e != nil {
return diagnosticResult{}, e
}
b, e := getBounded(ctx, client, base, "/api/v1/namespaces/"+url.PathEscape(ns)+"/pods?labelSelector=app%3Ddroboticflow-api&limit=10", t, 256*1024)
if e != nil {
return diagnosticResult{}, e
}
var list struct {
Items []struct {
Metadata struct {
Name string `json:"name"`
} `json:"metadata"`
Spec struct {
Containers []struct {
Name string `json:"name"`
} `json:"containers"`
} `json:"spec"`
} `json:"items"`
Metadata struct {
Continue string `json:"continue"`
} `json:"metadata"`
}
if json.Unmarshal(b, &list) != nil {
return diagnosticResult{}, errors.New("invalid pods")
}
evidence := []map[string]string{}
n := 0
for _, pod := range list.Items {
if n == 2 {
break
}
if len(pod.Spec.Containers) == 0 {
continue
}
n++
path := "/api/v1/namespaces/" + url.PathEscape(ns) + "/pods/" + url.PathEscape(pod.Metadata.Name) + "/log?timestamps=true&sinceSeconds=900&tailLines=200&limitBytes=65536&container=" + url.QueryEscape(pod.Spec.Containers[0].Name)
raw, e := getBounded(ctx, client, base, path, t, 65536)
if e != nil {
return diagnosticResult{}, e
}
for _, line := range parseTraces(string(raw), id) {
line["pod"] = pod.Metadata.Name
evidence = append(evidence, line)
}
}
limitations := []string{"Only structured request_trace records; raw logs and route parameters omitted", "Last 15 minutes, at most 200 lines/64KiB per pod; logs may be rotated", "No matching trace is insufficient evidence, not proof of success"}
if len(list.Items) > 2 || list.Metadata.Continue != "" {
limitations = append(limitations, "Partial pod coverage: only first two API pods inspected")
}
return diagnosticResult{Available: true, Evidence: evidence, Limitations: limitations}, nil
}
}

View File

@ -0,0 +1,132 @@
package main
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
)
func testGateway(d *diagnostics) *gateway {
return &gateway{diagnostics: d, keys: [][32]byte{sha256.Sum256([]byte("test-key"))}, snapshots: map[string]snapshot{}}
}
func request(g *gateway, path string) *httptest.ResponseRecorder {
r := httptest.NewRequest("GET", path, nil)
r.Header.Set("Authorization", "Bearer test-key")
w := httptest.NewRecorder()
g.ServeHTTP(w, r)
return w
}
func TestDiagnosticHTTPContracts(t *testing.T) {
d := newDiagnostics()
g := testGateway(d)
if w := request(g, "/diagnostics/v1/egress"); w.Code != 503 {
t.Fatal(w.Code)
}
if w := request(g, "/diagnostics/v1/tasks/annotation-task-test"); w.Code != 503 {
t.Fatal(w.Code)
}
for _, path := range []string{"tasks/$where", "tasks/annotation-task-x?filter=bad", "logs/api/../../../secrets", "logs/api/not-a-request-id"} {
if w := request(g, "/diagnostics/v1/"+path); w.Code != 400 && w.Code != 404 {
t.Fatalf("%s %d", path, w.Code)
}
}
var calls atomic.Int32
d.task = func(context.Context, string) (diagnosticResult, error) {
calls.Add(1)
return diagnosticResult{Available: true, Evidence: map[string]string{"finding": "task_currently_cancelled"}}, nil
}
for i := 0; i < 5; i++ {
if w := request(g, "/diagnostics/v1/tasks/annotation-task-test"); w.Code != 200 {
t.Fatal(w.Code)
}
}
if calls.Load() != 1 {
t.Fatal("polling repeated source reads")
}
d.task = func(context.Context, string) (diagnosticResult, error) {
return diagnosticResult{}, errors.New("mongodb://user:secret@host customer payload")
}
w := request(g, "/diagnostics/v1/tasks/annotation-task-other")
if w.Code != 503 || strings.Contains(w.Body.String(), "secret") {
t.Fatal("upstream details leaked")
}
}
func TestTraceFiltering(t *testing.T) {
id := strings.Repeat("a", 32)
raw := "2026-11-10T01:00:00Z request_trace id=" + id + " kind=slow method=GET path=/api/private/customer-secret status=504 elapsed_ms=12000 token=secret\npassword=secret\n"
out := parseTraces(raw, id)
b, _ := json.Marshal(out)
if len(out) != 1 || strings.Contains(string(b), "secret") || strings.Contains(string(b), "private") {
t.Fatal(string(b))
}
if len(parseTraces(raw, strings.Repeat("b", 32))) != 0 {
t.Fatal("cross-request trace")
}
}
func TestDiagnosticConcurrencyAndCancellation(t *testing.T) {
d := newDiagnostics()
g := testGateway(d)
entered := make(chan struct{}, 2)
release := make(chan struct{})
d.task = func(ctx context.Context, id string) (diagnosticResult, error) {
entered <- struct{}{}
select {
case <-release:
return diagnosticResult{Available: true}, nil
case <-ctx.Done():
return diagnosticResult{}, ctx.Err()
}
}
done := make(chan struct{}, 2)
for _, id := range []string{"a", "b"} {
go func(id string) { request(g, "/diagnostics/v1/tasks/annotation-task-"+id); done <- struct{}{} }(id)
}
for i := 0; i < 2; i++ {
select {
case <-entered:
case <-time.After(time.Second):
t.Fatal("not started")
}
}
if w := request(g, "/diagnostics/v1/tasks/annotation-task-c"); w.Code != 429 {
t.Fatal(w.Code)
}
close(release)
<-done
<-done
}
func TestLogLoaderBoundedAndNoRawOutput(t *testing.T) {
id := strings.Repeat("c", 32)
var calls atomic.Int32
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
if strings.HasSuffix(r.URL.Path, "/pods") {
if r.URL.Query().Get("labelSelector") != "app=droboticflow-api" {
t.Error("arbitrary selector")
}
w.Write([]byte(`{"items":[{"metadata":{"name":"api-1"},"spec":{"containers":[{"name":"api"}]}},{"metadata":{"name":"api-2"},"spec":{"containers":[{"name":"api"}]}},{"metadata":{"name":"api-3"}}]}`))
return
}
if r.URL.Query().Get("follow") != "" || r.URL.Query().Get("limitBytes") != "65536" {
t.Error("unbounded log query")
}
w.Write([]byte("2026-11-10T01:00:00Z request_trace id=" + id + " kind=diagnostic status=500 elapsed_ms=3000\nSECRET raw payload"))
}))
defer s.Close()
loader := logLoader(s.Client(), s.URL, "test", func() (string, error) { return "token", nil })
v, e := loader(context.Background(), id)
if e != nil || !v.Available || calls.Load() != 3 {
t.Fatalf("%v %v", v, e)
}
b, _ := json.Marshal(v)
if strings.Contains(string(b), "SECRET") || !strings.Contains(string(b), "Partial pod coverage") {
t.Fatal(string(b))
}
}

View File

@ -0,0 +1,69 @@
package main
import (
"encoding/json"
"errors"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/services/bcm"
"github.com/baidubce/bce-sdk-go/services/bcm/model"
"os"
"time"
)
type bcmConfiguration struct{ AK, SK, UserID, InstanceID, Scope string }
func newBCMCollector(file string) (func() diagnosticResult, error) {
raw, e := os.ReadFile(file)
if e != nil {
return nil, e
}
var cfg bcmConfiguration
if json.Unmarshal(raw, &cfg) != nil || cfg.AK == "" || cfg.SK == "" || cfg.UserID == "" || cfg.InstanceID == "" || cfg.Scope != "BCE_EIP" {
return nil, errors.New("invalid fixed EIP configuration")
}
c, e := bcm.NewClient(cfg.AK, cfg.SK, "https://bcm.bj.baidubce.com")
if e != nil {
return nil, e
}
c.Config.ConnectionTimeoutInMillis = 3000
c.Config.Retry = bce.NewNoRetryPolicy()
return func() diagnosticResult { return collectEgress(c, cfg, time.Now().UTC()) }, nil
}
type metricReader interface {
GetMetricData(*model.GetMetricDataRequest) (*model.GetMetricDataResponse, error)
}
func collectEgress(c metricReader, cfg bcmConfiguration, now time.Time) diagnosticResult {
evidence := map[string]any{"instanceId": cfg.InstanceID, "periodSeconds": 60, "configuredLimitMbps": 50, "windowStart": now.Add(-time.Hour).Format(time.RFC3339), "windowEnd": now.Format(time.RFC3339)}
for _, metric := range []string{"WebOutBitsPerSecond", "WebInBitsPerSecond"} {
r, e := c.GetMetricData(&model.GetMetricDataRequest{UserId: cfg.UserID, Scope: cfg.Scope, MetricName: metric, Dimensions: map[string]string{"InstanceId": cfg.InstanceID}, Statistics: []string{"average", "maximum"}, PeriodInSecond: 60, StartTime: now.Add(-time.Hour).Format(time.RFC3339), EndTime: now.Format(time.RFC3339)})
if e != nil || r == nil || r.Code != "OK" {
return diagnosticResult{Error: "bcm_unavailable", ObservedAt: now.Format(time.RFC3339)}
}
if len(r.DataPoints) == 0 || len(r.DataPoints) > 61 {
return diagnosticResult{Error: "bcm_missing_or_excessive_points", ObservedAt: now.Format(time.RFC3339)}
}
latest := time.Time{}
for _, point := range r.DataPoints {
if point == nil {
return diagnosticResult{Error: "bcm_invalid_points"}
}
stamp, e := time.Parse(time.RFC3339, point.Timestamp)
if e != nil || stamp.After(now.Add(time.Minute)) || stamp.Before(now.Add(-time.Hour-time.Minute)) {
return diagnosticResult{Error: "bcm_invalid_timestamp"}
}
if point.Average == nil || point.Maximum == nil || *point.Average < 0 || *point.Maximum < 0 {
return diagnosticResult{Error: "bcm_invalid_value"}
}
if stamp.After(latest) {
latest = stamp
}
}
if now.Sub(latest) > 5*time.Minute {
return diagnosticResult{Error: "bcm_stale_data", ObservedAt: now.Format(time.RFC3339)}
}
evidence[metric] = r.DataPoints
}
return diagnosticResult{Available: true, ObservedAt: now.Format(time.RFC3339), Evidence: evidence, Limitations: []string{"bps, one-minute averages/maxima; inspect source timestamps for delayed samples", "Bandwidth configuration is not measured utilization; no packet-drop metrics connected yet"}}
}

View File

@ -0,0 +1,55 @@
package main
import (
"github.com/baidubce/bce-sdk-go/services/bcm/model"
"net/http/httptest"
"testing"
"time"
)
type fakeMetrics struct {
response *model.GetMetricDataResponse
calls []*model.GetMetricDataRequest
}
func (f *fakeMetrics) GetMetricData(r *model.GetMetricDataRequest) (*model.GetMetricDataResponse, error) {
f.calls = append(f.calls, r)
return f.response, nil
}
func TestEgressEvidenceAndMissingStaleData(t *testing.T) {
now := time.Date(2026, 11, 10, 10, 0, 0, 0, time.UTC)
v := 12000000.0
f := &fakeMetrics{response: &model.GetMetricDataResponse{Code: "OK", DataPoints: []*model.DataPoints{{Timestamp: now.Add(-time.Minute).Format(time.RFC3339), Average: &v, Maximum: &v}}}}
cfg := bcmConfiguration{InstanceID: "fixed-instance", Scope: "BCE_EIP", UserID: "account"}
got := collectEgress(f, cfg, now)
if !got.Available || len(f.calls) != 2 {
t.Fatalf("%+v", got)
}
for _, r := range f.calls {
if r.Dimensions["InstanceId"] != "fixed-instance" || r.PeriodInSecond != 60 {
t.Fatal("metric scope escaped")
}
}
f.response.DataPoints[0].Timestamp = now.Add(-10 * time.Minute).Format(time.RFC3339)
if got = collectEgress(f, cfg, now); got.Available || got.Error != "bcm_stale_data" {
t.Fatal(got)
}
f.response.DataPoints = nil
if got = collectEgress(f, cfg, now); got.Available {
t.Fatal("missing data treated as zero")
}
}
func TestEgressHTTPRejectsStaleSnapshot(t *testing.T) {
d := newDiagnostics()
for _, sample := range []struct {
age time.Duration
status int
}{{time.Second, 200}, {3 * time.Minute, 503}} {
d.egress = diagnosticResult{Available: true, ObservedAt: time.Now().Add(-sample.age).UTC().Format(time.RFC3339), Evidence: map[string]int{"bps": 12000000}}
w := httptest.NewRecorder()
if !d.serve(w, httptest.NewRequest("GET", "/diagnostics/v1/egress", nil)) || w.Code != sample.status {
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
}
}
}

View File

@ -25,12 +25,13 @@ type snapshot struct {
Error string `json:"error,omitempty"`
}
type gateway struct {
mu sync.RWMutex
snapshots map[string]snapshot
keys [][32]byte
rateMu sync.Mutex
windows map[[32]byte]time.Time
hits map[[32]byte]int
diagnostics *diagnostics
mu sync.RWMutex
snapshots map[string]snapshot
keys [][32]byte
rateMu sync.Mutex
windows map[[32]byte]time.Time
hits map[[32]byte]int
}
type resource struct {
Metadata struct {
@ -154,8 +155,11 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"query_parameters_not_supported"}`, 400)
return
}
if g.diagnostics != nil && g.diagnostics.serve(w, r) {
return
}
if r.URL.Path == "/diagnostics/v1/capabilities" {
io.WriteString(w, `{"version":1,"transport":"http-json","tools":["nodes","pods","node-usage","pod-usage"],"refreshSeconds":30,"unavailable":["egress","task-diagnosis","logs","source-code","mcp"],"mutations":false}`)
json.NewEncoder(w).Encode(map[string]any{"version": 2, "transport": "http-json", "tools": []string{"nodes", "pods", "node-usage", "pod-usage"}, "diagnosticRoutes": []string{"tasks/{taskId}", "logs/api/{requestId}", "egress"}, "configured": map[string]bool{"tasks": g.diagnostics != nil && g.diagnostics.task != nil, "logs": g.diagnostics != nil && g.diagnostics.logs != nil}, "refreshSeconds": 30, "unavailable": []string{"source-code", "mcp"}, "mutations": false})
return
}
key := strings.TrimPrefix(r.URL.Path, "/diagnostics/v1/")
@ -210,6 +214,7 @@ func main() {
log.Fatal("invalid cluster CA")
}
client := &http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, MaxConnsPerHost: 1}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
configureDiagnostics(g, client, ns, sa+"token")
paths := map[string]string{"nodes": "/api/v1/nodes?limit=500", "pods": "/api/v1/namespaces/" + ns + "/pods?limit=500", "node-usage": "/apis/metrics.k8s.io/v1beta1/nodes", "pod-usage": "/apis/metrics.k8s.io/v1beta1/namespaces/" + ns + "/pods"}
for k := range paths {
g.snapshots[k] = snapshot{Error: "warming_up"}

View File

@ -0,0 +1,85 @@
package main
import (
"context"
"encoding/json"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"net/url"
"os"
"strings"
"testing"
"time"
)
func TestIsolatedMongoViewOnlyDiagnosis(t *testing.T) {
uri := os.Getenv("DIAGNOSTIC_TEST_MONGO_URI")
if uri == "" {
t.Skip("run deploy/diagnostic-gateway/test-isolated.py for authenticated fixture")
}
u, e := url.Parse(uri)
if e != nil || u.Hostname() != "127.0.0.1" || !strings.HasPrefix(u.Path, "/diagnostic_test_") {
t.Fatal("isolated localhost database required")
}
database := strings.TrimPrefix(u.Path, "/")
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(2))
if e != nil {
t.Fatal(e)
}
defer c.Disconnect(ctx)
if e = checkPrivileges(ctx, c, database); e != nil {
t.Fatal(e)
}
db := c.Database(database)
if e = db.Collection("annotationTasks").FindOne(ctx, bson.M{}).Err(); e == nil {
t.Fatal("reader can access source collection")
}
if _, e = db.Collection("diagnostic_tasks_v1").InsertOne(ctx, bson.M{"id": "illegal"}); e == nil {
t.Fatal("reader can write")
}
var view bson.M
if e = db.Collection("diagnostic_tasks_v1").FindOne(ctx, bson.M{"id": "annotation-task-failed"}).Decode(&view); e != nil {
t.Fatal(e)
}
raw, _ := json.Marshal(view)
if strings.Contains(string(raw), "CUSTOMER_SECRET") {
t.Fatal("view leaked payload")
}
if n, e := db.Collection("diagnostic_tasks_v1").CountDocuments(ctx, bson.M{"organizationId": "other-org"}); e != nil || n != 0 {
t.Fatal("organization leakage", e, n)
}
d := newDiagnostics()
d.task = mongoTaskLoader(c, database, "test-org")
g := testGateway(d)
cases := []struct{ id, needle string }{{"failed", "task_validation_record_failed"}, {"cancelled", "task_currently_cancelled"}, {"processing", "processing_job_failed"}, {"recovered", "current state cannot prove historical failure cause"}, {"foreign", "taskFound\":false"}}
for _, tc := range cases {
w := request(g, "/diagnostics/v1/tasks/annotation-task-"+tc.id)
if w.Code != 200 || !strings.Contains(w.Body.String(), tc.needle) || strings.Contains(w.Body.String(), "CUSTOMER_SECRET") {
t.Fatalf("%s: %d %s", tc.id, w.Code, w.Body.String())
}
}
start := time.Now()
for i := 0; i < 40; i++ {
w := request(g, "/diagnostics/v1/tasks/annotation-task-failed")
if w.Code != 200 {
t.Fatal(w.Code)
}
}
t.Logf("40 warm HTTP diagnostics: %s (bounded cache)", time.Since(start))
// Revocation invalidates source access after cached evidence expires; clear cache to test the next source read.
adminURI := os.Getenv("DIAGNOSTIC_TEST_ADMIN_URI")
a, e := mongo.Connect(ctx, options.Client().ApplyURI(adminURI))
if e != nil {
t.Fatal(e)
}
defer a.Disconnect(ctx)
if e = a.Database(database).RunCommand(ctx, bson.D{{Key: "grantPrivilegesToRole", Value: "diagnostic_gateway_views_v1"}, {Key: "privileges", Value: bson.A{bson.M{"resource": bson.M{"db": database, "collection": "annotationTasks"}, "actions": bson.A{"find"}}}}}).Err(); e != nil {
t.Fatal(e)
}
if e = checkPrivileges(ctx, c, database); e == nil {
t.Fatal("expanded privileges accepted")
}
}

View File

@ -0,0 +1,107 @@
package main
import (
"context"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
)
var diagnosticViews = []string{"diagnostic_tasks_v1", "diagnostic_items_v1", "diagnostic_validation_v1", "diagnostic_events_v1"}
func checkPrivileges(ctx context.Context, client *mongo.Client, dbName string) error {
var out struct {
AuthInfo struct {
Users []bson.M `bson:"authenticatedUsers"`
Privileges []struct {
Resource struct {
DB string `bson:"db"`
Collection string `bson:"collection"`
} `bson:"resource"`
Actions []string `bson:"actions"`
} `bson:"authenticatedUserPrivileges"`
} `bson:"authInfo"`
}
if e := client.Database("admin").RunCommand(ctx, bson.D{{Key: "connectionStatus", Value: 1}, {Key: "showPrivileges", Value: true}}).Decode(&out); e != nil {
return e
}
if len(out.AuthInfo.Users) != 1 || len(out.AuthInfo.Privileges) == 0 {
return errors.New("dedicated authenticated identity required")
}
for _, p := range out.AuthInfo.Privileges {
allowed := false
for _, v := range diagnosticViews {
if p.Resource.DB == dbName && p.Resource.Collection == v {
allowed = true
}
}
if !allowed {
return errors.New("privilege outside diagnostic views")
}
for _, a := range p.Actions {
if a != "find" {
return errors.New("non-find privilege")
}
}
}
return nil
}
func mongoTaskLoader(client *mongo.Client, dbName, org string) diagnosticLoader {
return func(ctx context.Context, id string) (diagnosticResult, error) {
if !taskIDPattern.MatchString(id) || org == "" {
return diagnosticResult{}, errors.New("scope required")
}
// Recheck effective grants so a later privilege expansion fails closed.
if e := checkPrivileges(ctx, client, dbName); e != nil {
return diagnosticResult{}, e
}
db := client.Database(dbName)
one := func(collection string, filter bson.M) (bson.M, error) {
var result bson.M
e := db.Collection(collection).FindOne(ctx, filter, options.FindOne().SetMaxTime(time.Second)).Decode(&result)
if e == mongo.ErrNoDocuments {
return nil, nil
}
return result, e
}
task, e := one("diagnostic_tasks_v1", bson.M{"id": id, "organizationId": org})
if e != nil {
return diagnosticResult{}, e
}
if task == nil {
return diagnosticResult{Available: true, Evidence: bson.M{"taskFound": false}, Limitations: []string{"Not found in authorized scope; no conclusion about other organizations"}}, nil
}
item, e := one("diagnostic_items_v1", bson.M{"legacyTaskId": id, "organizationId": org})
if e != nil {
return diagnosticResult{}, e
}
validation, e := one("diagnostic_validation_v1", bson.M{"id": id, "organizationId": org})
if e != nil {
return diagnosticResult{}, e
}
events := []bson.M{}
if item != nil {
cur, e := db.Collection("diagnostic_events_v1").Find(ctx, bson.M{"itemId": item["id"], "organizationId": org}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(20).SetMaxTime(time.Second))
if e != nil {
return diagnosticResult{}, e
}
defer cur.Close(ctx)
if e = cur.All(ctx, &events); e != nil {
return diagnosticResult{}, e
}
}
findings := []string{}
if item != nil && item["status"] == "cancelled" {
findings = append(findings, "task_currently_cancelled")
}
if validation != nil && validation["status"] == "failed" {
findings = append(findings, "task_validation_record_failed")
}
if job, ok := item["processingJob"].(bson.M); ok && job["status"] == "failed" {
findings = append(findings, "processing_job_failed")
}
return diagnosticResult{Available: true, Evidence: bson.M{"task": task, "item": item, "taskValidation": validation, "recentEvents": events, "findings": findings, "actorEligibility": "unknown"}, Limitations: []string{"State evidence only; no independent canClaim calculation", "Task-ID validation only; source-v1 certificate supersession is not fully diagnosed", "At most 20 recent events; current state cannot prove historical failure cause", "Deleted history cannot be reconstructed"}}, nil
}
}

View File

@ -3,14 +3,22 @@
import argparse,json,os,ssl,urllib.request,urllib.error
from pathlib import Path
p=argparse.ArgumentParser()
p.add_argument('tool',choices=['capabilities','nodes','pods','node-usage','pod-usage'])
p.add_argument('tool',choices=['capabilities','nodes','pods','node-usage','pod-usage','tasks','logs','egress'])
p.add_argument('--id', help='Task ID for tasks or 32-character request ID for logs')
a=p.parse_args()
import re
route=a.tool
if a.tool in ('tasks','logs'):
pattern=r'annotation-task-[A-Za-z0-9_-]{1,100}' if a.tool=='tasks' else r'[a-f0-9]{32}'
if not a.id or not re.fullmatch(pattern,a.id):p.error('A valid --id is required')
route=('tasks/' if a.tool=='tasks' else 'logs/api/')+a.id
elif a.id:p.error('--id is only valid for tasks/logs')
key=Path(os.environ['DIAGNOSTIC_KEY_FILE']).read_text().strip()
ctx=ssl.create_default_context(cafile=os.environ['DIAGNOSTIC_CA_FILE'])
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self,*args,**kwargs):return None
opener=urllib.request.build_opener(NoRedirect,urllib.request.HTTPSHandler(context=ctx))
req=urllib.request.Request('https://120.48.180.80/agent/diagnostics/v1/'+a.tool,headers={'Authorization':'Bearer '+key})
req=urllib.request.Request('https://120.48.180.80/agent/diagnostics/v1/'+route,headers={'Authorization':'Bearer '+key})
try:
with opener.open(req,timeout=15) as r:
data=r.read(1024*1024+1)

View File

@ -0,0 +1,27 @@
// Run only after explicit production authorization. Credentials are environment-only.
const fs = require('fs');
function commandWorked(result) { if (!result || result.ok !== 1) throw Error('Metadata creation failed; inspect partial state before retry'); }
const spec = JSON.parse(fs.readFileSync(process.env.DIAGNOSTIC_VIEW_SPEC, 'utf8'));
const database = process.env.DIAGNOSTIC_DATABASE;
const org = process.env.DIAGNOSTIC_ORGANIZATION;
const password = process.env.DIAGNOSTIC_READER_PASSWORD;
if (!database || !org || !password || password.length < 32) throw Error('Explicit database, organization and random reader password required');
const target = db.getSiblingDB(database);
const user = 'diagnostic_gateway_reader_v1';
const role = 'diagnostic_gateway_views_v1';
const names = Object.keys(spec);
if (names.length !== 4 || names.some(n => !/^diagnostic_(tasks|items|validation|events)_v1$/.test(n))) throw Error('Unexpected view scope');
if (target.getUser(user) || target.getRole(role) || target.getCollectionInfos().some(c => names.includes(c.name))) throw Error('Name collision: inspect existing state; no overwrite or automatic retry');
for (const [name, definition] of Object.entries(spec)) {
const project = {_id:0}; for (const f of definition.fields) project[f]=1;
if (definition.errorCategory) project.errorCategory = {$switch:{branches:[
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'imu|gyro|accel',options:'i'}},then:'imu_invalid'},
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'duration|timestamp|cadence',options:'i'}},then:'video_timing_mismatch'},
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'scope',options:'i'}},then:'storage_scope'},
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'not found|missing',options:'i'}},then:'source_missing'}],default:'unknown'}};
commandWorked(target.runCommand({create:name,viewOn:definition.source,pipeline:[{$match:{organizationId:org}},{$project:project}]}));
print('created view '+name);
}
commandWorked(target.runCommand({createRole:role,privileges:names.map(collection=>({resource:{db:database,collection},actions:['find']})),roles:[]}));
commandWorked(target.runCommand({createUser:user,pwd:password,roles:[{role,db:database}]}));
print('Created view-only role and reader. No source records modified. No automatic rollback on partial failure.');

View File

@ -0,0 +1,16 @@
# Local verification — 2026-11-10
Scope: diagnostic gateway extension only. No production connection or mutation, synthetic authenticated disposable Mongo 7, no persistent volumes.
Passed:
- `go test -race ./cmd/diagnostic-gateway`: HTTP contracts, bounded cache/concurrency, malformed identifiers, error redaction, structured logs and EIP missing/stale data, including stale HTTP snapshots.
- `go vet ./cmd/diagnostic-gateway` and local binary build.
- `python3 deploy/diagnostic-gateway/test-isolated.py`: actual view/account creation; collision refusal; raw-collection read and view writes denied; organization filtering and secret omission; HTTP failed/cancelled/processing/recovered/foreign task fixtures; expanded effective privileges rejected.
- 506 synthetic tasks including 501 larger payloads: exact view query returned 1, examined 1 key and 1 document. Forty warm HTTP handler calls took approximately 6 ms total; this is a local cache check, NOT production network/capacity acceptance.
- Python client/fixture syntax and Git whitespace checks.
Initial isolated runs found and fixed mongosh `assert.commandWorked` incompatibility and unordered multi-key Go test command construction. Final isolated run passed under race detection.
Not verified: live Mongo view performance, actual BCM IAM/metric compatibility, production traffic, full historical incidents, all API pods, processor logs, source-v1 supersession, real credential revocation, Docker image runtime and long-duration resource behavior. Missing credentials do not silently enable application/admin credential reuse. Task/log snapshots may remain cached for 15 seconds after permission changes.
Production enablement is separately described in `production-approval-checklist.md`; no push or deployment performed.

View File

@ -0,0 +1,24 @@
# Apply only after explicit approval; removal revokes this additional grant.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: diagnostic-gateway-log-reader
namespace: droboticflow-fanyu
rules:
- apiGroups: [""]
resources: [pods/log]
verbs: [get]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: diagnostic-gateway-log-reader
namespace: droboticflow-fanyu
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: diagnostic-gateway-log-reader
subjects:
- kind: ServiceAccount
name: diagnostic-gateway
namespace: droboticflow-fanyu

View File

@ -0,0 +1,6 @@
{
"diagnostic_tasks_v1":{"source":"annotationTasks","fields":["id","organizationId","projectId","runId","workflowId","status","currentRevision","supplierId","assignedUserId","createdAt","updatedAt"]},
"diagnostic_items_v1":{"source":"humanItems","fields":["id","legacyTaskId","organizationId","projectId","runId","currentStageId","status","version","reviewRound","assignedUserId","supplierId","submittedAt","updatedAt","cancelledAt","processingJob.id","processingJob.kind","processingJob.status","processingJob.stage","processingJob.attempt","processingJob.nextAttemptAt","processingJob.leaseUntil","processingJob.createdAt","processingJob.updatedAt","processingJob.completedSlices","processingJob.totalSlices"]},
"diagnostic_validation_v1":{"source":"annotationIndexJobs","fields":["id","taskId","organizationId","projectId","runId","status","attempt","createdAt","nextAttemptAt","revalidateSource","sourceFingerprint","supersededSourceFingerprint"],"errorCategory":true},
"diagnostic_events_v1":{"source":"humanEvents","fields":["id","organizationId","itemId","stageId","eventType","createdAt"]}
}

View File

@ -0,0 +1,20 @@
# Diagnostic extension: production approval checklist
Local implementation only; no production grants, accounts, views, Secrets or deployment changed by this extension.
| Change requiring approval | Exact bounded scope |
|---|---|
| Mongo metadata | Database `drobodata-fanyu`, organization `org-29cd7a684faba19f630f8ad5890e18bb`; four views in `mongo-views.json`, role `diagnostic_gateway_views_v1`, user `diagnostic_gateway_reader_v1`; only `find` on those views |
| Mongo Secret | New `diagnostic-gateway-mongo-readonly`, key `uri`; generated dedicated password, never reuse application credentials |
| Log RBAC | Apply `log-permission.yaml`: only `get pods/log` in `droboticflow-fanyu`; internal grant covers namespace logs, public API filters only structured API traces |
| Cloud identity | Proposed `diagnostic_gateway_bcm_reader_v1`; metric query only; exact IAM action/resource policy and EIP InstanceId still require authoritative verification before creation; no wildcard fallback |
| Cloud Secret | New `diagnostic-gateway-bcm-readonly`, key `config.json` containing AK, SK, UserID, InstanceID, Scope=BCE_EIP; not ready to create until previous row resolved |
| Gateway rollout | New verified image; mount the two new Secrets read-only; env `DIAGNOSTIC_MONGO_URI_FILE`, `DIAGNOSTIC_ORGANIZATION`, `DIAGNOSTIC_BCM_CONFIG_FILE`, `DIAGNOSTIC_LOGS_ENABLED=true`. Keep current CPU/memory/replicas/EIP/50 Mbps unchanged. Image push and deployment separately authorized |
Use `create-mongo-views.js` only after approval, supplying `DIAGNOSTIC_VIEW_SPEC`, `DIAGNOSTIC_DATABASE`, `DIAGNOSTIC_ORGANIZATION`, `DIAGNOSTIC_READER_PASSWORD` through private local execution. It refuses name collisions and never overwrites records. Partial creation stops; inspect before retry, no automatic rollback. It creates views, role, then user. No source collection, task, annotation, index or workflow writes.
Before rollout, confirm organization and existing source indexes with bounded read-only checks. Local exact-ID explain scanned one key/document; production performance remains unverified. Do not create indexes implicitly. If views scan excessively, leave the adapter disabled and request a separate index plan.
Disable adapters before rollback; remove only the new RoleBinding/Role and Secret mounts, then the dedicated user/role/four views/Secrets after confirming no consumer. Preserve all source data. Cloud credential revocation must be tested after its policy is verified.
Current coverage limits: first two API pods only; no generic logs, no source-v1 validation reconstruction, no display-name lookup, no cloud packet-drop counters, no historical deleted-record recovery. Broader coverage is pending, not a production acceptance claim.

View File

@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Disposable authenticated Mongo. No customer data and no production connection."""
import json,os,secrets,subprocess,tempfile,time
from pathlib import Path
root=Path(__file__).resolve().parents[2]
name='diagnostic-isolated-'+secrets.token_hex(4)
database='diagnostic_test_'+secrets.token_hex(4)
password=secrets.token_hex(24)
def run(args,**kwargs):
r=subprocess.run(args,**kwargs)
if r.returncode:raise RuntimeError('isolated fixture step failed; no credentials echoed')
return r
try:
run(['docker','run','-d','--rm','--name',name,'-p','127.0.0.1::27017','mongo:7.0','--auth','--bind_ip_all'],stdout=subprocess.DEVNULL)
for _ in range(30):
r=subprocess.run(['docker','exec',name,'mongosh','--quiet','--eval','db.adminCommand({ping:1})'],capture_output=True)
if r.returncode==0:break
time.sleep(1)
else:raise RuntimeError('isolated Mongo did not start')
run(['docker','exec',name,'mongosh','--quiet','--eval',f'db.getSiblingDB("admin").createUser({{user:"fixture_admin",pwd:"{password}",roles:["root"]}})'],stdout=subprocess.DEVNULL)
fixture='''const d=db.getSiblingDB(process.env.DIAGNOSTIC_DATABASE);
for(const c of ['annotationTasks','humanItems','annotationIndexJobs','humanEvents']) {d.createCollection(c);d[c].createIndex({id:1},{unique:true,name:'id_unique'});}
d.humanItems.createIndex({organizationId:1,legacyTaskId:1});d.humanEvents.createIndex({organizationId:1,itemId:1,createdAt:-1});
for(const kind of ['failed','cancelled','processing','recovered','foreign']) {
const org=kind==='foreign'?'other-org':'test-org';const id='annotation-task-'+kind;
d.annotationTasks.insertOne({id,organizationId:org,status:kind==='recovered'?'submitted':'pending_annotation',sourceVideo:'CUSTOMER_SECRET',labelSnapshot:['CUSTOMER_SECRET']});
d.humanItems.insertOne({id:'item-'+kind,legacyTaskId:id,organizationId:org,status:kind==='cancelled'?'cancelled':'ready',subject:{payload:'CUSTOMER_SECRET'},processingJob:kind==='processing'?{status:'failed',input:{secret:'CUSTOMER_SECRET'}}:undefined});
d.annotationIndexJobs.insertOne({id,taskId:id,organizationId:org,status:kind==='failed'?'failed':'completed',error:'invalid IMU CUSTOMEr_SECRET'});
}
for(let i=0;i<501;i++)d.annotationTasks.insertOne({id:'annotation-task-noise-'+i,organizationId:'test-org',payload:'x'.repeat(16384)});
'''
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);(p/'fixture.js').write_text(fixture)
for src,dest in [(p/'fixture.js','/tmp/fixture.js'),(root/'deploy/diagnostic-gateway/mongo-views.json','/tmp/views.json'),(root/'deploy/diagnostic-gateway/create-mongo-views.js','/tmp/create.js')]:run(['docker','cp',str(src),name+':'+dest],stdout=subprocess.DEVNULL)
prefix=['docker','exec','-e','DIAGNOSTIC_DATABASE='+database,'-e','DIAGNOSTIC_ORGANIZATION=test-org','-e','DIAGNOSTIC_VIEW_SPEC=/tmp/views.json','-e','DIAGNOSTIC_READER_PASSWORD='+password,name,'mongosh','--quiet','--username','fixture_admin','--password',password,'--authenticationDatabase','admin']
run(prefix+['/tmp/fixture.js'],stdout=subprocess.DEVNULL);run(prefix+['/tmp/create.js'],stdout=subprocess.DEVNULL)
# A second run must refuse to overwrite existing view/account state.
repeat=subprocess.run(prefix+['/tmp/create.js'],capture_output=True);assert repeat.returncode!=0
explain=f'const d=db.getSiblingDB("{database}");const e=d.diagnostic_tasks_v1.find({{id:"annotation-task-failed",organizationId:"test-org"}}).explain("executionStats");print(JSON.stringify(e));'
e=json.loads(subprocess.check_output(prefix+['--eval',explain]));stats=e.get('executionStats') or next(x['$cursor']['executionStats'] for x in e['stages'] if '$cursor' in x);assert stats['totalDocsExamined']<=2,stats
print('View predicate pushdown:',json.dumps({k:stats[k] for k in ['nReturned','totalKeysExamined','totalDocsExamined']}))
port=subprocess.check_output(['docker','port',name,'27017/tcp']).decode().strip().split(':')[-1]
env=os.environ.copy();env['DIAGNOSTIC_TEST_MONGO_URI']=f'mongodb://diagnostic_gateway_reader_v1:{password}@127.0.0.1:{port}/{database}?authSource={database}'
env['DIAGNOSTIC_TEST_ADMIN_URI']=f'mongodb://fixture_admin:{password}@127.0.0.1:{port}/admin'
run(['go','test','-race','./cmd/diagnostic-gateway','-run','TestIsolatedMongoViewOnlyDiagnosis','-count=1','-v'],cwd=root,env=env)
finally:
subprocess.run(['docker','rm','-f',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)

View File

@ -54,7 +54,7 @@ python3 client.py node-usage
python3 client.py pod-usage
```
The client honors normal HTTPS proxy environment settings. It never disables certificate verification and never follows redirects carrying the key. Endpoint responses are JSON. Only the five listed tool names are supported; this is HTTP, NOT an MCP server. Existing MCP-only agents must use their HTTP tool or invoke this client.
The client honors normal HTTPS proxy environment settings. It never disables certificate verification and never follows redirects carrying the key. Endpoint responses are JSON. The five commands above work with the initial deployment; extension commands below require v2 enablement; this is HTTP, NOT an MCP server. Existing MCP-only agents must use their HTTP tool or invoke this client.
Authenticated requests have a 60-per-minute per-key limit; the ingress also applies 2 requests/second per observed source IP with a burst of 10. A 429 requires waiting before retry. Load-balancer source NAT may make the ingress limit shared by clients. Do not run stress tests against production. The gateway logs a short key fingerprint, method and path, not key values. Public/private-CA certificate expires 90 days after issuance; the operator must renew it before expiration.
@ -65,3 +65,25 @@ Share only ca.crt, the assigned key file, this guide and client.py with an appro
Business task diagnosis, bounded structured logs and EIP monitoring are being prepared. Their permission boundary and acceptance criteria are in deploy/diagnostic-gateway/readonly-data-plan.md. Dedicated Mongo view-only and BCM monitor-only identities do not yet exist. Continue using capabilities as the authority; do not call undocumented endpoints or infer that these data sources are already connected.
When historical records have been deleted or logs rotated, the correct result is insufficient evidence. A current successful task cannot explain a historical failure by itself. Synthetic replay must always be labeled synthetic.
## Local v2 implementation (production enablement pending)
The client additionally supports:
```sh
python3 client.py tasks --id annotation-task-EXACT_ID
python3 client.py logs --id 0123456789abcdef0123456789abcdef
python3 client.py egress
```
These map to GET `/diagnostics/v1/tasks/{id}`, `/diagnostics/v1/logs/api/{requestId}`, and `/diagnostics/v1/egress` beneath `/agent`. Keys and CA configuration remain unchanged. No arbitrary query parameters, Mongo operators, pod names, URLs or shell commands are accepted. Task display names are not searchable; obtain the exact task ID from the platform. Do not enumerate IDs.
Always inspect `available`, `observedAt`, `limitations` and `error`. Unconfigured/unavailable adapters return 503, malformed IDs 400, overloaded diagnostics 429. Wait at least 15 seconds before a transient retry, at most twice. Never interpret unavailable/missing data as zero load or a healthy task. Task/log results cache for 15 seconds (including failures); permission changes may take this long to affect cached evidence. Whole source-read deadline is 3 seconds, concurrency is two, Mongo pool is two, query deadline is one second and events are capped at 20.
Task evidence contains projected task/item/validation state and recent event metadata, not annotation bodies or video links. `actorEligibility` remains unknown: this endpoint does not replace the platform's permission/claim rules. Historical source-level validation and supersession are not fully reconstructed. Error categories are clues, not proven root causes.
Logs inspect only the first two API pods, the last 15 minutes, at most 200 lines/64 KiB per pod. Only structured request traces matching the supplied request ID are returned; raw logs, customer parameters and unknown lines are omitted. This is partial coverage, especially with four API replicas. A missing match cannot rule out an incident. Processor logs, rotated logs and Mongo wait traces are not included.
Egress uses fixed EIP BCM metrics, one-minute samples over the last hour, refreshed once per minute. Values are bits/second; divide by 1,000,000 for Mbps. Configured 50 Mbps is separate from measured traffic. Samples older than five minutes or collector snapshots older than two minutes are unavailable. Throttling/drop counters are not connected. Cloud metric/IAM compatibility requires live validation after approval.
For an investigation, report the exact IDs, evidence time, observed state, missing evidence and confidence. A failed validation state can support isolation diagnosis; a recovered task cannot prove its initial failure cause. Current task data cannot prove a historical browser's first-500 search bug. Do not claim that API latency alone proves Mongo or bandwidth congestion. Current local replay uses synthetic records, not historical production confirmation.