forked from longtao.wu/droboticflow
🔎 Add bounded read-only diagnostics and isolated permission verification
This commit is contained in:
parent
e150a8a24c
commit
a98127c42d
@ -1,8 +1,11 @@
|
||||
FROM golang:1.24 AS build
|
||||
WORKDIR /build
|
||||
COPY cmd/diagnostic-gateway/main.go .
|
||||
RUN CGO_ENABLED=0 go build -trimpath -o /gateway main.go
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY cmd/diagnostic-gateway ./cmd/diagnostic-gateway
|
||||
RUN CGO_ENABLED=0 go build -trimpath -o /gateway ./cmd/diagnostic-gateway
|
||||
FROM scratch
|
||||
COPY --from=build /gateway /gateway
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
USER 65532:65532
|
||||
ENTRYPOINT ["/gateway"]
|
||||
|
||||
55
cmd/diagnostic-gateway/configure.go
Normal file
55
cmd/diagnostic-gateway/configure.go
Normal file
@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func configureDiagnostics(g *gateway, client *http.Client, ns, tokenFile string) {
|
||||
d := newDiagnostics()
|
||||
g.diagnostics = d
|
||||
if os.Getenv("DIAGNOSTIC_LOGS_ENABLED") == "true" {
|
||||
d.logs = logLoader(client, "https://kubernetes.default.svc", ns, func() (string, error) { b, e := os.ReadFile(tokenFile); return strings.TrimSpace(string(b)), e })
|
||||
}
|
||||
if file := os.Getenv("DIAGNOSTIC_MONGO_URI_FILE"); file != "" {
|
||||
raw, e := os.ReadFile(file)
|
||||
if e != nil {
|
||||
log.Fatal("diagnostic Mongo credential unavailable")
|
||||
}
|
||||
org := os.Getenv("DIAGNOSTIC_ORGANIZATION")
|
||||
if org == "" {
|
||||
log.Fatal("diagnostic organization required")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
c, e := mongo.Connect(ctx, options.Client().ApplyURI(strings.TrimSpace(string(raw))).SetMaxPoolSize(2).SetRetryReads(false).SetRetryWrites(false).SetServerSelectionTimeout(2*time.Second))
|
||||
if e != nil {
|
||||
log.Fatal("diagnostic Mongo connection failed")
|
||||
}
|
||||
if checkPrivileges(ctx, c, "drobodata-fanyu") != nil {
|
||||
log.Fatal("diagnostic Mongo identity must have find-only access to the four views")
|
||||
}
|
||||
d.task = mongoTaskLoader(c, "drobodata-fanyu", org)
|
||||
}
|
||||
if file := os.Getenv("DIAGNOSTIC_BCM_CONFIG_FILE"); file != "" {
|
||||
collect, e := newBCMCollector(file)
|
||||
if e != nil {
|
||||
log.Fatal("invalid BCM configuration")
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
v := collect()
|
||||
d.mu.Lock()
|
||||
d.egress = v
|
||||
d.mu.Unlock()
|
||||
time.Sleep(time.Minute)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
206
cmd/diagnostic-gateway/diagnostics.go
Normal file
206
cmd/diagnostic-gateway/diagnostics.go
Normal file
@ -0,0 +1,206 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type diagnosticResult struct {
|
||||
Available bool `json:"available"`
|
||||
ObservedAt string `json:"observedAt"`
|
||||
Evidence any `json:"evidence,omitempty"`
|
||||
Limitations []string `json:"limitations,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type diagnosticLoader func(context.Context, string) (diagnosticResult, error)
|
||||
type cachedDiagnostic struct {
|
||||
at time.Time
|
||||
result diagnosticResult
|
||||
}
|
||||
type diagnostics struct {
|
||||
task, logs diagnosticLoader
|
||||
mu sync.Mutex
|
||||
cache map[string]cachedDiagnostic
|
||||
busy chan struct{}
|
||||
egress diagnosticResult
|
||||
}
|
||||
|
||||
var taskIDPattern = regexp.MustCompile(`^annotation-task-[A-Za-z0-9_-]{1,100}$`)
|
||||
var requestIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
|
||||
|
||||
func newDiagnostics() *diagnostics {
|
||||
return &diagnostics{cache: map[string]cachedDiagnostic{}, busy: make(chan struct{}, 2), egress: diagnosticResult{Error: "bcm_not_configured"}}
|
||||
}
|
||||
func (d *diagnostics) serve(w http.ResponseWriter, r *http.Request) bool {
|
||||
path := strings.TrimPrefix(r.URL.Path, "/diagnostics/v1/")
|
||||
var loader diagnosticLoader
|
||||
arg := ""
|
||||
switch {
|
||||
case path == "egress":
|
||||
d.mu.Lock()
|
||||
v := d.egress
|
||||
d.mu.Unlock()
|
||||
if stamp, err := time.Parse(time.RFC3339, v.ObservedAt); v.Available && (err != nil || time.Since(stamp) > 2*time.Minute) {
|
||||
v = diagnosticResult{Error: "bcm_snapshot_stale"}
|
||||
}
|
||||
writeDiagnostic(w, v)
|
||||
return true
|
||||
case strings.HasPrefix(path, "tasks/"):
|
||||
arg = strings.TrimPrefix(path, "tasks/")
|
||||
if !taskIDPattern.MatchString(arg) {
|
||||
http.Error(w, `{"error":"invalid_task_id"}`, 400)
|
||||
return true
|
||||
}
|
||||
loader = d.task
|
||||
case strings.HasPrefix(path, "logs/api/"):
|
||||
arg = strings.TrimPrefix(path, "logs/api/")
|
||||
if !requestIDPattern.MatchString(arg) {
|
||||
http.Error(w, `{"error":"invalid_request_id"}`, 400)
|
||||
return true
|
||||
}
|
||||
loader = d.logs
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if loader == nil {
|
||||
writeDiagnostic(w, diagnosticResult{Error: "adapter_not_configured"})
|
||||
return true
|
||||
}
|
||||
d.mu.Lock()
|
||||
cached, ok := d.cache[path]
|
||||
d.mu.Unlock()
|
||||
if ok && time.Since(cached.at) < 15*time.Second {
|
||||
writeDiagnostic(w, cached.result)
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case d.busy <- struct{}{}:
|
||||
defer func() { <-d.busy }()
|
||||
default:
|
||||
http.Error(w, `{"error":"diagnostic_busy"}`, 429)
|
||||
return true
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
v, e := loader(ctx, arg)
|
||||
if e != nil {
|
||||
v = diagnosticResult{Error: "upstream_unavailable_or_timed_out"}
|
||||
}
|
||||
v.ObservedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
d.mu.Lock()
|
||||
if len(d.cache) >= 128 {
|
||||
d.cache = map[string]cachedDiagnostic{}
|
||||
}
|
||||
d.cache[path] = cachedDiagnostic{time.Now(), v}
|
||||
d.mu.Unlock()
|
||||
writeDiagnostic(w, v)
|
||||
return true
|
||||
}
|
||||
func writeDiagnostic(w http.ResponseWriter, v diagnosticResult) {
|
||||
if !v.Available {
|
||||
w.WriteHeader(503)
|
||||
}
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// Only already structured request traces are eligible. Unknown text never leaves the gateway.
|
||||
var tracePattern = regexp.MustCompile(`request_trace id=([a-f0-9]{32}) kind=(login|diagnostic|slow)(?: method=(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) path=\S+)? status=([0-9]{3}) elapsed_ms=([0-9]{1,9})(?:\s|$)`)
|
||||
|
||||
func parseTraces(raw, requestID string) []map[string]string {
|
||||
out := []map[string]string{}
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
m := tracePattern.FindStringSubmatch(line)
|
||||
if len(m) == 0 || m[1] != requestID {
|
||||
continue
|
||||
}
|
||||
stamp := strings.Fields(line)[0]
|
||||
if _, e := time.Parse(time.RFC3339Nano, stamp); e != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, map[string]string{"timestamp": stamp, "requestId": m[1], "kind": m[2], "method": m[3], "status": m[4], "elapsedMs": m[5]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
func getBounded(ctx context.Context, client *http.Client, base, path, token string, limit int) ([]byte, error) {
|
||||
req, e := http.NewRequestWithContext(ctx, "GET", base+path, nil)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
resp, e := client.Do(req)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, errors.New("upstream unavailable")
|
||||
}
|
||||
b, e := io.ReadAll(io.LimitReader(resp.Body, int64(limit+1)))
|
||||
if e != nil || len(b) > limit {
|
||||
return nil, errors.New("response limit")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
func logLoader(client *http.Client, base, ns string, token func() (string, error)) diagnosticLoader {
|
||||
return func(ctx context.Context, id string) (diagnosticResult, error) {
|
||||
t, e := token()
|
||||
if e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
b, e := getBounded(ctx, client, base, "/api/v1/namespaces/"+url.PathEscape(ns)+"/pods?labelSelector=app%3Ddroboticflow-api&limit=10", t, 256*1024)
|
||||
if e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
var list struct {
|
||||
Items []struct {
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"metadata"`
|
||||
Spec struct {
|
||||
Containers []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"containers"`
|
||||
} `json:"spec"`
|
||||
} `json:"items"`
|
||||
Metadata struct {
|
||||
Continue string `json:"continue"`
|
||||
} `json:"metadata"`
|
||||
}
|
||||
if json.Unmarshal(b, &list) != nil {
|
||||
return diagnosticResult{}, errors.New("invalid pods")
|
||||
}
|
||||
evidence := []map[string]string{}
|
||||
n := 0
|
||||
for _, pod := range list.Items {
|
||||
if n == 2 {
|
||||
break
|
||||
}
|
||||
if len(pod.Spec.Containers) == 0 {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
path := "/api/v1/namespaces/" + url.PathEscape(ns) + "/pods/" + url.PathEscape(pod.Metadata.Name) + "/log?timestamps=true&sinceSeconds=900&tailLines=200&limitBytes=65536&container=" + url.QueryEscape(pod.Spec.Containers[0].Name)
|
||||
raw, e := getBounded(ctx, client, base, path, t, 65536)
|
||||
if e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
for _, line := range parseTraces(string(raw), id) {
|
||||
line["pod"] = pod.Metadata.Name
|
||||
evidence = append(evidence, line)
|
||||
}
|
||||
}
|
||||
limitations := []string{"Only structured request_trace records; raw logs and route parameters omitted", "Last 15 minutes, at most 200 lines/64KiB per pod; logs may be rotated", "No matching trace is insufficient evidence, not proof of success"}
|
||||
if len(list.Items) > 2 || list.Metadata.Continue != "" {
|
||||
limitations = append(limitations, "Partial pod coverage: only first two API pods inspected")
|
||||
}
|
||||
return diagnosticResult{Available: true, Evidence: evidence, Limitations: limitations}, nil
|
||||
}
|
||||
}
|
||||
132
cmd/diagnostic-gateway/diagnostics_test.go
Normal file
132
cmd/diagnostic-gateway/diagnostics_test.go
Normal file
@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testGateway(d *diagnostics) *gateway {
|
||||
return &gateway{diagnostics: d, keys: [][32]byte{sha256.Sum256([]byte("test-key"))}, snapshots: map[string]snapshot{}}
|
||||
}
|
||||
func request(g *gateway, path string) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest("GET", path, nil)
|
||||
r.Header.Set("Authorization", "Bearer test-key")
|
||||
w := httptest.NewRecorder()
|
||||
g.ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
func TestDiagnosticHTTPContracts(t *testing.T) {
|
||||
d := newDiagnostics()
|
||||
g := testGateway(d)
|
||||
if w := request(g, "/diagnostics/v1/egress"); w.Code != 503 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
if w := request(g, "/diagnostics/v1/tasks/annotation-task-test"); w.Code != 503 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
for _, path := range []string{"tasks/$where", "tasks/annotation-task-x?filter=bad", "logs/api/../../../secrets", "logs/api/not-a-request-id"} {
|
||||
if w := request(g, "/diagnostics/v1/"+path); w.Code != 400 && w.Code != 404 {
|
||||
t.Fatalf("%s %d", path, w.Code)
|
||||
}
|
||||
}
|
||||
var calls atomic.Int32
|
||||
d.task = func(context.Context, string) (diagnosticResult, error) {
|
||||
calls.Add(1)
|
||||
return diagnosticResult{Available: true, Evidence: map[string]string{"finding": "task_currently_cancelled"}}, nil
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
if w := request(g, "/diagnostics/v1/tasks/annotation-task-test"); w.Code != 200 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatal("polling repeated source reads")
|
||||
}
|
||||
d.task = func(context.Context, string) (diagnosticResult, error) {
|
||||
return diagnosticResult{}, errors.New("mongodb://user:secret@host customer payload")
|
||||
}
|
||||
w := request(g, "/diagnostics/v1/tasks/annotation-task-other")
|
||||
if w.Code != 503 || strings.Contains(w.Body.String(), "secret") {
|
||||
t.Fatal("upstream details leaked")
|
||||
}
|
||||
}
|
||||
func TestTraceFiltering(t *testing.T) {
|
||||
id := strings.Repeat("a", 32)
|
||||
raw := "2026-11-10T01:00:00Z request_trace id=" + id + " kind=slow method=GET path=/api/private/customer-secret status=504 elapsed_ms=12000 token=secret\npassword=secret\n"
|
||||
out := parseTraces(raw, id)
|
||||
b, _ := json.Marshal(out)
|
||||
if len(out) != 1 || strings.Contains(string(b), "secret") || strings.Contains(string(b), "private") {
|
||||
t.Fatal(string(b))
|
||||
}
|
||||
if len(parseTraces(raw, strings.Repeat("b", 32))) != 0 {
|
||||
t.Fatal("cross-request trace")
|
||||
}
|
||||
}
|
||||
func TestDiagnosticConcurrencyAndCancellation(t *testing.T) {
|
||||
d := newDiagnostics()
|
||||
g := testGateway(d)
|
||||
entered := make(chan struct{}, 2)
|
||||
release := make(chan struct{})
|
||||
d.task = func(ctx context.Context, id string) (diagnosticResult, error) {
|
||||
entered <- struct{}{}
|
||||
select {
|
||||
case <-release:
|
||||
return diagnosticResult{Available: true}, nil
|
||||
case <-ctx.Done():
|
||||
return diagnosticResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
done := make(chan struct{}, 2)
|
||||
for _, id := range []string{"a", "b"} {
|
||||
go func(id string) { request(g, "/diagnostics/v1/tasks/annotation-task-"+id); done <- struct{}{} }(id)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("not started")
|
||||
}
|
||||
}
|
||||
if w := request(g, "/diagnostics/v1/tasks/annotation-task-c"); w.Code != 429 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
close(release)
|
||||
<-done
|
||||
<-done
|
||||
}
|
||||
func TestLogLoaderBoundedAndNoRawOutput(t *testing.T) {
|
||||
id := strings.Repeat("c", 32)
|
||||
var calls atomic.Int32
|
||||
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
if strings.HasSuffix(r.URL.Path, "/pods") {
|
||||
if r.URL.Query().Get("labelSelector") != "app=droboticflow-api" {
|
||||
t.Error("arbitrary selector")
|
||||
}
|
||||
w.Write([]byte(`{"items":[{"metadata":{"name":"api-1"},"spec":{"containers":[{"name":"api"}]}},{"metadata":{"name":"api-2"},"spec":{"containers":[{"name":"api"}]}},{"metadata":{"name":"api-3"}}]}`))
|
||||
return
|
||||
}
|
||||
if r.URL.Query().Get("follow") != "" || r.URL.Query().Get("limitBytes") != "65536" {
|
||||
t.Error("unbounded log query")
|
||||
}
|
||||
w.Write([]byte("2026-11-10T01:00:00Z request_trace id=" + id + " kind=diagnostic status=500 elapsed_ms=3000\nSECRET raw payload"))
|
||||
}))
|
||||
defer s.Close()
|
||||
loader := logLoader(s.Client(), s.URL, "test", func() (string, error) { return "token", nil })
|
||||
v, e := loader(context.Background(), id)
|
||||
if e != nil || !v.Available || calls.Load() != 3 {
|
||||
t.Fatalf("%v %v", v, e)
|
||||
}
|
||||
b, _ := json.Marshal(v)
|
||||
if strings.Contains(string(b), "SECRET") || !strings.Contains(string(b), "Partial pod coverage") {
|
||||
t.Fatal(string(b))
|
||||
}
|
||||
}
|
||||
69
cmd/diagnostic-gateway/egress.go
Normal file
69
cmd/diagnostic-gateway/egress.go
Normal file
@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/baidubce/bce-sdk-go/bce"
|
||||
"github.com/baidubce/bce-sdk-go/services/bcm"
|
||||
"github.com/baidubce/bce-sdk-go/services/bcm/model"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type bcmConfiguration struct{ AK, SK, UserID, InstanceID, Scope string }
|
||||
|
||||
func newBCMCollector(file string) (func() diagnosticResult, error) {
|
||||
raw, e := os.ReadFile(file)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
var cfg bcmConfiguration
|
||||
if json.Unmarshal(raw, &cfg) != nil || cfg.AK == "" || cfg.SK == "" || cfg.UserID == "" || cfg.InstanceID == "" || cfg.Scope != "BCE_EIP" {
|
||||
return nil, errors.New("invalid fixed EIP configuration")
|
||||
}
|
||||
c, e := bcm.NewClient(cfg.AK, cfg.SK, "https://bcm.bj.baidubce.com")
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
c.Config.ConnectionTimeoutInMillis = 3000
|
||||
c.Config.Retry = bce.NewNoRetryPolicy()
|
||||
return func() diagnosticResult { return collectEgress(c, cfg, time.Now().UTC()) }, nil
|
||||
}
|
||||
|
||||
type metricReader interface {
|
||||
GetMetricData(*model.GetMetricDataRequest) (*model.GetMetricDataResponse, error)
|
||||
}
|
||||
|
||||
func collectEgress(c metricReader, cfg bcmConfiguration, now time.Time) diagnosticResult {
|
||||
evidence := map[string]any{"instanceId": cfg.InstanceID, "periodSeconds": 60, "configuredLimitMbps": 50, "windowStart": now.Add(-time.Hour).Format(time.RFC3339), "windowEnd": now.Format(time.RFC3339)}
|
||||
for _, metric := range []string{"WebOutBitsPerSecond", "WebInBitsPerSecond"} {
|
||||
r, e := c.GetMetricData(&model.GetMetricDataRequest{UserId: cfg.UserID, Scope: cfg.Scope, MetricName: metric, Dimensions: map[string]string{"InstanceId": cfg.InstanceID}, Statistics: []string{"average", "maximum"}, PeriodInSecond: 60, StartTime: now.Add(-time.Hour).Format(time.RFC3339), EndTime: now.Format(time.RFC3339)})
|
||||
if e != nil || r == nil || r.Code != "OK" {
|
||||
return diagnosticResult{Error: "bcm_unavailable", ObservedAt: now.Format(time.RFC3339)}
|
||||
}
|
||||
if len(r.DataPoints) == 0 || len(r.DataPoints) > 61 {
|
||||
return diagnosticResult{Error: "bcm_missing_or_excessive_points", ObservedAt: now.Format(time.RFC3339)}
|
||||
}
|
||||
latest := time.Time{}
|
||||
for _, point := range r.DataPoints {
|
||||
if point == nil {
|
||||
return diagnosticResult{Error: "bcm_invalid_points"}
|
||||
}
|
||||
stamp, e := time.Parse(time.RFC3339, point.Timestamp)
|
||||
if e != nil || stamp.After(now.Add(time.Minute)) || stamp.Before(now.Add(-time.Hour-time.Minute)) {
|
||||
return diagnosticResult{Error: "bcm_invalid_timestamp"}
|
||||
}
|
||||
if point.Average == nil || point.Maximum == nil || *point.Average < 0 || *point.Maximum < 0 {
|
||||
return diagnosticResult{Error: "bcm_invalid_value"}
|
||||
}
|
||||
if stamp.After(latest) {
|
||||
latest = stamp
|
||||
}
|
||||
}
|
||||
if now.Sub(latest) > 5*time.Minute {
|
||||
return diagnosticResult{Error: "bcm_stale_data", ObservedAt: now.Format(time.RFC3339)}
|
||||
}
|
||||
evidence[metric] = r.DataPoints
|
||||
}
|
||||
return diagnosticResult{Available: true, ObservedAt: now.Format(time.RFC3339), Evidence: evidence, Limitations: []string{"bps, one-minute averages/maxima; inspect source timestamps for delayed samples", "Bandwidth configuration is not measured utilization; no packet-drop metrics connected yet"}}
|
||||
}
|
||||
55
cmd/diagnostic-gateway/egress_test.go
Normal file
55
cmd/diagnostic-gateway/egress_test.go
Normal file
@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/baidubce/bce-sdk-go/services/bcm/model"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeMetrics struct {
|
||||
response *model.GetMetricDataResponse
|
||||
calls []*model.GetMetricDataRequest
|
||||
}
|
||||
|
||||
func (f *fakeMetrics) GetMetricData(r *model.GetMetricDataRequest) (*model.GetMetricDataResponse, error) {
|
||||
f.calls = append(f.calls, r)
|
||||
return f.response, nil
|
||||
}
|
||||
func TestEgressEvidenceAndMissingStaleData(t *testing.T) {
|
||||
now := time.Date(2026, 11, 10, 10, 0, 0, 0, time.UTC)
|
||||
v := 12000000.0
|
||||
f := &fakeMetrics{response: &model.GetMetricDataResponse{Code: "OK", DataPoints: []*model.DataPoints{{Timestamp: now.Add(-time.Minute).Format(time.RFC3339), Average: &v, Maximum: &v}}}}
|
||||
cfg := bcmConfiguration{InstanceID: "fixed-instance", Scope: "BCE_EIP", UserID: "account"}
|
||||
got := collectEgress(f, cfg, now)
|
||||
if !got.Available || len(f.calls) != 2 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
for _, r := range f.calls {
|
||||
if r.Dimensions["InstanceId"] != "fixed-instance" || r.PeriodInSecond != 60 {
|
||||
t.Fatal("metric scope escaped")
|
||||
}
|
||||
}
|
||||
f.response.DataPoints[0].Timestamp = now.Add(-10 * time.Minute).Format(time.RFC3339)
|
||||
if got = collectEgress(f, cfg, now); got.Available || got.Error != "bcm_stale_data" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
f.response.DataPoints = nil
|
||||
if got = collectEgress(f, cfg, now); got.Available {
|
||||
t.Fatal("missing data treated as zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEgressHTTPRejectsStaleSnapshot(t *testing.T) {
|
||||
d := newDiagnostics()
|
||||
for _, sample := range []struct {
|
||||
age time.Duration
|
||||
status int
|
||||
}{{time.Second, 200}, {3 * time.Minute, 503}} {
|
||||
d.egress = diagnosticResult{Available: true, ObservedAt: time.Now().Add(-sample.age).UTC().Format(time.RFC3339), Evidence: map[string]int{"bps": 12000000}}
|
||||
w := httptest.NewRecorder()
|
||||
if !d.serve(w, httptest.NewRequest("GET", "/diagnostics/v1/egress", nil)) || w.Code != sample.status {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -25,12 +25,13 @@ type snapshot struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type gateway struct {
|
||||
mu sync.RWMutex
|
||||
snapshots map[string]snapshot
|
||||
keys [][32]byte
|
||||
rateMu sync.Mutex
|
||||
windows map[[32]byte]time.Time
|
||||
hits map[[32]byte]int
|
||||
diagnostics *diagnostics
|
||||
mu sync.RWMutex
|
||||
snapshots map[string]snapshot
|
||||
keys [][32]byte
|
||||
rateMu sync.Mutex
|
||||
windows map[[32]byte]time.Time
|
||||
hits map[[32]byte]int
|
||||
}
|
||||
type resource struct {
|
||||
Metadata struct {
|
||||
@ -154,8 +155,11 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"error":"query_parameters_not_supported"}`, 400)
|
||||
return
|
||||
}
|
||||
if g.diagnostics != nil && g.diagnostics.serve(w, r) {
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/diagnostics/v1/capabilities" {
|
||||
io.WriteString(w, `{"version":1,"transport":"http-json","tools":["nodes","pods","node-usage","pod-usage"],"refreshSeconds":30,"unavailable":["egress","task-diagnosis","logs","source-code","mcp"],"mutations":false}`)
|
||||
json.NewEncoder(w).Encode(map[string]any{"version": 2, "transport": "http-json", "tools": []string{"nodes", "pods", "node-usage", "pod-usage"}, "diagnosticRoutes": []string{"tasks/{taskId}", "logs/api/{requestId}", "egress"}, "configured": map[string]bool{"tasks": g.diagnostics != nil && g.diagnostics.task != nil, "logs": g.diagnostics != nil && g.diagnostics.logs != nil}, "refreshSeconds": 30, "unavailable": []string{"source-code", "mcp"}, "mutations": false})
|
||||
return
|
||||
}
|
||||
key := strings.TrimPrefix(r.URL.Path, "/diagnostics/v1/")
|
||||
@ -210,6 +214,7 @@ func main() {
|
||||
log.Fatal("invalid cluster CA")
|
||||
}
|
||||
client := &http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, MaxConnsPerHost: 1}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
configureDiagnostics(g, client, ns, sa+"token")
|
||||
paths := map[string]string{"nodes": "/api/v1/nodes?limit=500", "pods": "/api/v1/namespaces/" + ns + "/pods?limit=500", "node-usage": "/apis/metrics.k8s.io/v1beta1/nodes", "pod-usage": "/apis/metrics.k8s.io/v1beta1/namespaces/" + ns + "/pods"}
|
||||
for k := range paths {
|
||||
g.snapshots[k] = snapshot{Error: "warming_up"}
|
||||
|
||||
85
cmd/diagnostic-gateway/mongo_isolated_test.go
Normal file
85
cmd/diagnostic-gateway/mongo_isolated_test.go
Normal file
@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIsolatedMongoViewOnlyDiagnosis(t *testing.T) {
|
||||
uri := os.Getenv("DIAGNOSTIC_TEST_MONGO_URI")
|
||||
if uri == "" {
|
||||
t.Skip("run deploy/diagnostic-gateway/test-isolated.py for authenticated fixture")
|
||||
}
|
||||
u, e := url.Parse(uri)
|
||||
if e != nil || u.Hostname() != "127.0.0.1" || !strings.HasPrefix(u.Path, "/diagnostic_test_") {
|
||||
t.Fatal("isolated localhost database required")
|
||||
}
|
||||
database := strings.TrimPrefix(u.Path, "/")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(2))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer c.Disconnect(ctx)
|
||||
if e = checkPrivileges(ctx, c, database); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
db := c.Database(database)
|
||||
if e = db.Collection("annotationTasks").FindOne(ctx, bson.M{}).Err(); e == nil {
|
||||
t.Fatal("reader can access source collection")
|
||||
}
|
||||
if _, e = db.Collection("diagnostic_tasks_v1").InsertOne(ctx, bson.M{"id": "illegal"}); e == nil {
|
||||
t.Fatal("reader can write")
|
||||
}
|
||||
var view bson.M
|
||||
if e = db.Collection("diagnostic_tasks_v1").FindOne(ctx, bson.M{"id": "annotation-task-failed"}).Decode(&view); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
raw, _ := json.Marshal(view)
|
||||
if strings.Contains(string(raw), "CUSTOMER_SECRET") {
|
||||
t.Fatal("view leaked payload")
|
||||
}
|
||||
if n, e := db.Collection("diagnostic_tasks_v1").CountDocuments(ctx, bson.M{"organizationId": "other-org"}); e != nil || n != 0 {
|
||||
t.Fatal("organization leakage", e, n)
|
||||
}
|
||||
d := newDiagnostics()
|
||||
d.task = mongoTaskLoader(c, database, "test-org")
|
||||
g := testGateway(d)
|
||||
cases := []struct{ id, needle string }{{"failed", "task_validation_record_failed"}, {"cancelled", "task_currently_cancelled"}, {"processing", "processing_job_failed"}, {"recovered", "current state cannot prove historical failure cause"}, {"foreign", "taskFound\":false"}}
|
||||
for _, tc := range cases {
|
||||
w := request(g, "/diagnostics/v1/tasks/annotation-task-"+tc.id)
|
||||
if w.Code != 200 || !strings.Contains(w.Body.String(), tc.needle) || strings.Contains(w.Body.String(), "CUSTOMER_SECRET") {
|
||||
t.Fatalf("%s: %d %s", tc.id, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
start := time.Now()
|
||||
for i := 0; i < 40; i++ {
|
||||
w := request(g, "/diagnostics/v1/tasks/annotation-task-failed")
|
||||
if w.Code != 200 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
}
|
||||
t.Logf("40 warm HTTP diagnostics: %s (bounded cache)", time.Since(start))
|
||||
// Revocation invalidates source access after cached evidence expires; clear cache to test the next source read.
|
||||
adminURI := os.Getenv("DIAGNOSTIC_TEST_ADMIN_URI")
|
||||
a, e := mongo.Connect(ctx, options.Client().ApplyURI(adminURI))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer a.Disconnect(ctx)
|
||||
if e = a.Database(database).RunCommand(ctx, bson.D{{Key: "grantPrivilegesToRole", Value: "diagnostic_gateway_views_v1"}, {Key: "privileges", Value: bson.A{bson.M{"resource": bson.M{"db": database, "collection": "annotationTasks"}, "actions": bson.A{"find"}}}}}).Err(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = checkPrivileges(ctx, c, database); e == nil {
|
||||
t.Fatal("expanded privileges accepted")
|
||||
}
|
||||
}
|
||||
107
cmd/diagnostic-gateway/task_diagnostics.go
Normal file
107
cmd/diagnostic-gateway/task_diagnostics.go
Normal file
@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"time"
|
||||
)
|
||||
|
||||
var diagnosticViews = []string{"diagnostic_tasks_v1", "diagnostic_items_v1", "diagnostic_validation_v1", "diagnostic_events_v1"}
|
||||
|
||||
func checkPrivileges(ctx context.Context, client *mongo.Client, dbName string) error {
|
||||
var out struct {
|
||||
AuthInfo struct {
|
||||
Users []bson.M `bson:"authenticatedUsers"`
|
||||
Privileges []struct {
|
||||
Resource struct {
|
||||
DB string `bson:"db"`
|
||||
Collection string `bson:"collection"`
|
||||
} `bson:"resource"`
|
||||
Actions []string `bson:"actions"`
|
||||
} `bson:"authenticatedUserPrivileges"`
|
||||
} `bson:"authInfo"`
|
||||
}
|
||||
if e := client.Database("admin").RunCommand(ctx, bson.D{{Key: "connectionStatus", Value: 1}, {Key: "showPrivileges", Value: true}}).Decode(&out); e != nil {
|
||||
return e
|
||||
}
|
||||
if len(out.AuthInfo.Users) != 1 || len(out.AuthInfo.Privileges) == 0 {
|
||||
return errors.New("dedicated authenticated identity required")
|
||||
}
|
||||
for _, p := range out.AuthInfo.Privileges {
|
||||
allowed := false
|
||||
for _, v := range diagnosticViews {
|
||||
if p.Resource.DB == dbName && p.Resource.Collection == v {
|
||||
allowed = true
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
return errors.New("privilege outside diagnostic views")
|
||||
}
|
||||
for _, a := range p.Actions {
|
||||
if a != "find" {
|
||||
return errors.New("non-find privilege")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func mongoTaskLoader(client *mongo.Client, dbName, org string) diagnosticLoader {
|
||||
return func(ctx context.Context, id string) (diagnosticResult, error) {
|
||||
if !taskIDPattern.MatchString(id) || org == "" {
|
||||
return diagnosticResult{}, errors.New("scope required")
|
||||
}
|
||||
// Recheck effective grants so a later privilege expansion fails closed.
|
||||
if e := checkPrivileges(ctx, client, dbName); e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
db := client.Database(dbName)
|
||||
one := func(collection string, filter bson.M) (bson.M, error) {
|
||||
var result bson.M
|
||||
e := db.Collection(collection).FindOne(ctx, filter, options.FindOne().SetMaxTime(time.Second)).Decode(&result)
|
||||
if e == mongo.ErrNoDocuments {
|
||||
return nil, nil
|
||||
}
|
||||
return result, e
|
||||
}
|
||||
task, e := one("diagnostic_tasks_v1", bson.M{"id": id, "organizationId": org})
|
||||
if e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
if task == nil {
|
||||
return diagnosticResult{Available: true, Evidence: bson.M{"taskFound": false}, Limitations: []string{"Not found in authorized scope; no conclusion about other organizations"}}, nil
|
||||
}
|
||||
item, e := one("diagnostic_items_v1", bson.M{"legacyTaskId": id, "organizationId": org})
|
||||
if e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
validation, e := one("diagnostic_validation_v1", bson.M{"id": id, "organizationId": org})
|
||||
if e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
events := []bson.M{}
|
||||
if item != nil {
|
||||
cur, e := db.Collection("diagnostic_events_v1").Find(ctx, bson.M{"itemId": item["id"], "organizationId": org}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(20).SetMaxTime(time.Second))
|
||||
if e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
if e = cur.All(ctx, &events); e != nil {
|
||||
return diagnosticResult{}, e
|
||||
}
|
||||
}
|
||||
findings := []string{}
|
||||
if item != nil && item["status"] == "cancelled" {
|
||||
findings = append(findings, "task_currently_cancelled")
|
||||
}
|
||||
if validation != nil && validation["status"] == "failed" {
|
||||
findings = append(findings, "task_validation_record_failed")
|
||||
}
|
||||
if job, ok := item["processingJob"].(bson.M); ok && job["status"] == "failed" {
|
||||
findings = append(findings, "processing_job_failed")
|
||||
}
|
||||
return diagnosticResult{Available: true, Evidence: bson.M{"task": task, "item": item, "taskValidation": validation, "recentEvents": events, "findings": findings, "actorEligibility": "unknown"}, Limitations: []string{"State evidence only; no independent canClaim calculation", "Task-ID validation only; source-v1 certificate supersession is not fully diagnosed", "At most 20 recent events; current state cannot prove historical failure cause", "Deleted history cannot be reconstructed"}}, nil
|
||||
}
|
||||
}
|
||||
@ -3,14 +3,22 @@
|
||||
import argparse,json,os,ssl,urllib.request,urllib.error
|
||||
from pathlib import Path
|
||||
p=argparse.ArgumentParser()
|
||||
p.add_argument('tool',choices=['capabilities','nodes','pods','node-usage','pod-usage'])
|
||||
p.add_argument('tool',choices=['capabilities','nodes','pods','node-usage','pod-usage','tasks','logs','egress'])
|
||||
p.add_argument('--id', help='Task ID for tasks or 32-character request ID for logs')
|
||||
a=p.parse_args()
|
||||
import re
|
||||
route=a.tool
|
||||
if a.tool in ('tasks','logs'):
|
||||
pattern=r'annotation-task-[A-Za-z0-9_-]{1,100}' if a.tool=='tasks' else r'[a-f0-9]{32}'
|
||||
if not a.id or not re.fullmatch(pattern,a.id):p.error('A valid --id is required')
|
||||
route=('tasks/' if a.tool=='tasks' else 'logs/api/')+a.id
|
||||
elif a.id:p.error('--id is only valid for tasks/logs')
|
||||
key=Path(os.environ['DIAGNOSTIC_KEY_FILE']).read_text().strip()
|
||||
ctx=ssl.create_default_context(cafile=os.environ['DIAGNOSTIC_CA_FILE'])
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self,*args,**kwargs):return None
|
||||
opener=urllib.request.build_opener(NoRedirect,urllib.request.HTTPSHandler(context=ctx))
|
||||
req=urllib.request.Request('https://120.48.180.80/agent/diagnostics/v1/'+a.tool,headers={'Authorization':'Bearer '+key})
|
||||
req=urllib.request.Request('https://120.48.180.80/agent/diagnostics/v1/'+route,headers={'Authorization':'Bearer '+key})
|
||||
try:
|
||||
with opener.open(req,timeout=15) as r:
|
||||
data=r.read(1024*1024+1)
|
||||
|
||||
27
deploy/diagnostic-gateway/create-mongo-views.js
Normal file
27
deploy/diagnostic-gateway/create-mongo-views.js
Normal file
@ -0,0 +1,27 @@
|
||||
// Run only after explicit production authorization. Credentials are environment-only.
|
||||
const fs = require('fs');
|
||||
function commandWorked(result) { if (!result || result.ok !== 1) throw Error('Metadata creation failed; inspect partial state before retry'); }
|
||||
const spec = JSON.parse(fs.readFileSync(process.env.DIAGNOSTIC_VIEW_SPEC, 'utf8'));
|
||||
const database = process.env.DIAGNOSTIC_DATABASE;
|
||||
const org = process.env.DIAGNOSTIC_ORGANIZATION;
|
||||
const password = process.env.DIAGNOSTIC_READER_PASSWORD;
|
||||
if (!database || !org || !password || password.length < 32) throw Error('Explicit database, organization and random reader password required');
|
||||
const target = db.getSiblingDB(database);
|
||||
const user = 'diagnostic_gateway_reader_v1';
|
||||
const role = 'diagnostic_gateway_views_v1';
|
||||
const names = Object.keys(spec);
|
||||
if (names.length !== 4 || names.some(n => !/^diagnostic_(tasks|items|validation|events)_v1$/.test(n))) throw Error('Unexpected view scope');
|
||||
if (target.getUser(user) || target.getRole(role) || target.getCollectionInfos().some(c => names.includes(c.name))) throw Error('Name collision: inspect existing state; no overwrite or automatic retry');
|
||||
for (const [name, definition] of Object.entries(spec)) {
|
||||
const project = {_id:0}; for (const f of definition.fields) project[f]=1;
|
||||
if (definition.errorCategory) project.errorCategory = {$switch:{branches:[
|
||||
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'imu|gyro|accel',options:'i'}},then:'imu_invalid'},
|
||||
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'duration|timestamp|cadence',options:'i'}},then:'video_timing_mismatch'},
|
||||
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'scope',options:'i'}},then:'storage_scope'},
|
||||
{case:{$regexMatch:{input:{$ifNull:['$error','']},regex:'not found|missing',options:'i'}},then:'source_missing'}],default:'unknown'}};
|
||||
commandWorked(target.runCommand({create:name,viewOn:definition.source,pipeline:[{$match:{organizationId:org}},{$project:project}]}));
|
||||
print('created view '+name);
|
||||
}
|
||||
commandWorked(target.runCommand({createRole:role,privileges:names.map(collection=>({resource:{db:database,collection},actions:['find']})),roles:[]}));
|
||||
commandWorked(target.runCommand({createUser:user,pwd:password,roles:[{role,db:database}]}));
|
||||
print('Created view-only role and reader. No source records modified. No automatic rollback on partial failure.');
|
||||
16
deploy/diagnostic-gateway/local-verification.md
Normal file
16
deploy/diagnostic-gateway/local-verification.md
Normal file
@ -0,0 +1,16 @@
|
||||
# Local verification — 2026-11-10
|
||||
|
||||
Scope: diagnostic gateway extension only. No production connection or mutation, synthetic authenticated disposable Mongo 7, no persistent volumes.
|
||||
|
||||
Passed:
|
||||
- `go test -race ./cmd/diagnostic-gateway`: HTTP contracts, bounded cache/concurrency, malformed identifiers, error redaction, structured logs and EIP missing/stale data, including stale HTTP snapshots.
|
||||
- `go vet ./cmd/diagnostic-gateway` and local binary build.
|
||||
- `python3 deploy/diagnostic-gateway/test-isolated.py`: actual view/account creation; collision refusal; raw-collection read and view writes denied; organization filtering and secret omission; HTTP failed/cancelled/processing/recovered/foreign task fixtures; expanded effective privileges rejected.
|
||||
- 506 synthetic tasks including 501 larger payloads: exact view query returned 1, examined 1 key and 1 document. Forty warm HTTP handler calls took approximately 6 ms total; this is a local cache check, NOT production network/capacity acceptance.
|
||||
- Python client/fixture syntax and Git whitespace checks.
|
||||
|
||||
Initial isolated runs found and fixed mongosh `assert.commandWorked` incompatibility and unordered multi-key Go test command construction. Final isolated run passed under race detection.
|
||||
|
||||
Not verified: live Mongo view performance, actual BCM IAM/metric compatibility, production traffic, full historical incidents, all API pods, processor logs, source-v1 supersession, real credential revocation, Docker image runtime and long-duration resource behavior. Missing credentials do not silently enable application/admin credential reuse. Task/log snapshots may remain cached for 15 seconds after permission changes.
|
||||
|
||||
Production enablement is separately described in `production-approval-checklist.md`; no push or deployment performed.
|
||||
24
deploy/diagnostic-gateway/log-permission.yaml
Normal file
24
deploy/diagnostic-gateway/log-permission.yaml
Normal file
@ -0,0 +1,24 @@
|
||||
# Apply only after explicit approval; removal revokes this additional grant.
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: diagnostic-gateway-log-reader
|
||||
namespace: droboticflow-fanyu
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: [pods/log]
|
||||
verbs: [get]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: diagnostic-gateway-log-reader
|
||||
namespace: droboticflow-fanyu
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: diagnostic-gateway-log-reader
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: diagnostic-gateway
|
||||
namespace: droboticflow-fanyu
|
||||
6
deploy/diagnostic-gateway/mongo-views.json
Normal file
6
deploy/diagnostic-gateway/mongo-views.json
Normal file
@ -0,0 +1,6 @@
|
||||
{
|
||||
"diagnostic_tasks_v1":{"source":"annotationTasks","fields":["id","organizationId","projectId","runId","workflowId","status","currentRevision","supplierId","assignedUserId","createdAt","updatedAt"]},
|
||||
"diagnostic_items_v1":{"source":"humanItems","fields":["id","legacyTaskId","organizationId","projectId","runId","currentStageId","status","version","reviewRound","assignedUserId","supplierId","submittedAt","updatedAt","cancelledAt","processingJob.id","processingJob.kind","processingJob.status","processingJob.stage","processingJob.attempt","processingJob.nextAttemptAt","processingJob.leaseUntil","processingJob.createdAt","processingJob.updatedAt","processingJob.completedSlices","processingJob.totalSlices"]},
|
||||
"diagnostic_validation_v1":{"source":"annotationIndexJobs","fields":["id","taskId","organizationId","projectId","runId","status","attempt","createdAt","nextAttemptAt","revalidateSource","sourceFingerprint","supersededSourceFingerprint"],"errorCategory":true},
|
||||
"diagnostic_events_v1":{"source":"humanEvents","fields":["id","organizationId","itemId","stageId","eventType","createdAt"]}
|
||||
}
|
||||
20
deploy/diagnostic-gateway/production-approval-checklist.md
Normal file
20
deploy/diagnostic-gateway/production-approval-checklist.md
Normal file
@ -0,0 +1,20 @@
|
||||
# Diagnostic extension: production approval checklist
|
||||
|
||||
Local implementation only; no production grants, accounts, views, Secrets or deployment changed by this extension.
|
||||
|
||||
| Change requiring approval | Exact bounded scope |
|
||||
|---|---|
|
||||
| Mongo metadata | Database `drobodata-fanyu`, organization `org-29cd7a684faba19f630f8ad5890e18bb`; four views in `mongo-views.json`, role `diagnostic_gateway_views_v1`, user `diagnostic_gateway_reader_v1`; only `find` on those views |
|
||||
| Mongo Secret | New `diagnostic-gateway-mongo-readonly`, key `uri`; generated dedicated password, never reuse application credentials |
|
||||
| Log RBAC | Apply `log-permission.yaml`: only `get pods/log` in `droboticflow-fanyu`; internal grant covers namespace logs, public API filters only structured API traces |
|
||||
| Cloud identity | Proposed `diagnostic_gateway_bcm_reader_v1`; metric query only; exact IAM action/resource policy and EIP InstanceId still require authoritative verification before creation; no wildcard fallback |
|
||||
| Cloud Secret | New `diagnostic-gateway-bcm-readonly`, key `config.json` containing AK, SK, UserID, InstanceID, Scope=BCE_EIP; not ready to create until previous row resolved |
|
||||
| Gateway rollout | New verified image; mount the two new Secrets read-only; env `DIAGNOSTIC_MONGO_URI_FILE`, `DIAGNOSTIC_ORGANIZATION`, `DIAGNOSTIC_BCM_CONFIG_FILE`, `DIAGNOSTIC_LOGS_ENABLED=true`. Keep current CPU/memory/replicas/EIP/50 Mbps unchanged. Image push and deployment separately authorized |
|
||||
|
||||
Use `create-mongo-views.js` only after approval, supplying `DIAGNOSTIC_VIEW_SPEC`, `DIAGNOSTIC_DATABASE`, `DIAGNOSTIC_ORGANIZATION`, `DIAGNOSTIC_READER_PASSWORD` through private local execution. It refuses name collisions and never overwrites records. Partial creation stops; inspect before retry, no automatic rollback. It creates views, role, then user. No source collection, task, annotation, index or workflow writes.
|
||||
|
||||
Before rollout, confirm organization and existing source indexes with bounded read-only checks. Local exact-ID explain scanned one key/document; production performance remains unverified. Do not create indexes implicitly. If views scan excessively, leave the adapter disabled and request a separate index plan.
|
||||
|
||||
Disable adapters before rollback; remove only the new RoleBinding/Role and Secret mounts, then the dedicated user/role/four views/Secrets after confirming no consumer. Preserve all source data. Cloud credential revocation must be tested after its policy is verified.
|
||||
|
||||
Current coverage limits: first two API pods only; no generic logs, no source-v1 validation reconstruction, no display-name lookup, no cloud packet-drop counters, no historical deleted-record recovery. Broader coverage is pending, not a production acceptance claim.
|
||||
47
deploy/diagnostic-gateway/test-isolated.py
Normal file
47
deploy/diagnostic-gateway/test-isolated.py
Normal file
@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Disposable authenticated Mongo. No customer data and no production connection."""
|
||||
import json,os,secrets,subprocess,tempfile,time
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[2]
|
||||
name='diagnostic-isolated-'+secrets.token_hex(4)
|
||||
database='diagnostic_test_'+secrets.token_hex(4)
|
||||
password=secrets.token_hex(24)
|
||||
def run(args,**kwargs):
|
||||
r=subprocess.run(args,**kwargs)
|
||||
if r.returncode:raise RuntimeError('isolated fixture step failed; no credentials echoed')
|
||||
return r
|
||||
try:
|
||||
run(['docker','run','-d','--rm','--name',name,'-p','127.0.0.1::27017','mongo:7.0','--auth','--bind_ip_all'],stdout=subprocess.DEVNULL)
|
||||
for _ in range(30):
|
||||
r=subprocess.run(['docker','exec',name,'mongosh','--quiet','--eval','db.adminCommand({ping:1})'],capture_output=True)
|
||||
if r.returncode==0:break
|
||||
time.sleep(1)
|
||||
else:raise RuntimeError('isolated Mongo did not start')
|
||||
run(['docker','exec',name,'mongosh','--quiet','--eval',f'db.getSiblingDB("admin").createUser({{user:"fixture_admin",pwd:"{password}",roles:["root"]}})'],stdout=subprocess.DEVNULL)
|
||||
fixture='''const d=db.getSiblingDB(process.env.DIAGNOSTIC_DATABASE);
|
||||
for(const c of ['annotationTasks','humanItems','annotationIndexJobs','humanEvents']) {d.createCollection(c);d[c].createIndex({id:1},{unique:true,name:'id_unique'});}
|
||||
d.humanItems.createIndex({organizationId:1,legacyTaskId:1});d.humanEvents.createIndex({organizationId:1,itemId:1,createdAt:-1});
|
||||
for(const kind of ['failed','cancelled','processing','recovered','foreign']) {
|
||||
const org=kind==='foreign'?'other-org':'test-org';const id='annotation-task-'+kind;
|
||||
d.annotationTasks.insertOne({id,organizationId:org,status:kind==='recovered'?'submitted':'pending_annotation',sourceVideo:'CUSTOMER_SECRET',labelSnapshot:['CUSTOMER_SECRET']});
|
||||
d.humanItems.insertOne({id:'item-'+kind,legacyTaskId:id,organizationId:org,status:kind==='cancelled'?'cancelled':'ready',subject:{payload:'CUSTOMER_SECRET'},processingJob:kind==='processing'?{status:'failed',input:{secret:'CUSTOMER_SECRET'}}:undefined});
|
||||
d.annotationIndexJobs.insertOne({id,taskId:id,organizationId:org,status:kind==='failed'?'failed':'completed',error:'invalid IMU CUSTOMEr_SECRET'});
|
||||
}
|
||||
for(let i=0;i<501;i++)d.annotationTasks.insertOne({id:'annotation-task-noise-'+i,organizationId:'test-org',payload:'x'.repeat(16384)});
|
||||
'''
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);(p/'fixture.js').write_text(fixture)
|
||||
for src,dest in [(p/'fixture.js','/tmp/fixture.js'),(root/'deploy/diagnostic-gateway/mongo-views.json','/tmp/views.json'),(root/'deploy/diagnostic-gateway/create-mongo-views.js','/tmp/create.js')]:run(['docker','cp',str(src),name+':'+dest],stdout=subprocess.DEVNULL)
|
||||
prefix=['docker','exec','-e','DIAGNOSTIC_DATABASE='+database,'-e','DIAGNOSTIC_ORGANIZATION=test-org','-e','DIAGNOSTIC_VIEW_SPEC=/tmp/views.json','-e','DIAGNOSTIC_READER_PASSWORD='+password,name,'mongosh','--quiet','--username','fixture_admin','--password',password,'--authenticationDatabase','admin']
|
||||
run(prefix+['/tmp/fixture.js'],stdout=subprocess.DEVNULL);run(prefix+['/tmp/create.js'],stdout=subprocess.DEVNULL)
|
||||
# A second run must refuse to overwrite existing view/account state.
|
||||
repeat=subprocess.run(prefix+['/tmp/create.js'],capture_output=True);assert repeat.returncode!=0
|
||||
explain=f'const d=db.getSiblingDB("{database}");const e=d.diagnostic_tasks_v1.find({{id:"annotation-task-failed",organizationId:"test-org"}}).explain("executionStats");print(JSON.stringify(e));'
|
||||
e=json.loads(subprocess.check_output(prefix+['--eval',explain]));stats=e.get('executionStats') or next(x['$cursor']['executionStats'] for x in e['stages'] if '$cursor' in x);assert stats['totalDocsExamined']<=2,stats
|
||||
print('View predicate pushdown:',json.dumps({k:stats[k] for k in ['nReturned','totalKeysExamined','totalDocsExamined']}))
|
||||
port=subprocess.check_output(['docker','port',name,'27017/tcp']).decode().strip().split(':')[-1]
|
||||
env=os.environ.copy();env['DIAGNOSTIC_TEST_MONGO_URI']=f'mongodb://diagnostic_gateway_reader_v1:{password}@127.0.0.1:{port}/{database}?authSource={database}'
|
||||
env['DIAGNOSTIC_TEST_ADMIN_URI']=f'mongodb://fixture_admin:{password}@127.0.0.1:{port}/admin'
|
||||
run(['go','test','-race','./cmd/diagnostic-gateway','-run','TestIsolatedMongoViewOnlyDiagnosis','-count=1','-v'],cwd=root,env=env)
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
@ -54,7 +54,7 @@ python3 client.py node-usage
|
||||
python3 client.py pod-usage
|
||||
```
|
||||
|
||||
The client honors normal HTTPS proxy environment settings. It never disables certificate verification and never follows redirects carrying the key. Endpoint responses are JSON. Only the five listed tool names are supported; this is HTTP, NOT an MCP server. Existing MCP-only agents must use their HTTP tool or invoke this client.
|
||||
The client honors normal HTTPS proxy environment settings. It never disables certificate verification and never follows redirects carrying the key. Endpoint responses are JSON. The five commands above work with the initial deployment; extension commands below require v2 enablement; this is HTTP, NOT an MCP server. Existing MCP-only agents must use their HTTP tool or invoke this client.
|
||||
|
||||
Authenticated requests have a 60-per-minute per-key limit; the ingress also applies 2 requests/second per observed source IP with a burst of 10. A 429 requires waiting before retry. Load-balancer source NAT may make the ingress limit shared by clients. Do not run stress tests against production. The gateway logs a short key fingerprint, method and path, not key values. Public/private-CA certificate expires 90 days after issuance; the operator must renew it before expiration.
|
||||
|
||||
@ -65,3 +65,25 @@ Share only ca.crt, the assigned key file, this guide and client.py with an appro
|
||||
Business task diagnosis, bounded structured logs and EIP monitoring are being prepared. Their permission boundary and acceptance criteria are in deploy/diagnostic-gateway/readonly-data-plan.md. Dedicated Mongo view-only and BCM monitor-only identities do not yet exist. Continue using capabilities as the authority; do not call undocumented endpoints or infer that these data sources are already connected.
|
||||
|
||||
When historical records have been deleted or logs rotated, the correct result is insufficient evidence. A current successful task cannot explain a historical failure by itself. Synthetic replay must always be labeled synthetic.
|
||||
|
||||
## Local v2 implementation (production enablement pending)
|
||||
|
||||
The client additionally supports:
|
||||
|
||||
```sh
|
||||
python3 client.py tasks --id annotation-task-EXACT_ID
|
||||
python3 client.py logs --id 0123456789abcdef0123456789abcdef
|
||||
python3 client.py egress
|
||||
```
|
||||
|
||||
These map to GET `/diagnostics/v1/tasks/{id}`, `/diagnostics/v1/logs/api/{requestId}`, and `/diagnostics/v1/egress` beneath `/agent`. Keys and CA configuration remain unchanged. No arbitrary query parameters, Mongo operators, pod names, URLs or shell commands are accepted. Task display names are not searchable; obtain the exact task ID from the platform. Do not enumerate IDs.
|
||||
|
||||
Always inspect `available`, `observedAt`, `limitations` and `error`. Unconfigured/unavailable adapters return 503, malformed IDs 400, overloaded diagnostics 429. Wait at least 15 seconds before a transient retry, at most twice. Never interpret unavailable/missing data as zero load or a healthy task. Task/log results cache for 15 seconds (including failures); permission changes may take this long to affect cached evidence. Whole source-read deadline is 3 seconds, concurrency is two, Mongo pool is two, query deadline is one second and events are capped at 20.
|
||||
|
||||
Task evidence contains projected task/item/validation state and recent event metadata, not annotation bodies or video links. `actorEligibility` remains unknown: this endpoint does not replace the platform's permission/claim rules. Historical source-level validation and supersession are not fully reconstructed. Error categories are clues, not proven root causes.
|
||||
|
||||
Logs inspect only the first two API pods, the last 15 minutes, at most 200 lines/64 KiB per pod. Only structured request traces matching the supplied request ID are returned; raw logs, customer parameters and unknown lines are omitted. This is partial coverage, especially with four API replicas. A missing match cannot rule out an incident. Processor logs, rotated logs and Mongo wait traces are not included.
|
||||
|
||||
Egress uses fixed EIP BCM metrics, one-minute samples over the last hour, refreshed once per minute. Values are bits/second; divide by 1,000,000 for Mbps. Configured 50 Mbps is separate from measured traffic. Samples older than five minutes or collector snapshots older than two minutes are unavailable. Throttling/drop counters are not connected. Cloud metric/IAM compatibility requires live validation after approval.
|
||||
|
||||
For an investigation, report the exact IDs, evidence time, observed state, missing evidence and confidence. A failed validation state can support isolation diagnosis; a recovered task cannot prove its initial failure cause. Current task data cannot prove a historical browser's first-500 search bug. Do not claim that API latency alone proves Mongo or bandwidth congestion. Current local replay uses synthetic records, not historical production confirmation.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user