⚡ Preserve selective annotation bounds on Mongo 8.0.17
Some checks failed
web-ci / checks (push) Has been cancelled
web-ci / checks (pull_request) Has been cancelled

This commit is contained in:
d-robotics 2026-10-17 21:40:08 +08:00
parent b399f1855d
commit ad47b733e2
5 changed files with 183 additions and 22 deletions

View File

@ -13,11 +13,19 @@ It adds no authority cache, document migration, media processing or resource cha
## Implementation
The exception query is distributed into two OR branches, each with a same-stage
elemMatch. An OR inside one elemMatch prevented Mongo from bounding stage ID and
role fields even after adding a compound index. The equivalent outer OR allows
the index to narrow candidates before fetching documents. The existing domain
resolver still applies the first-stage-ID rule and exact role semantics in Go.
The first candidate (b399f185) distributed the exception query into two OR
branches. It passed Mongo 8.0.32 locally but failed the production Mongo
8.0.17-r1 gate: both the unhinted and hinted negative multikey predicates
still fetched about 22k documents. The image was not deployed.
The replacement uses 18 positive interval/null cases over stage kind and ID.
String ranges exclude annotation kind and the four stage IDs that always
resolve to review/acceptance; null cases preserve missing/null historical
fields. These are candidate bounds, not an authorization decision. The
unchanged Go resolver still enforces exact roles and first-stage-ID precedence.
Some custom review stages can enter this metadata superset, but cannot become
visible unless the existing resolver grants annotator access. No index hint,
cross-request authority cache or new production index is required.
New nonunique, nonsparse index on humanBatches:
organizationId ascending, stages.kind ascending, stages.id ascending,
@ -64,7 +72,40 @@ was opened with verify-existing, without installing the new index or changing
task data. The local Worker service was unavailable, so this verifies list/UI
reads rather than worker processing, media playback or end-to-end delivery.
## Proposed production sequence — not executed or authorized
## Actual production gate and revised evidence
The approved index was created successfully in 3,601 ms. Its exact ordered keys
and options were verified; all five previous indexes remained unchanged.
No customer document was modified. Image b399f185 was stopped before either
Deployment patch because the query gate failed. The old release journal retains
failed change-safety evidence; it must not be deployed or relabeled as accepted.
The production read-only positive-range prototype fetched 0 documents and
examined 24 index keys, versus 22,447 document fetches for the stopped candidate.
This one explain took 212 ms server time / 260 ms client time; it is not an
end-to-end response prediction or production concurrency acceptance.
The failure was reproduced locally using official Mongo 8.0.17 on isolated
loopback port 27038 before the fix: 25,000 documents fetched after index setup.
After the fix the same test fetched 0 documents / 12 keys. A matrix of 832
historical kind/ID/role combinations, including null, missing, empty and unknown
strings, verified exact resolved visibility with and without the index.
The load fixture can target only fixed loopback ports 27018 or 27038; it cannot
be redirected to a production URI.
On the same 8.0.17 fixture, 100 distinct actors completed 200/200 HTTP journeys
with p95 479.019 ms and maximum 512.659 ms; single-request p95 was 15.776 ms.
The revised Go gate still reports only the 70 known baseline failures; targeted
race/business checks passed. These local figures do not represent production
concurrency acceptance.
The revised candidate needs a new immutable image plan and explicit approval.
The index already exists, so the revised release must verify it read-only and
must not repeat the DDL. Keep the original index operation and failed query
evidence for audit. On rollback, retain the additional nonunique index.
## Original proposed production sequence (historical)
The current database bootstrap mode is verify-existing. An image deployment
alone does not install this index. Database DDL is outside scripts/release.py's

View File

@ -42,7 +42,14 @@ func TestAnnotationSidebar100ActorsMongoHTTP(t *testing.T) {
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second))
uri := os.Getenv("FANYU_SIDEBAR_MONGO_URI")
if uri == "" {
uri = "mongodb://127.0.0.1:27018/?directConnection=true"
}
if uri != "mongodb://127.0.0.1:27018/?directConnection=true" && uri != "mongodb://127.0.0.1:27038/?directConnection=true" {
t.Fatal("isolated loopback Mongo only")
}
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetServerSelectionTimeout(5*time.Second))
if err != nil {
t.Fatal(err)
}

View File

@ -0,0 +1,102 @@
package store
import (
"fmt"
"testing"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
)
// Null/missing historic fields decode as empty domain strings. Positive ranges
// must preserve those cases, unknown future names and the exact role resolver.
func TestBlindAnnotationHistoricalCandidateScopeMongo(t *testing.T) {
ctx, s := poolLocalMongo(t)
type field struct {
value interface{}
present bool
}
values := func(names ...string) []field {
result := []field{{nil, false}, {nil, true}}
for _, n := range names {
result = append(result, field{n, true})
}
return result
}
kinds := values("", "annotation", "review", "acceptance", "approval", "legacy-unknown")
ids := values("", "split_annotation", "cut_adjustment", "split_quality_review", "cut_review", "semantic_annotation", "quality_review", "semantic_review", "acceptance", "prepackage_acceptance", "custom-stage")
roles := values("", "annotator", "cutter", "quality_reviewer", "acceptance_reviewer", "historical-unknown")
docs := []interface{}{}
expected := map[string]bool{}
org := "historic-interval-org"
for _, kind := range kinds {
for _, id := range ids {
for _, role := range roles {
raw := bson.M{}
for key, f := range map[string]field{"kind": kind, "id": id, "requiredRole": role} {
if f.present {
raw[key] = f.value
}
}
bytes, err := bson.Marshal(raw)
if err != nil {
t.Fatal(err)
}
var stage domain.HumanStageDefinition
if err := bson.Unmarshal(bytes, &stage); err != nil {
t.Fatal(err)
}
name := fmt.Sprintf("historic-%04d", len(docs))
expected[name] = stage.Kind != domain.HumanStageAnnotation && domain.AnnotationRoleForHumanStage(stage) == domain.AnnotationRoleAnnotator
docs = append(docs, bson.M{"id": name, "organizationId": org, "runId": name, "gateNodeId": "gate", "stages": bson.A{raw}})
}
}
}
if _, err := s.humanBatches.InsertMany(ctx, docs); err != nil {
t.Fatal(err)
}
for _, indexed := range []bool{false, true} {
t.Run(fmt.Sprintf("indexed=%t", indexed), func(t *testing.T) {
if indexed {
if _, err := s.humanBatches.Indexes().CreateOne(ctx, blindAnnotationStageExceptionIndex()); err != nil {
t.Fatal(err)
}
}
filter, err := s.blindAnnotationMongoVisibility(ctx, org, domain.HumanItemQuery{PluginID: "annotation-qa", BlindAnnotationActorID: "actor"})
if err != nil {
t.Fatal(err)
}
// Query the generated actual item filter against one item per domain case.
// It must agree exactly after the candidate superset is resolved in Go.
col := s.db.Collection(fmt.Sprintf("syntheticItems%t", indexed))
items := make([]interface{}, 0, len(docs))
for _, doc := range docs {
b := doc.(bson.M)
raw := b["stages"].(bson.A)[0].(bson.M)
id, _ := raw["id"].(string)
items = append(items, bson.M{"id": b["id"], "batchId": b["id"], "pluginId": "annotation-qa", "currentStageId": id})
}
if _, err := col.InsertMany(ctx, items); err != nil {
t.Fatal(err)
}
cur, err := col.Find(ctx, filter)
if err != nil {
t.Fatal(err)
}
var rows []struct{ ID string }
if err := cur.All(ctx, &rows); err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, row := range rows {
seen[row.ID] = true
}
for id, want := range expected {
if seen[id] != want {
t.Errorf("case %s visibility=%t want=%t", id, seen[id], want)
}
}
t.Logf("%d historical kind/ID/role combinations preserve exact visibility", len(expected))
})
}
}

View File

@ -410,20 +410,31 @@ func blindAnnotationVisible(item domain.HumanItemRecord, batch domain.HumanBatch
// The predicate is deliberately a superset; the domain resolver and first-ID
// rule below remain authoritative, including historical role aliases.
func (s *MongoStore) blindAnnotationExceptionalBatches(ctx context.Context, org string) ([]domain.HumanBatchRecord, error) {
// Keep the two alternatives as separate elemMatch branches: an OR inside
// one elemMatch leaves stage ID/role index bounds unconstrained in Mongo.
// Each branch still binds all conditions to the same stage element.
filter := bson.M{"organizationId": normalizedOrganizationID(org), "$or": bson.A{
bson.M{"stages": bson.M{"$elemMatch": bson.M{
"kind": bson.M{"$ne": domain.HumanStageAnnotation},
"id": bson.M{"$in": bson.A{"split_annotation", domain.FanyuV5StageCutAdjustment, "split_quality_review", domain.FanyuV5StageCutReview, domain.FanyuV5StageSemanticAnnotation}},
}}},
bson.M{"stages": bson.M{"$elemMatch": bson.M{
"kind": bson.M{"$ne": domain.HumanStageAnnotation},
"id": bson.M{"$nin": bson.A{"quality_review", domain.FanyuV5StageSemanticReview, "acceptance", "prepackage_acceptance"}},
"requiredRole": bson.M{"$nin": bson.A{"quality_reviewer", "acceptance_reviewer"}},
}}},
}}
// Mongo 8.0.17 cannot bound the negative multikey predicates that newer
// versions optimize. Use positive string intervals plus null/missing cases.
// The four excluded IDs always resolve to review/acceptance; every other
// non-annotation stage is a candidate. Roles and first-ID precedence remain
// authoritative in blindAnnotationMongoVisibility below.
kinds := bson.A{bson.M{"$lt": domain.HumanStageAnnotation}, bson.M{"$gt": domain.HumanStageAnnotation}, nil}
excludedIDs := []string{"acceptance", "prepackage_acceptance", "quality_review", domain.FanyuV5StageSemanticReview}
alternatives := make(bson.A, 0, (len(excludedIDs)+2)*len(kinds))
for interval := 0; interval <= len(excludedIDs)+1; interval++ {
var id interface{}
if interval <= len(excludedIDs) {
bounds := bson.M{}
if interval > 0 {
bounds["$gt"] = excludedIDs[interval-1]
}
if interval < len(excludedIDs) {
bounds["$lt"] = excludedIDs[interval]
}
id = bounds
}
for _, kind := range kinds {
alternatives = append(alternatives, bson.M{"stages": bson.M{"$elemMatch": bson.M{"kind": kind, "id": id}}})
}
}
filter := bson.M{"organizationId": normalizedOrganizationID(org), "$or": alternatives}
cursor, err := s.humanBatches.Find(ctx, filter, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "stages.id": 1, "stages.kind": 1, "stages.requiredRole": 1}))
if err != nil {
return nil, err

View File

@ -156,7 +156,7 @@ func poolLocalMongo(t *testing.T, monitors ...*event.CommandMonitor) (context.Co
t.Skip("POOL_LOCAL_MONGO_URI required")
}
parsed, err := url.Parse(uri)
if err != nil || parsed.Scheme != "mongodb" || parsed.Host != "127.0.0.1:27018" {
if err != nil || parsed.Scheme != "mongodb" || (parsed.Host != "127.0.0.1:27018" && parsed.Host != "127.0.0.1:27038") {
t.Fatal("isolated local replica only")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)