⚡ Preserve selective annotation bounds on Mongo 8.0.17
This commit is contained in:
parent
b399f1855d
commit
ad47b733e2
@ -13,11 +13,19 @@ It adds no authority cache, document migration, media processing or resource cha
|
||||
|
||||
## Implementation
|
||||
|
||||
The exception query is distributed into two OR branches, each with a same-stage
|
||||
elemMatch. An OR inside one elemMatch prevented Mongo from bounding stage ID and
|
||||
role fields even after adding a compound index. The equivalent outer OR allows
|
||||
the index to narrow candidates before fetching documents. The existing domain
|
||||
resolver still applies the first-stage-ID rule and exact role semantics in Go.
|
||||
The first candidate (b399f185) distributed the exception query into two OR
|
||||
branches. It passed Mongo 8.0.32 locally but failed the production Mongo
|
||||
8.0.17-r1 gate: both the unhinted and hinted negative multikey predicates
|
||||
still fetched about 22k documents. The image was not deployed.
|
||||
|
||||
The replacement uses 18 positive interval/null cases over stage kind and ID.
|
||||
String ranges exclude annotation kind and the four stage IDs that always
|
||||
resolve to review/acceptance; null cases preserve missing/null historical
|
||||
fields. These are candidate bounds, not an authorization decision. The
|
||||
unchanged Go resolver still enforces exact roles and first-stage-ID precedence.
|
||||
Some custom review stages can enter this metadata superset, but cannot become
|
||||
visible unless the existing resolver grants annotator access. No index hint,
|
||||
cross-request authority cache or new production index is required.
|
||||
|
||||
New nonunique, nonsparse index on humanBatches:
|
||||
organizationId ascending, stages.kind ascending, stages.id ascending,
|
||||
@ -64,7 +72,40 @@ was opened with verify-existing, without installing the new index or changing
|
||||
task data. The local Worker service was unavailable, so this verifies list/UI
|
||||
reads rather than worker processing, media playback or end-to-end delivery.
|
||||
|
||||
## Proposed production sequence — not executed or authorized
|
||||
## Actual production gate and revised evidence
|
||||
|
||||
The approved index was created successfully in 3,601 ms. Its exact ordered keys
|
||||
and options were verified; all five previous indexes remained unchanged.
|
||||
No customer document was modified. Image b399f185 was stopped before either
|
||||
Deployment patch because the query gate failed. The old release journal retains
|
||||
failed change-safety evidence; it must not be deployed or relabeled as accepted.
|
||||
|
||||
The production read-only positive-range prototype fetched 0 documents and
|
||||
examined 24 index keys, versus 22,447 document fetches for the stopped candidate.
|
||||
This one explain took 212 ms server time / 260 ms client time; it is not an
|
||||
end-to-end response prediction or production concurrency acceptance.
|
||||
|
||||
The failure was reproduced locally using official Mongo 8.0.17 on isolated
|
||||
loopback port 27038 before the fix: 25,000 documents fetched after index setup.
|
||||
After the fix the same test fetched 0 documents / 12 keys. A matrix of 832
|
||||
historical kind/ID/role combinations, including null, missing, empty and unknown
|
||||
strings, verified exact resolved visibility with and without the index.
|
||||
The load fixture can target only fixed loopback ports 27018 or 27038; it cannot
|
||||
be redirected to a production URI.
|
||||
|
||||
On the same 8.0.17 fixture, 100 distinct actors completed 200/200 HTTP journeys
|
||||
with p95 479.019 ms and maximum 512.659 ms; single-request p95 was 15.776 ms.
|
||||
The revised Go gate still reports only the 70 known baseline failures; targeted
|
||||
race/business checks passed. These local figures do not represent production
|
||||
concurrency acceptance.
|
||||
|
||||
The revised candidate needs a new immutable image plan and explicit approval.
|
||||
The index already exists, so the revised release must verify it read-only and
|
||||
must not repeat the DDL. Keep the original index operation and failed query
|
||||
evidence for audit. On rollback, retain the additional nonunique index.
|
||||
|
||||
## Original proposed production sequence (historical)
|
||||
|
||||
|
||||
The current database bootstrap mode is verify-existing. An image deployment
|
||||
alone does not install this index. Database DDL is outside scripts/release.py's
|
||||
|
||||
@ -42,7 +42,14 @@ func TestAnnotationSidebar100ActorsMongoHTTP(t *testing.T) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second))
|
||||
uri := os.Getenv("FANYU_SIDEBAR_MONGO_URI")
|
||||
if uri == "" {
|
||||
uri = "mongodb://127.0.0.1:27018/?directConnection=true"
|
||||
}
|
||||
if uri != "mongodb://127.0.0.1:27018/?directConnection=true" && uri != "mongodb://127.0.0.1:27038/?directConnection=true" {
|
||||
t.Fatal("isolated loopback Mongo only")
|
||||
}
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetServerSelectionTimeout(5*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
102
internal/store/blind_annotation_scope_test.go
Normal file
102
internal/store/blind_annotation_scope_test.go
Normal file
@ -0,0 +1,102 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
// Null/missing historic fields decode as empty domain strings. Positive ranges
|
||||
// must preserve those cases, unknown future names and the exact role resolver.
|
||||
func TestBlindAnnotationHistoricalCandidateScopeMongo(t *testing.T) {
|
||||
ctx, s := poolLocalMongo(t)
|
||||
type field struct {
|
||||
value interface{}
|
||||
present bool
|
||||
}
|
||||
values := func(names ...string) []field {
|
||||
result := []field{{nil, false}, {nil, true}}
|
||||
for _, n := range names {
|
||||
result = append(result, field{n, true})
|
||||
}
|
||||
return result
|
||||
}
|
||||
kinds := values("", "annotation", "review", "acceptance", "approval", "legacy-unknown")
|
||||
ids := values("", "split_annotation", "cut_adjustment", "split_quality_review", "cut_review", "semantic_annotation", "quality_review", "semantic_review", "acceptance", "prepackage_acceptance", "custom-stage")
|
||||
roles := values("", "annotator", "cutter", "quality_reviewer", "acceptance_reviewer", "historical-unknown")
|
||||
docs := []interface{}{}
|
||||
expected := map[string]bool{}
|
||||
org := "historic-interval-org"
|
||||
for _, kind := range kinds {
|
||||
for _, id := range ids {
|
||||
for _, role := range roles {
|
||||
raw := bson.M{}
|
||||
for key, f := range map[string]field{"kind": kind, "id": id, "requiredRole": role} {
|
||||
if f.present {
|
||||
raw[key] = f.value
|
||||
}
|
||||
}
|
||||
bytes, err := bson.Marshal(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stage domain.HumanStageDefinition
|
||||
if err := bson.Unmarshal(bytes, &stage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := fmt.Sprintf("historic-%04d", len(docs))
|
||||
expected[name] = stage.Kind != domain.HumanStageAnnotation && domain.AnnotationRoleForHumanStage(stage) == domain.AnnotationRoleAnnotator
|
||||
docs = append(docs, bson.M{"id": name, "organizationId": org, "runId": name, "gateNodeId": "gate", "stages": bson.A{raw}})
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := s.humanBatches.InsertMany(ctx, docs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, indexed := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("indexed=%t", indexed), func(t *testing.T) {
|
||||
if indexed {
|
||||
if _, err := s.humanBatches.Indexes().CreateOne(ctx, blindAnnotationStageExceptionIndex()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
filter, err := s.blindAnnotationMongoVisibility(ctx, org, domain.HumanItemQuery{PluginID: "annotation-qa", BlindAnnotationActorID: "actor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Query the generated actual item filter against one item per domain case.
|
||||
// It must agree exactly after the candidate superset is resolved in Go.
|
||||
col := s.db.Collection(fmt.Sprintf("syntheticItems%t", indexed))
|
||||
items := make([]interface{}, 0, len(docs))
|
||||
for _, doc := range docs {
|
||||
b := doc.(bson.M)
|
||||
raw := b["stages"].(bson.A)[0].(bson.M)
|
||||
id, _ := raw["id"].(string)
|
||||
items = append(items, bson.M{"id": b["id"], "batchId": b["id"], "pluginId": "annotation-qa", "currentStageId": id})
|
||||
}
|
||||
if _, err := col.InsertMany(ctx, items); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cur, err := col.Find(ctx, filter)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var rows []struct{ ID string }
|
||||
if err := cur.All(ctx, &rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, row := range rows {
|
||||
seen[row.ID] = true
|
||||
}
|
||||
for id, want := range expected {
|
||||
if seen[id] != want {
|
||||
t.Errorf("case %s visibility=%t want=%t", id, seen[id], want)
|
||||
}
|
||||
}
|
||||
t.Logf("%d historical kind/ID/role combinations preserve exact visibility", len(expected))
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -410,20 +410,31 @@ func blindAnnotationVisible(item domain.HumanItemRecord, batch domain.HumanBatch
|
||||
// The predicate is deliberately a superset; the domain resolver and first-ID
|
||||
// rule below remain authoritative, including historical role aliases.
|
||||
func (s *MongoStore) blindAnnotationExceptionalBatches(ctx context.Context, org string) ([]domain.HumanBatchRecord, error) {
|
||||
// Keep the two alternatives as separate elemMatch branches: an OR inside
|
||||
// one elemMatch leaves stage ID/role index bounds unconstrained in Mongo.
|
||||
// Each branch still binds all conditions to the same stage element.
|
||||
filter := bson.M{"organizationId": normalizedOrganizationID(org), "$or": bson.A{
|
||||
bson.M{"stages": bson.M{"$elemMatch": bson.M{
|
||||
"kind": bson.M{"$ne": domain.HumanStageAnnotation},
|
||||
"id": bson.M{"$in": bson.A{"split_annotation", domain.FanyuV5StageCutAdjustment, "split_quality_review", domain.FanyuV5StageCutReview, domain.FanyuV5StageSemanticAnnotation}},
|
||||
}}},
|
||||
bson.M{"stages": bson.M{"$elemMatch": bson.M{
|
||||
"kind": bson.M{"$ne": domain.HumanStageAnnotation},
|
||||
"id": bson.M{"$nin": bson.A{"quality_review", domain.FanyuV5StageSemanticReview, "acceptance", "prepackage_acceptance"}},
|
||||
"requiredRole": bson.M{"$nin": bson.A{"quality_reviewer", "acceptance_reviewer"}},
|
||||
}}},
|
||||
}}
|
||||
// Mongo 8.0.17 cannot bound the negative multikey predicates that newer
|
||||
// versions optimize. Use positive string intervals plus null/missing cases.
|
||||
// The four excluded IDs always resolve to review/acceptance; every other
|
||||
// non-annotation stage is a candidate. Roles and first-ID precedence remain
|
||||
// authoritative in blindAnnotationMongoVisibility below.
|
||||
kinds := bson.A{bson.M{"$lt": domain.HumanStageAnnotation}, bson.M{"$gt": domain.HumanStageAnnotation}, nil}
|
||||
excludedIDs := []string{"acceptance", "prepackage_acceptance", "quality_review", domain.FanyuV5StageSemanticReview}
|
||||
alternatives := make(bson.A, 0, (len(excludedIDs)+2)*len(kinds))
|
||||
for interval := 0; interval <= len(excludedIDs)+1; interval++ {
|
||||
var id interface{}
|
||||
if interval <= len(excludedIDs) {
|
||||
bounds := bson.M{}
|
||||
if interval > 0 {
|
||||
bounds["$gt"] = excludedIDs[interval-1]
|
||||
}
|
||||
if interval < len(excludedIDs) {
|
||||
bounds["$lt"] = excludedIDs[interval]
|
||||
}
|
||||
id = bounds
|
||||
}
|
||||
for _, kind := range kinds {
|
||||
alternatives = append(alternatives, bson.M{"stages": bson.M{"$elemMatch": bson.M{"kind": kind, "id": id}}})
|
||||
}
|
||||
}
|
||||
filter := bson.M{"organizationId": normalizedOrganizationID(org), "$or": alternatives}
|
||||
cursor, err := s.humanBatches.Find(ctx, filter, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "stages.id": 1, "stages.kind": 1, "stages.requiredRole": 1}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -156,7 +156,7 @@ func poolLocalMongo(t *testing.T, monitors ...*event.CommandMonitor) (context.Co
|
||||
t.Skip("POOL_LOCAL_MONGO_URI required")
|
||||
}
|
||||
parsed, err := url.Parse(uri)
|
||||
if err != nil || parsed.Scheme != "mongodb" || parsed.Host != "127.0.0.1:27018" {
|
||||
if err != nil || parsed.Scheme != "mongodb" || (parsed.Host != "127.0.0.1:27018" && parsed.Host != "127.0.0.1:27038") {
|
||||
t.Fatal("isolated local replica only")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user