🧪 Add scoped live performance audit and frame verification
This commit is contained in:
parent
cc0202d40d
commit
2a19c943ef
@ -10,8 +10,11 @@ import (
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -40,6 +43,86 @@ func TestSeedAnnotationPerformanceSandbox(t *testing.T) {
|
||||
source := store.NewMongoStore(client.Database(os.Getenv("MONGO_DB")))
|
||||
target := store.NewMongoStore(client.Database(name))
|
||||
mode := os.Getenv("FANYU_PERF_MODE")
|
||||
if mode == "audit" {
|
||||
db := client.Database(name)
|
||||
indexes, e := db.Collection("annotationTimelineIndexes").CountDocuments(ctx, bson.M{})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cur, e := db.Collection("humanItems").Find(ctx, bson.M{"projectId": "project-perf-" + nonce}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "legacyTaskId": 1, "currentStageId": 1, "processingJob.status": 1, "processingJob.stage": 1, "processingJob.attempt": 1, "processingJob.error": 1}))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var rows []bson.M
|
||||
if e = cur.All(ctx, &rows); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
out, _ := json.Marshal(map[string]any{"indexes": indexes, "items": rows})
|
||||
fmt.Println("PERFORMANCE_AUDIT " + string(out))
|
||||
return
|
||||
}
|
||||
if mode == "verify" {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
tasks, e := target.ListAnnotationTasks(ctx, "org-29cd7a684faba19f630f8ad5890e18bb", "project-perf-"+nonce)
|
||||
if e != nil || len(tasks) == 0 {
|
||||
t.Fatal("no performance tasks")
|
||||
}
|
||||
sort.Slice(tasks, func(i, j int) bool { return tasks[i].ID < tasks[j].ID })
|
||||
task := tasks[0]
|
||||
run, e := target.GetRun(ctx, task.RunID)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var chosen *domain.ExecutionArtifact
|
||||
for i := range run.Artifacts {
|
||||
a := &run.Artifacts[i]
|
||||
if boolValue(a.Data["metadataOnly"]) {
|
||||
chosen = a
|
||||
}
|
||||
}
|
||||
if chosen == nil || !strings.HasPrefix(chosen.Path, "/workspace/system/runs/run-perf-"+nonce+"-") {
|
||||
t.Fatal("missing scoped metadata-only generation")
|
||||
}
|
||||
manifest, e := readVerifiedAnnotationJSON(filepath.Join(chosen.Path, "revision-manifest.json"), chosen.Metadata.SHA256)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
frames, files := 0, 0
|
||||
for _, raw := range listValue(manifest["items"]) {
|
||||
entry := mapValue(raw)
|
||||
directory := filepath.Join(chosen.Path, stringValue(entry["directory"]))
|
||||
for kind, rawFile := range mapValue(entry["files"]) {
|
||||
f := mapValue(rawFile)
|
||||
path := filepath.Join(directory, stringValue(f["name"]))
|
||||
if !strings.HasPrefix(path, chosen.Path+"/") {
|
||||
t.Fatal("unsafe member path")
|
||||
}
|
||||
sha, size, e := sha256File(path)
|
||||
if e != nil || sha != stringValue(f["sha256"]) || size != int64Value(f["bytes"]) {
|
||||
t.Fatal("file digest mismatch", kind, e)
|
||||
}
|
||||
files++
|
||||
if kind == "leftVideo" || kind == "rightVideo" {
|
||||
n, e := ffprobeFrameCount(ctx, path)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
ptsKey := "leftPts"
|
||||
if kind == "rightVideo" {
|
||||
ptsKey = "rightPts"
|
||||
}
|
||||
if n != intValue(mapValue(mapValue(entry["files"])[ptsKey])["rowCount"]) {
|
||||
t.Fatal("video/PTS mismatch")
|
||||
}
|
||||
frames += n
|
||||
}
|
||||
}
|
||||
}
|
||||
out, _ := json.Marshal(map[string]any{"taskId": task.ID, "files": files, "stereoFrames": frames, "metadataOnly": true, "sourceDuration": task.SourceDurationSeconds})
|
||||
fmt.Println("PERFORMANCE_VERIFIED " + string(out))
|
||||
return
|
||||
}
|
||||
if mode == "disable-users" {
|
||||
users, e := target.ListUsers(ctx)
|
||||
if e != nil {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user