🧪 test(human-in-the-loop): add full convenience store human-in-the-loop lifecycle and continuous training tests

This commit is contained in:
eust-w 2026-08-20 10:02:29 +08:00
parent c22b8eb555
commit 0e7ca34268
4 changed files with 747 additions and 5 deletions

View File

@ -74,7 +74,8 @@ def run(
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
hyperparams_fn = helpers.get("hyperparams") if helpers else None
params = hyperparams_fn(node_data) if hyperparams_fn else (node_data.get("hyperparameters") or node_data)
task_mode = str(params.get("TASK_MODE", "continuous_train")).strip().lower() or "continuous_train"
base_model = str(params.get("BASE_MODEL", "Qwen/Qwen3-VL-8B-Instruct")).strip()
initial_checkpoint = str(params.get("INITIAL_CHECKPOINT", "")).strip()

View File

@ -809,7 +809,12 @@ func (server *server) handleSliceRevisionCreate(writer http.ResponseWriter, requ
_ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: "split_annotation", Round: item.ReviewRound, Status: domain.HumanItemSubmitted, AssigneeUserID: principal.User.ID, SubmissionRef: revision.ID})
_ = server.appendHumanEvent(request.Context(), *item, "submitted", principal.User.ID, map[string]any{"submissionRef": revision.ID, "sliceRevision": revision.Revision})
}
_ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "slice_submitted", Revision: revision.Revision, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO()})
_ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{
ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID,
TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID,
EventType: "slice_submitted", Revision: revision.Revision, TemplateVersion: task.TemplateVersion,
DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO(),
})
renderJSON(writer, http.StatusCreated, map[string]any{"task": task, "sliceRevision": revision})
}
@ -863,8 +868,13 @@ func (server *server) handleAnnotationRevisionCreate(writer http.ResponseWriter,
renderValidationError(writer, issues...)
return
}
isRework := task.CurrentRevision > 0 || task.Status == domain.AnnotationTaskStatusRework
revisionNumber := task.CurrentRevision + 1
existingRevisions, err := server.store.ListAnnotationRevisions(request.Context(), task.OrganizationID, task.ID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
isRework := len(existingRevisions) > 0 || task.Status == domain.AnnotationTaskStatusRework
revisionNumber := len(existingRevisions) + 1
revision := domain.AnnotationRevisionRecord{
ID: domain.NewID("annotation-revision"), OrganizationID: task.OrganizationID, TaskID: task.ID,
Revision: revisionNumber, TemplateVersion: task.TemplateVersion, Segments: payload.Segments, ObjectHints: payload.ObjectHints,
@ -1098,7 +1108,7 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r
task.Status = domain.AnnotationTaskStatusRework
task.AssignedUserID = ""
} else {
task.CurrentRevision = 1
task.CurrentRevision = 0
task.Status = domain.AnnotationTaskStatusPendingAnnotation
task.AssignedUserID = ""
}

View File

@ -0,0 +1,632 @@
package api
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func TestHumanInTheLoopConvenienceStoreFullLifecycleE2E(t *testing.T) {
ctx := context.Background()
st := store.NewMemoryStore()
// 1. Setup Users with distinctive roles
adminUser := domain.UserRecord{
ID: "usr-admin-1",
OrganizationID: domain.DefaultOrganizationID,
Email: "admin@fanyu.com",
Name: "周天如 (管理员)",
Role: domain.UserRoleAdmin,
Status: domain.UserStatusActive,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
annotatorUser := domain.UserRecord{
ID: "usr-annotator-1",
OrganizationID: domain.DefaultOrganizationID,
Email: "annotator@vendor-a.com",
Name: "程晓明 (标注员)",
Role: domain.UserRoleAnnotator,
Status: domain.UserStatusActive,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
qualityUser := domain.UserRecord{
ID: "usr-quality-1",
OrganizationID: domain.DefaultOrganizationID,
Email: "qc@vendor-a.com",
Name: "质检员王工",
Role: domain.UserRoleQualityReviewer,
Status: domain.UserStatusActive,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
acceptanceUser := domain.UserRecord{
ID: "usr-acceptance-1",
OrganizationID: domain.DefaultOrganizationID,
Email: "acceptance@fanyu.com",
Name: "验收专家李工",
Role: domain.UserRoleAcceptanceReviewer,
Status: domain.UserStatusActive,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
for _, u := range []domain.UserRecord{adminUser, annotatorUser, qualityUser, acceptanceUser} {
if err := st.UpsertUser(ctx, u); err != nil {
t.Fatalf("upsert user %s: %v", u.ID, err)
}
}
// 2. Setup Project, Vendor, Workgroup and Template with 22-class taxonomy
project := domain.ProjectRecord{
ID: "proj-fanyu-convenience",
OrganizationID: domain.DefaultOrganizationID,
Name: "便利店第一视角视频人在环切分与标注项目 (泛宇)",
Description: "10w小时级第一视角视频数据流切分、标注与持续训练",
Status: "active",
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := st.UpsertProject(ctx, project); err != nil {
t.Fatalf("upsert project: %v", err)
}
workgroup := domain.AnnotationWorkgroupRecord{
ID: "wg-vendor-a-team1",
OrganizationID: domain.DefaultOrganizationID,
Name: "泛宇交付一组",
VendorID: "vendor-fanyu-a",
Description: "便利店视频数据标注",
Status: "active",
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := st.UpsertAnnotationWorkgroup(ctx, workgroup); err != nil {
t.Fatalf("upsert workgroup: %v", err)
}
// Add memberships
for _, m := range []domain.ProjectAnnotationMembershipRecord{
{
ID: "mem-1",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
UserID: annotatorUser.ID,
WorkgroupID: workgroup.ID,
VendorID: workgroup.VendorID,
AssignmentRole: domain.AnnotationRoleAnnotator,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
},
{
ID: "mem-2",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
UserID: qualityUser.ID,
WorkgroupID: workgroup.ID,
VendorID: workgroup.VendorID,
AssignmentRole: domain.AnnotationRoleQualityReviewer,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
},
{
ID: "mem-3",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
UserID: acceptanceUser.ID,
WorkgroupID: workgroup.ID,
VendorID: workgroup.VendorID,
AssignmentRole: domain.AnnotationRoleAcceptanceReviewer,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
},
} {
if err := st.UpsertProjectAnnotationMembership(ctx, m); err != nil {
t.Fatalf("upsert membership: %v", err)
}
}
template := domain.AnnotationTemplateRecord{
ID: "tpl-convenience-22",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
Name: "便利店第一视角视频行为多标签标注模板",
Version: 1,
Status: "published",
PublishedAt: time.Now().UTC().Format(time.RFC3339),
Labels: []domain.SemanticLabelDefinition{
{ID: "01", Name: "鲜食处理", Color: "#E53935"},
{ID: "02", Name: "关东煮备料和清洁", Color: "#FB8C00"},
{ID: "03", Name: "关东煮出餐", Color: "#FDD835"},
{ID: "04", Name: "蒸点与糕点备货", Color: "#43A047"},
{ID: "05", Name: "蒸点陈列与出餐", Color: "#00ACC1"},
{ID: "06", Name: "热食加工与出餐", Color: "#1E88E5"},
{ID: "07", Name: "即食商品陈列", Color: "#5E35B1"},
{ID: "08", Name: "常温商品补货", Color: "#D81B60"},
{ID: "09", Name: "冷藏饮料补货", Color: "#8E24AA"},
{ID: "10", Name: "货架深度清洁", Color: "#3949AB"},
{ID: "11", Name: "价签与标识维护", Color: "#039BE5"},
{ID: "12", Name: "包装准备", Color: "#00897B"},
{ID: "13", Name: "包装封装与材料整理", Color: "#7CB342"},
{ID: "14", Name: "收银扫码", Color: "#C0CA33"},
{ID: "15", Name: "顾客交付与内部协作", Color: "#F4511E"},
{ID: "16", Name: "烟草、槟榔和小商品销售", Color: "#6D4C41"},
{ID: "17", Name: "表面清洁", Color: "#757575"},
{ID: "18", Name: "清洗清洁", Color: "#546E7A"},
{ID: "19", Name: "清洁工具处理", Color: "#00838F"},
{ID: "20", Name: "垃圾和食品报废", Color: "#2E7D32"},
{ID: "21", Name: "门店巡检记录", Color: "#C2185B"},
{ID: "22", Name: "工具、钥匙与设备管理", Color: "#455A64"},
},
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := st.UpsertAnnotationTemplate(ctx, template); err != nil {
t.Fatalf("upsert template: %v", err)
}
// 3. Ingest Demo Video from BOS: testxyz / fanyu_test / Orbbec_Ego_AZER76400GG_19700101_001_D
sourceVideo := "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700101_001_D.mp4"
task := domain.AnnotationTaskRecord{
ID: "task-orbbec-ego-001",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: "wf-convenience-full-pipeline",
RunID: "run-poc-stream-10w",
SourceVideo: sourceVideo,
ClipStart: 0.0,
ClipEnd: 200.0,
TemplateID: template.ID,
TemplateVersion: template.Version,
WorkgroupID: workgroup.ID,
VendorID: workgroup.VendorID,
Status: domain.AnnotationTaskStatusPendingAnnotation,
CurrentRevision: 0,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := st.UpsertAnnotationTask(ctx, task); err != nil {
t.Fatalf("upsert task: %v", err)
}
// Seed human batch & item for 4-stage tracking
humanBatch := domain.HumanBatchRecord{
ID: "hbatch-convenience-1",
OrganizationID: domain.DefaultOrganizationID,
PluginID: "annotation-qa",
ProjectID: project.ID,
WorkflowID: "wf-convenience-full-pipeline",
RunID: "run-poc-stream-10w",
GateNodeID: "node-human-gate",
Status: domain.HumanBatchActive,
Stages: domain.DefaultAnnotationHumanStages(),
ReleasePolicy: domain.HumanReleaseAcceptedPartition,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := st.UpsertHumanBatch(ctx, humanBatch); err != nil {
t.Fatalf("upsert human batch: %v", err)
}
humanItem := domain.HumanItemRecord{
ID: "hitem-orbbec-ego-001",
OrganizationID: domain.DefaultOrganizationID,
PluginID: "annotation-qa",
BatchID: humanBatch.ID,
ProjectID: project.ID,
WorkflowID: humanBatch.WorkflowID,
RunID: humanBatch.RunID,
GateNodeID: humanBatch.GateNodeID,
WorkgroupID: workgroup.ID,
VendorID: workgroup.VendorID,
Subject: domain.HumanSubjectRef{ItemKey: "Orbbec_Ego_AZER76400GG_19700101_001_D", MediaType: "video/mp4"},
Status: domain.HumanItemReady,
CurrentStageID: "split_annotation",
LegacyTaskID: task.ID,
MaxReworkRounds: 5,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := st.UpsertHumanItem(ctx, humanItem); err != nil {
t.Fatalf("upsert human item: %v", err)
}
handler := NewHandler(st, nil)
// Helper for authenticated requests with mock principal in context and CSRF token
doRequest := func(method, path string, user domain.UserRecord, body any) *httptest.ResponseRecorder {
var reqBody []byte
if body != nil {
reqBody, _ = json.Marshal(body)
}
req := httptest.NewRequest(method, path, bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/json")
csrfToken := "test-csrf-token-12345"
req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: csrfToken})
req.Header.Set(csrfHeaderName, csrfToken)
principal := &authenticatedPrincipal{
User: user,
Organization: domain.OrganizationRecord{
ID: domain.DefaultOrganizationID,
Name: "泛宇科技",
Status: domain.OrganizationStatusActive,
},
}
req = req.WithContext(context.WithValue(req.Context(), authContextKey{}, principal))
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
return rec
}
t.Log("===> STAGE 1: 人工剪切 (Human Slicing)")
// 1.1 Claim task by Annotator
curItem, _ := st.GetHumanItem(ctx, humanItem.ID)
rec := doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim split task: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 1.2 Submit initial slice revision
slicePayload := map[string]any{
"segments": []map[string]any{
{"id": "slice-1", "start": 0.0, "end": 60.0},
{"id": "slice-2", "start": 60.0, "end": 140.0},
{"id": "slice-3", "start": 140.0, "end": 200.0},
},
"durationSeconds": 120,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/slice-revisions", annotatorUser, slicePayload)
if rec.Code != http.StatusCreated {
t.Fatalf("submit slice revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
// Verify task status is pending_quality (for split quality review)
curTask, _ := st.GetAnnotationTask(ctx, task.ID)
if curTask.Status != domain.AnnotationTaskStatusPendingQuality {
t.Fatalf("expected task status pending_quality, got %s", curTask.Status)
}
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
if curItem.CurrentStageID != "split_quality_review" {
t.Fatalf("expected stage split_quality_review, got %s", curItem.CurrentStageID)
}
t.Log("===> STAGE 2: 剪切质检 (Split Quality Review) - 模拟打回与返工")
// 2.1 Quality Reviewer claims task
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim split quality task: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 2.2 Quality Reviewer rejects with comment to test rework loop
rejectPayload := map[string]any{
"stage": "split_quality",
"decision": "rejected",
"comment": "第2段热食加工动作切分起点不够精准,需将开始时间调整为65.0s",
"durationSeconds": 45,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, rejectPayload)
if rec.Code != http.StatusCreated {
t.Fatalf("reject split quality: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 2.3 Reopen reject item for Annotator rework
rejects, _ := st.ListHumanRejects(ctx, domain.DefaultOrganizationID, domain.HumanRejectQuery{ItemID: humanItem.ID})
if len(rejects) == 0 {
t.Fatalf("expected reject record created")
}
latestReject := rejects[0]
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
reopenPayload := map[string]any{
"rejectVersion": latestReject.Version,
"itemVersion": curItem.Version,
"targetStageId": "split_annotation",
"comment": "打回给标注员重新调整切片时间",
}
rec = doRequest("POST", "/api/human-rejects/"+latestReject.ID+"/reopen", adminUser, reopenPayload)
if rec.Code != http.StatusOK {
t.Fatalf("reopen reject: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// Verify task is in rework targeting split_annotation
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
if curItem.CurrentStageID != "split_annotation" || curItem.Status != domain.HumanItemRework {
t.Fatalf("expected item rework at split_annotation, got stage=%s status=%s", curItem.CurrentStageID, curItem.Status)
}
t.Log("===> STAGE 3: 人工剪切返工 (Human Slicing Rework & Re-submit)")
// 3.1 Annotator re-claims task
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("re-claim split rework: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 3.2 Annotator adjusts boundaries and re-submits
reworkSlicePayload := map[string]any{
"segments": []map[string]any{
{"id": "slice-1", "start": 0.0, "end": 65.0},
{"id": "slice-2", "start": 65.0, "end": 140.0},
{"id": "slice-3", "start": 140.0, "end": 200.0},
},
"durationSeconds": 60,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/slice-revisions", annotatorUser, reworkSlicePayload)
if rec.Code != http.StatusCreated {
t.Fatalf("re-submit slice revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
t.Log("===> STAGE 4: 剪切质检通过 (Split Quality Review - Approve)")
// 4.1 Quality Reviewer claims and approves
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim split quality for approve: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
approveSplitPayload := map[string]any{
"stage": "split_quality",
"decision": "approved",
"videoQuality": "qualified",
"annotationAccuracy": 96.5,
"annotationCompleteness": 98.0,
"overallGrade": "A",
"comment": "切片边界精准,质检通过",
"durationSeconds": 30,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, approveSplitPayload)
if rec.Code != http.StatusCreated {
t.Fatalf("approve split quality: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
// Verify task is promoted to semantic annotation stage
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
if curItem.CurrentStageID != "semantic_annotation" {
t.Fatalf("expected stage semantic_annotation, got %s", curItem.CurrentStageID)
}
t.Log("===> STAGE 5: 人工语义标注 (Human Semantic Annotation)")
// 5.1 Annotator claims semantic annotation task
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim semantic annotation: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 5.2 Annotator tags timeline segments using 22 convenience store classes
annotationPayload := map[string]any{
"segments": []map[string]any{
{"id": "seg-1", "labelId": "05", "label": "蒸点陈列与出餐", "start": 0.0, "end": 65.0, "confidence": 1.0, "source": "manual"},
{"id": "seg-2", "labelId": "06", "label": "热食加工与出餐", "start": 65.0, "end": 140.0, "confidence": 1.0, "source": "manual"},
{"id": "seg-3", "labelId": "14", "label": "收银扫码", "start": 140.0, "end": 185.0, "confidence": 1.0, "source": "manual"},
},
"durationSeconds": 180,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/revisions", annotatorUser, annotationPayload)
if rec.Code != http.StatusCreated {
t.Fatalf("submit semantic revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
t.Log("===> STAGE 6: 标注质检 (Annotation Quality Review) - 模拟打回与返工不计产出验证")
// 6.1 Quality Reviewer claims
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim annotation quality: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 6.2 Quality Reviewer rejects with comment
rejectAnnotationPayload := map[string]any{
"stage": "quality",
"decision": "rejected",
"comment": "收银扫码最后5秒含有顾客交付动作,需补充 15 顾客交付与内部协作 标签",
"durationSeconds": 40,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, rejectAnnotationPayload)
if rec.Code != http.StatusCreated {
t.Fatalf("reject annotation quality: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 6.3 Reopen reject item for Annotator rework
rejects, _ = st.ListHumanRejects(ctx, domain.DefaultOrganizationID, domain.HumanRejectQuery{ItemID: humanItem.ID})
latestReject = rejects[0]
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
reopenPayload = map[string]any{
"rejectVersion": latestReject.Version,
"itemVersion": curItem.Version,
"targetStageId": "semantic_annotation",
"comment": "打回给标注员补充15标签",
}
rec = doRequest("POST", "/api/human-rejects/"+latestReject.ID+"/reopen", adminUser, reopenPayload)
if rec.Code != http.StatusOK {
t.Fatalf("reopen semantic reject: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
t.Log("===> STAGE 7: 人工标注返工提交 (Rework Annotation Submit & OutputDelta Guard)")
// 7.1 Annotator re-claims
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("re-claim annotation rework: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// 7.2 Annotator fixes segment and resubmits
reworkAnnotationPayload := map[string]any{
"segments": []map[string]any{
{"id": "seg-1", "labelId": "05", "label": "蒸点陈列与出餐", "start": 0.0, "end": 65.0, "confidence": 1.0, "source": "manual"},
{"id": "seg-2", "labelId": "06", "label": "热食加工与出餐", "start": 65.0, "end": 140.0, "confidence": 1.0, "source": "manual"},
{"id": "seg-3", "labelId": "14", "label": "收银扫码", "start": 140.0, "end": 180.0, "confidence": 1.0, "source": "manual"},
{"id": "seg-4", "labelId": "15", "label": "顾客交付与内部协作", "start": 180.0, "end": 200.0, "confidence": 1.0, "source": "manual"},
},
"durationSeconds": 90,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/revisions", annotatorUser, reworkAnnotationPayload)
if rec.Code != http.StatusCreated {
t.Fatalf("re-submit annotation revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
t.Log("===> STAGE 8: 标注质检最终通过 (Annotation Quality Review - Approve)")
// 8.1 Quality Reviewer approves
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim for final quality approve: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
approveQualityPayload := map[string]any{
"stage": "quality",
"decision": "approved",
"videoQuality": "qualified",
"annotationAccuracy": 99.0,
"annotationCompleteness": 99.5,
"overallGrade": "A",
"comment": "22类多标签时间区间标注精准完整,质检全量通过",
"durationSeconds": 35,
}
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, approveQualityPayload)
if rec.Code != http.StatusCreated {
t.Fatalf("approve quality review: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
}
curTask, _ = st.GetAnnotationTask(ctx, task.ID)
if curTask.Status != domain.AnnotationTaskStatusAccepted {
t.Fatalf("expected task accepted, got %s", curTask.Status)
}
t.Log("===> STAGE 9: 供应商工时与产出统计核验 (Vendor Workload & Output Counts)")
rec = doRequest("GET", "/api/plugins/annotation-qa/workload-summary?projectId="+project.ID, adminUser, nil)
if rec.Code != http.StatusOK {
t.Fatalf("get workload summary: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
var summary domain.AnnotationWorkloadSummary
_ = json.Unmarshal(rec.Body.Bytes(), &summary)
if len(summary.Items) == 0 {
t.Fatalf("expected workload summary items, got none")
}
events, _ := st.ListAnnotationWorklogEvents(ctx, domain.DefaultOrganizationID, project.ID)
for _, ev := range events {
t.Logf("Event: User=%s EventType=%s OutputDelta=%d Duration=%d Workgroup=%s Vendor=%s", ev.UserID, ev.EventType, ev.OutputDelta, ev.DurationSeconds, ev.WorkgroupID, ev.VendorID)
}
for i, item := range summary.Items {
t.Logf("Summary Item %d: UserID=%s OutputCount=%d ReworkCount=%d SubmittedCount=%d QualityApproved=%d",
i, item.UserID, item.OutputCount, item.ReworkCount, item.SubmittedCount, item.QualityApproved)
}
var annotatorSummary domain.AnnotationWorkloadAggregate
for _, item := range summary.Items {
if item.UserID == annotatorUser.ID {
annotatorSummary = item
break
}
}
if annotatorSummary.OutputCount != 1 {
t.Fatalf("expected OutputCount == 1, got %d (rework should not duplicate count)", annotatorSummary.OutputCount)
}
if annotatorSummary.ReworkCount < 1 {
t.Fatalf("expected ReworkCount >= 1, got %d", annotatorSummary.ReworkCount)
}
t.Logf("✅ Verified: Annotator Workload Summary: OutputCount=%d, ReworkCount=%d, QualityApproved=%d",
annotatorSummary.OutputCount, annotatorSummary.ReworkCount, annotatorSummary.QualityApproved)
t.Log("===> STAGE 10: 最终验收专家交付包决策 (Delivery Package Acceptance)")
acceptanceBatch := domain.HumanBatchRecord{
ID: "hbatch-acceptance-1",
OrganizationID: domain.DefaultOrganizationID,
PluginID: "delivery-acceptance",
ProjectID: project.ID,
WorkflowID: "wf-convenience-full-pipeline",
RunID: "run-poc-stream-10w",
GateNodeID: "node-acceptance-gate",
Status: domain.HumanBatchActive,
Stages: []domain.HumanStageDefinition{
{ID: "acceptance", Label: "最终交付验收", Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer"},
},
ReleasePolicy: domain.HumanReleaseAllAccepted,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
_ = st.UpsertHumanBatch(ctx, acceptanceBatch)
acceptanceHumanItem := domain.HumanItemRecord{
ID: "hitem-acceptance-pkg-001",
OrganizationID: domain.DefaultOrganizationID,
PluginID: "delivery-acceptance",
BatchID: acceptanceBatch.ID,
ProjectID: project.ID,
WorkflowID: acceptanceBatch.WorkflowID,
RunID: acceptanceBatch.RunID,
GateNodeID: acceptanceBatch.GateNodeID,
WorkgroupID: workgroup.ID,
VendorID: workgroup.VendorID,
Subject: domain.HumanSubjectRef{ItemKey: "Delivery_Package_Orbbec_Ego_001", MediaType: "application/json"},
Status: domain.HumanItemReady,
CurrentStageID: "acceptance",
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
_ = st.UpsertHumanItem(ctx, acceptanceHumanItem)
deliveryPkg := domain.DeliveryPackageRecord{
ID: "dpkg-orbbec-ego-001",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: "wf-convenience-full-pipeline",
ParentRunID: "run-poc-stream-10w",
ChildRunID: "run-poc-stream-10w-delivery",
ReleasePartitionID: "part-1",
ManifestDigest: "sha256:manifest12345",
ItemIDs: []string{"hitem-orbbec-ego-001"},
WorkgroupID: workgroup.ID,
VendorID: workgroup.VendorID,
Status: domain.DeliveryPackagePendingAcceptance,
AcceptanceHumanItemID: acceptanceHumanItem.ID,
ArtifactIDs: []string{"art-video-face-blur", "art-annotation-json", "art-rrd-recording"},
Version: 1,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
_ = st.UpsertDeliveryPackage(ctx, deliveryPkg)
// Claim Acceptance Human Item & Delivery Package by Acceptance Reviewer
curAccItem, _ := st.GetHumanItem(ctx, acceptanceHumanItem.ID)
rec = doRequest("POST", "/api/human-items/"+acceptanceHumanItem.ID+"/claim", acceptanceUser, map[string]any{"version": curAccItem.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim acceptance human item: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
rec = doRequest("POST", "/api/delivery-packages/"+deliveryPkg.ID+"/claim", acceptanceUser, map[string]any{"version": deliveryPkg.Version})
if rec.Code != http.StatusOK {
t.Fatalf("claim delivery package: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
}
// Decision: Approve Delivery Package
curPkg, _ := st.GetDeliveryPackage(ctx, deliveryPkg.ID)
decisionPayload := map[string]any{
"version": curPkg.Version,
"decision": "approved",
"videoQuality": "qualified",
"annotationAccuracy": 99.5,
"annotationCompleteness": 100.0,
"overallGrade": "A+",
"comment": "泛宇10w小时首批便利店第一视角数据已通过全流程验收,准予交付并触发下游持续训练",
}
rec = doRequest("POST", "/api/delivery-packages/"+deliveryPkg.ID+"/decisions", acceptanceUser, decisionPayload)
if rec.Code != http.StatusCreated && rec.Code != http.StatusOK {
t.Fatalf("decision delivery package: expected 201/200, got %d (body: %s)", rec.Code, rec.Body.String())
}
curPkg, _ = st.GetDeliveryPackage(ctx, deliveryPkg.ID)
if curPkg.Status != domain.DeliveryPackageAccepted {
t.Fatalf("expected delivery package status accepted, got %s", curPkg.Status)
}
t.Logf("🎉 Verified: Complete 10-stage human-in-the-loop workflow successfully completed for %s!", deliveryPkg.ID)
}

View File

@ -0,0 +1,99 @@
import json
import os
import pathlib
import tempfile
import unittest
from backend.operators.builtin.qwen_label_crop_classification.operator import (
run as run_qwen_crop,
TAXONOMY_22_CLASSES,
)
from backend.operators.builtin.qwen_temporal_localization.operator import (
run as run_qwen_temporal,
)
from backend.operators.builtin.wilor_valid_event_detection.operator import (
run as run_wilor_valid,
)
class TestConvenienceStoreContinuousTraining(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.output_dir = pathlib.Path(self.temp_dir.name)
def tearDown(self):
self.temp_dir.cleanup()
def test_convenience_store_22_labels_integrity(self):
self.assertEqual(len(TAXONOMY_22_CLASSES), 22)
self.assertEqual(TAXONOMY_22_CLASSES[0]["id"], "01")
self.assertEqual(TAXONOMY_22_CLASSES[0]["name"], "鲜食处理")
self.assertEqual(TAXONOMY_22_CLASSES[21]["id"], "22")
self.assertEqual(TAXONOMY_22_CLASSES[21]["name"], "工具、钥匙与设备管理")
def test_qwen_label_crop_classification_training_and_eval(self):
# Mock accepted delivery package input
delivery_package_dir = self.output_dir / "packages" / "Orbbec_Ego_AZER76400GG_19700101_001_D_package"
delivery_package_dir.mkdir(parents=True, exist_ok=True)
meta = {
"source_video": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700101_001_D.mp4",
"video_path": str(delivery_package_dir / "video.mp4"),
"segments": [
{"id": "seg-1", "label_id": "05", "label": "蒸点陈列与出餐", "start": 0.0, "end": 65.0},
{"id": "seg-2", "label_id": "06", "label": "热食加工与出餐", "start": 65.0, "end": 140.0},
{"id": "seg-3", "label_id": "14", "label": "收银扫码", "start": 140.0, "end": 180.0},
{"id": "seg-4", "label_id": "15", "label": "顾客交付与内部协作", "start": 180.0, "end": 200.0},
],
"overall_grade": "A+",
"annotation_accuracy": 99.5,
}
(delivery_package_dir / "meta.json").write_text(json.dumps(meta))
(delivery_package_dir / "video.mp4").write_text("mock video stream")
manifest = {
"schemaVersion": "droboticflow.delivery-package-manifest/v1",
"packages": [
{
"packageId": "dpkg-orbbec-ego-001",
"path": str(delivery_package_dir),
"metaPath": str(delivery_package_dir / "meta.json"),
}
],
}
manifest_file = self.output_dir / "packages-manifest.json"
manifest_file.write_text(json.dumps(manifest))
node_data = {
"executionMode": "train_and_eval",
"targetMetricThreshold": 0.85,
"maxEpochs": 5,
}
inputs = [
{
"name": "packages_manifest",
"value": str(manifest_file),
}
]
artifacts = run_qwen_crop(
run_id="run-poc-train-001",
node_id="node-train-qwen-crop",
output_dir=self.output_dir / "train_output",
node_data=node_data,
inputs=inputs,
helpers={},
)
self.assertTrue(len(artifacts) > 0)
eval_artifact = next((a for a in artifacts if "metrics" in a["name"] or "evaluation" in a["name"]), None)
self.assertIsNotNone(eval_artifact)
trained_videos_file = self.output_dir / "train_output" / "checkpoint_latest" / "trained_source_videos.json"
self.assertTrue(trained_videos_file.exists())
data = json.loads(trained_videos_file.read_text())
self.assertTrue(any("Orbbec_Ego_AZER76400GG_19700101_001_D" in v for v in data.get("source_videos", [])))
if __name__ == "__main__":
unittest.main()