forked from longtao.wu/droboticflow
🧪 test(human-in-the-loop): add full convenience store human-in-the-loop lifecycle and continuous training tests
This commit is contained in:
parent
c22b8eb555
commit
0e7ca34268
@ -74,7 +74,8 @@ def run(
|
||||
inputs: list[dict[str, Any]],
|
||||
helpers: HelperMap,
|
||||
) -> list[dict[str, Any]]:
|
||||
params = helpers["hyperparams"](node_data)
|
||||
hyperparams_fn = helpers.get("hyperparams") if helpers else None
|
||||
params = hyperparams_fn(node_data) if hyperparams_fn else (node_data.get("hyperparameters") or node_data)
|
||||
task_mode = str(params.get("TASK_MODE", "continuous_train")).strip().lower() or "continuous_train"
|
||||
base_model = str(params.get("BASE_MODEL", "Qwen/Qwen3-VL-8B-Instruct")).strip()
|
||||
initial_checkpoint = str(params.get("INITIAL_CHECKPOINT", "")).strip()
|
||||
|
||||
@ -809,7 +809,12 @@ func (server *server) handleSliceRevisionCreate(writer http.ResponseWriter, requ
|
||||
_ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: "split_annotation", Round: item.ReviewRound, Status: domain.HumanItemSubmitted, AssigneeUserID: principal.User.ID, SubmissionRef: revision.ID})
|
||||
_ = server.appendHumanEvent(request.Context(), *item, "submitted", principal.User.ID, map[string]any{"submissionRef": revision.ID, "sliceRevision": revision.Revision})
|
||||
}
|
||||
_ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "slice_submitted", Revision: revision.Revision, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO()})
|
||||
_ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{
|
||||
ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID,
|
||||
TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID,
|
||||
EventType: "slice_submitted", Revision: revision.Revision, TemplateVersion: task.TemplateVersion,
|
||||
DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO(),
|
||||
})
|
||||
renderJSON(writer, http.StatusCreated, map[string]any{"task": task, "sliceRevision": revision})
|
||||
}
|
||||
|
||||
@ -863,8 +868,13 @@ func (server *server) handleAnnotationRevisionCreate(writer http.ResponseWriter,
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
isRework := task.CurrentRevision > 0 || task.Status == domain.AnnotationTaskStatusRework
|
||||
revisionNumber := task.CurrentRevision + 1
|
||||
existingRevisions, err := server.store.ListAnnotationRevisions(request.Context(), task.OrganizationID, task.ID)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
isRework := len(existingRevisions) > 0 || task.Status == domain.AnnotationTaskStatusRework
|
||||
revisionNumber := len(existingRevisions) + 1
|
||||
revision := domain.AnnotationRevisionRecord{
|
||||
ID: domain.NewID("annotation-revision"), OrganizationID: task.OrganizationID, TaskID: task.ID,
|
||||
Revision: revisionNumber, TemplateVersion: task.TemplateVersion, Segments: payload.Segments, ObjectHints: payload.ObjectHints,
|
||||
@ -1098,7 +1108,7 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r
|
||||
task.Status = domain.AnnotationTaskStatusRework
|
||||
task.AssignedUserID = ""
|
||||
} else {
|
||||
task.CurrentRevision = 1
|
||||
task.CurrentRevision = 0
|
||||
task.Status = domain.AnnotationTaskStatusPendingAnnotation
|
||||
task.AssignedUserID = ""
|
||||
}
|
||||
|
||||
632
internal/api/human_in_the_loop_e2e_test.go
Normal file
632
internal/api/human_in_the_loop_e2e_test.go
Normal file
@ -0,0 +1,632 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func TestHumanInTheLoopConvenienceStoreFullLifecycleE2E(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := store.NewMemoryStore()
|
||||
|
||||
// 1. Setup Users with distinctive roles
|
||||
adminUser := domain.UserRecord{
|
||||
ID: "usr-admin-1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Email: "admin@fanyu.com",
|
||||
Name: "周天如 (管理员)",
|
||||
Role: domain.UserRoleAdmin,
|
||||
Status: domain.UserStatusActive,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
annotatorUser := domain.UserRecord{
|
||||
ID: "usr-annotator-1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Email: "annotator@vendor-a.com",
|
||||
Name: "程晓明 (标注员)",
|
||||
Role: domain.UserRoleAnnotator,
|
||||
Status: domain.UserStatusActive,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
qualityUser := domain.UserRecord{
|
||||
ID: "usr-quality-1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Email: "qc@vendor-a.com",
|
||||
Name: "质检员王工",
|
||||
Role: domain.UserRoleQualityReviewer,
|
||||
Status: domain.UserStatusActive,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
acceptanceUser := domain.UserRecord{
|
||||
ID: "usr-acceptance-1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Email: "acceptance@fanyu.com",
|
||||
Name: "验收专家李工",
|
||||
Role: domain.UserRoleAcceptanceReviewer,
|
||||
Status: domain.UserStatusActive,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
|
||||
for _, u := range []domain.UserRecord{adminUser, annotatorUser, qualityUser, acceptanceUser} {
|
||||
if err := st.UpsertUser(ctx, u); err != nil {
|
||||
t.Fatalf("upsert user %s: %v", u.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Setup Project, Vendor, Workgroup and Template with 22-class taxonomy
|
||||
project := domain.ProjectRecord{
|
||||
ID: "proj-fanyu-convenience",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Name: "便利店第一视角视频人在环切分与标注项目 (泛宇)",
|
||||
Description: "10w小时级第一视角视频数据流切分、标注与持续训练",
|
||||
Status: "active",
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := st.UpsertProject(ctx, project); err != nil {
|
||||
t.Fatalf("upsert project: %v", err)
|
||||
}
|
||||
|
||||
workgroup := domain.AnnotationWorkgroupRecord{
|
||||
ID: "wg-vendor-a-team1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Name: "泛宇交付一组",
|
||||
VendorID: "vendor-fanyu-a",
|
||||
Description: "便利店视频数据标注",
|
||||
Status: "active",
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := st.UpsertAnnotationWorkgroup(ctx, workgroup); err != nil {
|
||||
t.Fatalf("upsert workgroup: %v", err)
|
||||
}
|
||||
|
||||
// Add memberships
|
||||
for _, m := range []domain.ProjectAnnotationMembershipRecord{
|
||||
{
|
||||
ID: "mem-1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: project.ID,
|
||||
UserID: annotatorUser.ID,
|
||||
WorkgroupID: workgroup.ID,
|
||||
VendorID: workgroup.VendorID,
|
||||
AssignmentRole: domain.AnnotationRoleAnnotator,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
},
|
||||
{
|
||||
ID: "mem-2",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: project.ID,
|
||||
UserID: qualityUser.ID,
|
||||
WorkgroupID: workgroup.ID,
|
||||
VendorID: workgroup.VendorID,
|
||||
AssignmentRole: domain.AnnotationRoleQualityReviewer,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
},
|
||||
{
|
||||
ID: "mem-3",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: project.ID,
|
||||
UserID: acceptanceUser.ID,
|
||||
WorkgroupID: workgroup.ID,
|
||||
VendorID: workgroup.VendorID,
|
||||
AssignmentRole: domain.AnnotationRoleAcceptanceReviewer,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
},
|
||||
} {
|
||||
if err := st.UpsertProjectAnnotationMembership(ctx, m); err != nil {
|
||||
t.Fatalf("upsert membership: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
template := domain.AnnotationTemplateRecord{
|
||||
ID: "tpl-convenience-22",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: project.ID,
|
||||
Name: "便利店第一视角视频行为多标签标注模板",
|
||||
Version: 1,
|
||||
Status: "published",
|
||||
PublishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Labels: []domain.SemanticLabelDefinition{
|
||||
{ID: "01", Name: "鲜食处理", Color: "#E53935"},
|
||||
{ID: "02", Name: "关东煮备料和清洁", Color: "#FB8C00"},
|
||||
{ID: "03", Name: "关东煮出餐", Color: "#FDD835"},
|
||||
{ID: "04", Name: "蒸点与糕点备货", Color: "#43A047"},
|
||||
{ID: "05", Name: "蒸点陈列与出餐", Color: "#00ACC1"},
|
||||
{ID: "06", Name: "热食加工与出餐", Color: "#1E88E5"},
|
||||
{ID: "07", Name: "即食商品陈列", Color: "#5E35B1"},
|
||||
{ID: "08", Name: "常温商品补货", Color: "#D81B60"},
|
||||
{ID: "09", Name: "冷藏饮料补货", Color: "#8E24AA"},
|
||||
{ID: "10", Name: "货架深度清洁", Color: "#3949AB"},
|
||||
{ID: "11", Name: "价签与标识维护", Color: "#039BE5"},
|
||||
{ID: "12", Name: "包装准备", Color: "#00897B"},
|
||||
{ID: "13", Name: "包装封装与材料整理", Color: "#7CB342"},
|
||||
{ID: "14", Name: "收银扫码", Color: "#C0CA33"},
|
||||
{ID: "15", Name: "顾客交付与内部协作", Color: "#F4511E"},
|
||||
{ID: "16", Name: "烟草、槟榔和小商品销售", Color: "#6D4C41"},
|
||||
{ID: "17", Name: "表面清洁", Color: "#757575"},
|
||||
{ID: "18", Name: "清洗清洁", Color: "#546E7A"},
|
||||
{ID: "19", Name: "清洁工具处理", Color: "#00838F"},
|
||||
{ID: "20", Name: "垃圾和食品报废", Color: "#2E7D32"},
|
||||
{ID: "21", Name: "门店巡检记录", Color: "#C2185B"},
|
||||
{ID: "22", Name: "工具、钥匙与设备管理", Color: "#455A64"},
|
||||
},
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := st.UpsertAnnotationTemplate(ctx, template); err != nil {
|
||||
t.Fatalf("upsert template: %v", err)
|
||||
}
|
||||
|
||||
// 3. Ingest Demo Video from BOS: testxyz / fanyu_test / Orbbec_Ego_AZER76400GG_19700101_001_D
|
||||
sourceVideo := "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700101_001_D.mp4"
|
||||
task := domain.AnnotationTaskRecord{
|
||||
ID: "task-orbbec-ego-001",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: project.ID,
|
||||
WorkflowID: "wf-convenience-full-pipeline",
|
||||
RunID: "run-poc-stream-10w",
|
||||
SourceVideo: sourceVideo,
|
||||
ClipStart: 0.0,
|
||||
ClipEnd: 200.0,
|
||||
TemplateID: template.ID,
|
||||
TemplateVersion: template.Version,
|
||||
WorkgroupID: workgroup.ID,
|
||||
VendorID: workgroup.VendorID,
|
||||
Status: domain.AnnotationTaskStatusPendingAnnotation,
|
||||
CurrentRevision: 0,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := st.UpsertAnnotationTask(ctx, task); err != nil {
|
||||
t.Fatalf("upsert task: %v", err)
|
||||
}
|
||||
|
||||
// Seed human batch & item for 4-stage tracking
|
||||
humanBatch := domain.HumanBatchRecord{
|
||||
ID: "hbatch-convenience-1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
PluginID: "annotation-qa",
|
||||
ProjectID: project.ID,
|
||||
WorkflowID: "wf-convenience-full-pipeline",
|
||||
RunID: "run-poc-stream-10w",
|
||||
GateNodeID: "node-human-gate",
|
||||
Status: domain.HumanBatchActive,
|
||||
Stages: domain.DefaultAnnotationHumanStages(),
|
||||
ReleasePolicy: domain.HumanReleaseAcceptedPartition,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := st.UpsertHumanBatch(ctx, humanBatch); err != nil {
|
||||
t.Fatalf("upsert human batch: %v", err)
|
||||
}
|
||||
|
||||
humanItem := domain.HumanItemRecord{
|
||||
ID: "hitem-orbbec-ego-001",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
PluginID: "annotation-qa",
|
||||
BatchID: humanBatch.ID,
|
||||
ProjectID: project.ID,
|
||||
WorkflowID: humanBatch.WorkflowID,
|
||||
RunID: humanBatch.RunID,
|
||||
GateNodeID: humanBatch.GateNodeID,
|
||||
WorkgroupID: workgroup.ID,
|
||||
VendorID: workgroup.VendorID,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "Orbbec_Ego_AZER76400GG_19700101_001_D", MediaType: "video/mp4"},
|
||||
Status: domain.HumanItemReady,
|
||||
CurrentStageID: "split_annotation",
|
||||
LegacyTaskID: task.ID,
|
||||
MaxReworkRounds: 5,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := st.UpsertHumanItem(ctx, humanItem); err != nil {
|
||||
t.Fatalf("upsert human item: %v", err)
|
||||
}
|
||||
|
||||
handler := NewHandler(st, nil)
|
||||
|
||||
// Helper for authenticated requests with mock principal in context and CSRF token
|
||||
doRequest := func(method, path string, user domain.UserRecord, body any) *httptest.ResponseRecorder {
|
||||
var reqBody []byte
|
||||
if body != nil {
|
||||
reqBody, _ = json.Marshal(body)
|
||||
}
|
||||
req := httptest.NewRequest(method, path, bytes.NewReader(reqBody))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
csrfToken := "test-csrf-token-12345"
|
||||
req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: csrfToken})
|
||||
req.Header.Set(csrfHeaderName, csrfToken)
|
||||
|
||||
principal := &authenticatedPrincipal{
|
||||
User: user,
|
||||
Organization: domain.OrganizationRecord{
|
||||
ID: domain.DefaultOrganizationID,
|
||||
Name: "泛宇科技",
|
||||
Status: domain.OrganizationStatusActive,
|
||||
},
|
||||
}
|
||||
req = req.WithContext(context.WithValue(req.Context(), authContextKey{}, principal))
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 1: 人工剪切 (Human Slicing)")
|
||||
// 1.1 Claim task by Annotator
|
||||
curItem, _ := st.GetHumanItem(ctx, humanItem.ID)
|
||||
rec := doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim split task: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 1.2 Submit initial slice revision
|
||||
slicePayload := map[string]any{
|
||||
"segments": []map[string]any{
|
||||
{"id": "slice-1", "start": 0.0, "end": 60.0},
|
||||
{"id": "slice-2", "start": 60.0, "end": 140.0},
|
||||
{"id": "slice-3", "start": 140.0, "end": 200.0},
|
||||
},
|
||||
"durationSeconds": 120,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/slice-revisions", annotatorUser, slicePayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("submit slice revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Verify task status is pending_quality (for split quality review)
|
||||
curTask, _ := st.GetAnnotationTask(ctx, task.ID)
|
||||
if curTask.Status != domain.AnnotationTaskStatusPendingQuality {
|
||||
t.Fatalf("expected task status pending_quality, got %s", curTask.Status)
|
||||
}
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
if curItem.CurrentStageID != "split_quality_review" {
|
||||
t.Fatalf("expected stage split_quality_review, got %s", curItem.CurrentStageID)
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 2: 剪切质检 (Split Quality Review) - 模拟打回与返工")
|
||||
// 2.1 Quality Reviewer claims task
|
||||
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim split quality task: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 2.2 Quality Reviewer rejects with comment to test rework loop
|
||||
rejectPayload := map[string]any{
|
||||
"stage": "split_quality",
|
||||
"decision": "rejected",
|
||||
"comment": "第2段热食加工动作切分起点不够精准,需将开始时间调整为65.0s",
|
||||
"durationSeconds": 45,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, rejectPayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("reject split quality: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 2.3 Reopen reject item for Annotator rework
|
||||
rejects, _ := st.ListHumanRejects(ctx, domain.DefaultOrganizationID, domain.HumanRejectQuery{ItemID: humanItem.ID})
|
||||
if len(rejects) == 0 {
|
||||
t.Fatalf("expected reject record created")
|
||||
}
|
||||
latestReject := rejects[0]
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
reopenPayload := map[string]any{
|
||||
"rejectVersion": latestReject.Version,
|
||||
"itemVersion": curItem.Version,
|
||||
"targetStageId": "split_annotation",
|
||||
"comment": "打回给标注员重新调整切片时间",
|
||||
}
|
||||
rec = doRequest("POST", "/api/human-rejects/"+latestReject.ID+"/reopen", adminUser, reopenPayload)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("reopen reject: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Verify task is in rework targeting split_annotation
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
if curItem.CurrentStageID != "split_annotation" || curItem.Status != domain.HumanItemRework {
|
||||
t.Fatalf("expected item rework at split_annotation, got stage=%s status=%s", curItem.CurrentStageID, curItem.Status)
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 3: 人工剪切返工 (Human Slicing Rework & Re-submit)")
|
||||
// 3.1 Annotator re-claims task
|
||||
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("re-claim split rework: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 3.2 Annotator adjusts boundaries and re-submits
|
||||
reworkSlicePayload := map[string]any{
|
||||
"segments": []map[string]any{
|
||||
{"id": "slice-1", "start": 0.0, "end": 65.0},
|
||||
{"id": "slice-2", "start": 65.0, "end": 140.0},
|
||||
{"id": "slice-3", "start": 140.0, "end": 200.0},
|
||||
},
|
||||
"durationSeconds": 60,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/slice-revisions", annotatorUser, reworkSlicePayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("re-submit slice revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 4: 剪切质检通过 (Split Quality Review - Approve)")
|
||||
// 4.1 Quality Reviewer claims and approves
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim split quality for approve: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
approveSplitPayload := map[string]any{
|
||||
"stage": "split_quality",
|
||||
"decision": "approved",
|
||||
"videoQuality": "qualified",
|
||||
"annotationAccuracy": 96.5,
|
||||
"annotationCompleteness": 98.0,
|
||||
"overallGrade": "A",
|
||||
"comment": "切片边界精准,质检通过",
|
||||
"durationSeconds": 30,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, approveSplitPayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("approve split quality: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Verify task is promoted to semantic annotation stage
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
if curItem.CurrentStageID != "semantic_annotation" {
|
||||
t.Fatalf("expected stage semantic_annotation, got %s", curItem.CurrentStageID)
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 5: 人工语义标注 (Human Semantic Annotation)")
|
||||
// 5.1 Annotator claims semantic annotation task
|
||||
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim semantic annotation: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 5.2 Annotator tags timeline segments using 22 convenience store classes
|
||||
annotationPayload := map[string]any{
|
||||
"segments": []map[string]any{
|
||||
{"id": "seg-1", "labelId": "05", "label": "蒸点陈列与出餐", "start": 0.0, "end": 65.0, "confidence": 1.0, "source": "manual"},
|
||||
{"id": "seg-2", "labelId": "06", "label": "热食加工与出餐", "start": 65.0, "end": 140.0, "confidence": 1.0, "source": "manual"},
|
||||
{"id": "seg-3", "labelId": "14", "label": "收银扫码", "start": 140.0, "end": 185.0, "confidence": 1.0, "source": "manual"},
|
||||
},
|
||||
"durationSeconds": 180,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/revisions", annotatorUser, annotationPayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("submit semantic revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 6: 标注质检 (Annotation Quality Review) - 模拟打回与返工不计产出验证")
|
||||
// 6.1 Quality Reviewer claims
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim annotation quality: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 6.2 Quality Reviewer rejects with comment
|
||||
rejectAnnotationPayload := map[string]any{
|
||||
"stage": "quality",
|
||||
"decision": "rejected",
|
||||
"comment": "收银扫码最后5秒含有顾客交付动作,需补充 15 顾客交付与内部协作 标签",
|
||||
"durationSeconds": 40,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, rejectAnnotationPayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("reject annotation quality: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 6.3 Reopen reject item for Annotator rework
|
||||
rejects, _ = st.ListHumanRejects(ctx, domain.DefaultOrganizationID, domain.HumanRejectQuery{ItemID: humanItem.ID})
|
||||
latestReject = rejects[0]
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
reopenPayload = map[string]any{
|
||||
"rejectVersion": latestReject.Version,
|
||||
"itemVersion": curItem.Version,
|
||||
"targetStageId": "semantic_annotation",
|
||||
"comment": "打回给标注员补充15标签",
|
||||
}
|
||||
rec = doRequest("POST", "/api/human-rejects/"+latestReject.ID+"/reopen", adminUser, reopenPayload)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("reopen semantic reject: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 7: 人工标注返工提交 (Rework Annotation Submit & OutputDelta Guard)")
|
||||
// 7.1 Annotator re-claims
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", annotatorUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("re-claim annotation rework: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// 7.2 Annotator fixes segment and resubmits
|
||||
reworkAnnotationPayload := map[string]any{
|
||||
"segments": []map[string]any{
|
||||
{"id": "seg-1", "labelId": "05", "label": "蒸点陈列与出餐", "start": 0.0, "end": 65.0, "confidence": 1.0, "source": "manual"},
|
||||
{"id": "seg-2", "labelId": "06", "label": "热食加工与出餐", "start": 65.0, "end": 140.0, "confidence": 1.0, "source": "manual"},
|
||||
{"id": "seg-3", "labelId": "14", "label": "收银扫码", "start": 140.0, "end": 180.0, "confidence": 1.0, "source": "manual"},
|
||||
{"id": "seg-4", "labelId": "15", "label": "顾客交付与内部协作", "start": 180.0, "end": 200.0, "confidence": 1.0, "source": "manual"},
|
||||
},
|
||||
"durationSeconds": 90,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/revisions", annotatorUser, reworkAnnotationPayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("re-submit annotation revision: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 8: 标注质检最终通过 (Annotation Quality Review - Approve)")
|
||||
// 8.1 Quality Reviewer approves
|
||||
curItem, _ = st.GetHumanItem(ctx, humanItem.ID)
|
||||
rec = doRequest("POST", "/api/human-items/"+humanItem.ID+"/claim", qualityUser, map[string]any{"version": curItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim for final quality approve: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
approveQualityPayload := map[string]any{
|
||||
"stage": "quality",
|
||||
"decision": "approved",
|
||||
"videoQuality": "qualified",
|
||||
"annotationAccuracy": 99.0,
|
||||
"annotationCompleteness": 99.5,
|
||||
"overallGrade": "A",
|
||||
"comment": "22类多标签时间区间标注精准完整,质检全量通过",
|
||||
"durationSeconds": 35,
|
||||
}
|
||||
rec = doRequest("POST", "/api/plugins/annotation-qa/tasks/"+task.ID+"/reviews", qualityUser, approveQualityPayload)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("approve quality review: expected 201, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
curTask, _ = st.GetAnnotationTask(ctx, task.ID)
|
||||
if curTask.Status != domain.AnnotationTaskStatusAccepted {
|
||||
t.Fatalf("expected task accepted, got %s", curTask.Status)
|
||||
}
|
||||
|
||||
t.Log("===> STAGE 9: 供应商工时与产出统计核验 (Vendor Workload & Output Counts)")
|
||||
rec = doRequest("GET", "/api/plugins/annotation-qa/workload-summary?projectId="+project.ID, adminUser, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("get workload summary: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
var summary domain.AnnotationWorkloadSummary
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &summary)
|
||||
|
||||
if len(summary.Items) == 0 {
|
||||
t.Fatalf("expected workload summary items, got none")
|
||||
}
|
||||
|
||||
events, _ := st.ListAnnotationWorklogEvents(ctx, domain.DefaultOrganizationID, project.ID)
|
||||
for _, ev := range events {
|
||||
t.Logf("Event: User=%s EventType=%s OutputDelta=%d Duration=%d Workgroup=%s Vendor=%s", ev.UserID, ev.EventType, ev.OutputDelta, ev.DurationSeconds, ev.WorkgroupID, ev.VendorID)
|
||||
}
|
||||
|
||||
for i, item := range summary.Items {
|
||||
t.Logf("Summary Item %d: UserID=%s OutputCount=%d ReworkCount=%d SubmittedCount=%d QualityApproved=%d",
|
||||
i, item.UserID, item.OutputCount, item.ReworkCount, item.SubmittedCount, item.QualityApproved)
|
||||
}
|
||||
|
||||
var annotatorSummary domain.AnnotationWorkloadAggregate
|
||||
for _, item := range summary.Items {
|
||||
if item.UserID == annotatorUser.ID {
|
||||
annotatorSummary = item
|
||||
break
|
||||
}
|
||||
}
|
||||
if annotatorSummary.OutputCount != 1 {
|
||||
t.Fatalf("expected OutputCount == 1, got %d (rework should not duplicate count)", annotatorSummary.OutputCount)
|
||||
}
|
||||
if annotatorSummary.ReworkCount < 1 {
|
||||
t.Fatalf("expected ReworkCount >= 1, got %d", annotatorSummary.ReworkCount)
|
||||
}
|
||||
|
||||
t.Logf("✅ Verified: Annotator Workload Summary: OutputCount=%d, ReworkCount=%d, QualityApproved=%d",
|
||||
annotatorSummary.OutputCount, annotatorSummary.ReworkCount, annotatorSummary.QualityApproved)
|
||||
|
||||
t.Log("===> STAGE 10: 最终验收专家交付包决策 (Delivery Package Acceptance)")
|
||||
acceptanceBatch := domain.HumanBatchRecord{
|
||||
ID: "hbatch-acceptance-1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
PluginID: "delivery-acceptance",
|
||||
ProjectID: project.ID,
|
||||
WorkflowID: "wf-convenience-full-pipeline",
|
||||
RunID: "run-poc-stream-10w",
|
||||
GateNodeID: "node-acceptance-gate",
|
||||
Status: domain.HumanBatchActive,
|
||||
Stages: []domain.HumanStageDefinition{
|
||||
{ID: "acceptance", Label: "最终交付验收", Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer"},
|
||||
},
|
||||
ReleasePolicy: domain.HumanReleaseAllAccepted,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
_ = st.UpsertHumanBatch(ctx, acceptanceBatch)
|
||||
|
||||
acceptanceHumanItem := domain.HumanItemRecord{
|
||||
ID: "hitem-acceptance-pkg-001",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
PluginID: "delivery-acceptance",
|
||||
BatchID: acceptanceBatch.ID,
|
||||
ProjectID: project.ID,
|
||||
WorkflowID: acceptanceBatch.WorkflowID,
|
||||
RunID: acceptanceBatch.RunID,
|
||||
GateNodeID: acceptanceBatch.GateNodeID,
|
||||
WorkgroupID: workgroup.ID,
|
||||
VendorID: workgroup.VendorID,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "Delivery_Package_Orbbec_Ego_001", MediaType: "application/json"},
|
||||
Status: domain.HumanItemReady,
|
||||
CurrentStageID: "acceptance",
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
_ = st.UpsertHumanItem(ctx, acceptanceHumanItem)
|
||||
|
||||
deliveryPkg := domain.DeliveryPackageRecord{
|
||||
ID: "dpkg-orbbec-ego-001",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: project.ID,
|
||||
WorkflowID: "wf-convenience-full-pipeline",
|
||||
ParentRunID: "run-poc-stream-10w",
|
||||
ChildRunID: "run-poc-stream-10w-delivery",
|
||||
ReleasePartitionID: "part-1",
|
||||
ManifestDigest: "sha256:manifest12345",
|
||||
ItemIDs: []string{"hitem-orbbec-ego-001"},
|
||||
WorkgroupID: workgroup.ID,
|
||||
VendorID: workgroup.VendorID,
|
||||
Status: domain.DeliveryPackagePendingAcceptance,
|
||||
AcceptanceHumanItemID: acceptanceHumanItem.ID,
|
||||
ArtifactIDs: []string{"art-video-face-blur", "art-annotation-json", "art-rrd-recording"},
|
||||
Version: 1,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
_ = st.UpsertDeliveryPackage(ctx, deliveryPkg)
|
||||
|
||||
// Claim Acceptance Human Item & Delivery Package by Acceptance Reviewer
|
||||
curAccItem, _ := st.GetHumanItem(ctx, acceptanceHumanItem.ID)
|
||||
rec = doRequest("POST", "/api/human-items/"+acceptanceHumanItem.ID+"/claim", acceptanceUser, map[string]any{"version": curAccItem.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim acceptance human item: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
rec = doRequest("POST", "/api/delivery-packages/"+deliveryPkg.ID+"/claim", acceptanceUser, map[string]any{"version": deliveryPkg.Version})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("claim delivery package: expected 200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Decision: Approve Delivery Package
|
||||
curPkg, _ := st.GetDeliveryPackage(ctx, deliveryPkg.ID)
|
||||
decisionPayload := map[string]any{
|
||||
"version": curPkg.Version,
|
||||
"decision": "approved",
|
||||
"videoQuality": "qualified",
|
||||
"annotationAccuracy": 99.5,
|
||||
"annotationCompleteness": 100.0,
|
||||
"overallGrade": "A+",
|
||||
"comment": "泛宇10w小时首批便利店第一视角数据已通过全流程验收,准予交付并触发下游持续训练",
|
||||
}
|
||||
rec = doRequest("POST", "/api/delivery-packages/"+deliveryPkg.ID+"/decisions", acceptanceUser, decisionPayload)
|
||||
if rec.Code != http.StatusCreated && rec.Code != http.StatusOK {
|
||||
t.Fatalf("decision delivery package: expected 201/200, got %d (body: %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
curPkg, _ = st.GetDeliveryPackage(ctx, deliveryPkg.ID)
|
||||
if curPkg.Status != domain.DeliveryPackageAccepted {
|
||||
t.Fatalf("expected delivery package status accepted, got %s", curPkg.Status)
|
||||
}
|
||||
|
||||
t.Logf("🎉 Verified: Complete 10-stage human-in-the-loop workflow successfully completed for %s!", deliveryPkg.ID)
|
||||
}
|
||||
99
tests/backend/test_convenience_store_continuous_training.py
Normal file
99
tests/backend/test_convenience_store_continuous_training.py
Normal file
@ -0,0 +1,99 @@
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from backend.operators.builtin.qwen_label_crop_classification.operator import (
|
||||
run as run_qwen_crop,
|
||||
TAXONOMY_22_CLASSES,
|
||||
)
|
||||
from backend.operators.builtin.qwen_temporal_localization.operator import (
|
||||
run as run_qwen_temporal,
|
||||
)
|
||||
from backend.operators.builtin.wilor_valid_event_detection.operator import (
|
||||
run as run_wilor_valid,
|
||||
)
|
||||
|
||||
|
||||
class TestConvenienceStoreContinuousTraining(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.TemporaryDirectory()
|
||||
self.output_dir = pathlib.Path(self.temp_dir.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def test_convenience_store_22_labels_integrity(self):
|
||||
self.assertEqual(len(TAXONOMY_22_CLASSES), 22)
|
||||
self.assertEqual(TAXONOMY_22_CLASSES[0]["id"], "01")
|
||||
self.assertEqual(TAXONOMY_22_CLASSES[0]["name"], "鲜食处理")
|
||||
self.assertEqual(TAXONOMY_22_CLASSES[21]["id"], "22")
|
||||
self.assertEqual(TAXONOMY_22_CLASSES[21]["name"], "工具、钥匙与设备管理")
|
||||
|
||||
def test_qwen_label_crop_classification_training_and_eval(self):
|
||||
# Mock accepted delivery package input
|
||||
delivery_package_dir = self.output_dir / "packages" / "Orbbec_Ego_AZER76400GG_19700101_001_D_package"
|
||||
delivery_package_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
meta = {
|
||||
"source_video": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700101_001_D.mp4",
|
||||
"video_path": str(delivery_package_dir / "video.mp4"),
|
||||
"segments": [
|
||||
{"id": "seg-1", "label_id": "05", "label": "蒸点陈列与出餐", "start": 0.0, "end": 65.0},
|
||||
{"id": "seg-2", "label_id": "06", "label": "热食加工与出餐", "start": 65.0, "end": 140.0},
|
||||
{"id": "seg-3", "label_id": "14", "label": "收银扫码", "start": 140.0, "end": 180.0},
|
||||
{"id": "seg-4", "label_id": "15", "label": "顾客交付与内部协作", "start": 180.0, "end": 200.0},
|
||||
],
|
||||
"overall_grade": "A+",
|
||||
"annotation_accuracy": 99.5,
|
||||
}
|
||||
(delivery_package_dir / "meta.json").write_text(json.dumps(meta))
|
||||
(delivery_package_dir / "video.mp4").write_text("mock video stream")
|
||||
|
||||
manifest = {
|
||||
"schemaVersion": "droboticflow.delivery-package-manifest/v1",
|
||||
"packages": [
|
||||
{
|
||||
"packageId": "dpkg-orbbec-ego-001",
|
||||
"path": str(delivery_package_dir),
|
||||
"metaPath": str(delivery_package_dir / "meta.json"),
|
||||
}
|
||||
],
|
||||
}
|
||||
manifest_file = self.output_dir / "packages-manifest.json"
|
||||
manifest_file.write_text(json.dumps(manifest))
|
||||
|
||||
node_data = {
|
||||
"executionMode": "train_and_eval",
|
||||
"targetMetricThreshold": 0.85,
|
||||
"maxEpochs": 5,
|
||||
}
|
||||
inputs = [
|
||||
{
|
||||
"name": "packages_manifest",
|
||||
"value": str(manifest_file),
|
||||
}
|
||||
]
|
||||
|
||||
artifacts = run_qwen_crop(
|
||||
run_id="run-poc-train-001",
|
||||
node_id="node-train-qwen-crop",
|
||||
output_dir=self.output_dir / "train_output",
|
||||
node_data=node_data,
|
||||
inputs=inputs,
|
||||
helpers={},
|
||||
)
|
||||
|
||||
self.assertTrue(len(artifacts) > 0)
|
||||
eval_artifact = next((a for a in artifacts if "metrics" in a["name"] or "evaluation" in a["name"]), None)
|
||||
self.assertIsNotNone(eval_artifact)
|
||||
|
||||
trained_videos_file = self.output_dir / "train_output" / "checkpoint_latest" / "trained_source_videos.json"
|
||||
self.assertTrue(trained_videos_file.exists())
|
||||
data = json.loads(trained_videos_file.read_text())
|
||||
self.assertTrue(any("Orbbec_Ego_AZER76400GG_19700101_001_D" in v for v in data.get("source_videos", [])))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user