d-robotics efa8f50694 🐛 Fix reviewer single-task claim (attempt tuple upsert), focus-mode video fill, plugin migration guard, claim-author index, progressive task-pool rendering
- UpsertHumanStageAttempt now upserts on (organization, item, stage, round):
  review operations create attempt rows with operation-scoped ids, so the
  claim path's id-only upsert violated organizationId_itemId_stageId_round_unique
  (E11000 → 500 相同的记录已存在) and reviewers could not claim tasks whose
  round already had an attempt row. Verified against the 2026-08-31 snapshot
  replica with a real browser claim.
- Focus/compact workspace video: the fill rules targeted only <video>, but the
  runtime renders <movi-player> (custom element) whenever browser-transcode
  fallback is not confirmed, leaving the player at clamp-height with dead
  space. Both rules now also match movi-player.
- migrateLegacyAnnotationHumanItems: one-shot guard stamped into the plugin
  record when the sweep completes; GET /api/plugins no longer re-scans
  17k tasks + 17k humanItems every read (8s per call in production).
- New humanItems index organizationId_latestAuthorUserId: the mine-pool /
  work-buffer ActorUserID  previously had an unindexed latestAuthorUserId
  arm, forcing organizationId_legacyTaskId scans of 17k docs (6.2s) on every
  3s poll.
- Task pool progressive rendering (200-card windows with 显示更多): the pool
  state keeps full data, the DOM no longer mounts 17k cards at once.
2026-10-11 23:59:46 +05:30

500 lines
18 KiB
Go

package api
import (
"context"
"math"
"net/http"
"strings"
"droboticflow/internal/domain"
pluginregistry "droboticflow/internal/plugins"
"github.com/go-chi/chi/v5"
)
type pluginConfigUpdateRequest struct {
Enabled bool `json:"enabled"`
Config map[string]any `json:"config"`
}
type pluginStatusResponse struct {
Manifest domain.PluginManifest `json:"manifest"`
Enabled bool `json:"enabled"`
Config map[string]any `json:"config,omitempty"`
Record *domain.OrganizationPluginRecord `json:"record,omitempty"`
}
func (server *server) pluginStatus(ctx context.Context, organizationID string, manifest domain.PluginManifest) (pluginStatusResponse, error) {
response := pluginStatusResponse{Manifest: manifest, Enabled: manifest.System}
record, err := server.store.GetOrganizationPlugin(ctx, organizationID, manifest.ID)
if err != nil {
return response, err
}
if record != nil {
response.Record = record
response.Enabled = manifest.System || record.Enabled
response.Config = record.Config
if manifest.ID == pluginregistry.AnnotationQAID && record.Enabled {
if err := server.migrateLegacyAnnotationHumanItems(ctx, organizationID, record.Config); err != nil {
return response, err
}
}
return response, nil
}
if manifest.ID == pluginregistry.AnnotationQAID {
tasks, listErr := server.store.ListAnnotationTasks(ctx, organizationID, "")
if listErr != nil {
return response, listErr
}
if len(tasks) > 0 {
response.Enabled = true
migration := domain.OrganizationPluginRecord{
OrganizationID: organizationID,
PluginID: manifest.ID,
Enabled: true,
Config: defaultAnnotationPluginConfig(),
UpdatedBy: "migration",
}
if upsertErr := server.store.UpsertOrganizationPlugin(ctx, migration); upsertErr != nil {
return response, upsertErr
}
stored, getErr := server.store.GetOrganizationPlugin(ctx, organizationID, manifest.ID)
if getErr != nil {
return response, getErr
}
response.Record = stored
response.Config = migration.Config
if err := server.migrateLegacyAnnotationHumanItems(ctx, organizationID, migration.Config); err != nil {
return response, err
}
}
}
return response, nil
}
func humanItemStateFromLegacyTask(task domain.AnnotationTaskRecord) (string, domain.HumanItemStatus) {
switch task.Status {
case domain.AnnotationTaskStatusPendingQuality:
return "quality_review", domain.HumanItemSubmitted
case domain.AnnotationTaskStatusPendingAcceptance:
return "acceptance", domain.HumanItemSubmitted
case domain.AnnotationTaskStatusRework:
return "semantic_annotation", domain.HumanItemRework
case domain.AnnotationTaskStatusAccepted:
return "acceptance", domain.HumanItemAccepted
case domain.AnnotationTaskStatusRejected:
return "acceptance", domain.HumanItemEscalated
default:
return "split_annotation", domain.HumanItemReady
}
}
func pluginConfigInt(config map[string]any, key string, fallback int) int {
switch typed := config[key].(type) {
case int:
if typed > 0 {
return typed
}
case int32:
if typed > 0 {
return int(typed)
}
case int64:
if typed > 0 {
return int(typed)
}
case float64:
if typed > 0 && math.Trunc(typed) == typed {
return int(typed)
}
}
return fallback
}
func pluginReleasePolicy(config map[string]any) domain.HumanReleasePolicy {
if strings.TrimSpace(configString(config, "releasePolicy")) == string(domain.HumanReleaseAllAccepted) {
return domain.HumanReleaseAllAccepted
}
return domain.HumanReleaseAcceptedPartition
}
func configString(config map[string]any, key string) string {
value, _ := config[key].(string)
return value
}
func legacyHumanBatchID(task domain.AnnotationTaskRecord) string {
key := strings.Trim(strings.Join([]string{task.RunID, task.NodeID}, "-"), "-")
if key == "" {
key = task.ProjectID + "-manual"
}
return "human-batch-legacy-" + key
}
func (server *server) migrateLegacyAnnotationHumanItems(ctx context.Context, organizationID string, config map[string]any) error {
// One-shot guard: the sweep is idempotent but previously re-scanned every
// organization's tasks + humanItems on every GET /api/plugins (17k+17k
// records, ~95MB, 8s per read in production). Once the sweep finds nothing
// left to migrate the marker stops all future re-entry; config updates
// reset the marker via the plugin config handler.
if strings.TrimSpace(configString(config, "legacyMigrationDone")) == "1" {
return nil
}
tasks, err := server.store.ListAnnotationTasks(ctx, organizationID, "")
if err != nil || len(tasks) == 0 {
return err
}
batches, err := server.store.ListHumanBatches(ctx, organizationID, domain.HumanBatchQuery{PluginID: pluginregistry.AnnotationQAID})
if err != nil {
return err
}
batchByKey := make(map[string]domain.HumanBatchRecord)
for _, batch := range batches {
batchByKey[batch.RunID+"\x00"+batch.GateNodeID] = batch
}
existing, err := server.store.ListHumanItems(ctx, organizationID, domain.HumanItemQuery{PluginID: pluginregistry.AnnotationQAID})
if err != nil {
return err
}
existingLegacy := make(map[string]struct{}, len(existing))
for _, item := range existing {
existingLegacy[item.LegacyTaskID] = struct{}{}
}
groupCounts := make(map[string]int)
for _, task := range tasks {
groupCounts[task.RunID+"\x00"+task.NodeID]++
}
migrated := 0
for _, task := range tasks {
if _, ok := existingLegacy[task.ID]; ok {
continue
}
migrated++
groupKey := task.RunID + "\x00" + task.NodeID
batch, ok := batchByKey[groupKey]
if !ok {
stages := domain.DefaultAnnotationHumanStages()
if template, templateErr := server.store.GetAnnotationTemplate(ctx, task.TemplateID); templateErr != nil {
return templateErr
} else if template != nil && len(template.HumanStages) > 0 {
stages = template.HumanStages
}
batch = domain.HumanBatchRecord{
ID: legacyHumanBatchID(task), OrganizationID: task.OrganizationID, PluginID: pluginregistry.AnnotationQAID,
ProjectID: task.ProjectID, SupplierID: task.SupplierID, WorkflowID: task.WorkflowID, RunID: task.RunID, GateNodeID: task.NodeID,
Status: domain.HumanBatchActive, Stages: stages,
ReleasePolicy: pluginReleasePolicy(config),
PartitionMaxItems: pluginConfigInt(config, "partitionMaxItems", 32),
PartitionMaxWaitSeconds: pluginConfigInt(config, "partitionMaxWaitSeconds", 60),
MaxReworkRounds: pluginConfigInt(config, "maxReworkRounds", 5), ItemCount: groupCounts[groupKey],
CreatedAt: task.CreatedAt,
}
if err := server.store.UpsertHumanBatch(ctx, batch); err != nil {
return err
}
batchByKey[groupKey] = batch
}
stageID, status := humanItemStateFromLegacyTask(task)
if _, _, found := humanStage(&batch, stageID); !found && len(batch.Stages) > 0 {
stageID = batch.Stages[len(batch.Stages)-1].ID
}
authorID := ""
revisions, err := server.store.ListAnnotationRevisions(ctx, organizationID, task.ID)
if err != nil {
return err
}
for _, revision := range revisions {
if revision.Revision == task.CurrentRevision {
authorID = revision.AuthorUserID
}
}
item := domain.HumanItemRecord{
ID: "human-" + task.ID, OrganizationID: task.OrganizationID, PluginID: pluginregistry.AnnotationQAID,
BatchID: batch.ID, ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, RunID: task.RunID, GateNodeID: task.NodeID,
LegacyTaskID: task.ID,
Subject: domain.HumanSubjectRef{ArtifactID: task.ArtifactID, ItemKey: task.ID, MediaType: "video/mp4", Payload: map[string]any{
"sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "sourceIsFullVideo": task.SourceIsFullVideo, "sourceDurationSeconds": task.SourceDurationSeconds, "templateId": task.TemplateID,
}},
CurrentStageID: stageID, Status: status, AssignedUserID: task.AssignedUserID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID,
VendorID: task.VendorID, CurrentRevision: task.CurrentRevision, MaxReworkRounds: batch.MaxReworkRounds,
Version: 1, LatestAuthorUserID: authorID, CreatedAt: task.CreatedAt,
}
if status == domain.HumanItemAccepted {
item.AcceptedAt = task.UpdatedAt
}
if item.AssignedUserID != "" && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted) {
item.Status = domain.HumanItemClaimed
item.ClaimedAt = task.UpdatedAt
}
if err := server.store.UpsertHumanItem(ctx, item); err != nil {
return err
}
_ = server.store.AppendHumanEvent(ctx, domain.HumanEventRecord{
ID: "human-event-migrated-" + task.ID, OrganizationID: organizationID, BatchID: batch.ID, ItemID: item.ID,
StageID: stageID, EventType: "migrated", ActorUserID: "migration", ItemVersion: item.Version,
Payload: map[string]any{"legacyTaskId": task.ID, "legacyStatus": task.Status}, CreatedAt: nowISO(),
})
}
// Full sweep completed and found nothing left: stamp the organization's
// plugin record so subsequent status reads skip the rescan entirely.
if migrated == 0 {
if record, getErr := server.store.GetOrganizationPlugin(ctx, organizationID, pluginregistry.AnnotationQAID); getErr == nil && record != nil {
stamped := *record
if stamped.Config == nil {
stamped.Config = map[string]any{}
}
stamped.Config["legacyMigrationDone"] = "1"
if upsertErr := server.store.UpsertOrganizationPlugin(ctx, stamped); upsertErr != nil {
return upsertErr
}
}
}
return nil
}
func defaultAnnotationPluginConfig() map[string]any {
return map[string]any{
"releasePolicy": "accepted_partition",
"partitionMaxItems": 32,
"partitionMaxWaitSeconds": 60,
"maxReworkRounds": 5,
}
}
func normalizeAnnotationPluginConfig(values map[string]any) (map[string]any, []validationIssue) {
config := defaultAnnotationPluginConfig()
for key, value := range values {
config[key] = value
}
issues := make([]validationIssue, 0)
policy := configString(config, "releasePolicy")
if policy != string(domain.HumanReleaseAcceptedPartition) && policy != string(domain.HumanReleaseAllAccepted) {
issues = append(issues, validationIssue{Field: "config.releasePolicy", Message: "releasePolicy must be accepted_partition or all_accepted"})
}
limits := []struct {
key string
max int
}{{"partitionMaxItems", 1000}, {"partitionMaxWaitSeconds", 3600}, {"maxReworkRounds", 100}}
for _, limit := range limits {
value := pluginConfigInt(config, limit.key, -1)
if value < 1 || value > limit.max {
issues = append(issues, validationIssue{Field: "config." + limit.key, Message: limit.key + " is outside the supported range"})
continue
}
config[limit.key] = value
}
return config, issues
}
func (server *server) annotationPluginEnabled(ctx context.Context, organizationID string) (bool, error) {
manifest, _ := pluginregistry.Get(pluginregistry.AnnotationQAID)
status, err := server.pluginStatus(ctx, organizationID, manifest)
return status.Enabled, err
}
func (server *server) filterOperatorsForInstalledPlugins(ctx context.Context, organizationID string, items []domain.OperatorRecord) ([]domain.OperatorRecord, error) {
hiddenOperatorIDs := make(map[string]struct{})
for _, manifest := range pluginregistry.List() {
if len(manifest.OperatorIDs) == 0 {
continue
}
status, err := server.pluginStatus(ctx, organizationID, manifest)
if err != nil {
return nil, err
}
if status.Enabled {
continue
}
for _, operatorID := range manifest.OperatorIDs {
hiddenOperatorIDs[strings.TrimSpace(operatorID)] = struct{}{}
}
}
if len(hiddenOperatorIDs) == 0 {
return items, nil
}
filtered := make([]domain.OperatorRecord, 0, len(items))
for _, item := range items {
if _, hidden := hiddenOperatorIDs[strings.TrimSpace(item.ID)]; hidden {
continue
}
filtered = append(filtered, item)
}
return filtered, nil
}
func (server *server) enrichWorkflowCapabilities(ctx context.Context, item *domain.WorkflowRecord) error {
if item == nil {
return nil
}
enabled, err := server.annotationPluginEnabled(ctx, item.OrganizationID)
if err != nil {
return err
}
item.Capabilities = pluginregistry.WorkflowCapabilities(*item, enabled)
if len(item.Capabilities) == 0 {
item.HumanWorkSummary = nil
return nil
}
tasks, err := server.store.ListAnnotationTasks(ctx, item.OrganizationID, item.ProjectID)
if err != nil {
return err
}
summary := &domain.HumanWorkSummary{}
for _, task := range tasks {
if task.WorkflowID != item.ID {
continue
}
switch task.Status {
case domain.AnnotationTaskStatusPendingAnnotation:
summary.PendingAnnotation++
case domain.AnnotationTaskStatusPendingQuality:
summary.PendingQuality++
case domain.AnnotationTaskStatusPendingAcceptance:
summary.PendingAcceptance++
case domain.AnnotationTaskStatusRework:
summary.Rework++
}
}
humanItems, err := server.store.ListHumanItems(ctx, item.OrganizationID, domain.HumanItemQuery{ProjectID: item.ProjectID, WorkflowID: item.ID})
if err != nil {
return err
}
for _, humanItem := range humanItems {
if humanItem.Status == domain.HumanItemRejected {
summary.Rejected++
}
if humanItem.Status == domain.HumanItemEscalated {
summary.Escalated++
}
}
packages, err := server.store.ListDeliveryPackages(ctx, item.OrganizationID, domain.DeliveryPackageQuery{ProjectID: item.ProjectID, WorkflowID: item.ID, Status: domain.DeliveryPackagePendingAcceptance})
if err != nil {
return err
}
summary.PendingAcceptance = len(packages)
item.HumanWorkSummary = summary
for index := range item.Capabilities {
switch item.Capabilities[index].ID {
case "human_annotation":
item.Capabilities[index].PendingCount = summary.PendingAnnotation + summary.Rework
case "human_quality_review":
item.Capabilities[index].PendingCount = summary.PendingQuality
case "human_acceptance":
item.Capabilities[index].PendingCount = summary.PendingAcceptance
}
}
return nil
}
func (server *server) enrichWorkflowListCapabilities(ctx context.Context, items []domain.WorkflowRecord) error {
for index := range items {
if err := server.enrichWorkflowCapabilities(ctx, &items[index]); err != nil {
return err
}
}
return nil
}
func (server *server) handlePluginsList(writer http.ResponseWriter, request *http.Request) {
organizationID := server.requestOrganizationID(request)
items := make([]pluginStatusResponse, 0)
for _, manifest := range pluginregistry.List() {
status, err := server.pluginStatus(request.Context(), organizationID, manifest)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = append(items, status)
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleOrganizationPluginGet(writer http.ResponseWriter, request *http.Request) {
organizationID := strings.TrimSpace(chi.URLParam(request, "id"))
if !server.ensureTenantAccess(writer, request, organizationID, "Organization plugin") {
return
}
manifest, ok := pluginregistry.Get(chi.URLParam(request, "pluginId"))
if !ok {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Plugin not found"})
return
}
status, err := server.pluginStatus(request.Context(), organizationID, manifest)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, status)
}
func (server *server) handleOrganizationPluginUpdate(writer http.ResponseWriter, request *http.Request) {
organizationID := strings.TrimSpace(chi.URLParam(request, "id"))
if !server.ensureTenantAccess(writer, request, organizationID, "Organization plugin") {
return
}
manifest, ok := pluginregistry.Get(chi.URLParam(request, "pluginId"))
if !ok {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Plugin not found"})
return
}
if manifest.System {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "System plugin cannot be disabled", "code": "system_plugin_immutable"})
return
}
var payload pluginConfigUpdateRequest
if !decodeJSON(writer, request, &payload) {
return
}
config := payload.Config
if config == nil && manifest.ID == pluginregistry.AnnotationQAID {
config = defaultAnnotationPluginConfig()
}
if manifest.ID == pluginregistry.AnnotationQAID {
normalized, issues := normalizeAnnotationPluginConfig(config)
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
config = normalized
if !payload.Enabled {
items, err := server.store.ListHumanItems(request.Context(), organizationID, domain.HumanItemQuery{PluginID: pluginregistry.AnnotationQAID})
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
active := 0
for _, item := range items {
if humanItemPending(item.Status) || item.Status == domain.HumanItemEscalated {
active++
}
}
if active > 0 {
renderJSON(writer, http.StatusConflict, map[string]any{"error": "插件仍有未完成的人工任务,不能关闭", "code": "plugin_has_active_human_work", "activeItemCount": active})
return
}
}
}
principal, _ := currentPrincipal(request)
updatedBy := ""
if principal != nil {
updatedBy = principal.User.ID
}
record := domain.OrganizationPluginRecord{
OrganizationID: organizationID,
PluginID: manifest.ID,
Enabled: payload.Enabled,
Config: config,
UpdatedBy: updatedBy,
}
if err := server.store.UpsertOrganizationPlugin(request.Context(), record); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
status, err := server.pluginStatus(request.Context(), organizationID, manifest)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, status)
}