forked from longtao.wu/droboticflow
429 lines
15 KiB
Go
429 lines
15 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
pluginregistry "droboticflow/internal/plugins"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
)
|
|
|
|
type pluginConfigUpdateRequest struct {
|
|
Enabled bool `json:"enabled"`
|
|
Config map[string]any `json:"config"`
|
|
}
|
|
|
|
type pluginStatusResponse struct {
|
|
Manifest domain.PluginManifest `json:"manifest"`
|
|
Enabled bool `json:"enabled"`
|
|
Config map[string]any `json:"config,omitempty"`
|
|
Record *domain.OrganizationPluginRecord `json:"record,omitempty"`
|
|
}
|
|
|
|
func (server *server) pluginStatus(ctx context.Context, organizationID string, manifest domain.PluginManifest) (pluginStatusResponse, error) {
|
|
response := pluginStatusResponse{Manifest: manifest, Enabled: manifest.System}
|
|
record, err := server.store.GetOrganizationPlugin(ctx, organizationID, manifest.ID)
|
|
if err != nil {
|
|
return response, err
|
|
}
|
|
if record != nil {
|
|
response.Record = record
|
|
response.Enabled = manifest.System || record.Enabled
|
|
response.Config = record.Config
|
|
if manifest.ID == pluginregistry.AnnotationQAID && record.Enabled {
|
|
if err := server.migrateLegacyAnnotationHumanItems(ctx, organizationID, record.Config); err != nil {
|
|
return response, err
|
|
}
|
|
}
|
|
return response, nil
|
|
}
|
|
if manifest.ID == pluginregistry.AnnotationQAID {
|
|
tasks, listErr := server.store.ListAnnotationTasks(ctx, organizationID, "")
|
|
if listErr != nil {
|
|
return response, listErr
|
|
}
|
|
if len(tasks) > 0 {
|
|
response.Enabled = true
|
|
migration := domain.OrganizationPluginRecord{
|
|
OrganizationID: organizationID,
|
|
PluginID: manifest.ID,
|
|
Enabled: true,
|
|
Config: defaultAnnotationPluginConfig(),
|
|
UpdatedBy: "migration",
|
|
}
|
|
if upsertErr := server.store.UpsertOrganizationPlugin(ctx, migration); upsertErr != nil {
|
|
return response, upsertErr
|
|
}
|
|
stored, getErr := server.store.GetOrganizationPlugin(ctx, organizationID, manifest.ID)
|
|
if getErr != nil {
|
|
return response, getErr
|
|
}
|
|
response.Record = stored
|
|
response.Config = migration.Config
|
|
if err := server.migrateLegacyAnnotationHumanItems(ctx, organizationID, migration.Config); err != nil {
|
|
return response, err
|
|
}
|
|
}
|
|
}
|
|
return response, nil
|
|
}
|
|
|
|
func humanItemStateFromLegacyTask(task domain.AnnotationTaskRecord) (string, domain.HumanItemStatus) {
|
|
switch task.Status {
|
|
case domain.AnnotationTaskStatusPendingQuality:
|
|
return "quality_review", domain.HumanItemSubmitted
|
|
case domain.AnnotationTaskStatusPendingAcceptance:
|
|
return "acceptance", domain.HumanItemSubmitted
|
|
case domain.AnnotationTaskStatusRework:
|
|
return "semantic_annotation", domain.HumanItemRework
|
|
case domain.AnnotationTaskStatusAccepted:
|
|
return "acceptance", domain.HumanItemAccepted
|
|
case domain.AnnotationTaskStatusRejected:
|
|
return "acceptance", domain.HumanItemEscalated
|
|
default:
|
|
return "split_annotation", domain.HumanItemReady
|
|
}
|
|
}
|
|
|
|
func pluginConfigInt(config map[string]any, key string, fallback int) int {
|
|
switch typed := config[key].(type) {
|
|
case int:
|
|
if typed > 0 {
|
|
return typed
|
|
}
|
|
case int32:
|
|
if typed > 0 {
|
|
return int(typed)
|
|
}
|
|
case int64:
|
|
if typed > 0 {
|
|
return int(typed)
|
|
}
|
|
case float64:
|
|
if typed > 0 && math.Trunc(typed) == typed {
|
|
return int(typed)
|
|
}
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func pluginReleasePolicy(config map[string]any) domain.HumanReleasePolicy {
|
|
if strings.TrimSpace(configString(config, "releasePolicy")) == string(domain.HumanReleaseAllAccepted) {
|
|
return domain.HumanReleaseAllAccepted
|
|
}
|
|
return domain.HumanReleaseAcceptedPartition
|
|
}
|
|
|
|
func configString(config map[string]any, key string) string {
|
|
value, _ := config[key].(string)
|
|
return value
|
|
}
|
|
|
|
func legacyHumanBatchID(task domain.AnnotationTaskRecord) string {
|
|
key := strings.Trim(strings.Join([]string{task.RunID, task.NodeID}, "-"), "-")
|
|
if key == "" {
|
|
key = task.ProjectID + "-manual"
|
|
}
|
|
return "human-batch-legacy-" + key
|
|
}
|
|
|
|
func (server *server) migrateLegacyAnnotationHumanItems(ctx context.Context, organizationID string, config map[string]any) error {
|
|
tasks, err := server.store.ListAnnotationTasks(ctx, organizationID, "")
|
|
if err != nil || len(tasks) == 0 {
|
|
return err
|
|
}
|
|
batches, err := server.store.ListHumanBatches(ctx, organizationID, domain.HumanBatchQuery{PluginID: pluginregistry.AnnotationQAID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
batchByKey := make(map[string]domain.HumanBatchRecord)
|
|
for _, batch := range batches {
|
|
batchByKey[batch.RunID+"\x00"+batch.GateNodeID] = batch
|
|
}
|
|
existing, err := server.store.ListHumanItems(ctx, organizationID, domain.HumanItemQuery{PluginID: pluginregistry.AnnotationQAID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
existingLegacy := make(map[string]struct{}, len(existing))
|
|
for _, item := range existing {
|
|
existingLegacy[item.LegacyTaskID] = struct{}{}
|
|
}
|
|
groupCounts := make(map[string]int)
|
|
for _, task := range tasks {
|
|
groupCounts[task.RunID+"\x00"+task.NodeID]++
|
|
}
|
|
for _, task := range tasks {
|
|
if _, ok := existingLegacy[task.ID]; ok {
|
|
continue
|
|
}
|
|
groupKey := task.RunID + "\x00" + task.NodeID
|
|
batch, ok := batchByKey[groupKey]
|
|
if !ok {
|
|
stages := domain.DefaultAnnotationHumanStages()
|
|
if template, templateErr := server.store.GetAnnotationTemplate(ctx, task.TemplateID); templateErr != nil {
|
|
return templateErr
|
|
} else if template != nil && len(template.HumanStages) > 0 {
|
|
stages = template.HumanStages
|
|
}
|
|
batch = domain.HumanBatchRecord{
|
|
ID: legacyHumanBatchID(task), OrganizationID: task.OrganizationID, PluginID: pluginregistry.AnnotationQAID,
|
|
ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, RunID: task.RunID, GateNodeID: task.NodeID,
|
|
Status: domain.HumanBatchActive, Stages: stages,
|
|
ReleasePolicy: pluginReleasePolicy(config),
|
|
PartitionMaxItems: pluginConfigInt(config, "partitionMaxItems", 32),
|
|
PartitionMaxWaitSeconds: pluginConfigInt(config, "partitionMaxWaitSeconds", 60),
|
|
MaxReworkRounds: pluginConfigInt(config, "maxReworkRounds", 5), ItemCount: groupCounts[groupKey],
|
|
CreatedAt: task.CreatedAt,
|
|
}
|
|
if err := server.store.UpsertHumanBatch(ctx, batch); err != nil {
|
|
return err
|
|
}
|
|
batchByKey[groupKey] = batch
|
|
}
|
|
stageID, status := humanItemStateFromLegacyTask(task)
|
|
if _, _, found := humanStage(&batch, stageID); !found && len(batch.Stages) > 0 {
|
|
stageID = batch.Stages[len(batch.Stages)-1].ID
|
|
}
|
|
authorID := ""
|
|
revisions, err := server.store.ListAnnotationRevisions(ctx, organizationID, task.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, revision := range revisions {
|
|
if revision.Revision == task.CurrentRevision {
|
|
authorID = revision.AuthorUserID
|
|
}
|
|
}
|
|
item := domain.HumanItemRecord{
|
|
ID: "human-" + task.ID, OrganizationID: task.OrganizationID, PluginID: pluginregistry.AnnotationQAID,
|
|
BatchID: batch.ID, ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, RunID: task.RunID, GateNodeID: task.NodeID,
|
|
LegacyTaskID: task.ID,
|
|
Subject: domain.HumanSubjectRef{ArtifactID: task.ArtifactID, ItemKey: task.ID, MediaType: "video/mp4", Payload: map[string]any{
|
|
"sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "sourceIsFullVideo": task.SourceIsFullVideo, "sourceDurationSeconds": task.SourceDurationSeconds, "templateId": task.TemplateID,
|
|
}},
|
|
CurrentStageID: stageID, Status: status, AssignedUserID: task.AssignedUserID, WorkgroupID: task.WorkgroupID,
|
|
VendorID: task.VendorID, CurrentRevision: task.CurrentRevision, MaxReworkRounds: batch.MaxReworkRounds,
|
|
Version: 1, LatestAuthorUserID: authorID, CreatedAt: task.CreatedAt,
|
|
}
|
|
if status == domain.HumanItemAccepted {
|
|
item.AcceptedAt = task.UpdatedAt
|
|
}
|
|
if item.AssignedUserID != "" && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted) {
|
|
item.Status = domain.HumanItemClaimed
|
|
item.ClaimedAt = task.UpdatedAt
|
|
}
|
|
if err := server.store.UpsertHumanItem(ctx, item); err != nil {
|
|
return err
|
|
}
|
|
_ = server.store.AppendHumanEvent(ctx, domain.HumanEventRecord{
|
|
ID: "human-event-migrated-" + task.ID, OrganizationID: organizationID, BatchID: batch.ID, ItemID: item.ID,
|
|
StageID: stageID, EventType: "migrated", ActorUserID: "migration", ItemVersion: item.Version,
|
|
Payload: map[string]any{"legacyTaskId": task.ID, "legacyStatus": task.Status}, CreatedAt: nowISO(),
|
|
})
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func defaultAnnotationPluginConfig() map[string]any {
|
|
return map[string]any{
|
|
"releasePolicy": "accepted_partition",
|
|
"partitionMaxItems": 32,
|
|
"partitionMaxWaitSeconds": 60,
|
|
"maxReworkRounds": 5,
|
|
}
|
|
}
|
|
|
|
func normalizeAnnotationPluginConfig(values map[string]any) (map[string]any, []validationIssue) {
|
|
config := defaultAnnotationPluginConfig()
|
|
for key, value := range values {
|
|
config[key] = value
|
|
}
|
|
issues := make([]validationIssue, 0)
|
|
policy := configString(config, "releasePolicy")
|
|
if policy != string(domain.HumanReleaseAcceptedPartition) && policy != string(domain.HumanReleaseAllAccepted) {
|
|
issues = append(issues, validationIssue{Field: "config.releasePolicy", Message: "releasePolicy must be accepted_partition or all_accepted"})
|
|
}
|
|
limits := []struct {
|
|
key string
|
|
max int
|
|
}{{"partitionMaxItems", 1000}, {"partitionMaxWaitSeconds", 3600}, {"maxReworkRounds", 100}}
|
|
for _, limit := range limits {
|
|
value := pluginConfigInt(config, limit.key, -1)
|
|
if value < 1 || value > limit.max {
|
|
issues = append(issues, validationIssue{Field: "config." + limit.key, Message: limit.key + " is outside the supported range"})
|
|
continue
|
|
}
|
|
config[limit.key] = value
|
|
}
|
|
return config, issues
|
|
}
|
|
|
|
func (server *server) annotationPluginEnabled(ctx context.Context, organizationID string) (bool, error) {
|
|
manifest, _ := pluginregistry.Get(pluginregistry.AnnotationQAID)
|
|
status, err := server.pluginStatus(ctx, organizationID, manifest)
|
|
return status.Enabled, err
|
|
}
|
|
|
|
func (server *server) enrichWorkflowCapabilities(ctx context.Context, item *domain.WorkflowRecord) error {
|
|
if item == nil {
|
|
return nil
|
|
}
|
|
enabled, err := server.annotationPluginEnabled(ctx, item.OrganizationID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
item.Capabilities = pluginregistry.WorkflowCapabilities(*item, enabled)
|
|
if len(item.Capabilities) == 0 {
|
|
item.HumanWorkSummary = nil
|
|
return nil
|
|
}
|
|
tasks, err := server.store.ListAnnotationTasks(ctx, item.OrganizationID, item.ProjectID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
summary := &domain.HumanWorkSummary{}
|
|
for _, task := range tasks {
|
|
if task.WorkflowID != item.ID {
|
|
continue
|
|
}
|
|
switch task.Status {
|
|
case domain.AnnotationTaskStatusPendingAnnotation:
|
|
summary.PendingAnnotation++
|
|
case domain.AnnotationTaskStatusPendingQuality:
|
|
summary.PendingQuality++
|
|
case domain.AnnotationTaskStatusPendingAcceptance:
|
|
summary.PendingAcceptance++
|
|
case domain.AnnotationTaskStatusRework:
|
|
summary.Rework++
|
|
}
|
|
}
|
|
humanItems, err := server.store.ListHumanItems(ctx, item.OrganizationID, domain.HumanItemQuery{ProjectID: item.ProjectID, WorkflowID: item.ID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, humanItem := range humanItems {
|
|
if humanItem.Status == domain.HumanItemRejected {
|
|
summary.Rejected++
|
|
}
|
|
if humanItem.Status == domain.HumanItemEscalated {
|
|
summary.Escalated++
|
|
}
|
|
}
|
|
packages, err := server.store.ListDeliveryPackages(ctx, item.OrganizationID, domain.DeliveryPackageQuery{ProjectID: item.ProjectID, WorkflowID: item.ID, Status: domain.DeliveryPackagePendingAcceptance})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
summary.PendingAcceptance = len(packages)
|
|
item.HumanWorkSummary = summary
|
|
for index := range item.Capabilities {
|
|
switch item.Capabilities[index].ID {
|
|
case "human_annotation":
|
|
item.Capabilities[index].PendingCount = summary.PendingAnnotation + summary.Rework
|
|
case "human_quality_review":
|
|
item.Capabilities[index].PendingCount = summary.PendingQuality
|
|
case "human_acceptance":
|
|
item.Capabilities[index].PendingCount = summary.PendingAcceptance
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (server *server) enrichWorkflowListCapabilities(ctx context.Context, items []domain.WorkflowRecord) error {
|
|
for index := range items {
|
|
if err := server.enrichWorkflowCapabilities(ctx, &items[index]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (server *server) handlePluginsList(writer http.ResponseWriter, request *http.Request) {
|
|
organizationID := server.requestOrganizationID(request)
|
|
items := make([]pluginStatusResponse, 0)
|
|
for _, manifest := range pluginregistry.List() {
|
|
status, err := server.pluginStatus(request.Context(), organizationID, manifest)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
items = append(items, status)
|
|
}
|
|
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
|
}
|
|
|
|
func (server *server) handleOrganizationPluginGet(writer http.ResponseWriter, request *http.Request) {
|
|
organizationID := strings.TrimSpace(chi.URLParam(request, "id"))
|
|
if !server.ensureTenantAccess(writer, request, organizationID, "Organization plugin") {
|
|
return
|
|
}
|
|
manifest, ok := pluginregistry.Get(chi.URLParam(request, "pluginId"))
|
|
if !ok {
|
|
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Plugin not found"})
|
|
return
|
|
}
|
|
status, err := server.pluginStatus(request.Context(), organizationID, manifest)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, status)
|
|
}
|
|
|
|
func (server *server) handleOrganizationPluginUpdate(writer http.ResponseWriter, request *http.Request) {
|
|
organizationID := strings.TrimSpace(chi.URLParam(request, "id"))
|
|
if !server.ensureTenantAccess(writer, request, organizationID, "Organization plugin") {
|
|
return
|
|
}
|
|
manifest, ok := pluginregistry.Get(chi.URLParam(request, "pluginId"))
|
|
if !ok {
|
|
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Plugin not found"})
|
|
return
|
|
}
|
|
if manifest.System {
|
|
renderJSON(writer, http.StatusConflict, map[string]string{"error": "System plugin cannot be disabled", "code": "system_plugin_immutable"})
|
|
return
|
|
}
|
|
var payload pluginConfigUpdateRequest
|
|
if !decodeJSON(writer, request, &payload) {
|
|
return
|
|
}
|
|
config := payload.Config
|
|
if config == nil && manifest.ID == pluginregistry.AnnotationQAID {
|
|
config = defaultAnnotationPluginConfig()
|
|
}
|
|
if manifest.ID == pluginregistry.AnnotationQAID {
|
|
normalized, issues := normalizeAnnotationPluginConfig(config)
|
|
if len(issues) > 0 {
|
|
renderValidationError(writer, issues...)
|
|
return
|
|
}
|
|
config = normalized
|
|
}
|
|
principal, _ := currentPrincipal(request)
|
|
updatedBy := ""
|
|
if principal != nil {
|
|
updatedBy = principal.User.ID
|
|
}
|
|
record := domain.OrganizationPluginRecord{
|
|
OrganizationID: organizationID,
|
|
PluginID: manifest.ID,
|
|
Enabled: payload.Enabled,
|
|
Config: config,
|
|
UpdatedBy: updatedBy,
|
|
}
|
|
if err := server.store.UpsertOrganizationPlugin(request.Context(), record); err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
status, err := server.pluginStatus(request.Context(), organizationID, manifest)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, status)
|
|
}
|