droboticflow/internal/api/builtin_operators.go
2026-08-17 20:44:16 +08:00

415 lines
13 KiB
Go

package api
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"droboticflow/internal/domain"
_ "embed"
)
var reservedMongoCollections = map[string]struct{}{
"projects": {},
"operators": {},
"operatorTombstones": {},
"workflows": {},
"templates": {},
"runs": {},
"sourceStates": {},
"auditTasks": {},
"samplingTasks": {},
"annotationWorkgroups": {},
"annotationMemberships": {},
"annotationTemplates": {},
"annotationTasks": {},
"annotationRevisions": {},
"annotationReviews": {},
"annotationWorklogEvents": {},
"users": {},
"storageConnections": {},
"mongoConnections": {},
"settings": {},
}
//go:embed builtin_operators_catalog.json
var builtinOperatorCatalogJSON []byte
func builtinOperatorCatalogFingerprint() (string, int, error) {
var catalog any
if err := json.Unmarshal(builtinOperatorCatalogJSON, &catalog); err != nil {
return "", 0, fmt.Errorf("parse builtin operator catalog for fingerprint: %w", err)
}
records, ok := catalog.([]any)
if !ok || len(records) == 0 {
return "", 0, fmt.Errorf("builtin operator catalog is empty")
}
canonical, err := json.Marshal(catalog)
if err != nil {
return "", 0, fmt.Errorf("canonicalize builtin operator catalog: %w", err)
}
digest := sha256.Sum256(canonical)
return hex.EncodeToString(digest[:]), len(records), nil
}
func builtinOperatorRecords(now string, runnerImage string) ([]domain.OperatorRecord, error) {
var records []domain.OperatorRecord
if err := json.Unmarshal(builtinOperatorCatalogJSON, &records); err != nil {
return nil, fmt.Errorf("parse builtin operator catalog: %w", err)
}
if len(records) == 0 {
return nil, fmt.Errorf("builtin operator catalog is empty")
}
seenIDs := map[string]struct{}{}
seenNames := map[string]struct{}{}
for index := range records {
record := &records[index]
record.ID = strings.TrimSpace(record.ID)
record.Name = strings.TrimSpace(record.Name)
record.Type = strings.TrimSpace(record.Type)
record.Category = strings.TrimSpace(record.Category)
record.Description = strings.TrimSpace(record.Description)
if record.ID == "" {
return nil, fmt.Errorf("builtin operator catalog item %d has empty id", index)
}
if record.Name == "" {
return nil, fmt.Errorf("builtin operator catalog item %q has empty name", record.ID)
}
if !strings.EqualFold(record.Type, "builtin") {
return nil, fmt.Errorf("builtin operator catalog item %q has type %q", record.ID, record.Type)
}
if record.Category == "" {
return nil, fmt.Errorf("builtin operator catalog item %q has empty category", record.ID)
}
if record.Description == "" {
return nil, fmt.Errorf("builtin operator catalog item %q has empty description", record.ID)
}
normalizedID := normalizeBuiltinOperatorDeletionValue(record.ID)
if _, ok := seenIDs[normalizedID]; ok {
return nil, fmt.Errorf("builtin operator catalog has duplicate id %q", record.ID)
}
seenIDs[normalizedID] = struct{}{}
normalizedName := normalizeBuiltinOperatorDeletionValue(record.Name)
if _, ok := seenNames[normalizedName]; ok {
return nil, fmt.Errorf("builtin operator catalog has duplicate name %q", record.Name)
}
seenNames[normalizedName] = struct{}{}
record.Type = "builtin"
if strings.TrimSpace(record.Image) == "" {
record.Image = runnerImage
} else if derivedImage := deriveBuiltinOperatorImage(record.ID, record.Image, runnerImage); derivedImage != "" {
record.Image = derivedImage
}
if strings.TrimSpace(record.ImageID) == "" {
record.ImageID = record.Image
} else if !strings.Contains(record.ImageID, "@sha256:") || !sameImageRepository(record.Image, record.ImageID) {
record.ImageID = record.Image
}
record.CreatedAt = now
record.UpdatedAt = now
}
return records, nil
}
func deriveBuiltinOperatorImage(operatorID string, catalogImage string, runnerImage string) string {
if !hasIndependentBuiltinOperatorImage(domain.OperatorRecord{Image: catalogImage}) {
return ""
}
runner := strings.TrimSpace(runnerImage)
if runner == "" || !strings.Contains(runner, "/") || !strings.Contains(runner[strings.LastIndex(runner, "/")+1:], ":") {
return ""
}
registryPrefix := runner[:strings.LastIndex(runner, "/")]
tag := runner[strings.LastIndex(runner, ":")+1:]
slug := strings.TrimPrefix(strings.TrimSpace(operatorID), "operator-builtin-")
if slug == "" {
return ""
}
return registryPrefix + "/droboticflow-operator-" + slug + ":" + tag
}
func imageRepository(value string) string {
trimmed := strings.TrimSpace(value)
if at := strings.Index(trimmed, "@"); at >= 0 {
trimmed = trimmed[:at]
}
if slash := strings.LastIndex(trimmed, "/"); slash >= 0 {
if colon := strings.LastIndex(trimmed[slash+1:], ":"); colon >= 0 {
return trimmed[:slash+1+colon]
}
}
return trimmed
}
func sameImageRepository(image string, imageID string) bool {
return imageRepository(image) == imageRepository(imageID)
}
func findBuiltinOperator(items []domain.OperatorRecord, id string, name string) *domain.OperatorRecord {
for index := range items {
item := items[index]
if item.ID == id || strings.TrimSpace(item.Name) == strings.TrimSpace(name) {
copy := item
return &copy
}
}
return nil
}
type builtinOperatorDeletionSet struct {
ids map[string]struct{}
names map[string]struct{}
}
func normalizeBuiltinOperatorDeletionValue(value string) string {
return strings.ToLower(strings.TrimSpace(value))
}
func builtinOperatorDeletionKey(id string, name string) string {
if normalizedName := normalizeBuiltinOperatorDeletionValue(name); normalizedName != "" {
return "name:" + normalizedName
}
if normalizedID := normalizeBuiltinOperatorDeletionValue(id); normalizedID != "" {
return "id:" + normalizedID
}
return ""
}
func newBuiltinOperatorDeletionSet(items []domain.BuiltinOperatorDeletionRecord) builtinOperatorDeletionSet {
set := builtinOperatorDeletionSet{
ids: make(map[string]struct{}),
names: make(map[string]struct{}),
}
for _, item := range items {
if normalizedID := normalizeBuiltinOperatorDeletionValue(item.OperatorID); normalizedID != "" {
set.ids[normalizedID] = struct{}{}
}
if normalizedName := normalizeBuiltinOperatorDeletionValue(item.OperatorName); normalizedName != "" {
set.names[normalizedName] = struct{}{}
}
}
return set
}
func (set builtinOperatorDeletionSet) Has(item domain.OperatorRecord) bool {
if normalizedID := normalizeBuiltinOperatorDeletionValue(item.ID); normalizedID != "" {
if _, ok := set.ids[normalizedID]; ok {
return true
}
}
if normalizedName := normalizeBuiltinOperatorDeletionValue(item.Name); normalizedName != "" {
if _, ok := set.names[normalizedName]; ok {
return true
}
}
return false
}
func filterDeletedBuiltinOperators(items []domain.OperatorRecord, deletions builtinOperatorDeletionSet) []domain.OperatorRecord {
filtered := make([]domain.OperatorRecord, 0, len(items))
for _, item := range items {
if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") && deletions.Has(item) {
continue
}
filtered = append(filtered, item)
}
return filtered
}
func currentBuiltinOperatorIDSet() (map[string]struct{}, error) {
builtins, err := builtinOperatorRecords(nowISO(), "")
if err != nil {
return nil, err
}
ids := make(map[string]struct{}, len(builtins))
for _, item := range builtins {
ids[normalizeBuiltinOperatorDeletionValue(item.ID)] = struct{}{}
}
return ids, nil
}
func filterObsoleteBuiltinOperators(items []domain.OperatorRecord, currentBuiltinIDs map[string]struct{}) []domain.OperatorRecord {
filtered := make([]domain.OperatorRecord, 0, len(items))
for _, item := range items {
if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") {
id := normalizeBuiltinOperatorDeletionValue(item.ID)
if _, ok := currentBuiltinIDs[id]; !ok {
continue
}
}
filtered = append(filtered, item)
}
return filtered
}
func upsertBuiltinOperator(ctx context.Context, server *server, items []domain.OperatorRecord, builtin domain.OperatorRecord) ([]domain.OperatorRecord, error) {
existing := findBuiltinOperator(items, builtin.ID, builtin.Name)
if existing == nil {
if err := server.store.UpsertOperator(ctx, builtin); err != nil {
return items, err
}
return append(items, builtin), nil
}
updated := *existing
updated.ID = builtin.ID
updated.Name = builtin.Name
updated.Type = builtin.Type
updated.Category = builtin.Category
updated.Description = builtin.Description
updated.Inputs = builtin.Inputs
updated.Outputs = builtin.Outputs
updated.ConfigSchema = builtin.ConfigSchema
existingDigestPinned := strings.TrimSpace(existing.ImageID) != "" && strings.Contains(existing.ImageID, "@sha256:") && sameImageRepository(builtin.Image, existing.ImageID)
existingIndependentImage := hasIndependentBuiltinOperatorImage(*existing)
existingOnCurrentImage := strings.TrimSpace(existing.Image) == strings.TrimSpace(builtin.Image)
if existingDigestPinned && existingIndependentImage {
updated.Image = existing.Image
updated.ImageID = existing.ImageID
} else {
updated.Image = builtin.Image
if existingOnCurrentImage && existingDigestPinned {
updated.ImageID = existing.ImageID
} else if strings.TrimSpace(builtin.ImageID) != "" && strings.Contains(builtin.ImageID, "@sha256:") && sameImageRepository(builtin.Image, builtin.ImageID) {
updated.ImageID = builtin.ImageID
} else if existingDigestPinned {
updated.ImageID = existing.ImageID
} else {
updated.ImageID = builtin.ImageID
}
}
updated.ContainerCommand = builtin.ContainerCommand
updated.ContainerArgs = builtin.ContainerArgs
updated.RuntimeContract = builtin.RuntimeContract
updated.SkillDefinition = builtin.SkillDefinition
updated.Hyperparams = builtin.Hyperparams
updated.Resources = builtin.Resources
// Built-in metadata is reconciled from the source catalog at startup. Keep
// an explicit user SHM override so an API restart does not silently reset a
// per-operator /dev/shm setting; an empty value still inherits the catalog
// or platform default.
if existing.Resources != nil && strings.TrimSpace(existing.Resources.SharedMemorySize) != "" {
resources := domain.OperatorResources{}
if updated.Resources != nil {
resources = *updated.Resources
}
resources.SharedMemorySize = strings.TrimSpace(existing.Resources.SharedMemorySize)
updated.Resources = &resources
}
updated.UpdatedAt = builtin.UpdatedAt
if strings.TrimSpace(updated.CreatedAt) == "" {
updated.CreatedAt = builtin.CreatedAt
}
if err := server.store.UpsertOperator(ctx, updated); err != nil {
return items, err
}
for index := range items {
if items[index].ID == existing.ID || strings.TrimSpace(items[index].Name) == strings.TrimSpace(existing.Name) {
items[index] = updated
return items, nil
}
}
return append(items, updated), nil
}
func (server *server) ensureBuiltinOperators(ctx context.Context) error {
items, err := server.store.ListOperators(ctx)
if err != nil {
return err
}
deletions, err := server.store.ListBuiltinOperatorDeletions(ctx)
if err != nil {
return err
}
deletionSet := newBuiltinOperatorDeletionSet(deletions)
now := nowISO()
builtins, err := builtinOperatorRecords(now, server.operatorRunnerImage)
if err != nil {
return err
}
for _, builtin := range builtins {
if deletionSet.Has(builtin) {
continue
}
items, err = upsertBuiltinOperator(ctx, server, items, builtin)
if err != nil {
return err
}
}
return server.refreshBuiltinOperatorImages(ctx, deletionSet)
}
func hasIndependentBuiltinOperatorImage(item domain.OperatorRecord) bool {
image := strings.TrimSpace(item.Image)
if image == "" {
return false
}
return strings.Contains(image, "droboticflow-operator-") && !strings.Contains(image, "droboticflow-operator-runner")
}
func (server *server) refreshBuiltinOperatorImages(ctx context.Context, deletions builtinOperatorDeletionSet) error {
runnerImage := strings.TrimSpace(server.operatorRunnerImage)
if runnerImage == "" {
return nil
}
items, err := server.store.ListOperators(ctx)
if err != nil {
return err
}
now := nowISO()
for _, item := range items {
if strings.TrimSpace(strings.ToLower(item.Type)) != "builtin" {
continue
}
if deletions.Has(item) {
continue
}
if hasIndependentBuiltinOperatorImage(item) {
continue
}
if strings.TrimSpace(item.Image) == runnerImage {
continue
}
item.Image = runnerImage
item.UpdatedAt = now
if strings.TrimSpace(item.CreatedAt) == "" {
item.CreatedAt = now
}
if err := server.store.UpsertOperator(ctx, item); err != nil {
return err
}
}
return nil
}
func (server *server) listAvailableOperators(ctx context.Context) ([]domain.OperatorRecord, error) {
if err := server.ensureBuiltinOperators(ctx); err != nil {
return nil, err
}
items, err := server.listOperatorsForContext(ctx)
if err != nil {
return nil, err
}
deletions, err := server.store.ListBuiltinOperatorDeletions(ctx)
if err != nil {
return nil, err
}
currentIDs, err := currentBuiltinOperatorIDSet()
if err != nil {
return nil, err
}
return filterObsoleteBuiltinOperators(filterDeletedBuiltinOperators(items, newBuiltinOperatorDeletionSet(deletions)), currentIDs), nil
}