419 lines
13 KiB
Go
419 lines
13 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
|
|
_ "embed"
|
|
)
|
|
|
|
var reservedMongoCollections = map[string]struct{}{
|
|
"projects": {},
|
|
"operators": {},
|
|
"operatorTombstones": {},
|
|
"workflows": {},
|
|
"templates": {},
|
|
"runs": {},
|
|
"sourceStates": {},
|
|
"auditTasks": {},
|
|
"samplingTasks": {},
|
|
"annotationWorkgroups": {},
|
|
"annotationMemberships": {},
|
|
"annotationTemplates": {},
|
|
"annotationTasks": {},
|
|
"annotationRevisions": {},
|
|
"annotationReviews": {},
|
|
"annotationWorklogEvents": {},
|
|
"users": {},
|
|
"storageConnections": {},
|
|
"mongoConnections": {},
|
|
"settings": {},
|
|
}
|
|
|
|
//go:embed builtin_operators_catalog.json
|
|
var builtinOperatorCatalogJSON []byte
|
|
|
|
func builtinOperatorCatalogFingerprint() (string, int, error) {
|
|
var catalog any
|
|
if err := json.Unmarshal(builtinOperatorCatalogJSON, &catalog); err != nil {
|
|
return "", 0, fmt.Errorf("parse builtin operator catalog for fingerprint: %w", err)
|
|
}
|
|
records, ok := catalog.([]any)
|
|
if !ok || len(records) == 0 {
|
|
return "", 0, fmt.Errorf("builtin operator catalog is empty")
|
|
}
|
|
canonical, err := json.Marshal(catalog)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("canonicalize builtin operator catalog: %w", err)
|
|
}
|
|
digest := sha256.Sum256(canonical)
|
|
return hex.EncodeToString(digest[:]), len(records), nil
|
|
}
|
|
|
|
func builtinOperatorRecords(now string, runnerImage string) ([]domain.OperatorRecord, error) {
|
|
var records []domain.OperatorRecord
|
|
if err := json.Unmarshal(builtinOperatorCatalogJSON, &records); err != nil {
|
|
return nil, fmt.Errorf("parse builtin operator catalog: %w", err)
|
|
}
|
|
if len(records) == 0 {
|
|
return nil, fmt.Errorf("builtin operator catalog is empty")
|
|
}
|
|
|
|
seenIDs := map[string]struct{}{}
|
|
seenNames := map[string]struct{}{}
|
|
for index := range records {
|
|
record := &records[index]
|
|
record.ID = strings.TrimSpace(record.ID)
|
|
record.Name = strings.TrimSpace(record.Name)
|
|
record.Type = strings.TrimSpace(record.Type)
|
|
record.Category = strings.TrimSpace(record.Category)
|
|
record.Description = strings.TrimSpace(record.Description)
|
|
|
|
if record.ID == "" {
|
|
return nil, fmt.Errorf("builtin operator catalog item %d has empty id", index)
|
|
}
|
|
if record.Name == "" {
|
|
return nil, fmt.Errorf("builtin operator catalog item %q has empty name", record.ID)
|
|
}
|
|
if !strings.EqualFold(record.Type, "builtin") {
|
|
return nil, fmt.Errorf("builtin operator catalog item %q has type %q", record.ID, record.Type)
|
|
}
|
|
if record.Category == "" {
|
|
return nil, fmt.Errorf("builtin operator catalog item %q has empty category", record.ID)
|
|
}
|
|
if record.Description == "" {
|
|
return nil, fmt.Errorf("builtin operator catalog item %q has empty description", record.ID)
|
|
}
|
|
|
|
normalizedID := normalizeBuiltinOperatorDeletionValue(record.ID)
|
|
if _, ok := seenIDs[normalizedID]; ok {
|
|
return nil, fmt.Errorf("builtin operator catalog has duplicate id %q", record.ID)
|
|
}
|
|
seenIDs[normalizedID] = struct{}{}
|
|
|
|
normalizedName := normalizeBuiltinOperatorDeletionValue(record.Name)
|
|
if _, ok := seenNames[normalizedName]; ok {
|
|
return nil, fmt.Errorf("builtin operator catalog has duplicate name %q", record.Name)
|
|
}
|
|
seenNames[normalizedName] = struct{}{}
|
|
|
|
record.Type = "builtin"
|
|
if strings.TrimSpace(record.Image) == "" {
|
|
record.Image = runnerImage
|
|
} else if derivedImage := deriveBuiltinOperatorImage(record.ID, record.Image, runnerImage); derivedImage != "" {
|
|
record.Image = derivedImage
|
|
}
|
|
if strings.TrimSpace(record.ImageID) == "" {
|
|
record.ImageID = record.Image
|
|
} else if !strings.Contains(record.ImageID, "@sha256:") || !sameImageRepository(record.Image, record.ImageID) {
|
|
record.ImageID = record.Image
|
|
}
|
|
record.CreatedAt = now
|
|
record.UpdatedAt = now
|
|
}
|
|
|
|
return records, nil
|
|
}
|
|
|
|
func deriveBuiltinOperatorImage(operatorID string, catalogImage string, runnerImage string) string {
|
|
if !hasIndependentBuiltinOperatorImage(domain.OperatorRecord{Image: catalogImage}) {
|
|
return ""
|
|
}
|
|
runner := strings.TrimSpace(runnerImage)
|
|
if runner == "" || !strings.Contains(runner, "/") || !strings.Contains(runner[strings.LastIndex(runner, "/")+1:], ":") {
|
|
return ""
|
|
}
|
|
registryPrefix := runner[:strings.LastIndex(runner, "/")]
|
|
tag := runner[strings.LastIndex(runner, ":")+1:]
|
|
slug := strings.TrimPrefix(strings.TrimSpace(operatorID), "operator-builtin-")
|
|
if slug == "" {
|
|
return ""
|
|
}
|
|
return registryPrefix + "/droboticflow-operator-" + slug + ":" + tag
|
|
}
|
|
|
|
func imageRepository(value string) string {
|
|
trimmed := strings.TrimSpace(value)
|
|
if at := strings.Index(trimmed, "@"); at >= 0 {
|
|
trimmed = trimmed[:at]
|
|
}
|
|
if slash := strings.LastIndex(trimmed, "/"); slash >= 0 {
|
|
if colon := strings.LastIndex(trimmed[slash+1:], ":"); colon >= 0 {
|
|
return trimmed[:slash+1+colon]
|
|
}
|
|
}
|
|
return trimmed
|
|
}
|
|
|
|
func sameImageRepository(image string, imageID string) bool {
|
|
return imageRepository(image) == imageRepository(imageID)
|
|
}
|
|
|
|
func findBuiltinOperator(items []domain.OperatorRecord, id string, name string) *domain.OperatorRecord {
|
|
for index := range items {
|
|
item := items[index]
|
|
if item.ID == id || strings.TrimSpace(item.Name) == strings.TrimSpace(name) {
|
|
copy := item
|
|
return ©
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type builtinOperatorDeletionSet struct {
|
|
ids map[string]struct{}
|
|
names map[string]struct{}
|
|
}
|
|
|
|
func normalizeBuiltinOperatorDeletionValue(value string) string {
|
|
return strings.ToLower(strings.TrimSpace(value))
|
|
}
|
|
|
|
func builtinOperatorDeletionKey(id string, name string) string {
|
|
if normalizedName := normalizeBuiltinOperatorDeletionValue(name); normalizedName != "" {
|
|
return "name:" + normalizedName
|
|
}
|
|
if normalizedID := normalizeBuiltinOperatorDeletionValue(id); normalizedID != "" {
|
|
return "id:" + normalizedID
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func newBuiltinOperatorDeletionSet(items []domain.BuiltinOperatorDeletionRecord) builtinOperatorDeletionSet {
|
|
set := builtinOperatorDeletionSet{
|
|
ids: make(map[string]struct{}),
|
|
names: make(map[string]struct{}),
|
|
}
|
|
for _, item := range items {
|
|
if normalizedID := normalizeBuiltinOperatorDeletionValue(item.OperatorID); normalizedID != "" {
|
|
set.ids[normalizedID] = struct{}{}
|
|
}
|
|
if normalizedName := normalizeBuiltinOperatorDeletionValue(item.OperatorName); normalizedName != "" {
|
|
set.names[normalizedName] = struct{}{}
|
|
}
|
|
}
|
|
return set
|
|
}
|
|
|
|
func (set builtinOperatorDeletionSet) Has(item domain.OperatorRecord) bool {
|
|
if normalizedID := normalizeBuiltinOperatorDeletionValue(item.ID); normalizedID != "" {
|
|
if _, ok := set.ids[normalizedID]; ok {
|
|
return true
|
|
}
|
|
}
|
|
if normalizedName := normalizeBuiltinOperatorDeletionValue(item.Name); normalizedName != "" {
|
|
if _, ok := set.names[normalizedName]; ok {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func filterDeletedBuiltinOperators(items []domain.OperatorRecord, deletions builtinOperatorDeletionSet) []domain.OperatorRecord {
|
|
filtered := make([]domain.OperatorRecord, 0, len(items))
|
|
for _, item := range items {
|
|
if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") && deletions.Has(item) {
|
|
continue
|
|
}
|
|
filtered = append(filtered, item)
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
func currentBuiltinOperatorIDSet() (map[string]struct{}, error) {
|
|
builtins, err := builtinOperatorRecords(nowISO(), "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ids := make(map[string]struct{}, len(builtins))
|
|
for _, item := range builtins {
|
|
ids[normalizeBuiltinOperatorDeletionValue(item.ID)] = struct{}{}
|
|
}
|
|
return ids, nil
|
|
}
|
|
|
|
func filterObsoleteBuiltinOperators(items []domain.OperatorRecord, currentBuiltinIDs map[string]struct{}) []domain.OperatorRecord {
|
|
filtered := make([]domain.OperatorRecord, 0, len(items))
|
|
for _, item := range items {
|
|
if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") {
|
|
id := normalizeBuiltinOperatorDeletionValue(item.ID)
|
|
if _, ok := currentBuiltinIDs[id]; !ok {
|
|
continue
|
|
}
|
|
}
|
|
filtered = append(filtered, item)
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
func upsertBuiltinOperator(ctx context.Context, server *server, items []domain.OperatorRecord, builtin domain.OperatorRecord) ([]domain.OperatorRecord, error) {
|
|
existing := findBuiltinOperator(items, builtin.ID, builtin.Name)
|
|
if existing == nil {
|
|
if err := server.store.UpsertOperator(ctx, builtin); err != nil {
|
|
return items, err
|
|
}
|
|
return append(items, builtin), nil
|
|
}
|
|
|
|
updated := *existing
|
|
updated.ID = builtin.ID
|
|
updated.Name = builtin.Name
|
|
updated.Type = builtin.Type
|
|
updated.Category = builtin.Category
|
|
updated.Description = builtin.Description
|
|
updated.Inputs = builtin.Inputs
|
|
updated.Outputs = builtin.Outputs
|
|
updated.ConfigSchema = builtin.ConfigSchema
|
|
// An environment may mirror the same built-in operator under a private CCR
|
|
// namespace. Validate the existing tag/digest pair against itself and retain
|
|
// that independently published image; comparing it with the source catalog's
|
|
// POC namespace would silently replace a Fanyu digest on every API restart.
|
|
existingDigestPinned := strings.TrimSpace(existing.ImageID) != "" && strings.Contains(existing.ImageID, "@sha256:") && sameImageRepository(existing.Image, existing.ImageID)
|
|
existingIndependentImage := hasIndependentBuiltinOperatorImage(*existing)
|
|
existingOnCurrentImage := strings.TrimSpace(existing.Image) == strings.TrimSpace(builtin.Image)
|
|
if existingDigestPinned && existingIndependentImage {
|
|
updated.Image = existing.Image
|
|
updated.ImageID = existing.ImageID
|
|
} else {
|
|
updated.Image = builtin.Image
|
|
if existingOnCurrentImage && existingDigestPinned {
|
|
updated.ImageID = existing.ImageID
|
|
} else if strings.TrimSpace(builtin.ImageID) != "" && strings.Contains(builtin.ImageID, "@sha256:") && sameImageRepository(builtin.Image, builtin.ImageID) {
|
|
updated.ImageID = builtin.ImageID
|
|
} else if existingDigestPinned {
|
|
updated.ImageID = existing.ImageID
|
|
} else {
|
|
updated.ImageID = builtin.ImageID
|
|
}
|
|
}
|
|
updated.ContainerCommand = builtin.ContainerCommand
|
|
updated.ContainerArgs = builtin.ContainerArgs
|
|
updated.RuntimeContract = builtin.RuntimeContract
|
|
updated.SkillDefinition = builtin.SkillDefinition
|
|
updated.Hyperparams = builtin.Hyperparams
|
|
updated.Resources = builtin.Resources
|
|
// Built-in metadata is reconciled from the source catalog at startup. Keep
|
|
// an explicit user SHM override so an API restart does not silently reset a
|
|
// per-operator /dev/shm setting; an empty value still inherits the catalog
|
|
// or platform default.
|
|
if existing.Resources != nil && strings.TrimSpace(existing.Resources.SharedMemorySize) != "" {
|
|
resources := domain.OperatorResources{}
|
|
if updated.Resources != nil {
|
|
resources = *updated.Resources
|
|
}
|
|
resources.SharedMemorySize = strings.TrimSpace(existing.Resources.SharedMemorySize)
|
|
updated.Resources = &resources
|
|
}
|
|
updated.UpdatedAt = builtin.UpdatedAt
|
|
if strings.TrimSpace(updated.CreatedAt) == "" {
|
|
updated.CreatedAt = builtin.CreatedAt
|
|
}
|
|
if err := server.store.UpsertOperator(ctx, updated); err != nil {
|
|
return items, err
|
|
}
|
|
|
|
for index := range items {
|
|
if items[index].ID == existing.ID || strings.TrimSpace(items[index].Name) == strings.TrimSpace(existing.Name) {
|
|
items[index] = updated
|
|
return items, nil
|
|
}
|
|
}
|
|
return append(items, updated), nil
|
|
}
|
|
|
|
func (server *server) ensureBuiltinOperators(ctx context.Context) error {
|
|
items, err := server.store.ListOperators(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
deletions, err := server.store.ListBuiltinOperatorDeletions(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
deletionSet := newBuiltinOperatorDeletionSet(deletions)
|
|
|
|
now := nowISO()
|
|
builtins, err := builtinOperatorRecords(now, server.operatorRunnerImage)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, builtin := range builtins {
|
|
if deletionSet.Has(builtin) {
|
|
continue
|
|
}
|
|
items, err = upsertBuiltinOperator(ctx, server, items, builtin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return server.refreshBuiltinOperatorImages(ctx, deletionSet)
|
|
}
|
|
|
|
func hasIndependentBuiltinOperatorImage(item domain.OperatorRecord) bool {
|
|
image := strings.TrimSpace(item.Image)
|
|
if image == "" {
|
|
return false
|
|
}
|
|
return strings.Contains(image, "droboticflow-operator-") && !strings.Contains(image, "droboticflow-operator-runner")
|
|
}
|
|
|
|
func (server *server) refreshBuiltinOperatorImages(ctx context.Context, deletions builtinOperatorDeletionSet) error {
|
|
runnerImage := strings.TrimSpace(server.operatorRunnerImage)
|
|
if runnerImage == "" {
|
|
return nil
|
|
}
|
|
items, err := server.store.ListOperators(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := nowISO()
|
|
for _, item := range items {
|
|
if strings.TrimSpace(strings.ToLower(item.Type)) != "builtin" {
|
|
continue
|
|
}
|
|
if deletions.Has(item) {
|
|
continue
|
|
}
|
|
if hasIndependentBuiltinOperatorImage(item) {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(item.Image) == runnerImage {
|
|
continue
|
|
}
|
|
item.Image = runnerImage
|
|
item.UpdatedAt = now
|
|
if strings.TrimSpace(item.CreatedAt) == "" {
|
|
item.CreatedAt = now
|
|
}
|
|
if err := server.store.UpsertOperator(ctx, item); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (server *server) listAvailableOperators(ctx context.Context) ([]domain.OperatorRecord, error) {
|
|
if err := server.ensureBuiltinOperators(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := server.listOperatorsForContext(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
deletions, err := server.store.ListBuiltinOperatorDeletions(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
currentIDs, err := currentBuiltinOperatorIDSet()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return filterObsoleteBuiltinOperators(filterDeletedBuiltinOperators(items, newBuiltinOperatorDeletionSet(deletions)), currentIDs), nil
|
|
}
|