2026-08-22 19:46:05 +08:00

1759 lines
60 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package ai
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"sort"
"strings"
"time"
"unicode"
"droboticflow/internal/domain"
)
const (
defaultProvider = "openai-compatible"
defaultRequestTimeout = 240 * time.Second
defaultMaxAttempts = 2
operatorMaxTokens = 3200
workflowMaxTokens = 4096
maxPromptTextLength = 120
maxWorkflowCatalogOps = 16
)
var ErrUnavailable = errors.New("AI service is not configured")
type Status struct {
Enabled bool `json:"enabled"`
Provider string `json:"provider"`
BaseURL string `json:"baseURL,omitempty"`
Model string `json:"model,omitempty"`
Reason string `json:"reason,omitempty"`
OperatorGeneration *OperatorGenerationStatus `json:"operatorGeneration,omitempty"`
}
type OperatorGenerationStatus struct {
Enabled bool `json:"enabled"`
Ready bool `json:"ready"`
Reason string `json:"reason,omitempty"`
MissingFields []string `json:"missingFields,omitempty"`
}
type GeneratedWorkflow struct {
Name string `json:"name"`
Description string `json:"description"`
Nodes []domain.WorkflowNode `json:"nodes"`
Edges []domain.WorkflowEdge `json:"edges"`
}
type GeneratedOperatorPort struct {
Name string `json:"name"`
Type string `json:"type"`
Container string `json:"container,omitempty"`
PayloadMode string `json:"payloadMode,omitempty"`
Role string `json:"role,omitempty"`
Transmission string `json:"transmission"`
Description string `json:"description,omitempty"`
}
type GeneratedOperator struct {
Name string `json:"name"`
Category string `json:"category"`
Description string `json:"description"`
Version string `json:"version,omitempty"`
Image string `json:"image,omitempty"`
ImageID string `json:"imageId,omitempty"`
ContainerCommand []string `json:"containerCommand,omitempty"`
ContainerArgs []string `json:"containerArgs,omitempty"`
RuntimeContract map[string]any `json:"runtimeContract,omitempty"`
Resources *domain.OperatorResources `json:"resources,omitempty"`
Inputs []GeneratedOperatorPort `json:"inputs"`
Outputs []GeneratedOperatorPort `json:"outputs"`
Hyperparams []domain.HyperParam `json:"hyperparams"`
SkillDefinition string `json:"skillDefinition"`
PublicationStatus domain.OperatorPublicationStatus `json:"publicationStatus,omitempty"`
GenerationSource domain.OperatorGenerationSource `json:"generationSource,omitempty"`
PublicationEvidence *domain.OperatorPublicationEvidence `json:"publicationEvidence,omitempty"`
Implementation *GeneratedOperatorImplementation `json:"implementation,omitempty"`
}
type GeneratedOperatorImplementation struct {
RepositoryPath string `json:"repositoryPath"`
RepositoryName string `json:"repositoryName,omitempty"`
RepositoryURL string `json:"repositoryURL,omitempty"`
Files []GeneratedOperatorFile `json:"files"`
BuildCommands []string `json:"buildCommands"`
AcceptanceChecklist []string `json:"acceptanceChecklist"`
}
type GeneratedOperatorFile struct {
Path string `json:"path"`
Description string `json:"description"`
Content string `json:"content"`
}
type generatedHyperParamPayload struct {
Key string `json:"key"`
Value any `json:"value"`
Description string `json:"description,omitempty"`
Type string `json:"type,omitempty"`
}
func (generated *GeneratedOperator) UnmarshalJSON(data []byte) error {
type generatedOperatorPayload struct {
Name string `json:"name"`
Category string `json:"category"`
Description string `json:"description"`
Version string `json:"version,omitempty"`
Image string `json:"image,omitempty"`
ImageID string `json:"imageId,omitempty"`
ContainerCommand []string `json:"containerCommand,omitempty"`
ContainerArgs []string `json:"containerArgs,omitempty"`
RuntimeContract map[string]any `json:"runtimeContract,omitempty"`
Resources *domain.OperatorResources `json:"resources,omitempty"`
Inputs []GeneratedOperatorPort `json:"inputs"`
Outputs []GeneratedOperatorPort `json:"outputs"`
Hyperparams []generatedHyperParamPayload `json:"hyperparams"`
SkillDefinition string `json:"skillDefinition"`
PublicationStatus domain.OperatorPublicationStatus `json:"publicationStatus,omitempty"`
GenerationSource domain.OperatorGenerationSource `json:"generationSource,omitempty"`
PublicationEvidence *domain.OperatorPublicationEvidence `json:"publicationEvidence,omitempty"`
Implementation *GeneratedOperatorImplementation `json:"implementation,omitempty"`
}
var payload generatedOperatorPayload
if err := json.Unmarshal(data, &payload); err != nil {
return err
}
hyperparams := make([]domain.HyperParam, 0, len(payload.Hyperparams))
for _, item := range payload.Hyperparams {
hyperparams = append(hyperparams, domain.HyperParam{
Key: item.Key,
Value: stringifyGeneratedHyperParamValue(item.Value),
Description: item.Description,
Type: item.Type,
})
}
*generated = GeneratedOperator{
Name: payload.Name,
Category: payload.Category,
Description: payload.Description,
Version: payload.Version,
Image: payload.Image,
ImageID: payload.ImageID,
ContainerCommand: payload.ContainerCommand,
ContainerArgs: payload.ContainerArgs,
RuntimeContract: payload.RuntimeContract,
Resources: payload.Resources,
Inputs: payload.Inputs,
Outputs: payload.Outputs,
Hyperparams: hyperparams,
SkillDefinition: payload.SkillDefinition,
PublicationStatus: payload.PublicationStatus,
GenerationSource: payload.GenerationSource,
PublicationEvidence: payload.PublicationEvidence,
Implementation: payload.Implementation,
}
return nil
}
func stringifyGeneratedHyperParamValue(value any) string {
switch typed := value.(type) {
case nil:
return ""
case string:
return typed
case bool:
if typed {
return "true"
}
return "false"
case float64:
return strings.TrimSpace(fmt.Sprint(typed))
default:
payload, err := json.Marshal(typed)
if err != nil {
return strings.TrimSpace(fmt.Sprint(typed))
}
return string(payload)
}
}
type Service interface {
Status() Status
GenerateWorkflow(context.Context, string, []domain.OperatorRecord) (*GeneratedWorkflow, error)
GenerateOperator(context.Context, string) (*GeneratedOperator, error)
}
type disabledService struct {
status Status
}
func NewDisabledService(baseURL string, model string, reason string) Service {
if strings.TrimSpace(reason) == "" {
reason = "AI 服务未配置"
}
return &disabledService{
status: Status{
Enabled: false,
Provider: defaultProvider,
BaseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"),
Model: strings.TrimSpace(model),
Reason: reason,
},
}
}
func (service *disabledService) Status() Status {
return service.status
}
func (service *disabledService) GenerateWorkflow(context.Context, string, []domain.OperatorRecord) (*GeneratedWorkflow, error) {
return nil, ErrUnavailable
}
func (service *disabledService) GenerateOperator(context.Context, string) (*GeneratedOperator, error) {
return nil, ErrUnavailable
}
type client struct {
httpClient *http.Client
baseURL string
apiKey string
model string
requestTimeout time.Duration
maxAttempts int
}
func NewService(baseURL string, apiKey string, model string) Service {
normalizedBaseURL := strings.TrimRight(strings.TrimSpace(baseURL), "/")
normalizedKey := strings.TrimSpace(apiKey)
normalizedModel := strings.TrimSpace(model)
if normalizedKey == "" {
return NewDisabledService(normalizedBaseURL, normalizedModel, "服务端未配置 AI_API_KEY")
}
if normalizedBaseURL == "" {
return NewDisabledService("", normalizedModel, "服务端未配置 AI_API_BASE_URL")
}
if normalizedModel == "" {
return NewDisabledService(normalizedBaseURL, "", "服务端未配置 AI_MODEL")
}
return &client{
httpClient: &http.Client{},
baseURL: normalizedBaseURL,
apiKey: normalizedKey,
model: normalizedModel,
requestTimeout: defaultRequestTimeout,
maxAttempts: defaultMaxAttempts,
}
}
func (client *client) Status() Status {
return Status{
Enabled: true,
Provider: defaultProvider,
BaseURL: client.baseURL,
Model: client.model,
}
}
type chatMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type chatCompletionRequest struct {
Model string `json:"model"`
Messages []chatMessage `json:"messages"`
Temperature float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
ResponseFormat *chatResponseFormat `json:"response_format,omitempty"`
}
type chatResponseFormat struct {
Type string `json:"type"`
}
type chatCompletionResponse struct {
Choices []struct {
Message struct {
Content any `json:"content"`
} `json:"message"`
} `json:"choices"`
Error *struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
type workflowPlan struct {
Name string `json:"name"`
Description string `json:"description"`
Nodes []workflowPlanNode `json:"nodes"`
Edges []workflowPlanEdge `json:"edges"`
}
type workflowPlanNode struct {
ID string `json:"id"`
Label string `json:"label,omitempty"`
OperatorID string `json:"operatorId"`
Position domain.Position `json:"position"`
HyperparamOverrides []domain.HyperParam `json:"hyperparamOverrides,omitempty"`
Bucket string `json:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty"`
Pattern string `json:"pattern,omitempty"`
CPU string `json:"cpu,omitempty"`
Memory string `json:"memory,omitempty"`
GPU string `json:"gpu,omitempty"`
}
type workflowPlanEdge struct {
ID string `json:"id"`
Source string `json:"source"`
Target string `json:"target"`
SourcePort string `json:"sourcePort,omitempty"`
TargetPort string `json:"targetPort,omitempty"`
}
func (client *client) GenerateWorkflow(ctx context.Context, description string, operators []domain.OperatorRecord) (*GeneratedWorkflow, error) {
if strings.TrimSpace(description) == "" {
return nil, fmt.Errorf("description is required")
}
if len(operators) == 0 {
return nil, fmt.Errorf("no operators available")
}
type portContext struct {
Name string `json:"name"`
Type string `json:"type"`
Container string `json:"container,omitempty"`
PayloadMode string `json:"payloadMode,omitempty"`
Role string `json:"role,omitempty"`
Transmission string `json:"transmission,omitempty"`
}
type hyperparamContext struct {
Key string `json:"key"`
Description string `json:"description,omitempty"`
Type string `json:"type,omitempty"`
}
type operatorContext struct {
ID string `json:"id"`
Name string `json:"name"`
Category string `json:"category"`
Description string `json:"description,omitempty"`
Inputs []portContext `json:"inputs,omitempty"`
Outputs []portContext `json:"outputs,omitempty"`
Hyperparams []hyperparamContext `json:"hyperparams,omitempty"`
}
selectedOperators := selectWorkflowCatalogOperators(description, operators)
available := make([]operatorContext, 0, len(selectedOperators))
for _, operator := range selectedOperators {
inputs := make([]portContext, 0, len(operator.Inputs))
for _, port := range operator.Inputs {
inputs = append(inputs, portContext{
Name: strings.TrimSpace(port.Name),
Type: string(port.Type),
Container: string(domain.EffectivePortContract(port).Container),
PayloadMode: string(domain.EffectivePortContract(port).PayloadMode),
Role: strings.TrimSpace(port.Role),
Transmission: strings.TrimSpace(port.Transmission),
})
}
outputs := make([]portContext, 0, len(operator.Outputs))
for _, port := range operator.Outputs {
outputs = append(outputs, portContext{
Name: strings.TrimSpace(port.Name),
Type: string(port.Type),
Container: string(domain.EffectivePortContract(port).Container),
PayloadMode: string(domain.EffectivePortContract(port).PayloadMode),
Role: strings.TrimSpace(port.Role),
Transmission: strings.TrimSpace(port.Transmission),
})
}
hyperparams := make([]hyperparamContext, 0, len(operator.Hyperparams))
for _, item := range operator.Hyperparams {
hyperparams = append(hyperparams, hyperparamContext{
Key: strings.TrimSpace(item.Key),
Description: truncatePromptText(item.Description, 60),
Type: strings.TrimSpace(item.Type),
})
}
available = append(available, operatorContext{
ID: strings.TrimSpace(operator.ID),
Name: strings.TrimSpace(operator.Name),
Category: strings.TrimSpace(operator.Category),
Description: truncatePromptText(operator.Description, maxPromptTextLength),
Inputs: inputs,
Outputs: outputs,
Hyperparams: hyperparams,
})
}
contextJSON, err := json.Marshal(available)
if err != nil {
return nil, err
}
systemPrompt := strings.TrimSpace(`
You are a Drobodata workflow architect. Return compact JSON only, no markdown, no reasoning, no explanations.
Use only operatorId values from the operator catalog. Match sourcePort/targetPort to real port names.
Never connect multiple upstream edges into the same target input port.
Schema: {"name":"short name","description":"short summary","nodes":[{"id":"node-a","label":"optional","operatorId":"operator-id","position":{"x":120,"y":80},"hyperparamOverrides":[{"key":"ENV_KEY","value":"ENV_VALUE","description":"optional"}],"bucket":"optional","prefix":"optional","pattern":"optional","cpu":"optional","memory":"optional","gpu":"positive integer only when needed"}],"edges":[{"id":"edge-1","source":"node-a","target":"node-b","sourcePort":"output port name","targetPort":"input port name"}]}`)
userPrompt := fmt.Sprintf("User requirement:\n%s\n\nAvailable operators:\n%s", strings.TrimSpace(description), string(contextJSON))
var plan workflowPlan
if err := client.generateJSON(ctx, systemPrompt, userPrompt, workflowMaxTokens, &plan); err != nil {
return nil, err
}
if len(plan.Nodes) == 0 {
return nil, fmt.Errorf("AI did not return any workflow nodes")
}
operatorByID := make(map[string]domain.OperatorRecord, len(operators))
nodes := make([]domain.WorkflowNode, 0, len(plan.Nodes))
for _, operator := range operators {
operatorByID[operator.ID] = operator
}
nodeByID := make(map[string]domain.WorkflowNode, len(plan.Nodes))
for _, item := range plan.Nodes {
operator, ok := operatorByID[item.OperatorID]
if !ok {
return nil, fmt.Errorf("AI selected unknown operator %q", item.OperatorID)
}
if strings.TrimSpace(item.ID) == "" {
return nil, fmt.Errorf("AI returned a workflow node without id")
}
node := domain.WorkflowNode{
ID: strings.TrimSpace(item.ID),
Type: "operator",
Data: domain.WorkflowNodeData{
Label: chooseValue(item.Label, operator.Name),
Category: operator.Category,
Description: operator.Description,
Inputs: clonePorts(operator.Inputs),
Outputs: clonePorts(operator.Outputs),
Image: operator.Image,
SkillDefinition: operator.SkillDefinition,
Hyperparams: mergeHyperparams(operator.Hyperparams, item.HyperparamOverrides),
Bucket: strings.TrimSpace(item.Bucket),
Prefix: strings.TrimSpace(item.Prefix),
Pattern: strings.TrimSpace(item.Pattern),
CPU: strings.TrimSpace(item.CPU),
Memory: strings.TrimSpace(item.Memory),
GPU: strings.TrimSpace(item.GPU),
},
Position: item.Position,
}
nodes = append(nodes, node)
nodeByID[node.ID] = node
}
edges := make([]domain.WorkflowEdge, 0, len(plan.Edges))
for _, item := range plan.Edges {
if strings.TrimSpace(item.Source) == "" || strings.TrimSpace(item.Target) == "" {
return nil, fmt.Errorf("AI returned an edge without source/target")
}
sourceNode, sourceOK := nodeByID[item.Source]
targetNode, targetOK := nodeByID[item.Target]
if !sourceOK || !targetOK {
return nil, fmt.Errorf("AI returned an edge with unknown nodes: %s -> %s", item.Source, item.Target)
}
sourceHandle, err := resolvePortHandle(sourceNode.Data.Outputs, item.SourcePort)
if err != nil {
return nil, fmt.Errorf("resolve source port for edge %s: %w", chooseValue(item.ID, item.Source+"-"+item.Target), err)
}
targetHandle, err := resolvePortHandle(targetNode.Data.Inputs, item.TargetPort)
if err != nil {
return nil, fmt.Errorf("resolve target port for edge %s: %w", chooseValue(item.ID, item.Source+"-"+item.Target), err)
}
edges = append(edges, domain.WorkflowEdge{
ID: chooseValue(strings.TrimSpace(item.ID), "edge-"+item.Source+"-"+item.Target),
Source: item.Source,
Target: item.Target,
SourceHandle: sourceHandle,
TargetHandle: targetHandle,
})
}
if conflicts := domain.FindInputPortConflicts(nodes, edges); len(conflicts) > 0 {
return nil, fmt.Errorf("AI generated a workflow with multiple upstream edges feeding the same input port; split those branches or add separate upload operators for each result")
}
return &GeneratedWorkflow{
Name: chooseValue(strings.TrimSpace(plan.Name), "AI 生成工作流"),
Description: chooseValue(strings.TrimSpace(plan.Description), strings.TrimSpace(description)),
Nodes: nodes,
Edges: edges,
}, nil
}
func (client *client) GenerateOperator(ctx context.Context, description string) (*GeneratedOperator, error) {
if strings.TrimSpace(description) == "" {
return nil, fmt.Errorf("description is required")
}
systemPrompt := strings.TrimSpace(`
You are a Drobodata operator designer. Return one compact JSON object only, no markdown, no reasoning, no explanations. Keep skillDefinition under 600 Chinese chars and hyperparams under 10 items.
Allowed categories: IO, CV, AI, MATH, HUMAN, FLOW. Allowed port types: json, image, video, tensor, file. Port containers: single, collection, directory, dataset. Payload modes: inline, ref. Role is optional semantic metadata (video, pts, audio, imu, calibration, annotation, boundary, summary), never a transport type. Allowed legacy transmission values: bus, oss, shm, stream.
Design a production-ready external container operator contract, not just a card label.
Use ref for external/video/file/collection/directory/dataset artifacts and inline only for compact single JSON. Transmission is compatibility-only; prefer explicit container and payloadMode.
Leave image and imageId empty unless the user explicitly provided a registry image or digest; never invent a built image.
Add enough typed hyperparams and resource defaults for a real K8s Job. Hyperparam value must always be a string.
Schema: {"name":"short operator name","category":"AI|IO|CV|MATH|HUMAN|FLOW","description":"1 sentence","version":"v1.0.0","image":"optional existing registry image","imageId":"optional digest image","containerCommand":["python","/app/src/operator.py"],"containerArgs":[],"resources":{"cpuRequest":"500m","cpuLimit":"2","memoryRequest":"1Gi","memoryLimit":"4Gi","gpuType":"","gpuLimit":"","sharedMemorySize":"optional Kubernetes quantity such as 8Gi"},"runtimeContract":{"protocol":"droboticflow.external-operator/v1","implementationStatus":"requires-build-and-registry-push"},"inputs":[{"name":"input","type":"json|image|video|tensor|file","container":"single|collection|directory|dataset","payloadMode":"inline|ref","role":"optional semantic role","transmission":"bus|oss|shm|stream","description":"optional"}],"outputs":[{"name":"result","type":"json|image|video|tensor|file","container":"single|collection|directory|dataset","payloadMode":"inline|ref","role":"optional semantic role","transmission":"bus|oss|shm|stream","description":"optional"}],"hyperparams":[{"key":"ENV_KEY","value":"default","type":"string|number|integer|boolean|path|json","description":"what it controls"}],"skillDefinition":"detailed usage, inputs, outputs, limits, failure modes, acceptance"}`)
var generated GeneratedOperator
if err := client.generateJSON(ctx, systemPrompt, strings.TrimSpace(description), operatorMaxTokens, &generated); err != nil {
return nil, err
}
normalizeGeneratedOperator(&generated, description)
if strings.TrimSpace(generated.Name) == "" {
return nil, fmt.Errorf("AI did not return operator name")
}
if !validCategory(generated.Category) {
return nil, fmt.Errorf("AI returned unsupported operator category %q", generated.Category)
}
if err := validateGeneratedPorts(generated.Inputs); err != nil {
return nil, fmt.Errorf("invalid generated operator inputs: %w", err)
}
if err := validateGeneratedPorts(generated.Outputs); err != nil {
return nil, fmt.Errorf("invalid generated operator outputs: %w", err)
}
if strings.TrimSpace(generated.Description) == "" {
generated.Description = strings.TrimSpace(description)
}
if strings.TrimSpace(generated.SkillDefinition) == "" {
generated.SkillDefinition = generated.Description
}
if generated.RuntimeContract == nil {
generated.RuntimeContract = map[string]any{}
}
generated.RuntimeContract["implementationStatus"] = "draft-requires-real-logic"
generated.PublicationStatus = domain.OperatorPublicationStatusDraft
generated.GenerationSource = domain.OperatorGenerationSourceAI
if generated.Implementation == nil {
implementation := buildGeneratedOperatorImplementation(generated)
generated.Implementation = &implementation
}
generated.RuntimeContract["generatedRepository"] = generated.Implementation
return &generated, nil
}
func normalizeGeneratedOperator(generated *GeneratedOperator, description string) {
generated.Name = strings.TrimSpace(generated.Name)
generated.Category = strings.ToUpper(strings.TrimSpace(generated.Category))
generated.Description = strings.TrimSpace(generated.Description)
generated.Version = normalizeGeneratedVersion(generated.Version)
generated.Image = strings.TrimSpace(generated.Image)
generated.ImageID = strings.TrimSpace(generated.ImageID)
generated.PublicationStatus = domain.OperatorPublicationStatusDraft
generated.GenerationSource = domain.OperatorGenerationSourceAI
generated.ContainerCommand = normalizeGeneratedCommand(generated.ContainerCommand, []string{"python", "/app/src/operator.py"})
generated.ContainerArgs = normalizeGeneratedCommand(generated.ContainerArgs, nil)
normalizeGeneratedPorts(generated.Inputs)
normalizeGeneratedPorts(generated.Outputs)
generated.Hyperparams = normalizeGeneratedHyperparams(generated.Hyperparams)
if generated.Resources == nil {
generated.Resources = inferGeneratedResources(*generated, description)
} else {
generated.Resources = &domain.OperatorResources{
CPURequest: strings.TrimSpace(generated.Resources.CPURequest),
CPULimit: strings.TrimSpace(generated.Resources.CPULimit),
MemoryRequest: strings.TrimSpace(generated.Resources.MemoryRequest),
MemoryLimit: strings.TrimSpace(generated.Resources.MemoryLimit),
GPUType: strings.TrimSpace(generated.Resources.GPUType),
GPULimit: strings.TrimSpace(generated.Resources.GPULimit),
SharedMemorySize: strings.TrimSpace(generated.Resources.SharedMemorySize),
}
}
if generated.RuntimeContract == nil {
generated.RuntimeContract = map[string]any{}
}
mergeGeneratedRuntimeContract(generated.RuntimeContract, *generated)
}
func normalizeGeneratedVersion(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return "v1.0.0"
}
return normalizeOperatorVersionLike(trimmed)
}
func normalizeOperatorVersionLike(value string) string {
var builder strings.Builder
for _, character := range value {
switch {
case unicode.IsLetter(character) || unicode.IsDigit(character) || character == '_' || character == '.' || character == '-':
builder.WriteRune(character)
case unicode.IsSpace(character):
builder.WriteRune('-')
}
}
normalized := strings.Trim(builder.String(), "._-")
if normalized == "" {
return "v1.0.0"
}
if len([]rune(normalized)) > 128 {
runes := []rune(normalized)
normalized = string(runes[:128])
}
return normalized
}
func normalizeGeneratedCommand(values []string, fallback []string) []string {
normalized := make([]string, 0, len(values))
for _, value := range values {
item := strings.TrimSpace(value)
if item == "" || strings.ContainsAny(item, "\x00\r\n") {
continue
}
normalized = append(normalized, item)
}
if len(normalized) == 0 && fallback != nil {
normalized = append(normalized, fallback...)
}
return normalized
}
func normalizeGeneratedPorts(ports []GeneratedOperatorPort) {
for index := range ports {
ports[index].Name = strings.TrimSpace(ports[index].Name)
ports[index].Type = strings.ToLower(strings.TrimSpace(ports[index].Type))
ports[index].Container = strings.ToLower(strings.TrimSpace(ports[index].Container))
ports[index].PayloadMode = strings.ToLower(strings.TrimSpace(ports[index].PayloadMode))
ports[index].Role = strings.ToLower(strings.TrimSpace(ports[index].Role))
ports[index].Transmission = strings.ToLower(strings.TrimSpace(ports[index].Transmission))
ports[index].Description = strings.TrimSpace(ports[index].Description)
if ports[index].Transmission == "" {
ports[index].Transmission = defaultTransmissionForPortType(ports[index].Type)
}
if ports[index].Container == "" {
ports[index].Container = "single"
}
if ports[index].PayloadMode == "" {
if ports[index].Type == "json" && ports[index].Container == "single" {
ports[index].PayloadMode = "inline"
} else {
ports[index].PayloadMode = "ref"
}
}
}
}
func defaultTransmissionForPortType(portType string) string {
switch strings.ToLower(strings.TrimSpace(portType)) {
case "video", "file", "image":
return "oss"
case "tensor":
return "stream"
default:
return "bus"
}
}
func normalizeGeneratedHyperparams(items []domain.HyperParam) []domain.HyperParam {
normalized := make([]domain.HyperParam, 0, len(items))
seen := map[string]struct{}{}
for _, item := range items {
key := strings.ToUpper(strings.TrimSpace(item.Key))
key = normalizeEnvironmentKey(key)
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
paramType := strings.ToLower(strings.TrimSpace(item.Type))
if paramType == "" {
paramType = inferHyperparamType(item.Value)
}
normalized = append(normalized, domain.HyperParam{
Key: key,
Value: strings.TrimSpace(item.Value),
Description: strings.TrimSpace(item.Description),
Type: paramType,
})
}
if len(normalized) == 0 {
normalized = append(normalized, domain.HyperParam{
Key: "INPUT_MANIFEST_PATH",
Value: "",
Description: "上游输入 manifest 路径;平台运行时自动注入时可留空",
Type: "path",
})
}
return normalized
}
func normalizeEnvironmentKey(value string) string {
var builder strings.Builder
previousUnderscore := false
for _, character := range value {
switch {
case unicode.IsLetter(character) || unicode.IsDigit(character):
builder.WriteRune(unicode.ToUpper(character))
previousUnderscore = false
case character == '_', character == '-', character == ' ':
if !previousUnderscore && builder.Len() > 0 {
builder.WriteRune('_')
previousUnderscore = true
}
}
}
return strings.Trim(builder.String(), "_")
}
func inferHyperparamType(value string) string {
trimmed := strings.ToLower(strings.TrimSpace(value))
switch {
case trimmed == "true" || trimmed == "false":
return "boolean"
case strings.HasPrefix(trimmed, "/") || strings.HasPrefix(trimmed, "bos://") || strings.HasPrefix(trimmed, "s3://"):
return "path"
case strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "["):
return "json"
default:
if _, ok := parseInteger(trimmed); ok {
return "integer"
}
if _, ok := parseNumber(trimmed); ok {
return "number"
}
return "string"
}
}
func parseInteger(value string) (int, bool) {
if value == "" {
return 0, false
}
var parsed int
for _, character := range value {
if character < '0' || character > '9' {
return 0, false
}
parsed = parsed*10 + int(character-'0')
}
return parsed, true
}
func parseNumber(value string) (float64, bool) {
if value == "" {
return 0, false
}
var seenDigit, seenDot bool
for _, character := range value {
switch {
case character >= '0' && character <= '9':
seenDigit = true
case character == '.' && !seenDot:
seenDot = true
default:
return 0, false
}
}
return 0, seenDigit
}
func inferGeneratedResources(generated GeneratedOperator, description string) *domain.OperatorResources {
blob := strings.ToLower(strings.Join([]string{
generated.Name,
generated.Category,
generated.Description,
generated.SkillDefinition,
description,
}, "\n"))
needsGPU := containsAny(blob, []string{
"gpu", "cuda", "a800", "a100", "h100", "h800", "tensorRT", "tensorrt",
"openvla", "vla", "yolo", "diffusion", "act", "sam", "groundingdino",
"训练", "推理", "检测", "微调", "checkpoint",
})
if needsGPU {
return &domain.OperatorResources{
CPURequest: "4",
CPULimit: "16",
MemoryRequest: "32Gi",
MemoryLimit: "96Gi",
GPUType: "NVIDIA A800 80GB",
GPULimit: "1",
}
}
if containsAny(blob, []string{"video", "ffmpeg", "视频", "点云", "3d", "asset", "rosbag", "hdf5", "zarr"}) {
return &domain.OperatorResources{
CPURequest: "2",
CPULimit: "8",
MemoryRequest: "4Gi",
MemoryLimit: "16Gi",
}
}
return &domain.OperatorResources{
CPURequest: "500m",
CPULimit: "2",
MemoryRequest: "1Gi",
MemoryLimit: "4Gi",
}
}
func containsAny(value string, needles []string) bool {
for _, needle := range needles {
if strings.Contains(value, strings.ToLower(needle)) {
return true
}
}
return false
}
func mergeGeneratedRuntimeContract(contract map[string]any, generated GeneratedOperator) {
defaults := map[string]any{
"protocol": "droboticflow.external-operator/v1",
"implementationStatus": "requires-business-logic-and-registry-push",
"entrypoint": "src/operator.py",
"inputManifestEnv": "INPUT_MANIFEST_PATH",
"outputDirEnv": "OUTPUT_DIR",
"resultOutputPathEnv": "RESULT_OUTPUT_PATH",
"metricsOutputPathEnv": "RESULT_METRICS_PATH",
"stageManifestPathEnv": "STAGE_MANIFEST_PATH",
"containerCommand": generated.ContainerCommand,
"containerArgs": generated.ContainerArgs,
"operatorVersion": generated.Version,
"imageBuildRequired": strings.TrimSpace(generated.ImageID) == "",
"generatedBy": "ai-operator-generator/v2",
"generatedRepositoryNote": "AI 生成的是可运行协议骨架;生产通过前必须补齐真实业务处理、构建镜像、推送 CCR 并用真实 workflow 验收。",
}
for key, value := range defaults {
if _, exists := contract[key]; !exists {
contract[key] = value
}
}
if _, exists := contract["inputs"]; !exists {
contract["inputs"] = generated.Inputs
}
if _, exists := contract["outputs"]; !exists {
contract["outputs"] = generated.Outputs
}
if _, exists := contract["parameters"]; !exists {
contract["parameters"] = generated.Hyperparams
}
if _, exists := contract["resources"]; !exists {
contract["resources"] = generated.Resources
}
}
func buildGeneratedOperatorImplementation(generated GeneratedOperator) GeneratedOperatorImplementation {
return buildGeneratedOperatorImplementationWithSettings(generated, domain.OperatorGenerationSettings{})
}
func ApplyOperatorGenerationSettings(generated *GeneratedOperator, settings domain.OperatorGenerationSettings) {
if generated == nil {
return
}
if generated.RuntimeContract == nil {
generated.RuntimeContract = map[string]any{}
}
generated.PublicationStatus = domain.OperatorPublicationStatusDraft
generated.GenerationSource = domain.OperatorGenerationSourceAI
generated.RuntimeContract["implementationStatus"] = "draft-requires-real-logic"
generated.RuntimeContract["publicationStatus"] = string(domain.OperatorPublicationStatusDraft)
generated.RuntimeContract["generationSource"] = string(domain.OperatorGenerationSourceAI)
implementation := buildGeneratedOperatorImplementationWithSettings(*generated, settings)
generated.Implementation = &implementation
generated.RuntimeContract["generatedRepository"] = generated.Implementation
generated.RuntimeContract["codeRepository"] = map[string]any{
"provider": "gitea",
"baseURL": strings.TrimRight(strings.TrimSpace(settings.GiteaBaseURL), "/"),
"organization": strings.TrimSpace(settings.GiteaOrganization),
"mode": normalizeGeneratedRepositoryMode(settings.RepositoryMode),
"name": implementation.RepositoryName,
"url": implementation.RepositoryURL,
"defaultBranch": strings.TrimSpace(settings.DefaultBranch),
}
}
func buildGeneratedOperatorImplementationWithSettings(generated GeneratedOperator, settings domain.OperatorGenerationSettings) GeneratedOperatorImplementation {
slug := slugifyOperatorName(generated.Name)
repositoryPath := "operator-repos/external/" + slug
repositoryName := generatedRepositoryName(settings, slug)
repositoryURL := generatedRepositoryURL(settings, repositoryName)
suggestedImage := fmt.Sprintf("%s/%s:%s", generatedImageRegistry(settings), slug, generated.Version)
runtimeContract := cloneContractWithoutImplementation(generated.RuntimeContract)
files := []GeneratedOperatorFile{
{
Path: "Dockerfile",
Description: "每个外部算子的独立最终镜像定义",
Content: strings.TrimSpace(`
FROM python:3.11-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY src /app/src
COPY runtime_contract.json /app/runtime_contract.json
ENTRYPOINT ["python", "/app/src/operator.py"]`) + "\n",
},
{
Path: "requirements.txt",
Description: "最小运行依赖;按真实业务逻辑补充 opencv、torch、transformers 等依赖",
Content: generatedRequirements(generated) + "\n",
},
{
Path: "runtime_contract.json",
Description: "平台运行协议、输入输出、资源和验收契约",
Content: mustMarshalIndent(runtimeContract) + "\n",
},
{
Path: "operator.json",
Description: "可注册到 Drobodata 的算子元数据草案",
Content: mustMarshalIndent(generatedOperatorMetadata(generated, suggestedImage)) + "\n",
},
{
Path: "src/operator.py",
Description: "可运行的 Drobodata 协议适配入口;将 run_business_logic 替换为真实处理逻辑",
Content: generatedOperatorPython(generated) + "\n",
},
{
Path: "tests/acceptance/fixture_contract.json",
Description: "真实验收数据契约;禁止 mock、synthetic 和空文件",
Content: generatedFixtureContract(generated) + "\n",
},
{
Path: "README.md",
Description: "实现、构建、推送和验收说明",
Content: generatedOperatorReadme(generated, repositoryPath, suggestedImage) + "\n",
},
}
return GeneratedOperatorImplementation{
RepositoryPath: repositoryPath,
RepositoryName: repositoryName,
RepositoryURL: repositoryURL,
Files: files,
BuildCommands: []string{
fmt.Sprintf("cd %s", repositoryPath),
fmt.Sprintf("docker build -t %s .", suggestedImage),
fmt.Sprintf("docker push %s", suggestedImage),
fmt.Sprintf("docker buildx imagetools inspect %s", suggestedImage),
"把返回的 repo@sha256:... digest 填到算子 imageId 后再创建生产 workflow 验收",
},
AcceptanceChecklist: []string{
"使用公开或业务真实数据,不使用 mock、synthetic、空文件或 smoke-only 数据",
"workflow 至少包含 source -> 外部算子 -> verifier,验证上游 artifact 传输和下游消费",
"检查 RESULT_OUTPUT_PATH、RESULT_METRICS_PATH、STAGE_MANIFEST_PATH 均被写出",
"检查每个输出 artifact 可预览或可下载,并记录 sha256、大小和 content type",
"GPU 算子必须验证调度到目标 GPU,记录 GPU name、gpu count、显存和利用率采样",
},
}
}
func normalizeGeneratedRepositoryMode(value string) string {
switch strings.TrimSpace(value) {
case "monorepo":
return "monorepo"
default:
return "per_operator_repo"
}
}
func generatedRepositoryName(settings domain.OperatorGenerationSettings, slug string) string {
if normalizeGeneratedRepositoryMode(settings.RepositoryMode) == "monorepo" {
if value := strings.TrimSpace(settings.MonorepoName); value != "" {
return value
}
}
prefix := strings.TrimSpace(settings.RepositoryPrefix)
if prefix == "" {
prefix = "operator-external-"
}
return prefix + slug
}
func generatedRepositoryURL(settings domain.OperatorGenerationSettings, repositoryName string) string {
baseURL := strings.TrimRight(strings.TrimSpace(settings.GiteaBaseURL), "/")
organization := strings.Trim(strings.TrimSpace(settings.GiteaOrganization), "/")
if baseURL == "" || organization == "" || strings.TrimSpace(repositoryName) == "" {
return ""
}
return baseURL + "/" + organization + "/" + strings.TrimSpace(repositoryName)
}
func generatedImageRegistry(settings domain.OperatorGenerationSettings) string {
registry := strings.TrimRight(strings.TrimSpace(settings.ImageRegistry), "/")
if registry == "" {
registry = "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/custom-operators"
}
return registry
}
func slugifyOperatorName(value string) string {
var builder strings.Builder
previousDash := false
for _, character := range strings.ToLower(strings.TrimSpace(value)) {
switch {
case character >= 'a' && character <= 'z', character >= '0' && character <= '9':
builder.WriteRune(character)
previousDash = false
case unicode.IsLetter(character) || unicode.IsDigit(character):
if !previousDash && builder.Len() > 0 {
builder.WriteRune('-')
previousDash = true
}
case character == '-' || character == '_' || unicode.IsSpace(character):
if !previousDash && builder.Len() > 0 {
builder.WriteRune('-')
previousDash = true
}
}
}
slug := strings.Trim(builder.String(), "-")
if slug == "" {
slug = "custom-operator"
}
if len(slug) > 72 {
slug = strings.Trim(slug[:72], "-")
}
return slug
}
func cloneContractWithoutImplementation(contract map[string]any) map[string]any {
cloned := make(map[string]any, len(contract))
for key, value := range contract {
if key == "generatedRepository" {
continue
}
cloned[key] = value
}
return cloned
}
func generatedRequirements(generated GeneratedOperator) string {
needsOpenCV := false
for _, port := range append(append([]GeneratedOperatorPort{}, generated.Inputs...), generated.Outputs...) {
if port.Type == "video" || port.Type == "image" {
needsOpenCV = true
break
}
}
lines := []string{
"# Keep this file explicit. Add only dependencies required by the real operator.",
}
if needsOpenCV {
lines = append(lines, "opencv-python-headless>=4.10.0")
}
if generated.Resources != nil && strings.TrimSpace(generated.Resources.GPULimit) != "" {
lines = append(lines, "# GPU model dependencies are intentionally explicit; choose CUDA-compatible wheels in production.")
lines = append(lines, "# torch")
}
if len(lines) == 1 {
lines = append(lines, "# No third-party dependency is required for the generated protocol adapter.")
}
return strings.Join(lines, "\n")
}
func generatedOperatorMetadata(generated GeneratedOperator, suggestedImage string) map[string]any {
image := strings.TrimSpace(generated.Image)
imageID := strings.TrimSpace(generated.ImageID)
if image == "" {
image = suggestedImage
}
if imageID == "" && strings.Contains(image, "@sha256:") {
imageID = image
}
return map[string]any{
"name": generated.Name,
"type": "custom",
"category": generated.Category,
"description": generated.Description,
"version": generated.Version,
"image": image,
"imageId": imageID,
"containerCommand": generated.ContainerCommand,
"containerArgs": generated.ContainerArgs,
"runtimeContract": cloneContractWithoutImplementation(generated.RuntimeContract),
"inputs": generated.Inputs,
"outputs": generated.Outputs,
"hyperparams": generated.Hyperparams,
"resources": generated.Resources,
"skillDefinition": generated.SkillDefinition,
"publicationStatus": domain.OperatorPublicationStatusDraft,
"generationSource": domain.OperatorGenerationSourceAI,
"publicationEvidence": generated.PublicationEvidence,
}
}
func generatedOperatorPython(generated GeneratedOperator) string {
operatorName := quotePythonString(generated.Name)
inputsJSON := quotePythonString(mustMarshal(generated.Inputs))
outputsJSON := quotePythonString(mustMarshal(generated.Outputs))
hyperparamsJSON := quotePythonString(mustMarshal(generated.Hyperparams))
return strings.TrimSpace(fmt.Sprintf(`
import hashlib
import json
import os
import time
from pathlib import Path
OPERATOR_NAME = %s
INPUT_PORTS = json.loads(%s)
OUTPUT_PORTS = json.loads(%s)
HYPERPARAMS = json.loads(%s)
def _load_json_file(path):
if not path:
return {}
source = Path(path)
if not source.exists():
raise FileNotFoundError(f"manifest file does not exist: {source}")
return json.loads(source.read_text(encoding="utf-8"))
def _write_json(path, payload):
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return target
def _sha256(path):
digest = hashlib.sha256()
with Path(path).open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _runtime_params():
params = {}
for item in HYPERPARAMS:
key = item.get("key", "").strip()
if not key:
continue
params[key] = os.environ.get(key, item.get("value", ""))
return params
def run_business_logic(input_manifest, params, output_dir):
"""Draft-only protocol adapter.
Replace this function with real domain logic, tests and fixture-backed
acceptance before publishing the operator.
"""
summary_path = output_dir / "summary.json"
summary = {
"operator": OPERATOR_NAME,
"inputs": INPUT_PORTS,
"outputs": OUTPUT_PORTS,
"params": params,
"inputManifest": input_manifest,
"implementationStatus": "draft-requires-real-logic",
"publicationStatus": "draft",
}
_write_json(summary_path, summary)
raise RuntimeError(
f"{OPERATOR_NAME} is an AI-generated draft operator. "
"Implement real domain logic and publish verified CCR digest before production use."
)
def main():
started_at = time.time()
output_dir = Path(os.environ.get("OUTPUT_DIR", "/tmp/droboticflow-output"))
output_dir.mkdir(parents=True, exist_ok=True)
input_manifest = _load_json_file(os.environ.get("INPUT_MANIFEST_PATH", ""))
params = _runtime_params()
artifacts = run_business_logic(input_manifest, params, output_dir)
duration = time.time() - started_at
result = {
"status": "completed",
"operator": OPERATOR_NAME,
"artifacts": artifacts,
"outputs": {item["name"]: item["path"] for item in artifacts},
"durationSeconds": duration,
}
metrics = {
"duration_seconds": duration,
"input_port_count": len(INPUT_PORTS),
"output_port_count": len(OUTPUT_PORTS),
"artifact_count": len(artifacts),
}
stage_manifest = {
"operator": OPERATOR_NAME,
"inputs": input_manifest,
"outputs": artifacts,
"params": params,
"implementationStatus": "requires-business-logic",
}
result_path = os.environ.get("RESULT_OUTPUT_PATH", str(output_dir / "result.json"))
metrics_path = os.environ.get("RESULT_METRICS_PATH", str(output_dir / "metrics.json"))
stage_manifest_path = os.environ.get("STAGE_MANIFEST_PATH", str(output_dir / "stage_manifest.json"))
_write_json(result_path, result)
_write_json(metrics_path, metrics)
_write_json(stage_manifest_path, stage_manifest)
print(f"DROBOTICFLOW_RESULT_PATH={result_path}", flush=True)
print(f"DROBOTICFLOW_METRICS_PATH={metrics_path}", flush=True)
print(f"DROBOTICFLOW_STAGE_MANIFEST_PATH={stage_manifest_path}", flush=True)
if __name__ == "__main__":
main()
`, operatorName, inputsJSON, outputsJSON, hyperparamsJSON))
}
func generatedFixtureContract(generated GeneratedOperator) string {
payload := map[string]any{
"policy": "real-fixtures-only",
"forbidden": []string{
"mock",
"synthetic",
"generated-empty-file",
"smoke-only",
"hardcoded-output",
},
"operator": map[string]any{
"name": generated.Name,
"version": generated.Version,
"inputs": generated.Inputs,
"outputs": generated.Outputs,
},
"requiredEvidence": []string{
"source_url_or_business_dataset_id",
"license_or_internal_authorization",
"fixture_sha256",
"artifact_sha256",
"lineage_run_id",
"verifier_node_status",
},
}
return mustMarshalIndent(payload)
}
func generatedOperatorReadme(generated GeneratedOperator, repositoryPath string, suggestedImage string) string {
return strings.TrimSpace(fmt.Sprintf(`# %s
AI 生成的是外部算子工程草案,不是已经通过生产验收的镜像。把本目录落地到 %s 后,必须补齐真实业务逻辑、构建镜像、推送 CCR,并用真实 workflow 验收。
## 运行契约
- 入口:%s
- 结果:RESULT_OUTPUT_PATH
- 指标:RESULT_METRICS_PATH
- 阶段 manifest:STAGE_MANIFEST_PATH
- 上游输入 manifest:INPUT_MANIFEST_PATH
## 构建与推送
`+"```bash"+`
docker build -t %s .
docker push %s
docker buildx imagetools inspect %s
`+"```"+`
获取 digest 后,在 Drobodata 算子管理页填写 imageId,推荐使用 repo@sha256:... 运行生产 workflow。
## 验收
1. 使用真实数据 fixture。
2. 创建 source -> 本算子 -> verifier 的工作流。
3. 检查 result.json、metrics.json、stage_manifest.json。
4. 检查输出 artifact 可预览、可下载、sha256 可复核。
5. GPU 算子还要检查 GPU name、gpu count、显存与利用率采样。
`, generated.Name, repositoryPath, strings.Join(generated.ContainerCommand, " "), suggestedImage, suggestedImage, suggestedImage))
}
func quotePythonString(value string) string {
payload, err := json.Marshal(value)
if err != nil {
return `""`
}
return string(payload)
}
func mustMarshal(value any) string {
payload, err := json.Marshal(value)
if err != nil {
return "{}"
}
return string(payload)
}
func mustMarshalIndent(value any) string {
payload, err := json.MarshalIndent(value, "", " ")
if err != nil {
return "{}"
}
return string(payload)
}
func selectWorkflowCatalogOperators(description string, operators []domain.OperatorRecord) []domain.OperatorRecord {
if len(operators) <= maxWorkflowCatalogOps {
return operators
}
terms := promptMatchTerms(description)
type scoredOperator struct {
operator domain.OperatorRecord
score int
index int
}
scored := make([]scoredOperator, 0, len(operators))
for index, operator := range operators {
blob := workflowOperatorSearchText(operator)
score := 0
for _, term := range terms {
if strings.Contains(blob, term) {
score += len([]rune(term))
}
}
scored = append(scored, scoredOperator{operator: operator, score: score, index: index})
}
sort.SliceStable(scored, func(left int, right int) bool {
if scored[left].score != scored[right].score {
return scored[left].score > scored[right].score
}
return scored[left].index < scored[right].index
})
selected := make([]domain.OperatorRecord, 0, maxWorkflowCatalogOps)
seen := map[string]struct{}{}
for _, item := range scored {
if item.score <= 0 {
continue
}
if _, ok := seen[item.operator.ID]; ok {
continue
}
selected = append(selected, item.operator)
seen[item.operator.ID] = struct{}{}
if len(selected) >= maxWorkflowCatalogOps {
return selected
}
}
for _, operator := range operators {
if _, ok := seen[operator.ID]; ok {
continue
}
selected = append(selected, operator)
seen[operator.ID] = struct{}{}
if len(selected) >= maxWorkflowCatalogOps {
break
}
}
return selected
}
func workflowOperatorSearchText(operator domain.OperatorRecord) string {
parts := []string{
operator.ID,
operator.Name,
operator.Category,
operator.Description,
operator.SkillDefinition,
}
for _, port := range operator.Inputs {
parts = append(parts, port.Name, string(port.Type), port.Description, port.Transmission)
}
for _, port := range operator.Outputs {
parts = append(parts, port.Name, string(port.Type), port.Description, port.Transmission)
}
for _, item := range operator.Hyperparams {
parts = append(parts, item.Key, item.Description, item.Type)
}
return strings.ToLower(strings.Join(parts, "\n"))
}
func promptMatchTerms(description string) []string {
normalized := strings.ToLower(strings.TrimSpace(description))
seen := map[string]struct{}{}
terms := make([]string, 0)
addTerm := func(term string) {
term = strings.TrimSpace(term)
if term == "" {
return
}
runes := []rune(term)
if len(runes) < 2 && !isASCIIAlphaNumeric(term) {
return
}
if _, ok := seen[term]; ok {
return
}
seen[term] = struct{}{}
terms = append(terms, term)
}
for _, term := range strings.FieldsFunc(normalized, func(character rune) bool {
return !(unicode.IsLetter(character) || unicode.IsDigit(character))
}) {
addTerm(term)
}
cjk := make([]rune, 0)
flushCJK := func() {
for size := 2; size <= 3; size++ {
for index := 0; index+size <= len(cjk); index++ {
addTerm(string(cjk[index : index+size]))
}
}
cjk = cjk[:0]
}
for _, character := range normalized {
if unicode.In(character, unicode.Han) {
cjk = append(cjk, character)
continue
}
flushCJK()
}
flushCJK()
return terms
}
func isASCIIAlphaNumeric(value string) bool {
for _, character := range value {
if character > unicode.MaxASCII || !(unicode.IsLetter(character) || unicode.IsDigit(character)) {
return false
}
}
return value != ""
}
func (client *client) generateJSON(ctx context.Context, systemPrompt string, userPrompt string, maxTokens int, target any) error {
if maxTokens < 1 {
maxTokens = operatorMaxTokens
}
requestBody, err := json.Marshal(chatCompletionRequest{
Model: client.model,
Messages: []chatMessage{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: userPrompt},
},
Temperature: 0.1,
MaxTokens: maxTokens,
ResponseFormat: &chatResponseFormat{Type: "json_object"},
})
if err != nil {
return err
}
payload, err := client.chatCompletion(ctx, requestBody)
if err != nil {
return err
}
var completion chatCompletionResponse
if err := json.Unmarshal(payload, &completion); err != nil {
return fmt.Errorf("decode AI response: %w", err)
}
if completion.Error != nil && strings.TrimSpace(completion.Error.Message) != "" {
return errors.New(completion.Error.Message)
}
if len(completion.Choices) == 0 {
return fmt.Errorf("AI response does not contain choices")
}
content := normalizeMessageContent(completion.Choices[0].Message.Content)
if strings.TrimSpace(content) == "" {
return fmt.Errorf("AI response content is empty")
}
parsedJSON, err := extractJSONObject(content)
if err != nil {
return err
}
if err := json.Unmarshal([]byte(parsedJSON), target); err != nil {
return fmt.Errorf("decode AI JSON payload: %w", err)
}
return nil
}
type httpStatusError struct {
StatusCode int
Body string
}
func (err *httpStatusError) Error() string {
body := strings.TrimSpace(err.Body)
if body == "" {
return fmt.Sprintf("AI request failed with status %d", err.StatusCode)
}
return fmt.Sprintf("AI request failed with status %d: %s", err.StatusCode, body)
}
func (client *client) chatCompletion(ctx context.Context, requestBody []byte) ([]byte, error) {
attempts := client.maxAttempts
if attempts < 1 {
attempts = 1
}
var lastErr error
for attempt := 1; attempt <= attempts; attempt++ {
payload, err := client.chatCompletionOnce(ctx, requestBody)
if err == nil {
return payload, nil
}
lastErr = err
if !isRetriableAIError(err) || attempt == attempts || ctx.Err() != nil {
break
}
if err := sleepWithContext(ctx, time.Duration(attempt)*2*time.Second); err != nil {
lastErr = err
break
}
}
return nil, normalizeAIError(lastErr)
}
func (client *client) chatCompletionOnce(ctx context.Context, requestBody []byte) ([]byte, error) {
requestCtx := ctx
cancel := func() {}
if _, hasDeadline := ctx.Deadline(); !hasDeadline && client.requestTimeout > 0 {
requestCtx, cancel = context.WithTimeout(ctx, client.requestTimeout)
}
defer cancel()
request, err := http.NewRequestWithContext(requestCtx, http.MethodPost, client.baseURL+"/chat/completions", bytes.NewReader(requestBody))
if err != nil {
return nil, err
}
request.Header.Set("Authorization", "Bearer "+client.apiKey)
request.Header.Set("Content-Type", "application/json")
response, err := client.httpClient.Do(request)
if err != nil {
return nil, err
}
defer response.Body.Close()
payload, err := io.ReadAll(response.Body)
if err != nil {
return nil, err
}
if response.StatusCode >= 400 {
return nil, &httpStatusError{
StatusCode: response.StatusCode,
Body: string(payload),
}
}
return payload, nil
}
func isRetriableAIError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err) {
return true
}
var statusErr *httpStatusError
if errors.As(err, &statusErr) {
return statusErr.StatusCode == http.StatusTooManyRequests ||
statusErr.StatusCode == http.StatusRequestTimeout ||
statusErr.StatusCode >= http.StatusInternalServerError
}
return false
}
func isTimeoutError(err error) bool {
if err == nil {
return false
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
return true
}
return strings.Contains(strings.ToLower(err.Error()), "client.timeout")
}
func normalizeAIError(err error) error {
if err == nil {
return nil
}
if errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err) {
return fmt.Errorf("AI 生成超时,请稍后重试,或缩短描述范围后再试")
}
var statusErr *httpStatusError
if errors.As(err, &statusErr) {
switch {
case statusErr.StatusCode == http.StatusTooManyRequests:
return fmt.Errorf("AI 服务当前限流,请稍后重试")
case statusErr.StatusCode == http.StatusRequestTimeout:
return fmt.Errorf("AI 服务响应超时,请稍后重试")
case statusErr.StatusCode >= http.StatusInternalServerError:
return fmt.Errorf("AI 服务暂时不可用,请稍后重试")
}
}
return err
}
func sleepWithContext(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func normalizeMessageContent(content any) string {
switch value := content.(type) {
case string:
return value
case []any:
parts := make([]string, 0, len(value))
for _, item := range value {
typed, ok := item.(map[string]any)
if !ok {
continue
}
if text, ok := typed["text"].(string); ok {
parts = append(parts, text)
}
}
return strings.Join(parts, "\n")
default:
return ""
}
}
func extractJSONObject(raw string) (string, error) {
trimmed := strings.TrimSpace(raw)
trimmed = strings.TrimPrefix(trimmed, "```json")
trimmed = strings.TrimPrefix(trimmed, "```")
trimmed = strings.TrimSuffix(trimmed, "```")
trimmed = strings.TrimSpace(trimmed)
start := strings.Index(trimmed, "{")
end := strings.LastIndex(trimmed, "}")
if start < 0 || end < start {
return "", fmt.Errorf("AI response is not a JSON object")
}
return trimmed[start : end+1], nil
}
func truncatePromptText(value string, limit int) string {
trimmed := strings.TrimSpace(value)
if limit < 1 || len([]rune(trimmed)) <= limit {
return trimmed
}
runes := []rune(trimmed)
return strings.TrimSpace(string(runes[:limit])) + "..."
}
func clonePorts(ports []domain.PortDefinition) []domain.PortDefinition {
items := make([]domain.PortDefinition, len(ports))
copy(items, ports)
return items
}
func mergeHyperparams(base []domain.HyperParam, overrides []domain.HyperParam) []domain.HyperParam {
if len(base) == 0 && len(overrides) == 0 {
return []domain.HyperParam{}
}
merged := make([]domain.HyperParam, len(base))
copy(merged, base)
indexByKey := make(map[string]int, len(merged))
for index, item := range merged {
indexByKey[strings.ToUpper(strings.TrimSpace(item.Key))] = index
}
for _, item := range overrides {
key := strings.ToUpper(strings.TrimSpace(item.Key))
if key == "" {
continue
}
item.Key = strings.TrimSpace(item.Key)
if index, ok := indexByKey[key]; ok {
if strings.TrimSpace(item.Value) != "" {
merged[index].Value = item.Value
}
if strings.TrimSpace(item.Description) != "" {
merged[index].Description = item.Description
}
if strings.TrimSpace(item.Type) != "" {
merged[index].Type = item.Type
}
continue
}
merged = append(merged, item)
indexByKey[key] = len(merged) - 1
}
return merged
}
func resolvePortHandle(ports []domain.PortDefinition, name string) (string, error) {
normalizedName := strings.TrimSpace(strings.ToLower(name))
if normalizedName == "" {
if len(ports) == 1 {
return ports[0].ID, nil
}
return "", fmt.Errorf("port name is required when multiple ports are available")
}
for _, port := range ports {
if strings.ToLower(strings.TrimSpace(port.Name)) == normalizedName || strings.ToLower(strings.TrimSpace(port.ID)) == normalizedName {
return port.ID, nil
}
}
return "", fmt.Errorf("port %q not found", name)
}
func chooseValue(value string, fallback string) string {
if strings.TrimSpace(value) == "" {
return fallback
}
return strings.TrimSpace(value)
}
func validCategory(value string) bool {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "IO", "CV", "AI", "MATH", "HUMAN", "FLOW":
return true
default:
return false
}
}
func validPortType(value string) bool {
switch strings.ToLower(strings.TrimSpace(value)) {
case "json", "image", "video", "tensor", "file":
return true
default:
return false
}
}
func validTransmission(value string) bool {
switch strings.ToLower(strings.TrimSpace(value)) {
case "bus", "oss", "shm", "stream":
return true
default:
return false
}
}
func validateGeneratedPorts(ports []GeneratedOperatorPort) error {
for _, port := range ports {
if strings.TrimSpace(port.Name) == "" {
return fmt.Errorf("port name is required")
}
if !validPortType(port.Type) {
return fmt.Errorf("unsupported port type %q", port.Type)
}
if !domain.ValidPortContainer(domain.PortContainer(strings.ToLower(strings.TrimSpace(port.Container)))) {
return fmt.Errorf("unsupported port container %q", port.Container)
}
if !domain.ValidPortPayloadMode(domain.ArtifactTransferMode(strings.ToLower(strings.TrimSpace(port.PayloadMode)))) {
return fmt.Errorf("unsupported port payload mode %q", port.PayloadMode)
}
if !validTransmission(port.Transmission) {
return fmt.Errorf("unsupported transmission %q", port.Transmission)
}
}
return nil
}