🚦 converge worker runs and harden POC acceptance

This commit is contained in:
eust-w 2026-07-13 12:31:36 +08:00
parent c167ce095f
commit 6fe111021b
20 changed files with 367 additions and 114 deletions

View File

@ -43,15 +43,20 @@ type pocOperatorWorkflowImageEvidence struct {
}
type pocOperatorWorkflowAcceptanceSummary struct {
ContractVersion string `json:"contractVersion"`
OperatorCount int `json:"operatorCount"`
IndependentImageCoverage int `json:"independentImageCoverage"`
IndependentImageRequired int `json:"independentImageRequired"`
CommandWrapperOnlyOperatorCount int `json:"commandWrapperOnlyOperatorCount"`
GeneratedFixtureCount int `json:"generatedFixtureCount"`
MockFixtureCount int `json:"mockFixtureCount"`
GPUOperatorCount int `json:"gpuOperatorCount"`
StrictReady bool `json:"strictReady"`
ContractVersion string `json:"contractVersion"`
OperatorCount int `json:"operatorCount"`
CatalogHash string `json:"catalogHash"`
CatalogCount int `json:"catalogCount"`
IndependentImageCoverage int `json:"independentImageCoverage"`
IndependentImageRequired int `json:"independentImageRequired"`
CommandWrapperOnlyOperatorCount int `json:"commandWrapperOnlyOperatorCount"`
GeneratedFixtureCount int `json:"generatedFixtureCount"`
MockFixtureCount int `json:"mockFixtureCount"`
GPUOperatorCount int `json:"gpuOperatorCount"`
StrictReady bool `json:"strictReady"`
TrainingMode string `json:"trainingMode"`
FullTrainingRequired bool `json:"fullTrainingRequired"`
MissingOperatorIDs []string `json:"missingOperatorIds"`
}
type pocOperatorWorkflowAcceptanceResponse struct {
@ -724,9 +729,12 @@ func buildPocOperatorWorkflowImageEvidence(operator domain.OperatorRecord) pocOp
func buildPocOperatorWorkflowAcceptanceSummary(operators []domain.OperatorRecord) pocOperatorWorkflowAcceptanceSummary {
summary := pocOperatorWorkflowAcceptanceSummary{
ContractVersion: "poc-operator-workflows/v3",
ContractVersion: "poc-operator-workflows/v4",
OperatorCount: len(operators),
IndependentImageRequired: len(operators),
TrainingMode: "full",
FullTrainingRequired: true,
MissingOperatorIDs: []string{},
}
for _, operator := range operators {
if pocOperatorHasIndependentImage(operator) {
@ -1135,6 +1143,17 @@ func (server *server) handlePocOperatorWorkflowAcceptance(writer http.ResponseWr
Runs: []pocOperatorWorkflowAcceptanceRun{},
Batches: map[string][]string{},
}
if catalogHash, catalogCount, catalogErr := builtinOperatorCatalogFingerprint(); catalogErr == nil {
response.Summary.CatalogHash = catalogHash
response.Summary.CatalogCount = catalogCount
if catalogCount != len(operators) {
response.Summary.MissingOperatorIDs = []string{"catalog_count_mismatch"}
response.Summary.StrictReady = false
}
} else {
renderError(writer, http.StatusInternalServerError, catalogErr)
return
}
for _, batchName := range batchNames {
batchOperators := grouped[batchName]
if len(batchOperators) == 0 {

View File

@ -5800,7 +5800,7 @@ func runStatusIsValid(status domain.RunStatus) bool {
func runSummaryPaginationRequested(request *http.Request) bool {
query := request.URL.Query()
for _, key := range []string{"limit", "cursor", "q", "projectId", "workflowId"} {
for _, key := range []string{"limit", "cursor", "q", "projectId", "workflowId", "includeAggregates"} {
if _, ok := query[key]; ok {
return true
}
@ -5841,13 +5841,23 @@ func encodeRunSummaryCursor(cursor *domain.RunSummaryCursor) string {
func parseRunSummaryQuery(request *http.Request) (domain.RunSummaryQuery, []validationIssue) {
values := request.URL.Query()
query := domain.RunSummaryQuery{
Limit: 50,
Status: domain.RunStatus(strings.TrimSpace(values.Get("status"))),
Search: strings.TrimSpace(values.Get("q")),
ProjectID: strings.TrimSpace(values.Get("projectId")),
WorkflowID: strings.TrimSpace(values.Get("workflowId")),
Limit: 50,
IncludeAggregates: true,
Status: domain.RunStatus(strings.TrimSpace(values.Get("status"))),
Search: strings.TrimSpace(values.Get("q")),
ProjectID: strings.TrimSpace(values.Get("projectId")),
WorkflowID: strings.TrimSpace(values.Get("workflowId")),
}
issues := make([]validationIssue, 0, 4)
if raw, ok := values["includeAggregates"]; ok && len(raw) > 0 {
query.AggregatesExplicit = true
value, err := strconv.ParseBool(strings.TrimSpace(raw[0]))
if err != nil {
issues = append(issues, validationIssue{Field: "includeAggregates", Message: "includeAggregates must be true or false"})
} else {
query.IncludeAggregates = value
}
}
issues := make([]validationIssue, 0, 3)
if rawLimit, ok := values["limit"]; ok {
value := ""
if len(rawLimit) > 0 {
@ -5882,18 +5892,23 @@ func (server *server) handleRunsList(writer http.ResponseWriter, request *http.R
renderValidationError(writer, issues...)
return
}
startedAt := time.Now()
page, err := server.listRunSummaryPageForRequest(request.Context(), request, query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{
"items": page.Items,
"total": page.Total,
"statusCounts": page.StatusCounts,
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
"hasMore": page.HasMore,
})
writer.Header().Set("Server-Timing", fmt.Sprintf("runs;dur=%.1f", float64(time.Since(startedAt).Microseconds())/1000))
response := map[string]any{
"items": page.Items,
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
"hasMore": page.HasMore,
}
if query.IncludeAggregates {
response["total"] = page.Total
response["statusCounts"] = page.StatusCounts
}
renderJSON(writer, http.StatusOK, response)
return
}
items, err := server.listRunSummariesForRequest(request.Context(), request, request.URL.Query().Get("status"))

View File

@ -1,6 +1,12 @@
package domain
import "time"
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
)
type UserRole string
@ -374,17 +380,82 @@ type ArtifactMetadata struct {
}
type ExecutionArtifact struct {
ID string `json:"id" bson:"id"`
RunID string `json:"runId" bson:"runId"`
NodeID string `json:"nodeId" bson:"nodeId"`
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
PortName string `json:"portName,omitempty" bson:"portName,omitempty"`
Name string `json:"name" bson:"name"`
Type PortType `json:"type" bson:"type"`
Size string `json:"size" bson:"size"`
Path string `json:"path" bson:"path"`
Metadata *ArtifactMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"`
Data map[string]any `json:"data,omitempty" bson:"data,omitempty"`
ID string `json:"id" bson:"id"`
RunID string `json:"runId" bson:"runId"`
NodeID string `json:"nodeId" bson:"nodeId"`
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
PortName string `json:"portName,omitempty" bson:"portName,omitempty"`
Name string `json:"name" bson:"name"`
Type PortType `json:"type" bson:"type"`
Size string `json:"size" bson:"size"`
SizeBytes int64 `json:"sizeBytes,omitempty" bson:"sizeBytes,omitempty"`
Path string `json:"path" bson:"path"`
Metadata *ArtifactMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"`
Data map[string]any `json:"data,omitempty" bson:"data,omitempty"`
}
// UnmarshalJSON keeps the public artifact contract compatible with older
// runners that emitted size as a string while accepting newer runners that
// emit the exact byte count as a JSON number.
func (artifact *ExecutionArtifact) UnmarshalJSON(data []byte) error {
type plain ExecutionArtifact
var fields map[string]json.RawMessage
if err := json.Unmarshal(data, &fields); err != nil {
return err
}
sizeRaw := fields["size"]
delete(fields, "size")
withoutSize, err := json.Marshal(fields)
if err != nil {
return err
}
var parsed plain
if err := json.Unmarshal(withoutSize, &parsed); err != nil {
return err
}
*artifact = ExecutionArtifact(parsed)
if len(sizeRaw) == 0 || strings.TrimSpace(string(sizeRaw)) == "null" {
return nil
}
var sizeText string
if sizeRaw[0] == '"' {
if err := json.Unmarshal(sizeRaw, &sizeText); err != nil {
return fmt.Errorf("decode artifact size: %w", err)
}
artifact.Size = sizeText
if bytes, err := strconv.ParseInt(strings.TrimSpace(sizeText), 10, 64); err == nil && bytes >= 0 {
artifact.SizeBytes = bytes
}
return nil
}
var number json.Number
decoder := json.NewDecoder(strings.NewReader(string(sizeRaw)))
decoder.UseNumber()
if err := decoder.Decode(&number); err != nil {
return fmt.Errorf("decode artifact size: %w", err)
}
bytes, err := strconv.ParseInt(number.String(), 10, 64)
if err != nil || bytes < 0 {
return fmt.Errorf("artifact size must be a non-negative integer or string")
}
artifact.SizeBytes = bytes
artifact.Size = formatArtifactSize(bytes)
return nil
}
func formatArtifactSize(bytes int64) string {
if bytes < 1024 {
return strconv.FormatInt(bytes, 10) + " B"
}
value := float64(bytes)
units := []string{"KB", "MB", "GB", "TB"}
for _, unit := range units {
value /= 1024
if value < 1024 {
return strconv.FormatFloat(value, 'f', 1, 64) + " " + unit
}
}
return strconv.FormatFloat(value, 'f', 1, 64) + " PB"
}
type DataRefRecord struct {
@ -617,12 +688,14 @@ type RunSummaryCursor struct {
}
type RunSummaryQuery struct {
Limit int
Cursor *RunSummaryCursor
Status RunStatus
Search string
ProjectID string
WorkflowID string
Limit int
Cursor *RunSummaryCursor
IncludeAggregates bool
AggregatesExplicit bool
Status RunStatus
Search string
ProjectID string
WorkflowID string
}
type RunSummaryPage struct {

View File

@ -974,6 +974,10 @@ func (store *MemoryStore) listRunSummariesPageLocked(organizationID string, quer
last := pageItems[len(pageItems)-1]
nextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
if query.AggregatesExplicit && !query.IncludeAggregates {
statusCounts = nil
total = 0
}
return domain.RunSummaryPage{
Items: pageItems,
Total: total,

View File

@ -907,7 +907,7 @@ func (store *MongoStore) listRunSummariesPage(ctx context.Context, organizationI
var items []domain.RunSummary
var aggregates runSummaryAggregates
queryErrors := make(chan error, 2)
waitGroup.Add(2)
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
foundItems, err := store.findRunSummaryPageItems(queryCtx, itemFilter, limit)
@ -918,19 +918,23 @@ func (store *MongoStore) listRunSummariesPage(ctx context.Context, organizationI
}
items = foundItems
}()
go func() {
defer waitGroup.Done()
key := newRunSummaryAggregateCacheKey(organizationID, query)
foundAggregates, err := store.runSummaryAggregates.getOrLoad(queryCtx, key, func(loadCtx context.Context) (runSummaryAggregates, error) {
return store.loadRunSummaryAggregates(loadCtx, baseFilter)
})
if err != nil {
queryErrors <- err
cancel()
return
}
aggregates = foundAggregates
}()
includeAggregates := query.IncludeAggregates || !query.AggregatesExplicit
if includeAggregates {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
key := newRunSummaryAggregateCacheKey(organizationID, query)
foundAggregates, err := store.runSummaryAggregates.getOrLoad(queryCtx, key, func(loadCtx context.Context) (runSummaryAggregates, error) {
return store.loadRunSummaryAggregates(loadCtx, baseFilter)
})
if err != nil {
queryErrors <- err
cancel()
return
}
aggregates = foundAggregates
}()
}
waitGroup.Wait()
select {
case err := <-queryErrors:

View File

@ -70,6 +70,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
store.runs: {
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}),
index("organizationId_status_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
index("organizationId_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
index("createdAt_id", bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
index("status_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}),
@ -78,7 +79,9 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
index("status_lease_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "executionLeaseExpiresAt", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}),
index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}),
index("workflowId_status", bson.D{{Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}),
index("organizationId_workflowId_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
index("projectId_createdAt", bson.D{{Key: "projectId", Value: 1}, {Key: "createdAt", Value: -1}}),
index("organizationId_projectId_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
},
store.sourceStates: {
uniqueIndex("workflow_node_fingerprint_unique", bson.D{{Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}, {Key: "sourceFingerprint", Value: 1}}),

View File

@ -335,6 +335,20 @@ func TestParseExecutionOutputReadsResultFilePayload(t *testing.T) {
}
}
func TestParseExecutionOutputAcceptsNumericArtifactSize(t *testing.T) {
parsed, err := parseExecutionOutput(`DROBOTICFLOW_RESULT={"artifacts":[{"id":"artifact-size","name":"training_report","type":"file","size":1852,"path":"/workspace/report.json"}]}`, "训练")
if err != nil {
t.Fatalf("parseExecutionOutput returned error: %v", err)
}
if len(parsed.Artifacts) != 1 {
t.Fatalf("expected one artifact, got %#v", parsed.Artifacts)
}
artifact := parsed.Artifacts[0]
if artifact.SizeBytes != 1852 || artifact.Size != "1.8 KB" {
t.Fatalf("unexpected numeric artifact size: %#v", artifact)
}
}
func TestLoadStageManifestDataRefs(t *testing.T) {
stageManifestPath := filepath.Join(t.TempDir(), "stage-manifest.json")
payload := map[string]any{

View File

@ -106,6 +106,8 @@ func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http.
response["effectivePollIntervalMs"] = readiness.EffectivePollIntervalMS
response["stuckRunCount"] = readiness.StuckRunCount
response["orphanRuntimeCount"] = readiness.OrphanRuntimeCount
response["orphanRuntimeCleanedCount"] = readiness.OrphanRuntimeCleanedCount
response["orphanRuntimeCleanupErrorCount"] = readiness.OrphanRuntimeCleanupErrorCount
if organizationID != "" {
response["organizationId"] = organizationID
response["global"] = map[string]int{

View File

@ -33,6 +33,8 @@ type MetricsSnapshot struct {
CycleDurationMS uint64 `json:"cycleDurationMs"`
StuckRunCount uint64 `json:"stuckRunCount"`
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"`
OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"`
}
type WorkerMetrics struct {
@ -54,6 +56,8 @@ type WorkerMetrics struct {
cycleDurationMS atomic.Uint64
stuckRunCount atomic.Uint64
orphanRuntimeCount atomic.Uint64
orphanRuntimeCleanedCount atomic.Uint64
orphanRuntimeCleanupErrorCount atomic.Uint64
}
func (metrics *WorkerMetrics) RecordProcessAvailable(durationMS uint64, err error) {
@ -103,6 +107,11 @@ func (metrics *WorkerMetrics) RecordOrphanRuntimes(count int) {
metrics.orphanRuntimeCount.Store(uint64(max(count, 0)))
}
func (metrics *WorkerMetrics) RecordOrphanCleanup(cleaned int, errors int) {
metrics.orphanRuntimeCleanedCount.Add(uint64(max(cleaned, 0)))
metrics.orphanRuntimeCleanupErrorCount.Add(uint64(max(errors, 0)))
}
func metricTimestamp(value int64) string {
if value <= 0 {
return ""
@ -130,6 +139,8 @@ func (metrics *WorkerMetrics) Snapshot() MetricsSnapshot {
CycleDurationMS: metrics.cycleDurationMS.Load(),
StuckRunCount: metrics.stuckRunCount.Load(),
OrphanRuntimeCount: metrics.orphanRuntimeCount.Load(),
OrphanRuntimeCleanedCount: metrics.orphanRuntimeCleanedCount.Load(),
OrphanRuntimeCleanupErrorCount: metrics.orphanRuntimeCleanupErrorCount.Load(),
}
}

View File

@ -51,23 +51,25 @@ type BuildInfo struct {
}
type ReadinessSnapshot struct {
Ready bool `json:"ready"`
Status string `json:"status"`
Service string `json:"service"`
BuildCommit string `json:"buildCommit,omitempty"`
BuildTime string `json:"buildTime,omitempty"`
Image string `json:"image,omitempty"`
RunnerImage string `json:"runnerImage,omitempty"`
RunnerBuildCommit string `json:"runnerBuildCommit,omitempty"`
RunnerBuildTime string `json:"runnerBuildTime,omitempty"`
LastCycleAt string `json:"lastCycleAt,omitempty"`
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
CycleDurationMS uint64 `json:"cycleDurationMs"`
ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"`
EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"`
StuckRunCount uint64 `json:"stuckRunCount"`
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
Reason string `json:"reason,omitempty"`
Ready bool `json:"ready"`
Status string `json:"status"`
Service string `json:"service"`
BuildCommit string `json:"buildCommit,omitempty"`
BuildTime string `json:"buildTime,omitempty"`
Image string `json:"image,omitempty"`
RunnerImage string `json:"runnerImage,omitempty"`
RunnerBuildCommit string `json:"runnerBuildCommit,omitempty"`
RunnerBuildTime string `json:"runnerBuildTime,omitempty"`
LastCycleAt string `json:"lastCycleAt,omitempty"`
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
CycleDurationMS uint64 `json:"cycleDurationMs"`
ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"`
EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"`
StuckRunCount uint64 `json:"stuckRunCount"`
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"`
OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"`
Reason string `json:"reason,omitempty"`
}
type Runtime struct {
@ -456,22 +458,24 @@ func (runtime *Runtime) statusSnapshot() ReadinessSnapshot {
configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load())
effective := normalizedPollIntervalMS(runtime.effectivePollMS.Load())
snapshot := ReadinessSnapshot{
Ready: false,
Status: "not_ready",
Service: "droboticflow-worker",
BuildCommit: build.BuildCommit,
BuildTime: build.BuildTime,
Image: build.Image,
RunnerImage: firstNonEmptyRuntimeValue(runnerBuild.Image, runtime.operatorRunnerImage),
RunnerBuildCommit: runnerBuild.BuildCommit,
RunnerBuildTime: runnerBuild.BuildTime,
LastCycleAt: metrics.LastCycleAt,
LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt,
CycleDurationMS: metrics.CycleDurationMS,
ConfiguredPollIntervalMS: configured,
EffectivePollIntervalMS: effective,
StuckRunCount: metrics.StuckRunCount,
OrphanRuntimeCount: metrics.OrphanRuntimeCount,
Ready: false,
Status: "not_ready",
Service: "droboticflow-worker",
BuildCommit: build.BuildCommit,
BuildTime: build.BuildTime,
Image: build.Image,
RunnerImage: firstNonEmptyRuntimeValue(runnerBuild.Image, runtime.operatorRunnerImage),
RunnerBuildCommit: runnerBuild.BuildCommit,
RunnerBuildTime: runnerBuild.BuildTime,
LastCycleAt: metrics.LastCycleAt,
LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt,
CycleDurationMS: metrics.CycleDurationMS,
ConfiguredPollIntervalMS: configured,
EffectivePollIntervalMS: effective,
StuckRunCount: metrics.StuckRunCount,
OrphanRuntimeCount: metrics.OrphanRuntimeCount,
OrphanRuntimeCleanedCount: metrics.OrphanRuntimeCleanedCount,
OrphanRuntimeCleanupErrorCount: metrics.OrphanRuntimeCleanupErrorCount,
}
return snapshot
}
@ -1046,7 +1050,17 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
fmt.Printf("[worker] orphan runtime reconciliation failed: %v\n", err)
return
}
runtime.metrics.RecordOrphanRuntimes(len(orphans))
cleaned, cleanupErrors := 0, 0
for _, orphan := range orphans {
if orphan.Cleaned {
cleaned++
}
if strings.TrimSpace(orphan.CleanupError) != "" {
cleanupErrors++
}
}
runtime.metrics.RecordOrphanRuntimes(len(orphans) - cleaned)
runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors)
}
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
@ -1250,10 +1264,25 @@ func (runtime *Runtime) failReconciledRun(ctx context.Context, run *domain.RunRe
return err
}
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil || workflow == nil {
if err != nil {
return err
}
return runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO())
if workflow != nil {
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO()); err != nil {
return err
}
}
// Persist the terminal state before touching Kubernetes. A failed runtime is
// no longer useful to the scheduler, and cleanup is deliberately best-effort
// so a transient API error cannot leave the Run active again.
if runtime.launcher != nil {
cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
if cleanupErr := runtime.launcher.TerminateRun(cleanupCtx, "", run.ID); cleanupErr != nil {
fmt.Printf("[worker] cleanup failed runtime for terminal run %s: %v\n", run.ID, cleanupErr)
}
cancel()
}
return nil
}
func (runtime *Runtime) recoverMissingJob(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {

View File

@ -12,6 +12,8 @@ fi
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
# shellcheck source=scripts/lib/build-metadata.sh
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
REGISTRY_HOST="${REGISTRY_HOST:-}"
@ -43,10 +45,11 @@ POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}"
POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}"
if [[ -z "${IMAGE_TAG:-}" ]]; then
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)"
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)"
fi
POC_KANIKO_SOURCE_COMMIT="${POC_KANIKO_SOURCE_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-}"
POC_KANIKO_BUILD_TIME="$(droboticflow_normalize_build_time "$POC_KANIKO_BUILD_TIME")"
# shellcheck source=scripts/lib/image-config.sh
source "$ROOT_DIR/scripts/lib/image-config.sh"

View File

@ -15,6 +15,12 @@ export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
export WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-true}"
export WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
export WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"
export WORKER_PENDING_FAILURE_SECONDS="${WORKER_PENDING_FAILURE_SECONDS:-600}"
export WORKER_MISSING_JOB_MAX_RECOVERIES="${WORKER_MISSING_JOB_MAX_RECOVERIES:-3}"
export HOLOBRAIN_TRAINING_MODE="${HOLOBRAIN_TRAINING_MODE:-full}"
export K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
export WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"

View File

@ -13,6 +13,8 @@ export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
DEPLOY_ENV="${DEPLOY_ENV:-}"
# shellcheck source=scripts/lib/build-metadata.sh
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
@ -39,13 +41,19 @@ API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
API_REPLICAS="${API_REPLICAS:-2}"
WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-false}"
WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"
WORKER_PENDING_FAILURE_SECONDS="${WORKER_PENDING_FAILURE_SECONDS:-600}"
WORKER_MISSING_JOB_MAX_RECOVERIES="${WORKER_MISSING_JOB_MAX_RECOVERIES:-3}"
# shellcheck source=scripts/lib/image-config.sh
source "$ROOT_DIR/scripts/lib/image-config.sh"
droboticflow_resolve_image_configuration
DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}"
OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}"
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-$DROBOTICFLOW_BUILD_TIME}"
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-}"
droboticflow_resolve_build_metadata
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
@ -189,6 +197,11 @@ env \
API_SERVICE_TYPE="$API_SERVICE_TYPE" \
API_REPLICAS="$API_REPLICAS" \
WORKER_REPLICAS="$WORKER_REPLICAS" \
WORKER_ORPHAN_CLEANUP_ENABLED="$WORKER_ORPHAN_CLEANUP_ENABLED" \
WORKER_ORPHAN_MIN_AGE_SECONDS="$WORKER_ORPHAN_MIN_AGE_SECONDS" \
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="$WORKER_ORPHAN_CHECK_INTERVAL_SECONDS" \
WORKER_PENDING_FAILURE_SECONDS="$WORKER_PENDING_FAILURE_SECONDS" \
WORKER_MISSING_JOB_MAX_RECOVERIES="$WORKER_MISSING_JOB_MAX_RECOVERIES" \
AI_API_BASE_URL="$AI_API_BASE_URL" \
AI_MODEL="$AI_MODEL" \
AI_API_KEY="$AI_API_KEY" \

View File

@ -135,7 +135,7 @@ project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Cont
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
printf 'created project: %s\n' "$project_id"
acceptance_suffix="${HOLOBRAIN_ACCEPTANCE_SUFFIX:-$(date +%Y%m%d%H%M%S)-$$}"
acceptance_suffix="${HOLOBRAIN_ACCEPTANCE_SUFFIX:-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)-$$}"
workflow_payload="$(
ROOT_DIR="$ROOT_DIR" HOLOBRAIN_ACCEPTANCE_SUFFIX="$acceptance_suffix" python3 - <<'PY'
import json

View File

@ -0,0 +1,34 @@
#!/usr/bin/env bash
# Canonical build metadata is always RFC3339 UTC with a trailing Z. Human
# readable tags may use Asia/Shanghai, but must never be used as buildTime.
droboticflow_normalize_build_time() {
local raw="${1:-}"
if [[ -z "${raw//[[:space:]]/}" ]]; then
date -u +%Y-%m-%dT%H:%M:%SZ
return
fi
BUILD_TIME_INPUT="$raw" python3 - <<'PY'
from datetime import datetime, timezone, timedelta
import os
raw = os.environ.get("BUILD_TIME_INPUT", "").strip()
try:
value = datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError as exc:
raise SystemExit(f"invalid build time {raw!r}: expected RFC3339") from exc
if value.tzinfo is None:
raise SystemExit("build time must include a timezone offset or Z")
value = value.astimezone(timezone.utc)
if value > datetime.now(timezone.utc) + timedelta(minutes=5):
raise SystemExit("build time is more than five minutes in the future")
print(value.isoformat(timespec="seconds").replace("+00:00", "Z"))
PY
}
droboticflow_resolve_build_metadata() {
DROBOTICFLOW_BUILD_TIME="$(droboticflow_normalize_build_time "${DROBOTICFLOW_BUILD_TIME:-}")"
OPERATOR_RUNNER_BUILD_TIME="$(droboticflow_normalize_build_time "${OPERATOR_RUNNER_BUILD_TIME:-$DROBOTICFLOW_BUILD_TIME}")"
export DROBOTICFLOW_BUILD_TIME OPERATOR_RUNNER_BUILD_TIME
}

View File

@ -9,7 +9,7 @@ export DEPLOY_ENV
source "$ROOT_DIR/scripts/load-deploy-env.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-operator-acceptance-$(date +%Y%m%d%H%M%S)}"
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-operator-acceptance-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)}"
POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT="${POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT:-1800s}"
POC_ACCEPTANCE_REQUIRED_EXTERNALS="${POC_ACCEPTANCE_REQUIRED_EXTERNALS:-bos,mongo}"
POC_OPERATOR_ACCEPTANCE_CLEANUP_JOB="${POC_OPERATOR_ACCEPTANCE_CLEANUP_JOB:-false}"

View File

@ -17,8 +17,15 @@ API_TOKEN_SECRET_NAME="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticfl
API_TOKEN_SECRET_KEY="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-5400}"
POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="${POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json}"
POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="${POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT:-96}"
POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="${POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT:-$(python3 - "$ROOT_DIR/internal/api/builtin_operators_catalog.json" <<'PY'
import json
import os
import sys
print(len(json.load(open(sys.argv[1], encoding="utf-8"))))
PY
)}"
POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS:-15}"
POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION="${POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION:-false}"
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
DROBOTICFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-youbixuanpoc}"
@ -328,10 +335,16 @@ required = int(summary.get("independentImageRequired") or operator_count)
command_wrappers = int(summary.get("commandWrapperOnlyOperatorCount") or 0)
generated_fixtures = int(summary.get("generatedFixtureCount") or 0)
mock_fixtures = int(summary.get("mockFixtureCount") or 0)
if operator_count < minimum_operator_count:
raise SystemExit(f"operator workflow acceptance selected too few operators: {operator_count}/{minimum_operator_count}")
if summary.get("contractVersion") != "poc-operator-workflows/v3":
raise SystemExit(f"operator workflow contract version must be poc-operator-workflows/v3, got {summary.get('contractVersion')}")
if operator_count != minimum_operator_count:
raise SystemExit(f"operator workflow acceptance must cover the current catalog exactly: {operator_count}/{minimum_operator_count}")
if summary.get("contractVersion") != "poc-operator-workflows/v4":
raise SystemExit(f"operator workflow contract version must be poc-operator-workflows/v4, got {summary.get('contractVersion')}")
if int(summary.get("catalogCount") or 0) != operator_count or not summary.get("catalogHash"):
raise SystemExit("operator workflow acceptance catalog provenance is missing or inconsistent")
if summary.get("trainingMode") != "full" or summary.get("fullTrainingRequired") is not True:
raise SystemExit("operator workflow acceptance must require full training")
if summary.get("missingOperatorIds"):
raise SystemExit(f"operator workflow acceptance has missing operators: {summary.get('missingOperatorIds')}")
if coverage != required:
raise SystemExit(f"independent image coverage is incomplete: {coverage}/{required}")
if command_wrappers != 0:
@ -384,8 +397,9 @@ while (( SECONDS < deadline )); do
fetch_runs_json
fi
summary="$(
python3 - "$RUNS_JSON_FILE" <<'PY'
python3 - "$RUNS_JSON_FILE" <<'PY'
import json
import os
import sys
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
print(json.dumps([{"id": run.get("id"), "status": run.get("status"), "nodes": len(run.get("nodeExecutions") or [])} for run in runs], ensure_ascii=False))
@ -408,6 +422,7 @@ done
fetch_runs_json
python3 - "$RUNS_JSON_FILE" <<'PY'
import json
import os
import sys
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
@ -418,6 +433,7 @@ waiting = []
lineage_failures = []
production_failures = []
minimum_train_samples = int(__import__("os").environ.get("POC_OPERATOR_WORKFLOW_MIN_TRAIN_SAMPLE_COUNT", "128"))
require_audit_resolution = os.environ.get("POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION", "false").lower() in {"1", "true", "yes", "on"}
expected_edges_by_suffix = {
"外部存储与审计链路": [
@ -529,7 +545,7 @@ for run in runs:
"artifact": artifact.get("name"),
"error": "linear VLA policy is not accepted as production VLA training evidence",
})
if status in {"completed", "waiting_for_audit"}:
if status == "completed" or (status == "waiting_for_audit" and not require_audit_resolution):
covered.add(operator_id)
elif status == "failed":
failed.append({
@ -539,8 +555,8 @@ for run in runs:
"label": node.get("nodeLabel"),
"error": node.get("error"),
})
if run.get("status") == "waiting_for_audit":
waiting.append(run.get("id"))
if run.get("status") == "waiting_for_audit":
waiting.append(run.get("id"))
summary = {
"runCount": len(runs),
@ -557,11 +573,14 @@ summary = {
print("operator workflow acceptance summary:", json.dumps(summary, ensure_ascii=False))
if failed:
sys.exit(1)
if waiting and require_audit_resolution:
raise SystemExit(f"human review runs are still waiting for audit resolution: {waiting}")
if lineage_failures:
raise SystemExit("workflow data-transfer lineage validation failed")
if production_failures:
raise SystemExit("production-grade artifact validation failed")
non_terminal = [run.get("id") for run in runs if run.get("status") not in {"completed", "waiting_for_audit"}]
allowed_terminal = {"completed"} if require_audit_resolution else {"completed", "waiting_for_audit"}
non_terminal = [run.get("id") for run in runs if run.get("status") not in allowed_terminal]
if non_terminal:
raise SystemExit(f"non-terminal or unsuccessful runs: {non_terminal}")
PY

View File

@ -14,7 +14,7 @@ K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
API_TOKEN_SECRET_NAME="${POC_ACCEPTANCE_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
API_TOKEN_SECRET_KEY="${POC_ACCEPTANCE_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-strict-acceptance-$(date +%Y%m%d%H%M%S)}"
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-strict-acceptance-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)}"
POC_ACCEPTANCE_REQUIRED_EXTERNALS="${POC_ACCEPTANCE_REQUIRED_EXTERNALS:-bos,mongo}"
POC_ACCEPTANCE_IMPORT_RESULT="${POC_ACCEPTANCE_IMPORT_RESULT:-true}"
POC_ACCEPTANCE_IMPORT_API_BASE_URL="${POC_ACCEPTANCE_IMPORT_API_BASE_URL:-${API_BASE_URL:-${BASE_URL:-}}}"
@ -29,6 +29,7 @@ export POC_ACCEPTANCE_REQUIRED_EXTERNALS
export POC_ACCEPTANCE_IMPORT_RESULT
export POC_ACCEPTANCE_IMPORT_API_BASE_URL
export API_PUBLIC_SERVICE_NAME
export POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION=true
required_external_enabled() {
local target="$1"

View File

@ -14,6 +14,8 @@ export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
DEPLOY_ENV="${DEPLOY_ENV:-}"
# shellcheck source=scripts/lib/build-metadata.sh
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
MONGO_DB="${MONGO_DB:-droboticflow}"
@ -31,9 +33,10 @@ API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
API_REPLICAS="${API_REPLICAS:-2}"
WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}"
OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}"
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-$DROBOTICFLOW_BUILD_TIME}"
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-}"
droboticflow_resolve_build_metadata
WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-false}"
WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"

View File

@ -67,7 +67,7 @@ elif url.endswith("/api/acceptance/poc-operator-workflows"):
if not has_csrf:
raise SystemExit("missing csrf header")
pathlib.Path({str(captured)!r}).write_text(body, encoding="utf-8")
print(json.dumps({{"operatorCount": 2, "summary": {{"contractVersion": "poc-operator-workflows/v3", "operatorCount": 2, "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}}, "runs": [{{"batch": "数据接入与格式转换链路", "runId": "run-1", "operatorIds": ["operator-builtin-real-source", "operator-builtin-upload"], "imageEvidence": [
print(json.dumps({{"operatorCount": 2, "summary": {{"contractVersion": "poc-operator-workflows/v4", "operatorCount": 2, "catalogHash": "a"*64, "catalogCount": 2, "trainingMode": "full", "fullTrainingRequired": True, "missingOperatorIds": [], "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}}, "runs": [{{"batch": "数据接入与格式转换链路", "runId": "run-1", "operatorIds": ["operator-builtin-real-source", "operator-builtin-upload"], "imageEvidence": [
{{"operatorId": "operator-builtin-real-source", "image": "registry.example/droboticflow-operator-real-source:builtin", "imageId": "registry.example/droboticflow-operator-real-source@sha256:" + "a"*64}},
{{"operatorId": "operator-builtin-upload", "image": "registry.example/droboticflow-operator-upload:builtin", "imageId": "registry.example/droboticflow-operator-upload@sha256:" + "b"*64}}
]}}]}}))
@ -151,7 +151,7 @@ if url.endswith("/api/health") or url.endswith("/worker-api/health"):
elif url.endswith("/api/auth/login"):
print(json.dumps({"user": {"email": "poc-acceptance-admin@droboticflow.local"}}))
elif url.endswith("/api/acceptance/poc-operator-workflows"):
print(json.dumps({"operatorCount": 1, "summary": {"contractVersion": "poc-operator-workflows/v3", "operatorCount": 1, "independentImageCoverage": 1, "independentImageRequired": 1, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "人工审核与抽检链路", "runId": "run-1", "operatorIds": ["operator-builtin-human-review"], "imageEvidence": [
print(json.dumps({"operatorCount": 1, "summary": {"contractVersion": "poc-operator-workflows/v4", "operatorCount": 1, "catalogHash": "a"*64, "catalogCount": 1, "trainingMode": "full", "fullTrainingRequired": True, "missingOperatorIds": [], "independentImageCoverage": 1, "independentImageRequired": 1, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "人工审核与抽检链路", "runId": "run-1", "operatorIds": ["operator-builtin-human-review"], "imageEvidence": [
{"operatorId": "operator-builtin-human-review", "image": "registry.example/droboticflow-operator-human-review:builtin", "imageId": "registry.example/droboticflow-operator-human-review@sha256:" + "c"*64}
]}]}))
elif url.endswith("/worker-api/process-once"):
@ -231,7 +231,7 @@ if url.endswith("/api/health") or url.endswith("/worker-api/health"):
elif url.endswith("/api/auth/login"):
print(json.dumps({"user": {"email": "poc-acceptance-admin@droboticflow.local"}}))
elif url.endswith("/api/acceptance/poc-operator-workflows"):
print(json.dumps({"operatorCount": 2, "summary": {"contractVersion": "poc-operator-workflows/v3", "operatorCount": 2, "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "外部存储与审计链路", "runId": "run-1", "operatorIds": ["operator-builtin-shell-command", "operator-builtin-mongodb-write"], "imageEvidence": [
print(json.dumps({"operatorCount": 2, "summary": {"contractVersion": "poc-operator-workflows/v4", "operatorCount": 2, "catalogHash": "a"*64, "catalogCount": 2, "trainingMode": "full", "fullTrainingRequired": True, "missingOperatorIds": [], "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "外部存储与审计链路", "runId": "run-1", "operatorIds": ["operator-builtin-shell-command", "operator-builtin-mongodb-write"], "imageEvidence": [
{"operatorId": "operator-builtin-shell-command", "image": "registry.example/droboticflow-operator-shell-command:builtin", "imageId": "registry.example/droboticflow-operator-shell-command@sha256:" + "d"*64},
{"operatorId": "operator-builtin-mongodb-write", "image": "registry.example/droboticflow-operator-mongodb-write:builtin", "imageId": "registry.example/droboticflow-operator-mongodb-write@sha256:" + "e"*64}
]}]}))