forked from longtao.wu/droboticflow
🚦 converge worker runs and harden POC acceptance
This commit is contained in:
parent
c167ce095f
commit
6fe111021b
@ -43,15 +43,20 @@ type pocOperatorWorkflowImageEvidence struct {
|
||||
}
|
||||
|
||||
type pocOperatorWorkflowAcceptanceSummary struct {
|
||||
ContractVersion string `json:"contractVersion"`
|
||||
OperatorCount int `json:"operatorCount"`
|
||||
IndependentImageCoverage int `json:"independentImageCoverage"`
|
||||
IndependentImageRequired int `json:"independentImageRequired"`
|
||||
CommandWrapperOnlyOperatorCount int `json:"commandWrapperOnlyOperatorCount"`
|
||||
GeneratedFixtureCount int `json:"generatedFixtureCount"`
|
||||
MockFixtureCount int `json:"mockFixtureCount"`
|
||||
GPUOperatorCount int `json:"gpuOperatorCount"`
|
||||
StrictReady bool `json:"strictReady"`
|
||||
ContractVersion string `json:"contractVersion"`
|
||||
OperatorCount int `json:"operatorCount"`
|
||||
CatalogHash string `json:"catalogHash"`
|
||||
CatalogCount int `json:"catalogCount"`
|
||||
IndependentImageCoverage int `json:"independentImageCoverage"`
|
||||
IndependentImageRequired int `json:"independentImageRequired"`
|
||||
CommandWrapperOnlyOperatorCount int `json:"commandWrapperOnlyOperatorCount"`
|
||||
GeneratedFixtureCount int `json:"generatedFixtureCount"`
|
||||
MockFixtureCount int `json:"mockFixtureCount"`
|
||||
GPUOperatorCount int `json:"gpuOperatorCount"`
|
||||
StrictReady bool `json:"strictReady"`
|
||||
TrainingMode string `json:"trainingMode"`
|
||||
FullTrainingRequired bool `json:"fullTrainingRequired"`
|
||||
MissingOperatorIDs []string `json:"missingOperatorIds"`
|
||||
}
|
||||
|
||||
type pocOperatorWorkflowAcceptanceResponse struct {
|
||||
@ -724,9 +729,12 @@ func buildPocOperatorWorkflowImageEvidence(operator domain.OperatorRecord) pocOp
|
||||
|
||||
func buildPocOperatorWorkflowAcceptanceSummary(operators []domain.OperatorRecord) pocOperatorWorkflowAcceptanceSummary {
|
||||
summary := pocOperatorWorkflowAcceptanceSummary{
|
||||
ContractVersion: "poc-operator-workflows/v3",
|
||||
ContractVersion: "poc-operator-workflows/v4",
|
||||
OperatorCount: len(operators),
|
||||
IndependentImageRequired: len(operators),
|
||||
TrainingMode: "full",
|
||||
FullTrainingRequired: true,
|
||||
MissingOperatorIDs: []string{},
|
||||
}
|
||||
for _, operator := range operators {
|
||||
if pocOperatorHasIndependentImage(operator) {
|
||||
@ -1135,6 +1143,17 @@ func (server *server) handlePocOperatorWorkflowAcceptance(writer http.ResponseWr
|
||||
Runs: []pocOperatorWorkflowAcceptanceRun{},
|
||||
Batches: map[string][]string{},
|
||||
}
|
||||
if catalogHash, catalogCount, catalogErr := builtinOperatorCatalogFingerprint(); catalogErr == nil {
|
||||
response.Summary.CatalogHash = catalogHash
|
||||
response.Summary.CatalogCount = catalogCount
|
||||
if catalogCount != len(operators) {
|
||||
response.Summary.MissingOperatorIDs = []string{"catalog_count_mismatch"}
|
||||
response.Summary.StrictReady = false
|
||||
}
|
||||
} else {
|
||||
renderError(writer, http.StatusInternalServerError, catalogErr)
|
||||
return
|
||||
}
|
||||
for _, batchName := range batchNames {
|
||||
batchOperators := grouped[batchName]
|
||||
if len(batchOperators) == 0 {
|
||||
|
||||
@ -5800,7 +5800,7 @@ func runStatusIsValid(status domain.RunStatus) bool {
|
||||
|
||||
func runSummaryPaginationRequested(request *http.Request) bool {
|
||||
query := request.URL.Query()
|
||||
for _, key := range []string{"limit", "cursor", "q", "projectId", "workflowId"} {
|
||||
for _, key := range []string{"limit", "cursor", "q", "projectId", "workflowId", "includeAggregates"} {
|
||||
if _, ok := query[key]; ok {
|
||||
return true
|
||||
}
|
||||
@ -5841,13 +5841,23 @@ func encodeRunSummaryCursor(cursor *domain.RunSummaryCursor) string {
|
||||
func parseRunSummaryQuery(request *http.Request) (domain.RunSummaryQuery, []validationIssue) {
|
||||
values := request.URL.Query()
|
||||
query := domain.RunSummaryQuery{
|
||||
Limit: 50,
|
||||
Status: domain.RunStatus(strings.TrimSpace(values.Get("status"))),
|
||||
Search: strings.TrimSpace(values.Get("q")),
|
||||
ProjectID: strings.TrimSpace(values.Get("projectId")),
|
||||
WorkflowID: strings.TrimSpace(values.Get("workflowId")),
|
||||
Limit: 50,
|
||||
IncludeAggregates: true,
|
||||
Status: domain.RunStatus(strings.TrimSpace(values.Get("status"))),
|
||||
Search: strings.TrimSpace(values.Get("q")),
|
||||
ProjectID: strings.TrimSpace(values.Get("projectId")),
|
||||
WorkflowID: strings.TrimSpace(values.Get("workflowId")),
|
||||
}
|
||||
issues := make([]validationIssue, 0, 4)
|
||||
if raw, ok := values["includeAggregates"]; ok && len(raw) > 0 {
|
||||
query.AggregatesExplicit = true
|
||||
value, err := strconv.ParseBool(strings.TrimSpace(raw[0]))
|
||||
if err != nil {
|
||||
issues = append(issues, validationIssue{Field: "includeAggregates", Message: "includeAggregates must be true or false"})
|
||||
} else {
|
||||
query.IncludeAggregates = value
|
||||
}
|
||||
}
|
||||
issues := make([]validationIssue, 0, 3)
|
||||
if rawLimit, ok := values["limit"]; ok {
|
||||
value := ""
|
||||
if len(rawLimit) > 0 {
|
||||
@ -5882,18 +5892,23 @@ func (server *server) handleRunsList(writer http.ResponseWriter, request *http.R
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
startedAt := time.Now()
|
||||
page, err := server.listRunSummaryPageForRequest(request.Context(), request, query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{
|
||||
"items": page.Items,
|
||||
"total": page.Total,
|
||||
"statusCounts": page.StatusCounts,
|
||||
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
|
||||
"hasMore": page.HasMore,
|
||||
})
|
||||
writer.Header().Set("Server-Timing", fmt.Sprintf("runs;dur=%.1f", float64(time.Since(startedAt).Microseconds())/1000))
|
||||
response := map[string]any{
|
||||
"items": page.Items,
|
||||
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
|
||||
"hasMore": page.HasMore,
|
||||
}
|
||||
if query.IncludeAggregates {
|
||||
response["total"] = page.Total
|
||||
response["statusCounts"] = page.StatusCounts
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, response)
|
||||
return
|
||||
}
|
||||
items, err := server.listRunSummariesForRequest(request.Context(), request, request.URL.Query().Get("status"))
|
||||
|
||||
@ -1,6 +1,12 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type UserRole string
|
||||
|
||||
@ -374,17 +380,82 @@ type ArtifactMetadata struct {
|
||||
}
|
||||
|
||||
type ExecutionArtifact struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
RunID string `json:"runId" bson:"runId"`
|
||||
NodeID string `json:"nodeId" bson:"nodeId"`
|
||||
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
|
||||
PortName string `json:"portName,omitempty" bson:"portName,omitempty"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Type PortType `json:"type" bson:"type"`
|
||||
Size string `json:"size" bson:"size"`
|
||||
Path string `json:"path" bson:"path"`
|
||||
Metadata *ArtifactMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"`
|
||||
Data map[string]any `json:"data,omitempty" bson:"data,omitempty"`
|
||||
ID string `json:"id" bson:"id"`
|
||||
RunID string `json:"runId" bson:"runId"`
|
||||
NodeID string `json:"nodeId" bson:"nodeId"`
|
||||
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
|
||||
PortName string `json:"portName,omitempty" bson:"portName,omitempty"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Type PortType `json:"type" bson:"type"`
|
||||
Size string `json:"size" bson:"size"`
|
||||
SizeBytes int64 `json:"sizeBytes,omitempty" bson:"sizeBytes,omitempty"`
|
||||
Path string `json:"path" bson:"path"`
|
||||
Metadata *ArtifactMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"`
|
||||
Data map[string]any `json:"data,omitempty" bson:"data,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON keeps the public artifact contract compatible with older
|
||||
// runners that emitted size as a string while accepting newer runners that
|
||||
// emit the exact byte count as a JSON number.
|
||||
func (artifact *ExecutionArtifact) UnmarshalJSON(data []byte) error {
|
||||
type plain ExecutionArtifact
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &fields); err != nil {
|
||||
return err
|
||||
}
|
||||
sizeRaw := fields["size"]
|
||||
delete(fields, "size")
|
||||
withoutSize, err := json.Marshal(fields)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var parsed plain
|
||||
if err := json.Unmarshal(withoutSize, &parsed); err != nil {
|
||||
return err
|
||||
}
|
||||
*artifact = ExecutionArtifact(parsed)
|
||||
if len(sizeRaw) == 0 || strings.TrimSpace(string(sizeRaw)) == "null" {
|
||||
return nil
|
||||
}
|
||||
var sizeText string
|
||||
if sizeRaw[0] == '"' {
|
||||
if err := json.Unmarshal(sizeRaw, &sizeText); err != nil {
|
||||
return fmt.Errorf("decode artifact size: %w", err)
|
||||
}
|
||||
artifact.Size = sizeText
|
||||
if bytes, err := strconv.ParseInt(strings.TrimSpace(sizeText), 10, 64); err == nil && bytes >= 0 {
|
||||
artifact.SizeBytes = bytes
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var number json.Number
|
||||
decoder := json.NewDecoder(strings.NewReader(string(sizeRaw)))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&number); err != nil {
|
||||
return fmt.Errorf("decode artifact size: %w", err)
|
||||
}
|
||||
bytes, err := strconv.ParseInt(number.String(), 10, 64)
|
||||
if err != nil || bytes < 0 {
|
||||
return fmt.Errorf("artifact size must be a non-negative integer or string")
|
||||
}
|
||||
artifact.SizeBytes = bytes
|
||||
artifact.Size = formatArtifactSize(bytes)
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatArtifactSize(bytes int64) string {
|
||||
if bytes < 1024 {
|
||||
return strconv.FormatInt(bytes, 10) + " B"
|
||||
}
|
||||
value := float64(bytes)
|
||||
units := []string{"KB", "MB", "GB", "TB"}
|
||||
for _, unit := range units {
|
||||
value /= 1024
|
||||
if value < 1024 {
|
||||
return strconv.FormatFloat(value, 'f', 1, 64) + " " + unit
|
||||
}
|
||||
}
|
||||
return strconv.FormatFloat(value, 'f', 1, 64) + " PB"
|
||||
}
|
||||
|
||||
type DataRefRecord struct {
|
||||
@ -617,12 +688,14 @@ type RunSummaryCursor struct {
|
||||
}
|
||||
|
||||
type RunSummaryQuery struct {
|
||||
Limit int
|
||||
Cursor *RunSummaryCursor
|
||||
Status RunStatus
|
||||
Search string
|
||||
ProjectID string
|
||||
WorkflowID string
|
||||
Limit int
|
||||
Cursor *RunSummaryCursor
|
||||
IncludeAggregates bool
|
||||
AggregatesExplicit bool
|
||||
Status RunStatus
|
||||
Search string
|
||||
ProjectID string
|
||||
WorkflowID string
|
||||
}
|
||||
|
||||
type RunSummaryPage struct {
|
||||
|
||||
@ -974,6 +974,10 @@ func (store *MemoryStore) listRunSummariesPageLocked(organizationID string, quer
|
||||
last := pageItems[len(pageItems)-1]
|
||||
nextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
||||
}
|
||||
if query.AggregatesExplicit && !query.IncludeAggregates {
|
||||
statusCounts = nil
|
||||
total = 0
|
||||
}
|
||||
return domain.RunSummaryPage{
|
||||
Items: pageItems,
|
||||
Total: total,
|
||||
|
||||
@ -907,7 +907,7 @@ func (store *MongoStore) listRunSummariesPage(ctx context.Context, organizationI
|
||||
var items []domain.RunSummary
|
||||
var aggregates runSummaryAggregates
|
||||
queryErrors := make(chan error, 2)
|
||||
waitGroup.Add(2)
|
||||
waitGroup.Add(1)
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
foundItems, err := store.findRunSummaryPageItems(queryCtx, itemFilter, limit)
|
||||
@ -918,19 +918,23 @@ func (store *MongoStore) listRunSummariesPage(ctx context.Context, organizationI
|
||||
}
|
||||
items = foundItems
|
||||
}()
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
key := newRunSummaryAggregateCacheKey(organizationID, query)
|
||||
foundAggregates, err := store.runSummaryAggregates.getOrLoad(queryCtx, key, func(loadCtx context.Context) (runSummaryAggregates, error) {
|
||||
return store.loadRunSummaryAggregates(loadCtx, baseFilter)
|
||||
})
|
||||
if err != nil {
|
||||
queryErrors <- err
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
aggregates = foundAggregates
|
||||
}()
|
||||
includeAggregates := query.IncludeAggregates || !query.AggregatesExplicit
|
||||
if includeAggregates {
|
||||
waitGroup.Add(1)
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
key := newRunSummaryAggregateCacheKey(organizationID, query)
|
||||
foundAggregates, err := store.runSummaryAggregates.getOrLoad(queryCtx, key, func(loadCtx context.Context) (runSummaryAggregates, error) {
|
||||
return store.loadRunSummaryAggregates(loadCtx, baseFilter)
|
||||
})
|
||||
if err != nil {
|
||||
queryErrors <- err
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
aggregates = foundAggregates
|
||||
}()
|
||||
}
|
||||
waitGroup.Wait()
|
||||
select {
|
||||
case err := <-queryErrors:
|
||||
|
||||
@ -70,6 +70,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
store.runs: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("organizationId_status_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
|
||||
index("organizationId_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
|
||||
index("createdAt_id", bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
|
||||
index("status_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
@ -78,7 +79,9 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
index("status_lease_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "executionLeaseExpiresAt", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("workflowId_status", bson.D{{Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}),
|
||||
index("organizationId_workflowId_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
|
||||
index("projectId_createdAt", bson.D{{Key: "projectId", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("organizationId_projectId_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}),
|
||||
},
|
||||
store.sourceStates: {
|
||||
uniqueIndex("workflow_node_fingerprint_unique", bson.D{{Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}, {Key: "sourceFingerprint", Value: 1}}),
|
||||
|
||||
@ -335,6 +335,20 @@ func TestParseExecutionOutputReadsResultFilePayload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExecutionOutputAcceptsNumericArtifactSize(t *testing.T) {
|
||||
parsed, err := parseExecutionOutput(`DROBOTICFLOW_RESULT={"artifacts":[{"id":"artifact-size","name":"training_report","type":"file","size":1852,"path":"/workspace/report.json"}]}`, "训练")
|
||||
if err != nil {
|
||||
t.Fatalf("parseExecutionOutput returned error: %v", err)
|
||||
}
|
||||
if len(parsed.Artifacts) != 1 {
|
||||
t.Fatalf("expected one artifact, got %#v", parsed.Artifacts)
|
||||
}
|
||||
artifact := parsed.Artifacts[0]
|
||||
if artifact.SizeBytes != 1852 || artifact.Size != "1.8 KB" {
|
||||
t.Fatalf("unexpected numeric artifact size: %#v", artifact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStageManifestDataRefs(t *testing.T) {
|
||||
stageManifestPath := filepath.Join(t.TempDir(), "stage-manifest.json")
|
||||
payload := map[string]any{
|
||||
|
||||
@ -106,6 +106,8 @@ func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http.
|
||||
response["effectivePollIntervalMs"] = readiness.EffectivePollIntervalMS
|
||||
response["stuckRunCount"] = readiness.StuckRunCount
|
||||
response["orphanRuntimeCount"] = readiness.OrphanRuntimeCount
|
||||
response["orphanRuntimeCleanedCount"] = readiness.OrphanRuntimeCleanedCount
|
||||
response["orphanRuntimeCleanupErrorCount"] = readiness.OrphanRuntimeCleanupErrorCount
|
||||
if organizationID != "" {
|
||||
response["organizationId"] = organizationID
|
||||
response["global"] = map[string]int{
|
||||
|
||||
@ -33,6 +33,8 @@ type MetricsSnapshot struct {
|
||||
CycleDurationMS uint64 `json:"cycleDurationMs"`
|
||||
StuckRunCount uint64 `json:"stuckRunCount"`
|
||||
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
|
||||
OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"`
|
||||
OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"`
|
||||
}
|
||||
|
||||
type WorkerMetrics struct {
|
||||
@ -54,6 +56,8 @@ type WorkerMetrics struct {
|
||||
cycleDurationMS atomic.Uint64
|
||||
stuckRunCount atomic.Uint64
|
||||
orphanRuntimeCount atomic.Uint64
|
||||
orphanRuntimeCleanedCount atomic.Uint64
|
||||
orphanRuntimeCleanupErrorCount atomic.Uint64
|
||||
}
|
||||
|
||||
func (metrics *WorkerMetrics) RecordProcessAvailable(durationMS uint64, err error) {
|
||||
@ -103,6 +107,11 @@ func (metrics *WorkerMetrics) RecordOrphanRuntimes(count int) {
|
||||
metrics.orphanRuntimeCount.Store(uint64(max(count, 0)))
|
||||
}
|
||||
|
||||
func (metrics *WorkerMetrics) RecordOrphanCleanup(cleaned int, errors int) {
|
||||
metrics.orphanRuntimeCleanedCount.Add(uint64(max(cleaned, 0)))
|
||||
metrics.orphanRuntimeCleanupErrorCount.Add(uint64(max(errors, 0)))
|
||||
}
|
||||
|
||||
func metricTimestamp(value int64) string {
|
||||
if value <= 0 {
|
||||
return ""
|
||||
@ -130,6 +139,8 @@ func (metrics *WorkerMetrics) Snapshot() MetricsSnapshot {
|
||||
CycleDurationMS: metrics.cycleDurationMS.Load(),
|
||||
StuckRunCount: metrics.stuckRunCount.Load(),
|
||||
OrphanRuntimeCount: metrics.orphanRuntimeCount.Load(),
|
||||
OrphanRuntimeCleanedCount: metrics.orphanRuntimeCleanedCount.Load(),
|
||||
OrphanRuntimeCleanupErrorCount: metrics.orphanRuntimeCleanupErrorCount.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -51,23 +51,25 @@ type BuildInfo struct {
|
||||
}
|
||||
|
||||
type ReadinessSnapshot struct {
|
||||
Ready bool `json:"ready"`
|
||||
Status string `json:"status"`
|
||||
Service string `json:"service"`
|
||||
BuildCommit string `json:"buildCommit,omitempty"`
|
||||
BuildTime string `json:"buildTime,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
RunnerImage string `json:"runnerImage,omitempty"`
|
||||
RunnerBuildCommit string `json:"runnerBuildCommit,omitempty"`
|
||||
RunnerBuildTime string `json:"runnerBuildTime,omitempty"`
|
||||
LastCycleAt string `json:"lastCycleAt,omitempty"`
|
||||
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
|
||||
CycleDurationMS uint64 `json:"cycleDurationMs"`
|
||||
ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"`
|
||||
EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"`
|
||||
StuckRunCount uint64 `json:"stuckRunCount"`
|
||||
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Ready bool `json:"ready"`
|
||||
Status string `json:"status"`
|
||||
Service string `json:"service"`
|
||||
BuildCommit string `json:"buildCommit,omitempty"`
|
||||
BuildTime string `json:"buildTime,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
RunnerImage string `json:"runnerImage,omitempty"`
|
||||
RunnerBuildCommit string `json:"runnerBuildCommit,omitempty"`
|
||||
RunnerBuildTime string `json:"runnerBuildTime,omitempty"`
|
||||
LastCycleAt string `json:"lastCycleAt,omitempty"`
|
||||
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
|
||||
CycleDurationMS uint64 `json:"cycleDurationMs"`
|
||||
ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"`
|
||||
EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"`
|
||||
StuckRunCount uint64 `json:"stuckRunCount"`
|
||||
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
|
||||
OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"`
|
||||
OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
type Runtime struct {
|
||||
@ -456,22 +458,24 @@ func (runtime *Runtime) statusSnapshot() ReadinessSnapshot {
|
||||
configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load())
|
||||
effective := normalizedPollIntervalMS(runtime.effectivePollMS.Load())
|
||||
snapshot := ReadinessSnapshot{
|
||||
Ready: false,
|
||||
Status: "not_ready",
|
||||
Service: "droboticflow-worker",
|
||||
BuildCommit: build.BuildCommit,
|
||||
BuildTime: build.BuildTime,
|
||||
Image: build.Image,
|
||||
RunnerImage: firstNonEmptyRuntimeValue(runnerBuild.Image, runtime.operatorRunnerImage),
|
||||
RunnerBuildCommit: runnerBuild.BuildCommit,
|
||||
RunnerBuildTime: runnerBuild.BuildTime,
|
||||
LastCycleAt: metrics.LastCycleAt,
|
||||
LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt,
|
||||
CycleDurationMS: metrics.CycleDurationMS,
|
||||
ConfiguredPollIntervalMS: configured,
|
||||
EffectivePollIntervalMS: effective,
|
||||
StuckRunCount: metrics.StuckRunCount,
|
||||
OrphanRuntimeCount: metrics.OrphanRuntimeCount,
|
||||
Ready: false,
|
||||
Status: "not_ready",
|
||||
Service: "droboticflow-worker",
|
||||
BuildCommit: build.BuildCommit,
|
||||
BuildTime: build.BuildTime,
|
||||
Image: build.Image,
|
||||
RunnerImage: firstNonEmptyRuntimeValue(runnerBuild.Image, runtime.operatorRunnerImage),
|
||||
RunnerBuildCommit: runnerBuild.BuildCommit,
|
||||
RunnerBuildTime: runnerBuild.BuildTime,
|
||||
LastCycleAt: metrics.LastCycleAt,
|
||||
LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt,
|
||||
CycleDurationMS: metrics.CycleDurationMS,
|
||||
ConfiguredPollIntervalMS: configured,
|
||||
EffectivePollIntervalMS: effective,
|
||||
StuckRunCount: metrics.StuckRunCount,
|
||||
OrphanRuntimeCount: metrics.OrphanRuntimeCount,
|
||||
OrphanRuntimeCleanedCount: metrics.OrphanRuntimeCleanedCount,
|
||||
OrphanRuntimeCleanupErrorCount: metrics.OrphanRuntimeCleanupErrorCount,
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
@ -1046,7 +1050,17 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
|
||||
fmt.Printf("[worker] orphan runtime reconciliation failed: %v\n", err)
|
||||
return
|
||||
}
|
||||
runtime.metrics.RecordOrphanRuntimes(len(orphans))
|
||||
cleaned, cleanupErrors := 0, 0
|
||||
for _, orphan := range orphans {
|
||||
if orphan.Cleaned {
|
||||
cleaned++
|
||||
}
|
||||
if strings.TrimSpace(orphan.CleanupError) != "" {
|
||||
cleanupErrors++
|
||||
}
|
||||
}
|
||||
runtime.metrics.RecordOrphanRuntimes(len(orphans) - cleaned)
|
||||
runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors)
|
||||
}
|
||||
|
||||
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
||||
@ -1250,10 +1264,25 @@ func (runtime *Runtime) failReconciledRun(ctx context.Context, run *domain.RunRe
|
||||
return err
|
||||
}
|
||||
workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID)
|
||||
if err != nil || workflow == nil {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO())
|
||||
if workflow != nil {
|
||||
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Persist the terminal state before touching Kubernetes. A failed runtime is
|
||||
// no longer useful to the scheduler, and cleanup is deliberately best-effort
|
||||
// so a transient API error cannot leave the Run active again.
|
||||
if runtime.launcher != nil {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if cleanupErr := runtime.launcher.TerminateRun(cleanupCtx, "", run.ID); cleanupErr != nil {
|
||||
fmt.Printf("[worker] cleanup failed runtime for terminal run %s: %v\n", run.ID, cleanupErr)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runtime *Runtime) recoverMissingJob(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
|
||||
|
||||
@ -12,6 +12,8 @@ fi
|
||||
export DEPLOY_ENV
|
||||
# shellcheck source=scripts/load-deploy-env.sh
|
||||
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
||||
# shellcheck source=scripts/lib/build-metadata.sh
|
||||
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
|
||||
|
||||
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||||
REGISTRY_HOST="${REGISTRY_HOST:-}"
|
||||
@ -43,10 +45,11 @@ POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}"
|
||||
POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}"
|
||||
|
||||
if [[ -z "${IMAGE_TAG:-}" ]]; then
|
||||
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)"
|
||||
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)"
|
||||
fi
|
||||
POC_KANIKO_SOURCE_COMMIT="${POC_KANIKO_SOURCE_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
|
||||
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-}"
|
||||
POC_KANIKO_BUILD_TIME="$(droboticflow_normalize_build_time "$POC_KANIKO_BUILD_TIME")"
|
||||
|
||||
# shellcheck source=scripts/lib/image-config.sh
|
||||
source "$ROOT_DIR/scripts/lib/image-config.sh"
|
||||
|
||||
@ -15,6 +15,12 @@ export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
||||
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
||||
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
||||
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
||||
export WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-true}"
|
||||
export WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
|
||||
export WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"
|
||||
export WORKER_PENDING_FAILURE_SECONDS="${WORKER_PENDING_FAILURE_SECONDS:-600}"
|
||||
export WORKER_MISSING_JOB_MAX_RECOVERIES="${WORKER_MISSING_JOB_MAX_RECOVERIES:-3}"
|
||||
export HOLOBRAIN_TRAINING_MODE="${HOLOBRAIN_TRAINING_MODE:-full}"
|
||||
|
||||
export K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
||||
export WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
||||
|
||||
@ -13,6 +13,8 @@ export DEPLOY_ENV
|
||||
# shellcheck source=scripts/load-deploy-env.sh
|
||||
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
||||
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
||||
# shellcheck source=scripts/lib/build-metadata.sh
|
||||
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
|
||||
|
||||
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||||
|
||||
@ -39,13 +41,19 @@ API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
|
||||
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
|
||||
API_REPLICAS="${API_REPLICAS:-2}"
|
||||
WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
|
||||
WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-false}"
|
||||
WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
|
||||
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"
|
||||
WORKER_PENDING_FAILURE_SECONDS="${WORKER_PENDING_FAILURE_SECONDS:-600}"
|
||||
WORKER_MISSING_JOB_MAX_RECOVERIES="${WORKER_MISSING_JOB_MAX_RECOVERIES:-3}"
|
||||
# shellcheck source=scripts/lib/image-config.sh
|
||||
source "$ROOT_DIR/scripts/lib/image-config.sh"
|
||||
droboticflow_resolve_image_configuration
|
||||
DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
|
||||
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}"
|
||||
OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}"
|
||||
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-$DROBOTICFLOW_BUILD_TIME}"
|
||||
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-}"
|
||||
droboticflow_resolve_build_metadata
|
||||
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
|
||||
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
|
||||
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
|
||||
@ -189,6 +197,11 @@ env \
|
||||
API_SERVICE_TYPE="$API_SERVICE_TYPE" \
|
||||
API_REPLICAS="$API_REPLICAS" \
|
||||
WORKER_REPLICAS="$WORKER_REPLICAS" \
|
||||
WORKER_ORPHAN_CLEANUP_ENABLED="$WORKER_ORPHAN_CLEANUP_ENABLED" \
|
||||
WORKER_ORPHAN_MIN_AGE_SECONDS="$WORKER_ORPHAN_MIN_AGE_SECONDS" \
|
||||
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="$WORKER_ORPHAN_CHECK_INTERVAL_SECONDS" \
|
||||
WORKER_PENDING_FAILURE_SECONDS="$WORKER_PENDING_FAILURE_SECONDS" \
|
||||
WORKER_MISSING_JOB_MAX_RECOVERIES="$WORKER_MISSING_JOB_MAX_RECOVERIES" \
|
||||
AI_API_BASE_URL="$AI_API_BASE_URL" \
|
||||
AI_MODEL="$AI_MODEL" \
|
||||
AI_API_KEY="$AI_API_KEY" \
|
||||
|
||||
@ -135,7 +135,7 @@ project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Cont
|
||||
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
|
||||
printf 'created project: %s\n' "$project_id"
|
||||
|
||||
acceptance_suffix="${HOLOBRAIN_ACCEPTANCE_SUFFIX:-$(date +%Y%m%d%H%M%S)-$$}"
|
||||
acceptance_suffix="${HOLOBRAIN_ACCEPTANCE_SUFFIX:-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)-$$}"
|
||||
workflow_payload="$(
|
||||
ROOT_DIR="$ROOT_DIR" HOLOBRAIN_ACCEPTANCE_SUFFIX="$acceptance_suffix" python3 - <<'PY'
|
||||
import json
|
||||
|
||||
34
scripts/lib/build-metadata.sh
Normal file
34
scripts/lib/build-metadata.sh
Normal file
@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Canonical build metadata is always RFC3339 UTC with a trailing Z. Human
|
||||
# readable tags may use Asia/Shanghai, but must never be used as buildTime.
|
||||
|
||||
droboticflow_normalize_build_time() {
|
||||
local raw="${1:-}"
|
||||
if [[ -z "${raw//[[:space:]]/}" ]]; then
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ
|
||||
return
|
||||
fi
|
||||
BUILD_TIME_INPUT="$raw" python3 - <<'PY'
|
||||
from datetime import datetime, timezone, timedelta
|
||||
import os
|
||||
|
||||
raw = os.environ.get("BUILD_TIME_INPUT", "").strip()
|
||||
try:
|
||||
value = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise SystemExit(f"invalid build time {raw!r}: expected RFC3339") from exc
|
||||
if value.tzinfo is None:
|
||||
raise SystemExit("build time must include a timezone offset or Z")
|
||||
value = value.astimezone(timezone.utc)
|
||||
if value > datetime.now(timezone.utc) + timedelta(minutes=5):
|
||||
raise SystemExit("build time is more than five minutes in the future")
|
||||
print(value.isoformat(timespec="seconds").replace("+00:00", "Z"))
|
||||
PY
|
||||
}
|
||||
|
||||
droboticflow_resolve_build_metadata() {
|
||||
DROBOTICFLOW_BUILD_TIME="$(droboticflow_normalize_build_time "${DROBOTICFLOW_BUILD_TIME:-}")"
|
||||
OPERATOR_RUNNER_BUILD_TIME="$(droboticflow_normalize_build_time "${OPERATOR_RUNNER_BUILD_TIME:-$DROBOTICFLOW_BUILD_TIME}")"
|
||||
export DROBOTICFLOW_BUILD_TIME OPERATOR_RUNNER_BUILD_TIME
|
||||
}
|
||||
@ -9,7 +9,7 @@ export DEPLOY_ENV
|
||||
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
||||
|
||||
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||||
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-operator-acceptance-$(date +%Y%m%d%H%M%S)}"
|
||||
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-operator-acceptance-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)}"
|
||||
POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT="${POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT:-1800s}"
|
||||
POC_ACCEPTANCE_REQUIRED_EXTERNALS="${POC_ACCEPTANCE_REQUIRED_EXTERNALS:-bos,mongo}"
|
||||
POC_OPERATOR_ACCEPTANCE_CLEANUP_JOB="${POC_OPERATOR_ACCEPTANCE_CLEANUP_JOB:-false}"
|
||||
|
||||
@ -17,8 +17,15 @@ API_TOKEN_SECRET_NAME="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticfl
|
||||
API_TOKEN_SECRET_KEY="${POC_OPERATOR_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
|
||||
POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_RUN_TIMEOUT_SECONDS:-5400}"
|
||||
POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH="${POC_OPERATOR_WORKFLOW_FIXTURE_MANIFEST_PATH:-/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json}"
|
||||
POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="${POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT:-96}"
|
||||
POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT="${POC_OPERATOR_WORKFLOW_MIN_OPERATOR_COUNT:-$(python3 - "$ROOT_DIR/internal/api/builtin_operators_catalog.json" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
print(len(json.load(open(sys.argv[1], encoding="utf-8"))))
|
||||
PY
|
||||
)}"
|
||||
POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS="${POC_OPERATOR_WORKFLOW_PROCESS_ONCE_TIMEOUT_SECONDS:-15}"
|
||||
POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION="${POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION:-false}"
|
||||
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
|
||||
DROBOTICFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-youbixuanpoc}"
|
||||
|
||||
@ -328,10 +335,16 @@ required = int(summary.get("independentImageRequired") or operator_count)
|
||||
command_wrappers = int(summary.get("commandWrapperOnlyOperatorCount") or 0)
|
||||
generated_fixtures = int(summary.get("generatedFixtureCount") or 0)
|
||||
mock_fixtures = int(summary.get("mockFixtureCount") or 0)
|
||||
if operator_count < minimum_operator_count:
|
||||
raise SystemExit(f"operator workflow acceptance selected too few operators: {operator_count}/{minimum_operator_count}")
|
||||
if summary.get("contractVersion") != "poc-operator-workflows/v3":
|
||||
raise SystemExit(f"operator workflow contract version must be poc-operator-workflows/v3, got {summary.get('contractVersion')}")
|
||||
if operator_count != minimum_operator_count:
|
||||
raise SystemExit(f"operator workflow acceptance must cover the current catalog exactly: {operator_count}/{minimum_operator_count}")
|
||||
if summary.get("contractVersion") != "poc-operator-workflows/v4":
|
||||
raise SystemExit(f"operator workflow contract version must be poc-operator-workflows/v4, got {summary.get('contractVersion')}")
|
||||
if int(summary.get("catalogCount") or 0) != operator_count or not summary.get("catalogHash"):
|
||||
raise SystemExit("operator workflow acceptance catalog provenance is missing or inconsistent")
|
||||
if summary.get("trainingMode") != "full" or summary.get("fullTrainingRequired") is not True:
|
||||
raise SystemExit("operator workflow acceptance must require full training")
|
||||
if summary.get("missingOperatorIds"):
|
||||
raise SystemExit(f"operator workflow acceptance has missing operators: {summary.get('missingOperatorIds')}")
|
||||
if coverage != required:
|
||||
raise SystemExit(f"independent image coverage is incomplete: {coverage}/{required}")
|
||||
if command_wrappers != 0:
|
||||
@ -384,8 +397,9 @@ while (( SECONDS < deadline )); do
|
||||
fetch_runs_json
|
||||
fi
|
||||
summary="$(
|
||||
python3 - "$RUNS_JSON_FILE" <<'PY'
|
||||
python3 - "$RUNS_JSON_FILE" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
|
||||
print(json.dumps([{"id": run.get("id"), "status": run.get("status"), "nodes": len(run.get("nodeExecutions") or [])} for run in runs], ensure_ascii=False))
|
||||
@ -408,6 +422,7 @@ done
|
||||
fetch_runs_json
|
||||
python3 - "$RUNS_JSON_FILE" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
runs = json.loads(open(sys.argv[1], encoding="utf-8").read())
|
||||
@ -418,6 +433,7 @@ waiting = []
|
||||
lineage_failures = []
|
||||
production_failures = []
|
||||
minimum_train_samples = int(__import__("os").environ.get("POC_OPERATOR_WORKFLOW_MIN_TRAIN_SAMPLE_COUNT", "128"))
|
||||
require_audit_resolution = os.environ.get("POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION", "false").lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
expected_edges_by_suffix = {
|
||||
"外部存储与审计链路": [
|
||||
@ -529,7 +545,7 @@ for run in runs:
|
||||
"artifact": artifact.get("name"),
|
||||
"error": "linear VLA policy is not accepted as production VLA training evidence",
|
||||
})
|
||||
if status in {"completed", "waiting_for_audit"}:
|
||||
if status == "completed" or (status == "waiting_for_audit" and not require_audit_resolution):
|
||||
covered.add(operator_id)
|
||||
elif status == "failed":
|
||||
failed.append({
|
||||
@ -539,8 +555,8 @@ for run in runs:
|
||||
"label": node.get("nodeLabel"),
|
||||
"error": node.get("error"),
|
||||
})
|
||||
if run.get("status") == "waiting_for_audit":
|
||||
waiting.append(run.get("id"))
|
||||
if run.get("status") == "waiting_for_audit":
|
||||
waiting.append(run.get("id"))
|
||||
|
||||
summary = {
|
||||
"runCount": len(runs),
|
||||
@ -557,11 +573,14 @@ summary = {
|
||||
print("operator workflow acceptance summary:", json.dumps(summary, ensure_ascii=False))
|
||||
if failed:
|
||||
sys.exit(1)
|
||||
if waiting and require_audit_resolution:
|
||||
raise SystemExit(f"human review runs are still waiting for audit resolution: {waiting}")
|
||||
if lineage_failures:
|
||||
raise SystemExit("workflow data-transfer lineage validation failed")
|
||||
if production_failures:
|
||||
raise SystemExit("production-grade artifact validation failed")
|
||||
non_terminal = [run.get("id") for run in runs if run.get("status") not in {"completed", "waiting_for_audit"}]
|
||||
allowed_terminal = {"completed"} if require_audit_resolution else {"completed", "waiting_for_audit"}
|
||||
non_terminal = [run.get("id") for run in runs if run.get("status") not in allowed_terminal]
|
||||
if non_terminal:
|
||||
raise SystemExit(f"non-terminal or unsuccessful runs: {non_terminal}")
|
||||
PY
|
||||
|
||||
@ -14,7 +14,7 @@ K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||||
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
|
||||
API_TOKEN_SECRET_NAME="${POC_ACCEPTANCE_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
|
||||
API_TOKEN_SECRET_KEY="${POC_ACCEPTANCE_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
|
||||
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-strict-acceptance-$(date +%Y%m%d%H%M%S)}"
|
||||
POC_OPERATOR_ACCEPTANCE_JOB_NAME="${POC_OPERATOR_ACCEPTANCE_JOB_NAME:-droboticflow-strict-acceptance-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)}"
|
||||
POC_ACCEPTANCE_REQUIRED_EXTERNALS="${POC_ACCEPTANCE_REQUIRED_EXTERNALS:-bos,mongo}"
|
||||
POC_ACCEPTANCE_IMPORT_RESULT="${POC_ACCEPTANCE_IMPORT_RESULT:-true}"
|
||||
POC_ACCEPTANCE_IMPORT_API_BASE_URL="${POC_ACCEPTANCE_IMPORT_API_BASE_URL:-${API_BASE_URL:-${BASE_URL:-}}}"
|
||||
@ -29,6 +29,7 @@ export POC_ACCEPTANCE_REQUIRED_EXTERNALS
|
||||
export POC_ACCEPTANCE_IMPORT_RESULT
|
||||
export POC_ACCEPTANCE_IMPORT_API_BASE_URL
|
||||
export API_PUBLIC_SERVICE_NAME
|
||||
export POC_OPERATOR_WORKFLOW_REQUIRE_AUDIT_RESOLUTION=true
|
||||
|
||||
required_external_enabled() {
|
||||
local target="$1"
|
||||
|
||||
@ -14,6 +14,8 @@ export DEPLOY_ENV
|
||||
# shellcheck source=scripts/load-deploy-env.sh
|
||||
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
||||
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
||||
# shellcheck source=scripts/lib/build-metadata.sh
|
||||
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
|
||||
|
||||
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||||
MONGO_DB="${MONGO_DB:-droboticflow}"
|
||||
@ -31,9 +33,10 @@ API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
|
||||
API_REPLICAS="${API_REPLICAS:-2}"
|
||||
WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
|
||||
DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
|
||||
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}"
|
||||
OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}"
|
||||
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-$DROBOTICFLOW_BUILD_TIME}"
|
||||
OPERATOR_RUNNER_BUILD_TIME="${OPERATOR_RUNNER_BUILD_TIME:-}"
|
||||
droboticflow_resolve_build_metadata
|
||||
WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-false}"
|
||||
WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}"
|
||||
WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}"
|
||||
|
||||
@ -67,7 +67,7 @@ elif url.endswith("/api/acceptance/poc-operator-workflows"):
|
||||
if not has_csrf:
|
||||
raise SystemExit("missing csrf header")
|
||||
pathlib.Path({str(captured)!r}).write_text(body, encoding="utf-8")
|
||||
print(json.dumps({{"operatorCount": 2, "summary": {{"contractVersion": "poc-operator-workflows/v3", "operatorCount": 2, "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}}, "runs": [{{"batch": "数据接入与格式转换链路", "runId": "run-1", "operatorIds": ["operator-builtin-real-source", "operator-builtin-upload"], "imageEvidence": [
|
||||
print(json.dumps({{"operatorCount": 2, "summary": {{"contractVersion": "poc-operator-workflows/v4", "operatorCount": 2, "catalogHash": "a"*64, "catalogCount": 2, "trainingMode": "full", "fullTrainingRequired": True, "missingOperatorIds": [], "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}}, "runs": [{{"batch": "数据接入与格式转换链路", "runId": "run-1", "operatorIds": ["operator-builtin-real-source", "operator-builtin-upload"], "imageEvidence": [
|
||||
{{"operatorId": "operator-builtin-real-source", "image": "registry.example/droboticflow-operator-real-source:builtin", "imageId": "registry.example/droboticflow-operator-real-source@sha256:" + "a"*64}},
|
||||
{{"operatorId": "operator-builtin-upload", "image": "registry.example/droboticflow-operator-upload:builtin", "imageId": "registry.example/droboticflow-operator-upload@sha256:" + "b"*64}}
|
||||
]}}]}}))
|
||||
@ -151,7 +151,7 @@ if url.endswith("/api/health") or url.endswith("/worker-api/health"):
|
||||
elif url.endswith("/api/auth/login"):
|
||||
print(json.dumps({"user": {"email": "poc-acceptance-admin@droboticflow.local"}}))
|
||||
elif url.endswith("/api/acceptance/poc-operator-workflows"):
|
||||
print(json.dumps({"operatorCount": 1, "summary": {"contractVersion": "poc-operator-workflows/v3", "operatorCount": 1, "independentImageCoverage": 1, "independentImageRequired": 1, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "人工审核与抽检链路", "runId": "run-1", "operatorIds": ["operator-builtin-human-review"], "imageEvidence": [
|
||||
print(json.dumps({"operatorCount": 1, "summary": {"contractVersion": "poc-operator-workflows/v4", "operatorCount": 1, "catalogHash": "a"*64, "catalogCount": 1, "trainingMode": "full", "fullTrainingRequired": True, "missingOperatorIds": [], "independentImageCoverage": 1, "independentImageRequired": 1, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "人工审核与抽检链路", "runId": "run-1", "operatorIds": ["operator-builtin-human-review"], "imageEvidence": [
|
||||
{"operatorId": "operator-builtin-human-review", "image": "registry.example/droboticflow-operator-human-review:builtin", "imageId": "registry.example/droboticflow-operator-human-review@sha256:" + "c"*64}
|
||||
]}]}))
|
||||
elif url.endswith("/worker-api/process-once"):
|
||||
@ -231,7 +231,7 @@ if url.endswith("/api/health") or url.endswith("/worker-api/health"):
|
||||
elif url.endswith("/api/auth/login"):
|
||||
print(json.dumps({"user": {"email": "poc-acceptance-admin@droboticflow.local"}}))
|
||||
elif url.endswith("/api/acceptance/poc-operator-workflows"):
|
||||
print(json.dumps({"operatorCount": 2, "summary": {"contractVersion": "poc-operator-workflows/v3", "operatorCount": 2, "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "外部存储与审计链路", "runId": "run-1", "operatorIds": ["operator-builtin-shell-command", "operator-builtin-mongodb-write"], "imageEvidence": [
|
||||
print(json.dumps({"operatorCount": 2, "summary": {"contractVersion": "poc-operator-workflows/v4", "operatorCount": 2, "catalogHash": "a"*64, "catalogCount": 2, "trainingMode": "full", "fullTrainingRequired": True, "missingOperatorIds": [], "independentImageCoverage": 2, "independentImageRequired": 2, "commandWrapperOnlyOperatorCount": 0, "generatedFixtureCount": 0, "mockFixtureCount": 0}, "runs": [{"batch": "外部存储与审计链路", "runId": "run-1", "operatorIds": ["operator-builtin-shell-command", "operator-builtin-mongodb-write"], "imageEvidence": [
|
||||
{"operatorId": "operator-builtin-shell-command", "image": "registry.example/droboticflow-operator-shell-command:builtin", "imageId": "registry.example/droboticflow-operator-shell-command@sha256:" + "d"*64},
|
||||
{"operatorId": "operator-builtin-mongodb-write", "image": "registry.example/droboticflow-operator-mongodb-write:builtin", "imageId": "registry.example/droboticflow-operator-mongodb-write@sha256:" + "e"*64}
|
||||
]}]}))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user