🛠️ Refresh builtin runtime images safely

This commit is contained in:
eust-w 2026-08-17 16:04:44 +08:00
parent e199687a11
commit 4afdc1c869
4 changed files with 175 additions and 1 deletions

View File

@ -1738,11 +1738,15 @@ func stringList(value any) []string {
}
func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
copy, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
if err != nil {
return node, err
}
effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow)
if err != nil {
return node, err
}
copy, err := scanExecutionNode(effectiveWorkflow, node)
copy, err = scanExecutionNode(effectiveWorkflow, copy)
if err != nil {
return node, err
}
@ -1811,6 +1815,44 @@ func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *doma
return copy, nil
}
// resolveBuiltinNodeRuntime refreshes platform-owned runtime fields immediately
// before execution. Workflow nodes keep their user configuration and graph
// shape, while existing workflows automatically consume a newly published
// builtin image instead of retaining a stale or deleted registry digest.
func (runtime *Runtime) resolveBuiltinNodeRuntime(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
operatorID := strings.TrimSpace(node.Data.OperatorID)
if operatorID == "" || runtime == nil || runtime.store == nil {
return node, nil
}
operator, err := runtime.store.GetOperator(ctx, operatorID)
if err != nil {
return node, fmt.Errorf("resolve builtin operator %s: %w", operatorID, err)
}
if operator == nil || !strings.EqualFold(strings.TrimSpace(operator.Type), "builtin") {
return node, nil
}
image := strings.TrimSpace(operator.Image)
imageID := strings.TrimSpace(operator.ImageID)
if image == "" {
return node, fmt.Errorf("builtin operator %s has no published image", operatorID)
}
if imageID == "" {
imageID = image
}
node.Data.Image = image
node.Data.OperatorImageID = imageID
node.Data.ContainerCommand = cloneStringSlice(operator.ContainerCommand)
node.Data.ContainerArgs = cloneStringSlice(operator.ContainerArgs)
node.Data.RuntimeContract = operator.RuntimeContract
node.Data.PublicationStatus = operator.PublicationStatus
node.Data.GenerationSource = operator.GenerationSource
node.Data.PublicationEvidence = operator.PublicationEvidence
if strings.TrimSpace(operator.Version) != "" {
node.Data.OperatorVersion = strings.TrimSpace(operator.Version)
}
return node, nil
}
func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) {
if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" {
return workflow, nil

View File

@ -1116,6 +1116,71 @@ func TestChooseImagePrefersUntaggedDigestPinnedOperatorImageID(t *testing.T) {
}
}
func TestResolveBuiltinNodeRuntimeRefreshesStaleWorkflowImage(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
currentImage := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:builtin"
currentImageID := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("c", 64)
if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{
ID: "operator-builtin-bos-incremental-scan",
Name: "BOS 增量扫描",
Type: "builtin",
Category: "IO",
Description: "scan",
Image: currentImage,
ImageID: currentImageID,
ContainerCommand: []string{"python", "/app/backend/operator-runner/main.py"},
RuntimeContract: map[string]any{"handler": "builtin_source"},
}); err != nil {
t.Fatalf("upsert builtin operator: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{
OperatorID: "operator-builtin-bos-incremental-scan",
Image: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:deleted",
OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("a", 64),
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: "bos://bucket/prefix"}},
}}
resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
if err != nil {
t.Fatalf("resolve builtin runtime: %v", err)
}
if resolved.Data.Image != currentImage || resolved.Data.OperatorImageID != currentImageID {
t.Fatalf("expected current builtin image, got image=%q imageId=%q", resolved.Data.Image, resolved.Data.OperatorImageID)
}
if strings.Join(resolved.Data.ContainerCommand, " ") != "python /app/backend/operator-runner/main.py" {
t.Fatalf("expected current container command, got %#v", resolved.Data.ContainerCommand)
}
if len(resolved.Data.Hyperparams) != 1 || resolved.Data.Hyperparams[0].Value != "bos://bucket/prefix" {
t.Fatalf("expected workflow parameters to be preserved, got %#v", resolved.Data.Hyperparams)
}
}
func TestResolveBuiltinNodeRuntimeLeavesCustomOperatorSnapshotUntouched(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{
ID: "operator-custom", Name: "Custom", Type: "custom", Category: "IO", Description: "custom",
Image: "registry.example/custom:new", ImageID: "registry.example/custom@sha256:" + strings.Repeat("d", 64),
}); err != nil {
t.Fatalf("upsert custom operator: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{
OperatorID: "operator-custom", Image: "registry.example/custom:pinned",
OperatorImageID: "registry.example/custom@sha256:" + strings.Repeat("e", 64),
}}
resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
if err != nil {
t.Fatalf("resolve custom runtime: %v", err)
}
if resolved.Data.Image != node.Data.Image || resolved.Data.OperatorImageID != node.Data.OperatorImageID {
t.Fatalf("custom operator snapshot changed: %#v", resolved.Data)
}
}
func createTestVideoMP4(t *testing.T, path string, pattern string, fps int, frames int, width int, height int) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {

View File

@ -337,6 +337,7 @@ def sync_builtin_operators(
operator_runner_image: str,
*,
image_digest_resolver: ImageDigestResolver | None = None,
operator_ids: set[str] | None = None,
) -> int:
operators = request_json(client, "GET", "/api/operators")["items"]
builtin_operators = [item for item in operators if str(item.get("type") or "").strip().lower() == "builtin"]
@ -348,6 +349,8 @@ def sync_builtin_operators(
identifier = item_id(operator)
if not identifier:
raise ReconcileError(f"builtin operator {operator.get('name')} is missing id")
if operator_ids and identifier not in operator_ids:
continue
payload = builtin_operator_patch(operator, operator_runner_image, image_digest_resolver=image_digest_resolver)
payload_image = str(payload.get("image") or "")
payload_image_id = str(payload.get("imageId") or "")
@ -458,6 +461,12 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--k8s-endpoint", default=os.getenv("K8S_ENDPOINT", ""))
parser.add_argument("--k8s-namespace", default=os.getenv("K8S_NAMESPACE", "droboticflow"))
parser.add_argument("--operator-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE", ""))
parser.add_argument(
"--operator-id",
action="append",
default=[],
help="Only validate and reconcile the selected builtin operator; may be repeated.",
)
return parser.parse_args()
@ -478,6 +487,16 @@ def main() -> None:
os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip(),
os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""),
)
selected_operator_ids = {value.strip() for value in args.operator_id if value.strip()}
if selected_operator_ids:
updated = sync_builtin_operators(
client,
args.operator_runner_image,
image_digest_resolver=registry_digest_resolver_from_env(),
operator_ids=selected_operator_ids,
)
print(f"reconciled selected builtin operators: {updated} updated")
return
reconcile(
client,
k8s_endpoint=args.k8s_endpoint,

View File

@ -208,6 +208,54 @@ class ReconcileSystemDataTest(unittest.TestCase):
image_digest_resolver=lambda image: reconcile.image_ref_with_digest(image, registry_digest),
)
def test_sync_builtin_operators_can_target_one_operator(self):
release_image, release_image_id = reconcile.release_image_reference("real_source")
client = FakeClient(
{
("GET", "/api/operators"): {
"items": [
{
"id": "operator-builtin-real-source",
"name": "真实数据源",
"type": "builtin",
"category": "IO",
"description": "source",
"inputs": [],
"outputs": [],
"image": release_image,
"imageId": release_image,
"containerCommand": [],
},
{
"id": "operator-builtin-mongodb-write",
"name": "MongoDB 写入",
"type": "builtin",
"category": "DATA",
"description": "mongo",
"inputs": [],
"outputs": [],
"image": "runner:old",
"imageId": "runner:old",
"containerCommand": [],
},
]
},
("PUT", "/api/operators/operator-builtin-real-source"): {"id": "operator-builtin-real-source"},
}
)
updated = reconcile.sync_builtin_operators(
client,
"runner:new",
image_digest_resolver=lambda image: release_image_id,
operator_ids={"operator-builtin-real-source"},
)
self.assertEqual(updated, 1)
self.assertEqual(len(client.requests), 2)
self.assertEqual(client.requests[1][1], "/api/operators/operator-builtin-real-source")
self.assertEqual(client.requests[1][2]["image"], release_image)
def test_image_ref_with_digest_replaces_tag_with_digest(self):
self.assertEqual(
reconcile.image_ref_with_digest("ccr.example/ns/droboticflow-operator-yolo-train:v1", "sha256:" + "c" * 64),