🛠️ Refresh builtin runtime images safely
This commit is contained in:
parent
e199687a11
commit
4afdc1c869
@ -1738,11 +1738,15 @@ func stringList(value any) []string {
|
||||
}
|
||||
|
||||
func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
||||
copy, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
|
||||
if err != nil {
|
||||
return node, err
|
||||
}
|
||||
effectiveWorkflow, err := runtime.resolveScanWorkflow(ctx, workflow)
|
||||
if err != nil {
|
||||
return node, err
|
||||
}
|
||||
copy, err := scanExecutionNode(effectiveWorkflow, node)
|
||||
copy, err = scanExecutionNode(effectiveWorkflow, copy)
|
||||
if err != nil {
|
||||
return node, err
|
||||
}
|
||||
@ -1811,6 +1815,44 @@ func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *doma
|
||||
return copy, nil
|
||||
}
|
||||
|
||||
// resolveBuiltinNodeRuntime refreshes platform-owned runtime fields immediately
|
||||
// before execution. Workflow nodes keep their user configuration and graph
|
||||
// shape, while existing workflows automatically consume a newly published
|
||||
// builtin image instead of retaining a stale or deleted registry digest.
|
||||
func (runtime *Runtime) resolveBuiltinNodeRuntime(ctx context.Context, node domain.WorkflowNode) (domain.WorkflowNode, error) {
|
||||
operatorID := strings.TrimSpace(node.Data.OperatorID)
|
||||
if operatorID == "" || runtime == nil || runtime.store == nil {
|
||||
return node, nil
|
||||
}
|
||||
operator, err := runtime.store.GetOperator(ctx, operatorID)
|
||||
if err != nil {
|
||||
return node, fmt.Errorf("resolve builtin operator %s: %w", operatorID, err)
|
||||
}
|
||||
if operator == nil || !strings.EqualFold(strings.TrimSpace(operator.Type), "builtin") {
|
||||
return node, nil
|
||||
}
|
||||
image := strings.TrimSpace(operator.Image)
|
||||
imageID := strings.TrimSpace(operator.ImageID)
|
||||
if image == "" {
|
||||
return node, fmt.Errorf("builtin operator %s has no published image", operatorID)
|
||||
}
|
||||
if imageID == "" {
|
||||
imageID = image
|
||||
}
|
||||
node.Data.Image = image
|
||||
node.Data.OperatorImageID = imageID
|
||||
node.Data.ContainerCommand = cloneStringSlice(operator.ContainerCommand)
|
||||
node.Data.ContainerArgs = cloneStringSlice(operator.ContainerArgs)
|
||||
node.Data.RuntimeContract = operator.RuntimeContract
|
||||
node.Data.PublicationStatus = operator.PublicationStatus
|
||||
node.Data.GenerationSource = operator.GenerationSource
|
||||
node.Data.PublicationEvidence = operator.PublicationEvidence
|
||||
if strings.TrimSpace(operator.Version) != "" {
|
||||
node.Data.OperatorVersion = strings.TrimSpace(operator.Version)
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func (runtime *Runtime) resolveScanWorkflow(ctx context.Context, workflow *domain.WorkflowRecord) (*domain.WorkflowRecord, error) {
|
||||
if workflow == nil || workflow.ScanConfig == nil || strings.TrimSpace(workflow.ScanConfig.ConnectionID) == "" {
|
||||
return workflow, nil
|
||||
|
||||
@ -1116,6 +1116,71 @@ func TestChooseImagePrefersUntaggedDigestPinnedOperatorImageID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBuiltinNodeRuntimeRefreshesStaleWorkflowImage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
currentImage := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:builtin"
|
||||
currentImageID := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("c", 64)
|
||||
if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{
|
||||
ID: "operator-builtin-bos-incremental-scan",
|
||||
Name: "BOS 增量扫描",
|
||||
Type: "builtin",
|
||||
Category: "IO",
|
||||
Description: "scan",
|
||||
Image: currentImage,
|
||||
ImageID: currentImageID,
|
||||
ContainerCommand: []string{"python", "/app/backend/operator-runner/main.py"},
|
||||
RuntimeContract: map[string]any{"handler": "builtin_source"},
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert builtin operator: %v", err)
|
||||
}
|
||||
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
|
||||
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{
|
||||
OperatorID: "operator-builtin-bos-incremental-scan",
|
||||
Image: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:deleted",
|
||||
OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("a", 64),
|
||||
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: "bos://bucket/prefix"}},
|
||||
}}
|
||||
|
||||
resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve builtin runtime: %v", err)
|
||||
}
|
||||
if resolved.Data.Image != currentImage || resolved.Data.OperatorImageID != currentImageID {
|
||||
t.Fatalf("expected current builtin image, got image=%q imageId=%q", resolved.Data.Image, resolved.Data.OperatorImageID)
|
||||
}
|
||||
if strings.Join(resolved.Data.ContainerCommand, " ") != "python /app/backend/operator-runner/main.py" {
|
||||
t.Fatalf("expected current container command, got %#v", resolved.Data.ContainerCommand)
|
||||
}
|
||||
if len(resolved.Data.Hyperparams) != 1 || resolved.Data.Hyperparams[0].Value != "bos://bucket/prefix" {
|
||||
t.Fatalf("expected workflow parameters to be preserved, got %#v", resolved.Data.Hyperparams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBuiltinNodeRuntimeLeavesCustomOperatorSnapshotUntouched(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{
|
||||
ID: "operator-custom", Name: "Custom", Type: "custom", Category: "IO", Description: "custom",
|
||||
Image: "registry.example/custom:new", ImageID: "registry.example/custom@sha256:" + strings.Repeat("d", 64),
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert custom operator: %v", err)
|
||||
}
|
||||
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
|
||||
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{
|
||||
OperatorID: "operator-custom", Image: "registry.example/custom:pinned",
|
||||
OperatorImageID: "registry.example/custom@sha256:" + strings.Repeat("e", 64),
|
||||
}}
|
||||
|
||||
resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve custom runtime: %v", err)
|
||||
}
|
||||
if resolved.Data.Image != node.Data.Image || resolved.Data.OperatorImageID != node.Data.OperatorImageID {
|
||||
t.Fatalf("custom operator snapshot changed: %#v", resolved.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func createTestVideoMP4(t *testing.T, path string, pattern string, fps int, frames int, width int, height int) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
|
||||
@ -337,6 +337,7 @@ def sync_builtin_operators(
|
||||
operator_runner_image: str,
|
||||
*,
|
||||
image_digest_resolver: ImageDigestResolver | None = None,
|
||||
operator_ids: set[str] | None = None,
|
||||
) -> int:
|
||||
operators = request_json(client, "GET", "/api/operators")["items"]
|
||||
builtin_operators = [item for item in operators if str(item.get("type") or "").strip().lower() == "builtin"]
|
||||
@ -348,6 +349,8 @@ def sync_builtin_operators(
|
||||
identifier = item_id(operator)
|
||||
if not identifier:
|
||||
raise ReconcileError(f"builtin operator {operator.get('name')} is missing id")
|
||||
if operator_ids and identifier not in operator_ids:
|
||||
continue
|
||||
payload = builtin_operator_patch(operator, operator_runner_image, image_digest_resolver=image_digest_resolver)
|
||||
payload_image = str(payload.get("image") or "")
|
||||
payload_image_id = str(payload.get("imageId") or "")
|
||||
@ -458,6 +461,12 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--k8s-endpoint", default=os.getenv("K8S_ENDPOINT", ""))
|
||||
parser.add_argument("--k8s-namespace", default=os.getenv("K8S_NAMESPACE", "droboticflow"))
|
||||
parser.add_argument("--operator-runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE", ""))
|
||||
parser.add_argument(
|
||||
"--operator-id",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Only validate and reconcile the selected builtin operator; may be repeated.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@ -478,6 +487,16 @@ def main() -> None:
|
||||
os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL", "").strip(),
|
||||
os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD", ""),
|
||||
)
|
||||
selected_operator_ids = {value.strip() for value in args.operator_id if value.strip()}
|
||||
if selected_operator_ids:
|
||||
updated = sync_builtin_operators(
|
||||
client,
|
||||
args.operator_runner_image,
|
||||
image_digest_resolver=registry_digest_resolver_from_env(),
|
||||
operator_ids=selected_operator_ids,
|
||||
)
|
||||
print(f"reconciled selected builtin operators: {updated} updated")
|
||||
return
|
||||
reconcile(
|
||||
client,
|
||||
k8s_endpoint=args.k8s_endpoint,
|
||||
|
||||
@ -208,6 +208,54 @@ class ReconcileSystemDataTest(unittest.TestCase):
|
||||
image_digest_resolver=lambda image: reconcile.image_ref_with_digest(image, registry_digest),
|
||||
)
|
||||
|
||||
def test_sync_builtin_operators_can_target_one_operator(self):
|
||||
release_image, release_image_id = reconcile.release_image_reference("real_source")
|
||||
client = FakeClient(
|
||||
{
|
||||
("GET", "/api/operators"): {
|
||||
"items": [
|
||||
{
|
||||
"id": "operator-builtin-real-source",
|
||||
"name": "真实数据源",
|
||||
"type": "builtin",
|
||||
"category": "IO",
|
||||
"description": "source",
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"image": release_image,
|
||||
"imageId": release_image,
|
||||
"containerCommand": [],
|
||||
},
|
||||
{
|
||||
"id": "operator-builtin-mongodb-write",
|
||||
"name": "MongoDB 写入",
|
||||
"type": "builtin",
|
||||
"category": "DATA",
|
||||
"description": "mongo",
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"image": "runner:old",
|
||||
"imageId": "runner:old",
|
||||
"containerCommand": [],
|
||||
},
|
||||
]
|
||||
},
|
||||
("PUT", "/api/operators/operator-builtin-real-source"): {"id": "operator-builtin-real-source"},
|
||||
}
|
||||
)
|
||||
|
||||
updated = reconcile.sync_builtin_operators(
|
||||
client,
|
||||
"runner:new",
|
||||
image_digest_resolver=lambda image: release_image_id,
|
||||
operator_ids={"operator-builtin-real-source"},
|
||||
)
|
||||
|
||||
self.assertEqual(updated, 1)
|
||||
self.assertEqual(len(client.requests), 2)
|
||||
self.assertEqual(client.requests[1][1], "/api/operators/operator-builtin-real-source")
|
||||
self.assertEqual(client.requests[1][2]["image"], release_image)
|
||||
|
||||
def test_image_ref_with_digest_replaces_tag_with_digest(self):
|
||||
self.assertEqual(
|
||||
reconcile.image_ref_with_digest("ccr.example/ns/droboticflow-operator-yolo-train:v1", "sha256:" + "c" * 64),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user