✅ Harden POC acceptance preflight import

This commit is contained in:
eust-w 2026-06-23 09:27:42 +08:00
parent a6a9fa9d04
commit 3c539bc82a
5 changed files with 306 additions and 9 deletions

View File

@ -242,10 +242,12 @@ By default `POC_ACCEPTANCE_REQUIRED_EXTERNALS=bos,dcloud,mongo`, so missing `BOS
The result JSON exposes `acceptanceScope`, `fullExternalAcceptance`, `fixtureSources`, `operatorPassedCount`, `allOperatorCoverageComplete`, `blockingOperatorCoverageComplete`, `statusCounts`, and skipped-count fields. Treat `fullExternalAcceptance=true` plus `status=passed` as the strict full-acceptance signal only when `fixtureSources.items.*.generated=false`; a `dependency-isolation` result can prove the configured dependencies but not the omitted external operators.
When the Job emits a `DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=<json>` marker, the wrapper extracts it from the Job logs even if the Job exits failed. If `POC_ACCEPTANCE_IMPORT_API_BASE_URL`, `API_BASE_URL`, or `BASE_URL` is set, it posts that result to `/api/acceptance/poc-operator-result` and records it as a normal platform run under the `POC 算子验收` project. The imported run contains a `poc-operator-full-acceptance-result.json` artifact, so the Monitoring page can show the same strict-vs-dependency-isolation summary that appears in the raw Job log. Set `POC_ACCEPTANCE_IMPORT_RESULT=false` for one-off diagnostics that should not create or update platform acceptance history.
When the Job emits a `DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=<json>` marker, the wrapper extracts it from the Job logs even if the Job exits failed. If `POC_ACCEPTANCE_IMPORT_API_BASE_URL`, `API_BASE_URL`, or `BASE_URL` is set, it posts that result to `/api/acceptance/poc-operator-result` and records it as a normal platform run under the `POC 算子验收` project. In session-auth POC deployments, also provide `DROBOTICFLOW_ACCEPTANCE_EMAIL` and `DROBOTICFLOW_ACCEPTANCE_PASSWORD` so the import client can log in before posting the result; the API token alone only passes the perimeter guard and does not create a user session. The imported run contains a `poc-operator-full-acceptance-result.json` artifact, so the Monitoring page can show the same strict-vs-dependency-isolation summary that appears in the raw Job log. Set `POC_ACCEPTANCE_IMPORT_RESULT=false` for one-off diagnostics that should not create or update platform acceptance history.
If strict acceptance cannot even create the Kubernetes Job because the required D-Cloud FAT secret is missing or incomplete, the wrapper still emits a standard failed `DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=<json>` preflight result. When platform import is enabled, that failure is imported as a failed POC acceptance run with `preflightFailure=true`, `acceptanceScope=full`, `fullExternalAcceptance=false`, and blocking coverage tied to the D-Cloud operators derived from their repository manifests. This keeps the platform history honest: a missing D-Cloud secret is visible as the current full-acceptance blocker instead of being hidden by the last dependency-isolation success.
After D-Cloud readiness passes, the wrapper also refuses to create the Job when generated fixtures are disabled and no real fixture contract is configured. Provide either `POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH` or all explicit real fixture inputs, including `POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH`, `POC_ACCEPTANCE_CSV_INPUT_PATH`, `POC_ACCEPTANCE_JSON_INPUT_PATH`, `POC_ACCEPTANCE_LEROBOT_SOURCE_PATH`, `POC_ACCEPTANCE_ROSBAG_SOURCE_PATH`, `POC_ACCEPTANCE_MODEL_JSON_PATH`, and `POC_ACCEPTANCE_SHELL_COMMAND`. Missing fixture configuration is imported with `preflightStage=fixture-readiness`, `fullExternalAcceptance=false`, and blocking coverage for the full builtin operator set, so the platform history distinguishes "D-Cloud credentials missing" from "real acceptance data missing" without launching a Job that can only fail later.
The import endpoint validates full-acceptance consistency before it creates or updates a platform run. A payload with `fullExternalAcceptance=true` must also have `status=passed`, `acceptanceScope=full`, complete operator coverage, zero failed or required-skipped operators, and empty missing-coverage lists. Dependency-isolation evidence can still be imported, but it cannot be mislabeled as full external acceptance.
Configure or refresh the POC D-Cloud FAT secret from the current shell environment:

View File

@ -26,6 +26,13 @@ def load_result(path: pathlib.Path) -> dict[str, Any]:
return payload
def login_if_configured(client: DroboticflowClient, email: str, password: str) -> None:
if bool(email) != bool(password):
raise SystemExit("DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together")
if email:
client.login_if_configured(email, password, label="acceptance import user")
def main() -> int:
parser = argparse.ArgumentParser(description="Import a POC operator acceptance result into the platform run history")
parser.add_argument("--api-base-url", default=os.getenv("API_BASE_URL") or os.getenv("BASE_URL") or "")
@ -36,6 +43,8 @@ def main() -> int:
parser.add_argument("--runner-image", default=os.getenv("OPERATOR_RUNNER_IMAGE") or "")
parser.add_argument("--runner-image-id", default=os.getenv("POC_ACCEPTANCE_RUNNER_IMAGE_ID") or "")
parser.add_argument("--api-token", default=os.getenv("DROBOTICFLOW_API_TOKEN") or "")
parser.add_argument("--acceptance-email", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_EMAIL") or "")
parser.add_argument("--acceptance-password", default=os.getenv("DROBOTICFLOW_ACCEPTANCE_PASSWORD") or "")
args = parser.parse_args()
api_base_url = args.api_base_url.rstrip("/")
@ -54,6 +63,7 @@ def main() -> int:
},
}
client = DroboticflowClient(api_base_url, api_token=args.api_token)
login_if_configured(client, args.acceptance_email.strip(), args.acceptance_password)
response = client.request_json("POST", "/api/acceptance/poc-operator-result", request, expected=(201,))
run = response.get("run") if isinstance(response, dict) else {}
run_id = run.get("id") if isinstance(run, dict) else ""

View File

@ -103,6 +103,19 @@ required_external_enabled() {
return 1
}
bool_is_true() {
local value
value="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')"
case "$value" in
true|1|yes|on)
return 0
;;
*)
return 1
;;
esac
}
dcloud_secret_readiness_error() {
local secret_name="$1"
local secret_json
@ -157,11 +170,40 @@ if missing:
PY
}
fixture_readiness_error() {
if bool_is_true "$POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES"; then
return 0
fi
if [[ -n "$POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" ]]; then
return 0
fi
local missing=()
local key
for key in \
POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH \
POC_ACCEPTANCE_CSV_INPUT_PATH \
POC_ACCEPTANCE_JSON_INPUT_PATH \
POC_ACCEPTANCE_LEROBOT_SOURCE_PATH \
POC_ACCEPTANCE_ROSBAG_SOURCE_PATH \
POC_ACCEPTANCE_MODEL_JSON_PATH \
POC_ACCEPTANCE_SHELL_COMMAND
do
if [[ -z "${!key:-}" ]]; then
missing+=("$key")
fi
done
if [[ "${#missing[@]}" -gt 0 ]]; then
printf 'POC acceptance real fixtures are incomplete: set POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH or provide %s; generated fixtures are disabled' "$(IFS=', '; printf '%s' "${missing[*]}")"
fi
}
write_preflight_acceptance_failure_result() {
local result_file="$1"
local stage="$2"
local reason="$3"
python3 - "$ROOT_DIR" "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" "$WORKSPACE_MOUNT_PATH" "$POC_ACCEPTANCE_REQUIRED_EXTERNALS" "$stage" "$reason" "$POC_ACCEPTANCE_DCLOUD_SECRET_NAME" >"$result_file" <<'PY'
python3 - "$ROOT_DIR" "$POC_OPERATOR_ACCEPTANCE_JOB_NAME" "$WORKSPACE_MOUNT_PATH" "$POC_ACCEPTANCE_REQUIRED_EXTERNALS" "$stage" "$reason" "$POC_ACCEPTANCE_DCLOUD_SECRET_NAME" "$POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" "$POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES" >"$result_file" <<'PY'
from __future__ import annotations
import datetime
@ -177,6 +219,8 @@ required_raw = sys.argv[4]
stage = sys.argv[5]
reason = sys.argv[6]
dcloud_secret_name = sys.argv[7]
fixture_manifest_path = sys.argv[8]
allow_generated_fixtures = sys.argv[9]
required = sorted({item.strip().lower() for item in required_raw.split(",") if item.strip()})
scope = "full" if {"bos", "dcloud", "mongo"}.issubset(set(required)) else "dependency-isolation"
@ -198,7 +242,22 @@ for manifest_path in sorted((root / "backend" / "operators" / "builtin").glob("*
dcloud_operator_slugs.add(slug)
now = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
missing_coverage = sorted(dcloud_operator_slugs)
if stage == "fixture-readiness":
missing_coverage = sorted(operator_slugs)
handler = "fixture_readiness"
evidence = {
"fixtureManifestPath": fixture_manifest_path,
"allowGeneratedFixtures": allow_generated_fixtures,
"missingOperatorCoverage": missing_coverage,
}
else:
missing_coverage = sorted(dcloud_operator_slugs)
handler = "dcloud_secret_readiness"
evidence = {
"requiredExternal": "dcloud",
"secretName": dcloud_secret_name,
"missingOperatorCoverage": missing_coverage,
}
payload = {
"runId": run_id,
"workspace": f"{workspace_mount}/acceptance/{run_id}",
@ -224,14 +283,10 @@ payload = {
"records": [
{
"operator": "poc_acceptance_preflight",
"handler": "dcloud_secret_readiness",
"handler": handler,
"status": "failed",
"error": reason,
"evidence": {
"requiredExternal": "dcloud",
"secretName": dcloud_secret_name,
"missingOperatorCoverage": missing_coverage,
},
"evidence": evidence,
}
],
"failures": [reason],
@ -335,6 +390,16 @@ if [[ "$POC_OPERATOR_ACCEPTANCE_DRY_RUN" != "true" ]] && required_external_enabl
fi
fi
if [[ "$POC_OPERATOR_ACCEPTANCE_DRY_RUN" != "true" ]]; then
fixture_readiness_error="$(fixture_readiness_error)"
if [[ -n "$fixture_readiness_error" ]]; then
echo "ERROR: $fixture_readiness_error." >&2
echo "Provide a real fixture manifest under the shared workspace PVC or set all explicit POC_ACCEPTANCE_* fixture paths before final full acceptance." >&2
import_preflight_acceptance_failure "fixture-readiness" "$fixture_readiness_error"
exit 1
fi
fi
job_name_json="$(json_string "$POC_OPERATOR_ACCEPTANCE_JOB_NAME")"
namespace_json="$(json_string "$K8S_NAMESPACE")"
image_json="$(json_string "$OPERATOR_RUNNER_IMAGE")"

View File

@ -101,3 +101,92 @@ def test_import_poc_acceptance_result_requires_api_base_url(tmp_path):
assert result.returncode != 0
assert "--api-base-url or API_BASE_URL is required" in result.stderr
def test_import_poc_acceptance_result_logs_in_when_session_credentials_are_configured(tmp_path):
received = {}
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
length = int(self.headers.get("Content-Length", "0"))
if self.path == "/api/auth/login":
received["login"] = json.loads(self.rfile.read(length).decode("utf-8"))
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Set-Cookie", "droboticflow_session=session-1; Path=/; HttpOnly")
self.end_headers()
self.wfile.write(json.dumps({"email": received["login"]["email"]}).encode("utf-8"))
return
if self.path == "/api/acceptance/poc-operator-result":
received["acceptanceCookie"] = self.headers.get("Cookie", "")
received["acceptanceBody"] = json.loads(self.rfile.read(length).decode("utf-8"))
self.send_response(201)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps({"run": {"id": "run-with-session"}}).encode("utf-8"))
return
self.send_response(404)
self.end_headers()
def log_message(self, *_args):
return
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
result_file = tmp_path / "result.json"
result_file.write_text(json.dumps({"runId": "poc-session-import", "status": "failed"}), encoding="utf-8")
result = subprocess.run(
[
"python3",
str(SCRIPT),
"--api-base-url",
f"http://127.0.0.1:{server.server_port}",
"--result-file",
str(result_file),
"--acceptance-email",
"admin@example.test",
"--acceptance-password",
"password",
],
cwd=ROOT,
env={**os.environ, "NO_PROXY": "127.0.0.1,localhost", "no_proxy": "127.0.0.1,localhost"},
text=True,
capture_output=True,
check=False,
)
finally:
server.shutdown()
thread.join(timeout=5)
assert result.returncode == 0, result.stderr
assert json.loads(result.stdout.splitlines()[-1]) == {"imported": True, "runId": "run-with-session"}
assert received["login"] == {"email": "admin@example.test", "password": "password"}
assert "droboticflow_session=session-1" in received["acceptanceCookie"]
assert received["acceptanceBody"]["result"]["runId"] == "poc-session-import"
def test_import_poc_acceptance_result_rejects_unpaired_session_credentials(tmp_path):
result_file = tmp_path / "result.json"
result_file.write_text("{}", encoding="utf-8")
result = subprocess.run(
[
"python3",
str(SCRIPT),
"--api-base-url",
"http://127.0.0.1:9",
"--result-file",
str(result_file),
"--acceptance-email",
"admin@example.test",
],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert "DROBOTICFLOW_ACCEPTANCE_EMAIL and DROBOTICFLOW_ACCEPTANCE_PASSWORD must be set together" in result.stderr

View File

@ -193,6 +193,38 @@ def write_fake_kubectl_incomplete_dcloud_secret(tmp_path: pathlib.Path, calls_pa
return fake_bin
def write_fake_kubectl_complete_dcloud_secret(tmp_path: pathlib.Path, calls_path: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import base64\n"
"import json\n"
"import pathlib\n"
"import sys\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"if len(args) >= 5 and args[:4] == ['-n', 'droboticflow', 'get', 'secret']:\n"
" def enc(value):\n"
" return base64.b64encode(value.encode('utf-8')).decode('ascii')\n"
" print(json.dumps({'data': {\n"
" 'DCLOUD_FAT_BASE_URL': enc('https://cloud-fat.example'),\n"
" 'DCLOUD_FAT_AUTH_TOKEN': enc('token'),\n"
" 'DCLOUD_ORG_SYMBOL': enc('org-1'),\n"
" 'DCLOUD_SOURCEAPP': enc('dataCopilot'),\n"
" }}))\n"
" raise SystemExit(0)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def run_script_with_missing_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
@ -223,6 +255,7 @@ def run_script_with_missing_dcloud_secret(
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
@ -277,6 +310,7 @@ def run_script_with_incomplete_dcloud_secret(tmp_path: pathlib.Path) -> tuple[su
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
@ -307,6 +341,64 @@ def run_script_with_incomplete_dcloud_secret(tmp_path: pathlib.Path) -> tuple[su
return result, calls_path
def run_script_with_complete_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
fake_bin = write_fake_kubectl_complete_dcloud_secret(tmp_path, calls_path)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"DCLOUD_FAT_BASE_URL",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "false",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,dcloud,mongo",
"POC_ACCEPTANCE_IMPORT_RESULT": "false",
}
)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path
def test_poc_operator_acceptance_rejects_invalid_image_before_kubectl():
result = run_script({"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/operator runner:test"})
@ -464,6 +556,45 @@ def test_poc_operator_acceptance_rejects_dcloud_secret_without_base_url_before_j
]
def test_poc_operator_acceptance_rejects_missing_real_fixture_config_before_job(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(tmp_path)
assert result.returncode == 1
assert "POC acceptance real fixtures are incomplete" in result.stderr
assert "POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stderr
assert "POC_ACCEPTANCE_SHELL_COMMAND" in result.stderr
assert "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" in result.stdout
marker = next(line for line in result.stdout.splitlines() if line.startswith("DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="))
payload = json.loads(marker.split("=", 1)[1])
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "fixture-readiness"
assert payload["acceptanceScope"] == "full"
assert payload["fullExternalAcceptance"] is False
assert payload["operatorCount"] > 20
assert payload["blockingMissingCoverage"] == payload["missingCoverage"]
assert len(payload["blockingMissingCoverage"]) == payload["operatorCount"]
assert payload["records"][0]["handler"] == "fixture_readiness"
assert calls_path.read_text(encoding="utf-8").splitlines() == [
"-n droboticflow get secret droboticflow-dcloud-secret -o json",
]
def test_poc_operator_acceptance_skips_fixture_preflight_when_manifest_is_configured(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "true",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/poc-operator-acceptance.json",
},
)
assert result.returncode == 0, result.stderr + result.stdout
assert "fixture-readiness" not in result.stdout
assert not calls_path.exists()
assert "name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stdout
assert 'value: "/workspace/fixtures/poc-operator-acceptance.json"' in result.stdout
def test_poc_operator_acceptance_imports_missing_dcloud_secret_preflight_failure(tmp_path):
received: dict[str, object] = {}