forked from longtao.wu/droboticflow
890 lines
30 KiB
JavaScript
890 lines
30 KiB
JavaScript
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { execFileSync } from "node:child_process";
|
|
|
|
import { chromium } from "playwright";
|
|
import { authenticationHeaders, createAcceptanceAPIClient, positiveIntegerEnv } from "./lib/acceptance-client.mjs";
|
|
import { apiBaseURLFromEnv } from "./lib/service-endpoints.mjs";
|
|
|
|
const baseURL = apiBaseURLFromEnv({ env: process.env, cliArgs: process.argv });
|
|
const apiToken = (process.env.DROBOTICFLOW_API_TOKEN || "").trim();
|
|
const apiClient = createAcceptanceAPIClient({ baseURL, apiToken });
|
|
const suffix = Date.now().toString(36);
|
|
const prefix = `PWX ${suffix}`;
|
|
const maxPayloadBytes = 1024 * 1024;
|
|
|
|
const previewCsvPayload = readRequiredPayloadFile(
|
|
"PREVIEW_CHECK_CSV_PAYLOAD_FILE",
|
|
"CSV payload for report preview and CSV conversion checks",
|
|
);
|
|
const previewTextPayload = readRequiredPayloadFile(
|
|
"PREVIEW_CHECK_TEXT_PAYLOAD_FILE",
|
|
"text payload for artifact preview checks",
|
|
);
|
|
const dataOpsJsonPayload = readRequiredJSONPayload("PREVIEW_CHECK_JSON_PAYLOAD_FILE");
|
|
const dataOpsExtractField = requiredEnv("PREVIEW_CHECK_JSON_EXTRACT_FIELD");
|
|
const dataOpsFilterField = requiredEnv("PREVIEW_CHECK_JSON_FILTER_FIELD");
|
|
const dataOpsItems = dataOpsPayloadItems(dataOpsJsonPayload.value);
|
|
const dataOpsFieldMap = parseRequiredFieldMapEnv("PREVIEW_CHECK_JSON_FIELD_MAP_JSON", dataOpsItems);
|
|
const dataOpsFilterExpected = process.env.PREVIEW_CHECK_JSON_FILTER_EXPECTED_JSON
|
|
? parseJSONEnv("PREVIEW_CHECK_JSON_FILTER_EXPECTED_JSON")
|
|
: valueAtPath(dataOpsItems[0], dataOpsFilterField);
|
|
const dataOpsExtractExpected = valueAtPath(dataOpsJsonPayload.value, dataOpsExtractField);
|
|
const dataOpsFilterExpectedCount = dataOpsItems.filter((item) =>
|
|
jsonEqual(valueAtPath(item, dataOpsFilterField), dataOpsFilterExpected),
|
|
).length;
|
|
const scanSourcePath = requiredEnv("PREVIEW_CHECK_SCAN_SOURCE_PATH");
|
|
const rosbagSourcePath = requiredEnv("PREVIEW_CHECK_ROSBAG_SOURCE_PATH");
|
|
const lerobotSourcePath = requiredEnv("PREVIEW_CHECK_LEROBOT_SOURCE_PATH");
|
|
const scanMinFileCount = positiveIntegerEnv("PREVIEW_CHECK_SCAN_MIN_FILES", 1);
|
|
const workflowRunTimeoutMs = positiveIntegerEnv("PREVIEW_CHECK_RUN_TIMEOUT_SECONDS", 600) * 1000;
|
|
const k8sPayloadNamespace = (process.env.PREVIEW_CHECK_K8S_NAMESPACE || "").trim();
|
|
const k8sPayloadReadTarget = (process.env.PREVIEW_CHECK_K8S_PAYLOAD_READ_TARGET || "deployment/droboticflow-api").trim();
|
|
const rosbagExpectedTopic = (process.env.PREVIEW_CHECK_ROSBAG_TOPIC || "").trim();
|
|
const lerobotExpectedRobotType = (process.env.PREVIEW_CHECK_LEROBOT_ROBOT_TYPE || "").trim();
|
|
|
|
const names = {
|
|
user: `PWX Admin ${suffix}`,
|
|
email: `pwx-admin-${suffix}@playwright.local`,
|
|
project: `${prefix} Project`,
|
|
previewWorkflow: `${prefix} Preview`,
|
|
dataOpsWorkflow: `${prefix} DataOps`,
|
|
csvWorkflow: `${prefix} CSV`,
|
|
scanWorkflow: `${prefix} Scan`,
|
|
rosbagWorkflow: `${prefix} ROSBag`,
|
|
lerobotWorkflow: `${prefix} LeRobot`,
|
|
};
|
|
|
|
const requiredOperators = [
|
|
"目录扫描",
|
|
"CSV 转 JSON",
|
|
"JSON 转 CSV",
|
|
"JSON 字段提取",
|
|
"JSON 过滤",
|
|
"JSON 映射重命名",
|
|
"文本模板渲染",
|
|
"条件分流",
|
|
"LeRobot 数据集概览",
|
|
"ROSBag 数据概览",
|
|
"真实数据源",
|
|
];
|
|
|
|
function requiredEnv(name) {
|
|
const value = (process.env[name] || "").trim();
|
|
if (!value) {
|
|
throw new Error(`${name} is required for preview/operator acceptance`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function readRequiredPayloadFile(envName, description) {
|
|
const filePath = requiredEnv(envName);
|
|
const resolvedPath = path.resolve(filePath);
|
|
let stat;
|
|
try {
|
|
stat = fs.statSync(resolvedPath);
|
|
} catch (error) {
|
|
throw new Error(`${envName} is not readable: ${resolvedPath} (${error instanceof Error ? error.message : error})`);
|
|
}
|
|
if (!stat.isFile()) {
|
|
throw new Error(`${envName} must point to a file: ${resolvedPath}`);
|
|
}
|
|
if (stat.size === 0) {
|
|
throw new Error(`${envName} must not be empty: ${resolvedPath}`);
|
|
}
|
|
if (stat.size > maxPayloadBytes) {
|
|
throw new Error(`${envName} is ${stat.size} bytes; keep ${description} at or below ${maxPayloadBytes} bytes`);
|
|
}
|
|
const content = fs.readFileSync(resolvedPath);
|
|
const text = content.toString("utf8");
|
|
const previewNeedle = firstVisibleToken(text);
|
|
if (!previewNeedle) {
|
|
throw new Error(`${envName} must contain visible UTF-8 text: ${resolvedPath}`);
|
|
}
|
|
return {
|
|
path: resolvedPath,
|
|
bytes: content,
|
|
text,
|
|
previewNeedle,
|
|
};
|
|
}
|
|
|
|
function readRequiredJSONPayload(envName) {
|
|
const payload = readRequiredPayloadFile(envName, "JSON payload for data operator checks");
|
|
let value;
|
|
try {
|
|
value = JSON.parse(payload.text);
|
|
} catch (error) {
|
|
throw new Error(`${envName} must contain valid JSON: ${error instanceof Error ? error.message : error}`);
|
|
}
|
|
return { ...payload, value };
|
|
}
|
|
|
|
function parseJSONEnv(name) {
|
|
try {
|
|
return JSON.parse(process.env[name] || "");
|
|
} catch (error) {
|
|
throw new Error(`${name} must contain valid JSON: ${error instanceof Error ? error.message : error}`);
|
|
}
|
|
}
|
|
|
|
function parseRequiredFieldMapEnv(name, rows) {
|
|
const parsed = parseJSONEnv(name);
|
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || Object.keys(parsed).length === 0) {
|
|
throw new Error(`${name} must be a non-empty JSON object`);
|
|
}
|
|
const normalized = {};
|
|
const destinations = new Set();
|
|
for (const [source, destination] of Object.entries(parsed)) {
|
|
const sourceField = source.trim();
|
|
if (!sourceField) {
|
|
throw new Error(`${name} source field names must be non-empty`);
|
|
}
|
|
if (sourceField !== source) {
|
|
throw new Error(`${name} source field ${JSON.stringify(source)} must not have leading or trailing whitespace`);
|
|
}
|
|
if (typeof destination !== "string") {
|
|
throw new Error(`${name}.${sourceField} destination field must be a string`);
|
|
}
|
|
const destinationField = destination.trim();
|
|
if (!destinationField) {
|
|
throw new Error(`${name}.${sourceField} destination field must be non-empty`);
|
|
}
|
|
if (destinationField !== destination) {
|
|
throw new Error(`${name}.${sourceField} destination field must not have leading or trailing whitespace`);
|
|
}
|
|
if (destinations.has(destinationField)) {
|
|
throw new Error(`${name} maps multiple source fields to destination field ${JSON.stringify(destinationField)}`);
|
|
}
|
|
destinations.add(destinationField);
|
|
normalized[sourceField] = destinationField;
|
|
}
|
|
validateFieldMapCollisions(name, normalized, rows);
|
|
return normalized;
|
|
}
|
|
|
|
function validateFieldMapCollisions(name, fieldMap, rows) {
|
|
for (const [rowIndex, row] of rows.entries()) {
|
|
const mappedSources = new Map();
|
|
for (const sourceField of Object.keys(row)) {
|
|
const outputField = fieldMap[sourceField] || sourceField;
|
|
const previousSource = mappedSources.get(outputField);
|
|
if (previousSource) {
|
|
throw new Error(
|
|
`${name} maps row ${rowIndex} source fields ${JSON.stringify(previousSource)} and ${JSON.stringify(sourceField)} to the same output field ${JSON.stringify(outputField)}`,
|
|
);
|
|
}
|
|
mappedSources.set(outputField, sourceField);
|
|
}
|
|
}
|
|
}
|
|
|
|
function firstVisibleToken(text) {
|
|
for (const line of text.split(/\r?\n/)) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed) {
|
|
continue;
|
|
}
|
|
const csvCell = trimmed.split(/[,\t]/).map((item) => item.trim()).find(Boolean);
|
|
return (csvCell || trimmed).slice(0, 120);
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function dataOpsPayloadItems(payload) {
|
|
const items = Array.isArray(payload) ? payload : payload?.items;
|
|
if (!Array.isArray(items) || items.length === 0 || !items.every((item) => item && typeof item === "object" && !Array.isArray(item))) {
|
|
throw new Error("PREVIEW_CHECK_JSON_PAYLOAD_FILE must contain a non-empty object array or an object with items[]");
|
|
}
|
|
return items;
|
|
}
|
|
|
|
function valueAtPath(payload, fieldPath) {
|
|
let current = payload;
|
|
for (const segment of fieldPath.split(".").filter(Boolean)) {
|
|
if (current === null || typeof current !== "object" || !(segment in current)) {
|
|
throw new Error(`Field path ${fieldPath} is missing in PREVIEW_CHECK_JSON_PAYLOAD_FILE`);
|
|
}
|
|
current = current[segment];
|
|
}
|
|
if (current === undefined) {
|
|
throw new Error(`Field path ${fieldPath} resolved to undefined`);
|
|
}
|
|
return current;
|
|
}
|
|
|
|
function jsonEqual(left, right) {
|
|
return JSON.stringify(left) === JSON.stringify(right);
|
|
}
|
|
|
|
function expectedCsvRowCount(text) {
|
|
const rows = text
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
if (rows.length < 2) {
|
|
throw new Error("PREVIEW_CHECK_CSV_PAYLOAD_FILE must include a header and at least one data row");
|
|
}
|
|
return rows.length - 1;
|
|
}
|
|
|
|
function firstCsvDataToken(text) {
|
|
const rows = text
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
if (rows.length >= 2) {
|
|
return firstVisibleToken(rows.slice(1).join("\n"));
|
|
}
|
|
return firstVisibleToken(text);
|
|
}
|
|
|
|
function log(step) {
|
|
process.stdout.write(`[preview-check] ${step}\n`);
|
|
}
|
|
|
|
function assert(condition, message) {
|
|
if (!condition) {
|
|
throw new Error(message);
|
|
}
|
|
}
|
|
|
|
async function api(path, init = {}) {
|
|
return apiClient.request(path, init);
|
|
}
|
|
|
|
async function poll(fn, predicate, label, timeoutMs = 180000, intervalMs = 2000) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastValue;
|
|
while (Date.now() < deadline) {
|
|
lastValue = await fn();
|
|
if (predicate(lastValue)) {
|
|
return lastValue;
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
}
|
|
throw new Error(`Timed out waiting for ${label}. Last value: ${JSON.stringify(lastValue)}`);
|
|
}
|
|
|
|
async function gotoWithRetry(page, url) {
|
|
let lastError;
|
|
for (let attempt = 1; attempt <= 6; attempt += 1) {
|
|
try {
|
|
await page.goto(url, { waitUntil: "networkidle" });
|
|
return;
|
|
} catch (error) {
|
|
lastError = error;
|
|
if (attempt === 6) {
|
|
break;
|
|
}
|
|
await page.waitForTimeout(Math.min(attempt, 3) * 1000);
|
|
}
|
|
}
|
|
throw lastError;
|
|
}
|
|
|
|
async function cleanupTestData() {
|
|
log("cleanup previous PWX test data");
|
|
const projects = (await api("/api/projects")).items;
|
|
for (const project of projects) {
|
|
if (String(project.name || "").startsWith("PWX ")) {
|
|
await api(`/api/projects/${encodeURIComponent(project.id)}`, { method: "DELETE" });
|
|
}
|
|
}
|
|
}
|
|
|
|
async function ensureUIReady(page) {
|
|
await gotoWithRetry(page, baseURL);
|
|
const bootstrapVisible = await page
|
|
.getByText("新增真实平台用户")
|
|
.isVisible({ timeout: 5000 })
|
|
.catch(() => false);
|
|
if (bootstrapVisible) {
|
|
await page.getByPlaceholder("输入真实姓名或账号名").fill(names.user);
|
|
await page.getByPlaceholder("name@example.com").fill(names.email);
|
|
await page.locator("select").first().selectOption("admin");
|
|
await page.getByRole("button", { name: "创建并进入系统" }).click();
|
|
}
|
|
await page.getByRole("heading", { name: "Drobodata 仪表盘" }).waitFor();
|
|
}
|
|
|
|
async function createProject() {
|
|
return api("/api/projects", {
|
|
method: "POST",
|
|
body: JSON.stringify({
|
|
name: names.project,
|
|
description: "Created by preview/operator Playwright check",
|
|
}),
|
|
});
|
|
}
|
|
|
|
async function createWorkflow(projectId, name, nodes, edges) {
|
|
return api(`/api/projects/${projectId}/workflows`, {
|
|
method: "POST",
|
|
body: JSON.stringify({
|
|
name,
|
|
description: `Created by ${name}`,
|
|
type: "batch",
|
|
nodes,
|
|
edges,
|
|
}),
|
|
});
|
|
}
|
|
|
|
async function runWorkflow(workflowId) {
|
|
const queued = await api(`/api/workflows/${workflowId}/run`, {
|
|
method: "POST",
|
|
body: JSON.stringify({ trigger: "manual" }),
|
|
});
|
|
const finished = await poll(
|
|
async () => api(`/api/runs/${queued.id}`),
|
|
(run) => ["completed", "failed", "cancelled", "waiting_for_audit"].includes(run.status),
|
|
`run ${queued.id} to finish`,
|
|
workflowRunTimeoutMs,
|
|
3000,
|
|
);
|
|
assert(
|
|
finished.status === "completed",
|
|
`Run ${queued.id} finished with ${finished.status}: ${finished.error || "unknown error"}`,
|
|
);
|
|
return finished;
|
|
}
|
|
|
|
async function waitForArtifacts(runId, requiredNames) {
|
|
const required = [...new Set(requiredNames)];
|
|
const result = await poll(
|
|
async () => {
|
|
const [run, artifactPayload] = await Promise.all([
|
|
api(`/api/runs/${runId}`),
|
|
api(`/api/runs/${runId}/artifacts`),
|
|
]);
|
|
const items = Array.isArray(artifactPayload.items) ? artifactPayload.items : [];
|
|
const names = new Set(items.map((artifact) => artifact.name));
|
|
const missing = required.filter((name) => !names.has(name));
|
|
return { run, items, missing };
|
|
},
|
|
({ run, missing }) => missing.length === 0 || ["failed", "cancelled"].includes(run.status),
|
|
`artifacts ${required.join(", ")} for run ${runId}`,
|
|
workflowRunTimeoutMs,
|
|
3000,
|
|
);
|
|
assert(
|
|
result.missing.length === 0,
|
|
`Run ${runId} missing artifacts ${result.missing.join(", ")}; status ${result.run.status}: ${result.run.error || "unknown error"}`,
|
|
);
|
|
return result.items;
|
|
}
|
|
|
|
async function createWorkflows(projectId) {
|
|
const previewWorkflow = await createWorkflow(
|
|
projectId,
|
|
names.previewWorkflow,
|
|
[
|
|
{
|
|
id: "csv-source",
|
|
type: "operator",
|
|
data: {
|
|
label: "真实数据源",
|
|
category: "IO",
|
|
description: "Materialize supplied CSV preview payload without shell execution",
|
|
inputs: [],
|
|
outputs: [
|
|
{ id: "manifest", name: "manifest", type: "json" },
|
|
{ id: "csv", name: "csv", type: "file" },
|
|
],
|
|
hyperparams: [
|
|
{ key: "SOURCE_TEXT", value: previewCsvPayload.text },
|
|
{ key: "OUTPUT_NAME", value: "report.csv" },
|
|
{ key: "PAYLOAD_OUTPUT_ID", value: "csv" },
|
|
],
|
|
},
|
|
position: { x: 0, y: -80 },
|
|
},
|
|
{
|
|
id: "notes-source",
|
|
type: "operator",
|
|
data: {
|
|
label: "真实数据源",
|
|
category: "IO",
|
|
description: "Materialize supplied text preview payload without shell execution",
|
|
inputs: [],
|
|
outputs: [
|
|
{ id: "manifest", name: "manifest", type: "json" },
|
|
{ id: "notes", name: "notes", type: "file" },
|
|
],
|
|
hyperparams: [
|
|
{ key: "SOURCE_TEXT", value: previewTextPayload.text },
|
|
{ key: "OUTPUT_NAME", value: "notes.txt" },
|
|
{ key: "PAYLOAD_OUTPUT_ID", value: "notes" },
|
|
],
|
|
},
|
|
position: { x: 0, y: 80 },
|
|
},
|
|
],
|
|
[],
|
|
);
|
|
|
|
const dataOpsWorkflow = await createWorkflow(
|
|
projectId,
|
|
names.dataOpsWorkflow,
|
|
[
|
|
{
|
|
id: "generate",
|
|
type: "operator",
|
|
data: {
|
|
label: "真实数据源",
|
|
category: "IO",
|
|
description: "Materialize supplied JSON payload without shell execution",
|
|
inputs: [],
|
|
outputs: [
|
|
{ id: "manifest", name: "manifest", type: "json" },
|
|
{ id: "payload", name: "payload", type: "json" },
|
|
],
|
|
hyperparams: [
|
|
{ key: "SOURCE_JSON", value: JSON.stringify(dataOpsJsonPayload.value) },
|
|
{ key: "OUTPUT_NAME", value: "payload.json" },
|
|
{ key: "PAYLOAD_OUTPUT_ID", value: "payload" },
|
|
],
|
|
},
|
|
position: { x: 0, y: 0 },
|
|
},
|
|
{
|
|
id: "extract",
|
|
type: "operator",
|
|
data: {
|
|
label: "JSON 字段提取",
|
|
category: "DATA",
|
|
description: "Extract configured field",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [{ id: "out-1", name: "extracted", type: "json" }],
|
|
hyperparams: [{ key: "FIELD_PATH", value: dataOpsExtractField }],
|
|
},
|
|
position: { x: 220, y: -160 },
|
|
},
|
|
{
|
|
id: "filter",
|
|
type: "operator",
|
|
data: {
|
|
label: "JSON 过滤",
|
|
category: "DATA",
|
|
description: "Filter configured rows",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [{ id: "out-1", name: "filtered", type: "json" }],
|
|
hyperparams: [
|
|
{ key: "FIELD_PATH", value: dataOpsFilterField },
|
|
{ key: "OPERATOR", value: "eq" },
|
|
{ key: "EXPECTED_JSON", value: JSON.stringify(dataOpsFilterExpected) },
|
|
],
|
|
},
|
|
position: { x: 220, y: -20 },
|
|
},
|
|
{
|
|
id: "map",
|
|
type: "operator",
|
|
data: {
|
|
label: "JSON 映射重命名",
|
|
category: "DATA",
|
|
description: "Rename fields",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [{ id: "out-1", name: "mapped", type: "json" }],
|
|
hyperparams: [{ key: "FIELD_MAP_JSON", value: JSON.stringify(dataOpsFieldMap) }],
|
|
},
|
|
position: { x: 220, y: 120 },
|
|
},
|
|
{
|
|
id: "csv",
|
|
type: "operator",
|
|
data: {
|
|
label: "JSON 转 CSV",
|
|
category: "DATA",
|
|
description: "Export csv",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [{ id: "out-1", name: "csv", type: "file" }],
|
|
},
|
|
position: { x: 460, y: 120 },
|
|
},
|
|
{
|
|
id: "route",
|
|
type: "operator",
|
|
data: {
|
|
label: "条件分流",
|
|
category: "LOGIC",
|
|
description: "Route rows",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [
|
|
{ id: "matched", name: "matched", type: "json" },
|
|
{ id: "unmatched", name: "unmatched", type: "json" },
|
|
],
|
|
hyperparams: [
|
|
{ key: "FIELD_PATH", value: dataOpsFilterField },
|
|
{ key: "OPERATOR", value: "eq" },
|
|
{ key: "EXPECTED_JSON", value: JSON.stringify(dataOpsFilterExpected) },
|
|
],
|
|
},
|
|
position: { x: 220, y: 260 },
|
|
},
|
|
{
|
|
id: "keep-upload",
|
|
type: "operator",
|
|
data: {
|
|
label: "结果上传",
|
|
category: "IO",
|
|
description: "matched sink",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [],
|
|
},
|
|
position: { x: 500, y: 220 },
|
|
},
|
|
{
|
|
id: "drop-upload",
|
|
type: "operator",
|
|
data: {
|
|
label: "结果上传",
|
|
category: "IO",
|
|
description: "unmatched sink",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [],
|
|
},
|
|
position: { x: 500, y: 320 },
|
|
},
|
|
],
|
|
[
|
|
{ id: "generate-extract", source: "generate", sourceHandle: "payload", target: "extract", targetHandle: "in-1" },
|
|
{ id: "generate-filter", source: "generate", sourceHandle: "payload", target: "filter", targetHandle: "in-1" },
|
|
{ id: "generate-map", source: "generate", sourceHandle: "payload", target: "map", targetHandle: "in-1" },
|
|
{ id: "generate-route", source: "generate", sourceHandle: "payload", target: "route", targetHandle: "in-1" },
|
|
{ id: "map-csv", source: "map", sourceHandle: "out-1", target: "csv", targetHandle: "in-1" },
|
|
{ id: "route-keep", source: "route", sourceHandle: "matched", target: "keep-upload", targetHandle: "in-1" },
|
|
{ id: "route-drop", source: "route", sourceHandle: "unmatched", target: "drop-upload", targetHandle: "in-1" },
|
|
],
|
|
);
|
|
|
|
const csvWorkflow = await createWorkflow(
|
|
projectId,
|
|
names.csvWorkflow,
|
|
[
|
|
{
|
|
id: "generate",
|
|
type: "operator",
|
|
data: {
|
|
label: "真实数据源",
|
|
category: "IO",
|
|
description: "Materialize supplied CSV payload without shell execution",
|
|
inputs: [],
|
|
outputs: [
|
|
{ id: "manifest", name: "manifest", type: "json" },
|
|
{ id: "table", name: "table", type: "file" },
|
|
],
|
|
hyperparams: [
|
|
{ key: "SOURCE_TEXT", value: previewCsvPayload.text },
|
|
{ key: "OUTPUT_NAME", value: "table.csv" },
|
|
{ key: "PAYLOAD_OUTPUT_ID", value: "table" },
|
|
],
|
|
},
|
|
position: { x: 0, y: 0 },
|
|
},
|
|
{
|
|
id: "convert",
|
|
type: "operator",
|
|
data: {
|
|
label: "CSV 转 JSON",
|
|
category: "DATA",
|
|
description: "csv to json",
|
|
inputs: [{ id: "in-1", name: "input", type: "json" }],
|
|
outputs: [{ id: "out-1", name: "table", type: "json" }],
|
|
},
|
|
position: { x: 240, y: 0 },
|
|
},
|
|
],
|
|
[{ id: "generate-convert", source: "generate", sourceHandle: "table", target: "convert", targetHandle: "in-1" }],
|
|
);
|
|
|
|
const scanWorkflow = await createWorkflow(
|
|
projectId,
|
|
names.scanWorkflow,
|
|
[
|
|
{
|
|
id: "scan",
|
|
type: "operator",
|
|
data: {
|
|
label: "目录扫描",
|
|
category: "IO",
|
|
description: "directory scan",
|
|
inputs: [],
|
|
outputs: [{ id: "out-1", name: "summary", type: "json" }],
|
|
hyperparams: [{ key: "SOURCE_PATH", value: scanSourcePath }],
|
|
},
|
|
position: { x: 0, y: 0 },
|
|
},
|
|
],
|
|
[],
|
|
);
|
|
|
|
const rosbagWorkflow = await createWorkflow(
|
|
projectId,
|
|
names.rosbagWorkflow,
|
|
[
|
|
{
|
|
id: "summary",
|
|
type: "operator",
|
|
data: {
|
|
label: "ROSBag 数据概览",
|
|
category: "ROBOTICS",
|
|
description: "rosbag summary",
|
|
inputs: [],
|
|
outputs: [{ id: "out-1", name: "summary", type: "json" }],
|
|
hyperparams: [{ key: "SOURCE_PATH", value: rosbagSourcePath }],
|
|
},
|
|
position: { x: 0, y: 0 },
|
|
},
|
|
],
|
|
[],
|
|
);
|
|
|
|
const lerobotWorkflow = await createWorkflow(
|
|
projectId,
|
|
names.lerobotWorkflow,
|
|
[
|
|
{
|
|
id: "summary",
|
|
type: "operator",
|
|
data: {
|
|
label: "LeRobot 数据集概览",
|
|
category: "ROBOTICS",
|
|
description: "lerobot summary",
|
|
inputs: [],
|
|
outputs: [{ id: "out-1", name: "summary", type: "json" }],
|
|
hyperparams: [{ key: "SOURCE_PATH", value: lerobotSourcePath }],
|
|
},
|
|
position: { x: 0, y: 0 },
|
|
},
|
|
],
|
|
[],
|
|
);
|
|
|
|
return {
|
|
previewWorkflow,
|
|
dataOpsWorkflow,
|
|
csvWorkflow,
|
|
scanWorkflow,
|
|
rosbagWorkflow,
|
|
lerobotWorkflow,
|
|
};
|
|
}
|
|
|
|
function artifactByName(items, name) {
|
|
const item = items.find((artifact) => artifact.name === name);
|
|
assert(item, `Missing artifact ${name}`);
|
|
return item;
|
|
}
|
|
|
|
function artifactItemCount(artifact) {
|
|
const data = artifact?.data || {};
|
|
const dataRef = data.dataRef || {};
|
|
return data.itemsCount ?? data.count ?? data.rowCount ?? dataRef.itemCount;
|
|
}
|
|
|
|
function artifactPayloadPath(artifact) {
|
|
return artifact?.data?.dataRef?.payloadPath || artifact?.path || "";
|
|
}
|
|
|
|
function readJSONArtifactPayload(artifact) {
|
|
const payloadPath = artifactPayloadPath(artifact);
|
|
if (!k8sPayloadNamespace || !payloadPath.startsWith("/workspace/")) {
|
|
return artifact?.data || {};
|
|
}
|
|
try {
|
|
return JSON.parse(
|
|
execFileSync(
|
|
"kubectl",
|
|
["-n", k8sPayloadNamespace, "exec", k8sPayloadReadTarget, "--", "cat", payloadPath],
|
|
{ encoding: "utf8", maxBuffer: maxPayloadBytes },
|
|
),
|
|
);
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
throw new Error(`Failed to read JSON artifact payload ${artifact?.name || ""} from ${payloadPath}: ${message}`);
|
|
}
|
|
}
|
|
|
|
async function openArtifactModal(page, workflowName) {
|
|
await page.getByRole("link", { name: "执行监控" }).click();
|
|
await page.getByRole("heading", { name: "执行监控" }).waitFor();
|
|
const search = page.getByPlaceholder("搜索运行 ID、工作流或项目...");
|
|
await search.fill(workflowName);
|
|
const row = page.locator("tr").filter({ hasText: workflowName }).first();
|
|
await row.waitFor({ state: "visible" });
|
|
await row.locator('button[title="查看数据"]').click();
|
|
await page.getByText("产出数据").waitFor();
|
|
}
|
|
|
|
async function closeArtifactModal(page) {
|
|
await page.getByRole("button", { name: "关闭" }).click();
|
|
await page.getByRole("button", { name: "关闭" }).waitFor({ state: "hidden" });
|
|
}
|
|
|
|
async function main() {
|
|
await cleanupTestData();
|
|
|
|
log("verify operator catalog");
|
|
const operators = (await api("/api/operators")).items;
|
|
for (const name of requiredOperators) {
|
|
assert(
|
|
operators.some((item) => item.name === name),
|
|
`Missing built-in operator ${name}`,
|
|
);
|
|
}
|
|
|
|
log("create workflows");
|
|
const project = await createProject();
|
|
const workflows = await createWorkflows(project.id);
|
|
|
|
log("run preview workflow");
|
|
const previewRun = await runWorkflow(workflows.previewWorkflow.id);
|
|
const previewArtifacts = await waitForArtifacts(previewRun.id, ["report.csv", "notes.txt"]);
|
|
assert(
|
|
artifactByName(previewArtifacts, "report.csv").data.preview.kind === "table",
|
|
"report.csv preview should be table",
|
|
);
|
|
assert(
|
|
artifactByName(previewArtifacts, "notes.txt").data.preview.kind === "text",
|
|
"notes.txt preview should be text",
|
|
);
|
|
|
|
log("run data ops workflow");
|
|
const dataOpsRun = await runWorkflow(workflows.dataOpsWorkflow.id);
|
|
const dataOpsArtifacts = await waitForArtifacts(dataOpsRun.id, [
|
|
"extracted.json",
|
|
"filtered.json",
|
|
"mapped.json",
|
|
"rows.csv",
|
|
"matched.json",
|
|
"unmatched.json",
|
|
]);
|
|
assert(
|
|
jsonEqual(artifactByName(dataOpsArtifacts, "extracted.json").data.value, dataOpsExtractExpected),
|
|
`JSON extract should return value from ${dataOpsExtractField}`,
|
|
);
|
|
assert(
|
|
artifactByName(dataOpsArtifacts, "filtered.json").data.count === dataOpsFilterExpectedCount,
|
|
"JSON filter should keep the expected number of rows from supplied payload",
|
|
);
|
|
const mappedArtifact = artifactByName(dataOpsArtifacts, "mapped.json");
|
|
assert(artifactItemCount(mappedArtifact) === dataOpsItems.length, "JSON map should preserve row count");
|
|
const csvRowsArtifact = artifactByName(dataOpsArtifacts, "rows.csv");
|
|
const csvPreview = csvRowsArtifact.data.preview;
|
|
for (const [sourceField, destinationField] of Object.entries(dataOpsFieldMap)) {
|
|
assert(
|
|
csvPreview.columns.includes(destinationField),
|
|
`JSON map should expose renamed field ${destinationField} in downstream CSV preview`,
|
|
);
|
|
assert(
|
|
String(csvPreview.rows[0]?.[destinationField]) === String(dataOpsItems[0]?.[sourceField]),
|
|
`JSON map should rename ${sourceField} to ${destinationField}`,
|
|
);
|
|
}
|
|
assert(
|
|
csvPreview.kind === "table",
|
|
"JSON to CSV should produce previewable csv",
|
|
);
|
|
assert(artifactByName(dataOpsArtifacts, "matched.json").portId === "matched", "matched artifact should carry portId");
|
|
assert(
|
|
artifactByName(dataOpsArtifacts, "unmatched.json").portId === "unmatched",
|
|
"unmatched artifact should carry portId",
|
|
);
|
|
|
|
log("run csv conversion workflow");
|
|
const csvRun = await runWorkflow(workflows.csvWorkflow.id);
|
|
const csvArtifacts = await waitForArtifacts(csvRun.id, ["table.json"]);
|
|
assert(
|
|
artifactItemCount(artifactByName(csvArtifacts, "table.json")) === expectedCsvRowCount(previewCsvPayload.text),
|
|
"CSV to JSON should convert the supplied CSV row count",
|
|
);
|
|
|
|
log("run directory scan workflow");
|
|
const scanRun = await runWorkflow(workflows.scanWorkflow.id);
|
|
const scanArtifacts = await waitForArtifacts(scanRun.id, ["directory-scan.json"]);
|
|
assert(
|
|
artifactByName(scanArtifacts, "directory-scan.json").data.summary.fileCount >= scanMinFileCount,
|
|
"Directory scan should report the minimum file count for the supplied source path",
|
|
);
|
|
|
|
log("run rosbag workflow");
|
|
const rosbagRun = await runWorkflow(workflows.rosbagWorkflow.id);
|
|
const rosbagArtifacts = await waitForArtifacts(rosbagRun.id, ["rosbag-summary.json"]);
|
|
const rosbagPayload = readJSONArtifactPayload(artifactByName(rosbagArtifacts, "rosbag-summary.json"));
|
|
const rosbagPreview = rosbagPayload.databasePreview || rosbagPayload.preview || {};
|
|
assert(
|
|
rosbagPayload.datasetType === "rosbag2" || rosbagPreview.format === "rosbag2",
|
|
"ROSBag summary should detect rosbag2 sqlite",
|
|
);
|
|
assert(Array.isArray(rosbagPreview.topics) && rosbagPreview.topics.length > 0, "ROSBag summary should include topics");
|
|
if (rosbagExpectedTopic) {
|
|
assert(
|
|
rosbagPreview.topics.some((topic) => topic.name === rosbagExpectedTopic),
|
|
`ROSBag summary should include configured topic ${rosbagExpectedTopic}`,
|
|
);
|
|
}
|
|
|
|
log("run lerobot workflow");
|
|
const lerobotRun = await runWorkflow(workflows.lerobotWorkflow.id);
|
|
const lerobotArtifacts = await waitForArtifacts(lerobotRun.id, ["lerobot-summary.json"]);
|
|
const lerobotPayload = readJSONArtifactPayload(artifactByName(lerobotArtifacts, "lerobot-summary.json"));
|
|
const lerobotPreview = lerobotPayload.preview || {};
|
|
assert(lerobotPreview.format === "lerobot", "LeRobot summary should detect dataset");
|
|
assert(lerobotPreview.info && Object.keys(lerobotPreview.info).length > 0, "LeRobot summary should expose info.json");
|
|
if (lerobotExpectedRobotType) {
|
|
assert(
|
|
lerobotPreview.info.robot_type === lerobotExpectedRobotType,
|
|
`LeRobot summary should expose robot_type ${lerobotExpectedRobotType}`,
|
|
);
|
|
}
|
|
|
|
log("open browser and verify UI previews");
|
|
const browser = await chromium.launch({ headless: true });
|
|
const context = await browser.newContext({
|
|
extraHTTPHeaders: authenticationHeaders(apiToken),
|
|
viewport: { width: 1440, height: 1100 },
|
|
});
|
|
const page = await context.newPage();
|
|
try {
|
|
await ensureUIReady(page);
|
|
|
|
await page.getByRole("link", { name: "算子管理" }).click();
|
|
await page.getByRole("heading", { name: "算子管理" }).waitFor();
|
|
await page.getByPlaceholder("搜索算子...").fill("ROSBag 数据概览");
|
|
await page.getByText("ROSBag 数据概览").waitFor();
|
|
await page.getByPlaceholder("搜索算子...").fill("LeRobot 数据集概览");
|
|
await page.getByText("LeRobot 数据集概览").waitFor();
|
|
|
|
await openArtifactModal(page, names.previewWorkflow);
|
|
await page.getByText("report.csv").first().click();
|
|
await page.getByText(firstCsvDataToken(previewCsvPayload.text)).first().waitFor();
|
|
await page.getByText("notes.txt").first().click();
|
|
await page.getByText(previewTextPayload.previewNeedle).waitFor();
|
|
await closeArtifactModal(page);
|
|
|
|
await openArtifactModal(page, names.rosbagWorkflow);
|
|
await page.getByText("rosbag-summary.json").first().click();
|
|
await page.locator("pre").filter({ hasText: '"datasetType": "rosbag2"' }).first().waitFor();
|
|
await closeArtifactModal(page);
|
|
|
|
await openArtifactModal(page, names.lerobotWorkflow);
|
|
await page.getByText("lerobot-summary.json").first().click();
|
|
if (lerobotExpectedRobotType) {
|
|
await page.locator("pre").filter({ hasText: JSON.stringify(lerobotExpectedRobotType) }).first().waitFor();
|
|
}
|
|
await page.locator("pre").filter({ hasText: '"format": "lerobot"' }).first().waitFor();
|
|
await closeArtifactModal(page);
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
|
|
log("preview and operator check completed");
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exitCode = 1;
|
|
});
|