droboticflow-new/scripts/playwright-preview-operators-check.mjs
2026-06-23 01:43:37 +08:00

890 lines
30 KiB
JavaScript

import fs from "node:fs";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { chromium } from "playwright";
import { authenticationHeaders, createAcceptanceAPIClient, positiveIntegerEnv } from "./lib/acceptance-client.mjs";
import { apiBaseURLFromEnv } from "./lib/service-endpoints.mjs";
const baseURL = apiBaseURLFromEnv({ env: process.env, cliArgs: process.argv });
const apiToken = (process.env.DROBOTICFLOW_API_TOKEN || "").trim();
const apiClient = createAcceptanceAPIClient({ baseURL, apiToken });
const suffix = Date.now().toString(36);
const prefix = `PWX ${suffix}`;
const maxPayloadBytes = 1024 * 1024;
const previewCsvPayload = readRequiredPayloadFile(
"PREVIEW_CHECK_CSV_PAYLOAD_FILE",
"CSV payload for report preview and CSV conversion checks",
);
const previewTextPayload = readRequiredPayloadFile(
"PREVIEW_CHECK_TEXT_PAYLOAD_FILE",
"text payload for artifact preview checks",
);
const dataOpsJsonPayload = readRequiredJSONPayload("PREVIEW_CHECK_JSON_PAYLOAD_FILE");
const dataOpsExtractField = requiredEnv("PREVIEW_CHECK_JSON_EXTRACT_FIELD");
const dataOpsFilterField = requiredEnv("PREVIEW_CHECK_JSON_FILTER_FIELD");
const dataOpsItems = dataOpsPayloadItems(dataOpsJsonPayload.value);
const dataOpsFieldMap = parseRequiredFieldMapEnv("PREVIEW_CHECK_JSON_FIELD_MAP_JSON", dataOpsItems);
const dataOpsFilterExpected = process.env.PREVIEW_CHECK_JSON_FILTER_EXPECTED_JSON
? parseJSONEnv("PREVIEW_CHECK_JSON_FILTER_EXPECTED_JSON")
: valueAtPath(dataOpsItems[0], dataOpsFilterField);
const dataOpsExtractExpected = valueAtPath(dataOpsJsonPayload.value, dataOpsExtractField);
const dataOpsFilterExpectedCount = dataOpsItems.filter((item) =>
jsonEqual(valueAtPath(item, dataOpsFilterField), dataOpsFilterExpected),
).length;
const scanSourcePath = requiredEnv("PREVIEW_CHECK_SCAN_SOURCE_PATH");
const rosbagSourcePath = requiredEnv("PREVIEW_CHECK_ROSBAG_SOURCE_PATH");
const lerobotSourcePath = requiredEnv("PREVIEW_CHECK_LEROBOT_SOURCE_PATH");
const scanMinFileCount = positiveIntegerEnv("PREVIEW_CHECK_SCAN_MIN_FILES", 1);
const workflowRunTimeoutMs = positiveIntegerEnv("PREVIEW_CHECK_RUN_TIMEOUT_SECONDS", 600) * 1000;
const k8sPayloadNamespace = (process.env.PREVIEW_CHECK_K8S_NAMESPACE || "").trim();
const k8sPayloadReadTarget = (process.env.PREVIEW_CHECK_K8S_PAYLOAD_READ_TARGET || "deployment/droboticflow-api").trim();
const rosbagExpectedTopic = (process.env.PREVIEW_CHECK_ROSBAG_TOPIC || "").trim();
const lerobotExpectedRobotType = (process.env.PREVIEW_CHECK_LEROBOT_ROBOT_TYPE || "").trim();
const names = {
user: `PWX Admin ${suffix}`,
email: `pwx-admin-${suffix}@playwright.local`,
project: `${prefix} Project`,
previewWorkflow: `${prefix} Preview`,
dataOpsWorkflow: `${prefix} DataOps`,
csvWorkflow: `${prefix} CSV`,
scanWorkflow: `${prefix} Scan`,
rosbagWorkflow: `${prefix} ROSBag`,
lerobotWorkflow: `${prefix} LeRobot`,
};
const requiredOperators = [
"目录扫描",
"CSV 转 JSON",
"JSON 转 CSV",
"JSON 字段提取",
"JSON 过滤",
"JSON 映射重命名",
"文本模板渲染",
"条件分流",
"LeRobot 数据集概览",
"ROSBag 数据概览",
"真实数据源",
];
function requiredEnv(name) {
const value = (process.env[name] || "").trim();
if (!value) {
throw new Error(`${name} is required for preview/operator acceptance`);
}
return value;
}
function readRequiredPayloadFile(envName, description) {
const filePath = requiredEnv(envName);
const resolvedPath = path.resolve(filePath);
let stat;
try {
stat = fs.statSync(resolvedPath);
} catch (error) {
throw new Error(`${envName} is not readable: ${resolvedPath} (${error instanceof Error ? error.message : error})`);
}
if (!stat.isFile()) {
throw new Error(`${envName} must point to a file: ${resolvedPath}`);
}
if (stat.size === 0) {
throw new Error(`${envName} must not be empty: ${resolvedPath}`);
}
if (stat.size > maxPayloadBytes) {
throw new Error(`${envName} is ${stat.size} bytes; keep ${description} at or below ${maxPayloadBytes} bytes`);
}
const content = fs.readFileSync(resolvedPath);
const text = content.toString("utf8");
const previewNeedle = firstVisibleToken(text);
if (!previewNeedle) {
throw new Error(`${envName} must contain visible UTF-8 text: ${resolvedPath}`);
}
return {
path: resolvedPath,
bytes: content,
text,
previewNeedle,
};
}
function readRequiredJSONPayload(envName) {
const payload = readRequiredPayloadFile(envName, "JSON payload for data operator checks");
let value;
try {
value = JSON.parse(payload.text);
} catch (error) {
throw new Error(`${envName} must contain valid JSON: ${error instanceof Error ? error.message : error}`);
}
return { ...payload, value };
}
function parseJSONEnv(name) {
try {
return JSON.parse(process.env[name] || "");
} catch (error) {
throw new Error(`${name} must contain valid JSON: ${error instanceof Error ? error.message : error}`);
}
}
function parseRequiredFieldMapEnv(name, rows) {
const parsed = parseJSONEnv(name);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || Object.keys(parsed).length === 0) {
throw new Error(`${name} must be a non-empty JSON object`);
}
const normalized = {};
const destinations = new Set();
for (const [source, destination] of Object.entries(parsed)) {
const sourceField = source.trim();
if (!sourceField) {
throw new Error(`${name} source field names must be non-empty`);
}
if (sourceField !== source) {
throw new Error(`${name} source field ${JSON.stringify(source)} must not have leading or trailing whitespace`);
}
if (typeof destination !== "string") {
throw new Error(`${name}.${sourceField} destination field must be a string`);
}
const destinationField = destination.trim();
if (!destinationField) {
throw new Error(`${name}.${sourceField} destination field must be non-empty`);
}
if (destinationField !== destination) {
throw new Error(`${name}.${sourceField} destination field must not have leading or trailing whitespace`);
}
if (destinations.has(destinationField)) {
throw new Error(`${name} maps multiple source fields to destination field ${JSON.stringify(destinationField)}`);
}
destinations.add(destinationField);
normalized[sourceField] = destinationField;
}
validateFieldMapCollisions(name, normalized, rows);
return normalized;
}
function validateFieldMapCollisions(name, fieldMap, rows) {
for (const [rowIndex, row] of rows.entries()) {
const mappedSources = new Map();
for (const sourceField of Object.keys(row)) {
const outputField = fieldMap[sourceField] || sourceField;
const previousSource = mappedSources.get(outputField);
if (previousSource) {
throw new Error(
`${name} maps row ${rowIndex} source fields ${JSON.stringify(previousSource)} and ${JSON.stringify(sourceField)} to the same output field ${JSON.stringify(outputField)}`,
);
}
mappedSources.set(outputField, sourceField);
}
}
}
function firstVisibleToken(text) {
for (const line of text.split(/\r?\n/)) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
const csvCell = trimmed.split(/[,\t]/).map((item) => item.trim()).find(Boolean);
return (csvCell || trimmed).slice(0, 120);
}
return "";
}
function dataOpsPayloadItems(payload) {
const items = Array.isArray(payload) ? payload : payload?.items;
if (!Array.isArray(items) || items.length === 0 || !items.every((item) => item && typeof item === "object" && !Array.isArray(item))) {
throw new Error("PREVIEW_CHECK_JSON_PAYLOAD_FILE must contain a non-empty object array or an object with items[]");
}
return items;
}
function valueAtPath(payload, fieldPath) {
let current = payload;
for (const segment of fieldPath.split(".").filter(Boolean)) {
if (current === null || typeof current !== "object" || !(segment in current)) {
throw new Error(`Field path ${fieldPath} is missing in PREVIEW_CHECK_JSON_PAYLOAD_FILE`);
}
current = current[segment];
}
if (current === undefined) {
throw new Error(`Field path ${fieldPath} resolved to undefined`);
}
return current;
}
function jsonEqual(left, right) {
return JSON.stringify(left) === JSON.stringify(right);
}
function expectedCsvRowCount(text) {
const rows = text
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
if (rows.length < 2) {
throw new Error("PREVIEW_CHECK_CSV_PAYLOAD_FILE must include a header and at least one data row");
}
return rows.length - 1;
}
function firstCsvDataToken(text) {
const rows = text
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
if (rows.length >= 2) {
return firstVisibleToken(rows.slice(1).join("\n"));
}
return firstVisibleToken(text);
}
function log(step) {
process.stdout.write(`[preview-check] ${step}\n`);
}
function assert(condition, message) {
if (!condition) {
throw new Error(message);
}
}
async function api(path, init = {}) {
return apiClient.request(path, init);
}
async function poll(fn, predicate, label, timeoutMs = 180000, intervalMs = 2000) {
const deadline = Date.now() + timeoutMs;
let lastValue;
while (Date.now() < deadline) {
lastValue = await fn();
if (predicate(lastValue)) {
return lastValue;
}
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
throw new Error(`Timed out waiting for ${label}. Last value: ${JSON.stringify(lastValue)}`);
}
async function gotoWithRetry(page, url) {
let lastError;
for (let attempt = 1; attempt <= 6; attempt += 1) {
try {
await page.goto(url, { waitUntil: "networkidle" });
return;
} catch (error) {
lastError = error;
if (attempt === 6) {
break;
}
await page.waitForTimeout(Math.min(attempt, 3) * 1000);
}
}
throw lastError;
}
async function cleanupTestData() {
log("cleanup previous PWX test data");
const projects = (await api("/api/projects")).items;
for (const project of projects) {
if (String(project.name || "").startsWith("PWX ")) {
await api(`/api/projects/${encodeURIComponent(project.id)}`, { method: "DELETE" });
}
}
}
async function ensureUIReady(page) {
await gotoWithRetry(page, baseURL);
const bootstrapVisible = await page
.getByText("新增真实平台用户")
.isVisible({ timeout: 5000 })
.catch(() => false);
if (bootstrapVisible) {
await page.getByPlaceholder("输入真实姓名或账号名").fill(names.user);
await page.getByPlaceholder("name@example.com").fill(names.email);
await page.locator("select").first().selectOption("admin");
await page.getByRole("button", { name: "创建并进入系统" }).click();
}
await page.getByRole("heading", { name: "Drobodata 仪表盘" }).waitFor();
}
async function createProject() {
return api("/api/projects", {
method: "POST",
body: JSON.stringify({
name: names.project,
description: "Created by preview/operator Playwright check",
}),
});
}
async function createWorkflow(projectId, name, nodes, edges) {
return api(`/api/projects/${projectId}/workflows`, {
method: "POST",
body: JSON.stringify({
name,
description: `Created by ${name}`,
type: "batch",
nodes,
edges,
}),
});
}
async function runWorkflow(workflowId) {
const queued = await api(`/api/workflows/${workflowId}/run`, {
method: "POST",
body: JSON.stringify({ trigger: "manual" }),
});
const finished = await poll(
async () => api(`/api/runs/${queued.id}`),
(run) => ["completed", "failed", "cancelled", "waiting_for_audit"].includes(run.status),
`run ${queued.id} to finish`,
workflowRunTimeoutMs,
3000,
);
assert(
finished.status === "completed",
`Run ${queued.id} finished with ${finished.status}: ${finished.error || "unknown error"}`,
);
return finished;
}
async function waitForArtifacts(runId, requiredNames) {
const required = [...new Set(requiredNames)];
const result = await poll(
async () => {
const [run, artifactPayload] = await Promise.all([
api(`/api/runs/${runId}`),
api(`/api/runs/${runId}/artifacts`),
]);
const items = Array.isArray(artifactPayload.items) ? artifactPayload.items : [];
const names = new Set(items.map((artifact) => artifact.name));
const missing = required.filter((name) => !names.has(name));
return { run, items, missing };
},
({ run, missing }) => missing.length === 0 || ["failed", "cancelled"].includes(run.status),
`artifacts ${required.join(", ")} for run ${runId}`,
workflowRunTimeoutMs,
3000,
);
assert(
result.missing.length === 0,
`Run ${runId} missing artifacts ${result.missing.join(", ")}; status ${result.run.status}: ${result.run.error || "unknown error"}`,
);
return result.items;
}
async function createWorkflows(projectId) {
const previewWorkflow = await createWorkflow(
projectId,
names.previewWorkflow,
[
{
id: "csv-source",
type: "operator",
data: {
label: "真实数据源",
category: "IO",
description: "Materialize supplied CSV preview payload without shell execution",
inputs: [],
outputs: [
{ id: "manifest", name: "manifest", type: "json" },
{ id: "csv", name: "csv", type: "file" },
],
hyperparams: [
{ key: "SOURCE_TEXT", value: previewCsvPayload.text },
{ key: "OUTPUT_NAME", value: "report.csv" },
{ key: "PAYLOAD_OUTPUT_ID", value: "csv" },
],
},
position: { x: 0, y: -80 },
},
{
id: "notes-source",
type: "operator",
data: {
label: "真实数据源",
category: "IO",
description: "Materialize supplied text preview payload without shell execution",
inputs: [],
outputs: [
{ id: "manifest", name: "manifest", type: "json" },
{ id: "notes", name: "notes", type: "file" },
],
hyperparams: [
{ key: "SOURCE_TEXT", value: previewTextPayload.text },
{ key: "OUTPUT_NAME", value: "notes.txt" },
{ key: "PAYLOAD_OUTPUT_ID", value: "notes" },
],
},
position: { x: 0, y: 80 },
},
],
[],
);
const dataOpsWorkflow = await createWorkflow(
projectId,
names.dataOpsWorkflow,
[
{
id: "generate",
type: "operator",
data: {
label: "真实数据源",
category: "IO",
description: "Materialize supplied JSON payload without shell execution",
inputs: [],
outputs: [
{ id: "manifest", name: "manifest", type: "json" },
{ id: "payload", name: "payload", type: "json" },
],
hyperparams: [
{ key: "SOURCE_JSON", value: JSON.stringify(dataOpsJsonPayload.value) },
{ key: "OUTPUT_NAME", value: "payload.json" },
{ key: "PAYLOAD_OUTPUT_ID", value: "payload" },
],
},
position: { x: 0, y: 0 },
},
{
id: "extract",
type: "operator",
data: {
label: "JSON 字段提取",
category: "DATA",
description: "Extract configured field",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [{ id: "out-1", name: "extracted", type: "json" }],
hyperparams: [{ key: "FIELD_PATH", value: dataOpsExtractField }],
},
position: { x: 220, y: -160 },
},
{
id: "filter",
type: "operator",
data: {
label: "JSON 过滤",
category: "DATA",
description: "Filter configured rows",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [{ id: "out-1", name: "filtered", type: "json" }],
hyperparams: [
{ key: "FIELD_PATH", value: dataOpsFilterField },
{ key: "OPERATOR", value: "eq" },
{ key: "EXPECTED_JSON", value: JSON.stringify(dataOpsFilterExpected) },
],
},
position: { x: 220, y: -20 },
},
{
id: "map",
type: "operator",
data: {
label: "JSON 映射重命名",
category: "DATA",
description: "Rename fields",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [{ id: "out-1", name: "mapped", type: "json" }],
hyperparams: [{ key: "FIELD_MAP_JSON", value: JSON.stringify(dataOpsFieldMap) }],
},
position: { x: 220, y: 120 },
},
{
id: "csv",
type: "operator",
data: {
label: "JSON 转 CSV",
category: "DATA",
description: "Export csv",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [{ id: "out-1", name: "csv", type: "file" }],
},
position: { x: 460, y: 120 },
},
{
id: "route",
type: "operator",
data: {
label: "条件分流",
category: "LOGIC",
description: "Route rows",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [
{ id: "matched", name: "matched", type: "json" },
{ id: "unmatched", name: "unmatched", type: "json" },
],
hyperparams: [
{ key: "FIELD_PATH", value: dataOpsFilterField },
{ key: "OPERATOR", value: "eq" },
{ key: "EXPECTED_JSON", value: JSON.stringify(dataOpsFilterExpected) },
],
},
position: { x: 220, y: 260 },
},
{
id: "keep-upload",
type: "operator",
data: {
label: "结果上传",
category: "IO",
description: "matched sink",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [],
},
position: { x: 500, y: 220 },
},
{
id: "drop-upload",
type: "operator",
data: {
label: "结果上传",
category: "IO",
description: "unmatched sink",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [],
},
position: { x: 500, y: 320 },
},
],
[
{ id: "generate-extract", source: "generate", sourceHandle: "payload", target: "extract", targetHandle: "in-1" },
{ id: "generate-filter", source: "generate", sourceHandle: "payload", target: "filter", targetHandle: "in-1" },
{ id: "generate-map", source: "generate", sourceHandle: "payload", target: "map", targetHandle: "in-1" },
{ id: "generate-route", source: "generate", sourceHandle: "payload", target: "route", targetHandle: "in-1" },
{ id: "map-csv", source: "map", sourceHandle: "out-1", target: "csv", targetHandle: "in-1" },
{ id: "route-keep", source: "route", sourceHandle: "matched", target: "keep-upload", targetHandle: "in-1" },
{ id: "route-drop", source: "route", sourceHandle: "unmatched", target: "drop-upload", targetHandle: "in-1" },
],
);
const csvWorkflow = await createWorkflow(
projectId,
names.csvWorkflow,
[
{
id: "generate",
type: "operator",
data: {
label: "真实数据源",
category: "IO",
description: "Materialize supplied CSV payload without shell execution",
inputs: [],
outputs: [
{ id: "manifest", name: "manifest", type: "json" },
{ id: "table", name: "table", type: "file" },
],
hyperparams: [
{ key: "SOURCE_TEXT", value: previewCsvPayload.text },
{ key: "OUTPUT_NAME", value: "table.csv" },
{ key: "PAYLOAD_OUTPUT_ID", value: "table" },
],
},
position: { x: 0, y: 0 },
},
{
id: "convert",
type: "operator",
data: {
label: "CSV 转 JSON",
category: "DATA",
description: "csv to json",
inputs: [{ id: "in-1", name: "input", type: "json" }],
outputs: [{ id: "out-1", name: "table", type: "json" }],
},
position: { x: 240, y: 0 },
},
],
[{ id: "generate-convert", source: "generate", sourceHandle: "table", target: "convert", targetHandle: "in-1" }],
);
const scanWorkflow = await createWorkflow(
projectId,
names.scanWorkflow,
[
{
id: "scan",
type: "operator",
data: {
label: "目录扫描",
category: "IO",
description: "directory scan",
inputs: [],
outputs: [{ id: "out-1", name: "summary", type: "json" }],
hyperparams: [{ key: "SOURCE_PATH", value: scanSourcePath }],
},
position: { x: 0, y: 0 },
},
],
[],
);
const rosbagWorkflow = await createWorkflow(
projectId,
names.rosbagWorkflow,
[
{
id: "summary",
type: "operator",
data: {
label: "ROSBag 数据概览",
category: "ROBOTICS",
description: "rosbag summary",
inputs: [],
outputs: [{ id: "out-1", name: "summary", type: "json" }],
hyperparams: [{ key: "SOURCE_PATH", value: rosbagSourcePath }],
},
position: { x: 0, y: 0 },
},
],
[],
);
const lerobotWorkflow = await createWorkflow(
projectId,
names.lerobotWorkflow,
[
{
id: "summary",
type: "operator",
data: {
label: "LeRobot 数据集概览",
category: "ROBOTICS",
description: "lerobot summary",
inputs: [],
outputs: [{ id: "out-1", name: "summary", type: "json" }],
hyperparams: [{ key: "SOURCE_PATH", value: lerobotSourcePath }],
},
position: { x: 0, y: 0 },
},
],
[],
);
return {
previewWorkflow,
dataOpsWorkflow,
csvWorkflow,
scanWorkflow,
rosbagWorkflow,
lerobotWorkflow,
};
}
function artifactByName(items, name) {
const item = items.find((artifact) => artifact.name === name);
assert(item, `Missing artifact ${name}`);
return item;
}
function artifactItemCount(artifact) {
const data = artifact?.data || {};
const dataRef = data.dataRef || {};
return data.itemsCount ?? data.count ?? data.rowCount ?? dataRef.itemCount;
}
function artifactPayloadPath(artifact) {
return artifact?.data?.dataRef?.payloadPath || artifact?.path || "";
}
function readJSONArtifactPayload(artifact) {
const payloadPath = artifactPayloadPath(artifact);
if (!k8sPayloadNamespace || !payloadPath.startsWith("/workspace/")) {
return artifact?.data || {};
}
try {
return JSON.parse(
execFileSync(
"kubectl",
["-n", k8sPayloadNamespace, "exec", k8sPayloadReadTarget, "--", "cat", payloadPath],
{ encoding: "utf8", maxBuffer: maxPayloadBytes },
),
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Failed to read JSON artifact payload ${artifact?.name || ""} from ${payloadPath}: ${message}`);
}
}
async function openArtifactModal(page, workflowName) {
await page.getByRole("link", { name: "执行监控" }).click();
await page.getByRole("heading", { name: "执行监控" }).waitFor();
const search = page.getByPlaceholder("搜索运行 ID、工作流或项目...");
await search.fill(workflowName);
const row = page.locator("tr").filter({ hasText: workflowName }).first();
await row.waitFor({ state: "visible" });
await row.locator('button[title="查看数据"]').click();
await page.getByText("产出数据").waitFor();
}
async function closeArtifactModal(page) {
await page.getByRole("button", { name: "关闭" }).click();
await page.getByRole("button", { name: "关闭" }).waitFor({ state: "hidden" });
}
async function main() {
await cleanupTestData();
log("verify operator catalog");
const operators = (await api("/api/operators")).items;
for (const name of requiredOperators) {
assert(
operators.some((item) => item.name === name),
`Missing built-in operator ${name}`,
);
}
log("create workflows");
const project = await createProject();
const workflows = await createWorkflows(project.id);
log("run preview workflow");
const previewRun = await runWorkflow(workflows.previewWorkflow.id);
const previewArtifacts = await waitForArtifacts(previewRun.id, ["report.csv", "notes.txt"]);
assert(
artifactByName(previewArtifacts, "report.csv").data.preview.kind === "table",
"report.csv preview should be table",
);
assert(
artifactByName(previewArtifacts, "notes.txt").data.preview.kind === "text",
"notes.txt preview should be text",
);
log("run data ops workflow");
const dataOpsRun = await runWorkflow(workflows.dataOpsWorkflow.id);
const dataOpsArtifacts = await waitForArtifacts(dataOpsRun.id, [
"extracted.json",
"filtered.json",
"mapped.json",
"rows.csv",
"matched.json",
"unmatched.json",
]);
assert(
jsonEqual(artifactByName(dataOpsArtifacts, "extracted.json").data.value, dataOpsExtractExpected),
`JSON extract should return value from ${dataOpsExtractField}`,
);
assert(
artifactByName(dataOpsArtifacts, "filtered.json").data.count === dataOpsFilterExpectedCount,
"JSON filter should keep the expected number of rows from supplied payload",
);
const mappedArtifact = artifactByName(dataOpsArtifacts, "mapped.json");
assert(artifactItemCount(mappedArtifact) === dataOpsItems.length, "JSON map should preserve row count");
const csvRowsArtifact = artifactByName(dataOpsArtifacts, "rows.csv");
const csvPreview = csvRowsArtifact.data.preview;
for (const [sourceField, destinationField] of Object.entries(dataOpsFieldMap)) {
assert(
csvPreview.columns.includes(destinationField),
`JSON map should expose renamed field ${destinationField} in downstream CSV preview`,
);
assert(
String(csvPreview.rows[0]?.[destinationField]) === String(dataOpsItems[0]?.[sourceField]),
`JSON map should rename ${sourceField} to ${destinationField}`,
);
}
assert(
csvPreview.kind === "table",
"JSON to CSV should produce previewable csv",
);
assert(artifactByName(dataOpsArtifacts, "matched.json").portId === "matched", "matched artifact should carry portId");
assert(
artifactByName(dataOpsArtifacts, "unmatched.json").portId === "unmatched",
"unmatched artifact should carry portId",
);
log("run csv conversion workflow");
const csvRun = await runWorkflow(workflows.csvWorkflow.id);
const csvArtifacts = await waitForArtifacts(csvRun.id, ["table.json"]);
assert(
artifactItemCount(artifactByName(csvArtifacts, "table.json")) === expectedCsvRowCount(previewCsvPayload.text),
"CSV to JSON should convert the supplied CSV row count",
);
log("run directory scan workflow");
const scanRun = await runWorkflow(workflows.scanWorkflow.id);
const scanArtifacts = await waitForArtifacts(scanRun.id, ["directory-scan.json"]);
assert(
artifactByName(scanArtifacts, "directory-scan.json").data.summary.fileCount >= scanMinFileCount,
"Directory scan should report the minimum file count for the supplied source path",
);
log("run rosbag workflow");
const rosbagRun = await runWorkflow(workflows.rosbagWorkflow.id);
const rosbagArtifacts = await waitForArtifacts(rosbagRun.id, ["rosbag-summary.json"]);
const rosbagPayload = readJSONArtifactPayload(artifactByName(rosbagArtifacts, "rosbag-summary.json"));
const rosbagPreview = rosbagPayload.databasePreview || rosbagPayload.preview || {};
assert(
rosbagPayload.datasetType === "rosbag2" || rosbagPreview.format === "rosbag2",
"ROSBag summary should detect rosbag2 sqlite",
);
assert(Array.isArray(rosbagPreview.topics) && rosbagPreview.topics.length > 0, "ROSBag summary should include topics");
if (rosbagExpectedTopic) {
assert(
rosbagPreview.topics.some((topic) => topic.name === rosbagExpectedTopic),
`ROSBag summary should include configured topic ${rosbagExpectedTopic}`,
);
}
log("run lerobot workflow");
const lerobotRun = await runWorkflow(workflows.lerobotWorkflow.id);
const lerobotArtifacts = await waitForArtifacts(lerobotRun.id, ["lerobot-summary.json"]);
const lerobotPayload = readJSONArtifactPayload(artifactByName(lerobotArtifacts, "lerobot-summary.json"));
const lerobotPreview = lerobotPayload.preview || {};
assert(lerobotPreview.format === "lerobot", "LeRobot summary should detect dataset");
assert(lerobotPreview.info && Object.keys(lerobotPreview.info).length > 0, "LeRobot summary should expose info.json");
if (lerobotExpectedRobotType) {
assert(
lerobotPreview.info.robot_type === lerobotExpectedRobotType,
`LeRobot summary should expose robot_type ${lerobotExpectedRobotType}`,
);
}
log("open browser and verify UI previews");
const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({
extraHTTPHeaders: authenticationHeaders(apiToken),
viewport: { width: 1440, height: 1100 },
});
const page = await context.newPage();
try {
await ensureUIReady(page);
await page.getByRole("link", { name: "算子管理" }).click();
await page.getByRole("heading", { name: "算子管理" }).waitFor();
await page.getByPlaceholder("搜索算子...").fill("ROSBag 数据概览");
await page.getByText("ROSBag 数据概览").waitFor();
await page.getByPlaceholder("搜索算子...").fill("LeRobot 数据集概览");
await page.getByText("LeRobot 数据集概览").waitFor();
await openArtifactModal(page, names.previewWorkflow);
await page.getByText("report.csv").first().click();
await page.getByText(firstCsvDataToken(previewCsvPayload.text)).first().waitFor();
await page.getByText("notes.txt").first().click();
await page.getByText(previewTextPayload.previewNeedle).waitFor();
await closeArtifactModal(page);
await openArtifactModal(page, names.rosbagWorkflow);
await page.getByText("rosbag-summary.json").first().click();
await page.locator("pre").filter({ hasText: '"datasetType": "rosbag2"' }).first().waitFor();
await closeArtifactModal(page);
await openArtifactModal(page, names.lerobotWorkflow);
await page.getByText("lerobot-summary.json").first().click();
if (lerobotExpectedRobotType) {
await page.locator("pre").filter({ hasText: JSON.stringify(lerobotExpectedRobotType) }).first().waitFor();
}
await page.locator("pre").filter({ hasText: '"format": "lerobot"' }).first().waitFor();
await closeArtifactModal(page);
} finally {
await browser.close();
}
log("preview and operator check completed");
}
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});