forked from longtao.wu/droboticflow
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
321 lines
8.8 KiB
JavaScript
321 lines
8.8 KiB
JavaScript
const BASE = "http://114.111.24.223:30080";
|
|
const SOURCE_PATH =
|
|
"bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43";
|
|
const CONNECTION_ID = "storage-d69ba0e9824a9abd216eced490f0aed3";
|
|
const BUCKET = "datastorage";
|
|
const PREFIX = "algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43";
|
|
|
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
async function request(method, path, payload, expected = [200, 201, 202, 204]) {
|
|
let lastError;
|
|
for (let attempt = 1; attempt <= 6; attempt += 1) {
|
|
try {
|
|
const response = await fetch(`${BASE}${path}`, {
|
|
method,
|
|
headers: { "Content-Type": "application/json" },
|
|
body: payload === undefined ? undefined : JSON.stringify(payload),
|
|
});
|
|
const text = await response.text();
|
|
if (!expected.includes(response.status)) {
|
|
throw new Error(`${method} ${path} failed ${response.status}: ${text}`);
|
|
}
|
|
return text ? JSON.parse(text) : null;
|
|
} catch (error) {
|
|
lastError = error;
|
|
if (attempt === 6) {
|
|
throw error;
|
|
}
|
|
await sleep(Math.min(attempt, 3) * 1000);
|
|
}
|
|
}
|
|
throw lastError;
|
|
}
|
|
|
|
function buildNode(operator, nodeId, x, y, overrides = {}, extra = {}) {
|
|
const hyper = (operator.hyperparams || []).map((item) => ({ ...item }));
|
|
const seen = new Set();
|
|
for (const item of hyper) {
|
|
if (Object.prototype.hasOwnProperty.call(overrides, item.key)) {
|
|
item.value = overrides[item.key];
|
|
seen.add(item.key);
|
|
}
|
|
}
|
|
for (const [key, value] of Object.entries(overrides)) {
|
|
if (seen.has(key)) {
|
|
continue;
|
|
}
|
|
hyper.push({ key, value });
|
|
}
|
|
return {
|
|
id: nodeId,
|
|
type: "operator",
|
|
data: {
|
|
label: operator.name,
|
|
category: operator.category,
|
|
description: operator.description,
|
|
inputs: operator.inputs || [],
|
|
outputs: operator.outputs || [],
|
|
image: operator.image || "",
|
|
skillDefinition: operator.skillDefinition || "",
|
|
hyperparams: hyper,
|
|
...extra,
|
|
},
|
|
position: { x, y },
|
|
};
|
|
}
|
|
|
|
const interestingArtifactNames = new Set([
|
|
"source-manifest.json",
|
|
"dataset-index.json",
|
|
"dataset-validation.json",
|
|
"quality-report.json",
|
|
"filtered-videos.json",
|
|
"clips-manifest.json",
|
|
"annotations-manifest.json",
|
|
"packages-manifest.json",
|
|
"audit-samples.json",
|
|
"uploaded-results.json",
|
|
]);
|
|
|
|
async function cleanupAll() {
|
|
const scanWorkflows = (await request("GET", "/api/scan-configs")).items;
|
|
for (const workflow of scanWorkflows) {
|
|
await request("DELETE", `/api/scan-configs/${encodeURIComponent(workflow.id)}`);
|
|
}
|
|
const projects = (await request("GET", "/api/projects")).items;
|
|
for (const project of projects) {
|
|
const workflows = (
|
|
await request("GET", `/api/projects/${encodeURIComponent(project.id)}/workflows`)
|
|
).items;
|
|
for (const workflow of workflows) {
|
|
await request("DELETE", `/api/workflows/${encodeURIComponent(workflow.id)}`);
|
|
}
|
|
await request("DELETE", `/api/projects/${encodeURIComponent(project.id)}`);
|
|
}
|
|
const templates = (await request("GET", "/api/templates")).items;
|
|
for (const template of templates) {
|
|
await request("DELETE", `/api/templates/${encodeURIComponent(template.id)}`);
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
await cleanupAll();
|
|
|
|
const operators = (await request("GET", "/api/operators")).items;
|
|
const byName = Object.fromEntries(operators.map((item) => [item.name, item]));
|
|
const needed = [
|
|
"BOS 增量扫描",
|
|
"X 平台数据集索引",
|
|
"采集目录校验",
|
|
"视频质检",
|
|
"数据过滤",
|
|
"视频切片",
|
|
"视频标注",
|
|
"成品打包",
|
|
"抽检采样",
|
|
"结果上传",
|
|
];
|
|
for (const name of needed) {
|
|
if (!byName[name]) {
|
|
throw new Error(`missing operator: ${name}`);
|
|
}
|
|
}
|
|
|
|
const ts = new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 14);
|
|
const outputPrefix = `factory/output/xplatform-real-data/${ts}`;
|
|
const outputPath = `bos://${BUCKET}/${outputPrefix}`;
|
|
const project = await request("POST", "/api/projects", {
|
|
name: `XPlatform Real Data ${ts}`,
|
|
description: "真实 BOS 数据验收项目",
|
|
});
|
|
|
|
const nodes = [
|
|
buildNode(
|
|
byName["BOS 增量扫描"],
|
|
"source",
|
|
0,
|
|
0,
|
|
{
|
|
DATA_SOURCE: "BOS",
|
|
SOURCE_PATH: SOURCE_PATH,
|
|
SCAN_MODE: "FULL",
|
|
},
|
|
{ connectionId: CONNECTION_ID, bucket: BUCKET, prefix: PREFIX },
|
|
),
|
|
buildNode(byName["X 平台数据集索引"], "index", 260, 0),
|
|
buildNode(byName["采集目录校验"], "validate", 520, 0, {
|
|
REQUIRE_INTRINSICS: "false",
|
|
}),
|
|
buildNode(byName["视频质检"], "quality", 780, 0, {
|
|
MIN_CLARITY: "0.8",
|
|
MAX_SHAKE: "0.2",
|
|
MAX_HAND_OCCLUSION: "0.2",
|
|
CLARITY_SCALE: "35",
|
|
}),
|
|
buildNode(byName["数据过滤"], "filter", 1040, 0, {
|
|
DATE_START: "2026-02-01",
|
|
DATE_END: "2026-04-05",
|
|
PLACE: "蓟门里小区",
|
|
TYPE: "家庭",
|
|
MIN_CLARITY: "0.8",
|
|
MAX_SHAKE: "0.2",
|
|
MAX_HAND_OCCLUSION: "0.2",
|
|
}),
|
|
buildNode(byName["视频切片"], "slice", 1300, 0, {
|
|
SEGMENT_DURATION_SECONDS: "300",
|
|
}),
|
|
buildNode(byName["视频标注"], "annotate", 1560, 0, {
|
|
FRAME_INTERVAL_SECONDS: "1.0",
|
|
}),
|
|
buildNode(byName["成品打包"], "package", 1820, 0),
|
|
buildNode(byName["抽检采样"], "sample", 2080, -100, {
|
|
RULES_JSON: JSON.stringify([
|
|
{
|
|
id: "clarity-check",
|
|
field: "quality.clarity",
|
|
operator: "lt",
|
|
value: 1.0,
|
|
samplePercent: 100,
|
|
},
|
|
]),
|
|
}),
|
|
buildNode(
|
|
byName["结果上传"],
|
|
"upload",
|
|
2080,
|
|
120,
|
|
{
|
|
DATA_SOURCE: "BOS",
|
|
OUTPUT_PATH: outputPath,
|
|
SOURCE_PATH: outputPath,
|
|
},
|
|
{ connectionId: CONNECTION_ID, bucket: BUCKET, prefix: outputPrefix },
|
|
),
|
|
];
|
|
|
|
const edges = [
|
|
{
|
|
id: "source-index",
|
|
source: "source",
|
|
sourceHandle: "out-1",
|
|
target: "index",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "index-validate",
|
|
source: "index",
|
|
sourceHandle: "out-1",
|
|
target: "validate",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "validate-quality",
|
|
source: "validate",
|
|
sourceHandle: "out-1",
|
|
target: "quality",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "quality-filter",
|
|
source: "quality",
|
|
sourceHandle: "out-1",
|
|
target: "filter",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "filter-slice",
|
|
source: "filter",
|
|
sourceHandle: "out-1",
|
|
target: "slice",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "slice-annotate",
|
|
source: "slice",
|
|
sourceHandle: "out-1",
|
|
target: "annotate",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "annotate-package",
|
|
source: "annotate",
|
|
sourceHandle: "out-1",
|
|
target: "package",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "package-sample",
|
|
source: "package",
|
|
sourceHandle: "out-1",
|
|
target: "sample",
|
|
targetHandle: "in-1",
|
|
},
|
|
{
|
|
id: "package-upload",
|
|
source: "package",
|
|
sourceHandle: "out-1",
|
|
target: "upload",
|
|
targetHandle: "in-1",
|
|
},
|
|
];
|
|
|
|
const workflow = await request(
|
|
"POST",
|
|
`/api/projects/${encodeURIComponent(project.id)}/workflows`,
|
|
{
|
|
name: `XPlatform Real Pipeline ${ts}`,
|
|
description: "基于真实 BOS 数据的 X 平台处理流程",
|
|
type: "batch",
|
|
nodes,
|
|
edges,
|
|
},
|
|
);
|
|
const run = await request("POST", `/api/workflows/${encodeURIComponent(workflow.id)}/run`, {
|
|
trigger: "manual",
|
|
});
|
|
|
|
let terminal = null;
|
|
for (let index = 0; index < 900; index += 1) {
|
|
const current = await request("GET", `/api/runs/${encodeURIComponent(run.id)}`);
|
|
if (["completed", "failed", "cancelled", "waiting_for_audit"].includes(current.status)) {
|
|
terminal = current;
|
|
break;
|
|
}
|
|
await sleep(2000);
|
|
}
|
|
if (!terminal) {
|
|
throw new Error(`timeout waiting for run ${run.id}`);
|
|
}
|
|
|
|
const artifacts = (await request("GET", `/api/runs/${encodeURIComponent(run.id)}/artifacts`))
|
|
.items;
|
|
const logs = (await request("GET", `/api/runs/${encodeURIComponent(run.id)}/logs`)).items;
|
|
const interesting = {};
|
|
for (const item of artifacts) {
|
|
if (interestingArtifactNames.has(item.name)) {
|
|
interesting[item.name] = item.data;
|
|
}
|
|
}
|
|
|
|
const summary = {
|
|
projectId: project.id,
|
|
projectName: project.name,
|
|
workflowId: workflow.id,
|
|
workflowName: workflow.name,
|
|
runId: run.id,
|
|
status: terminal.status,
|
|
nodeStatuses: Object.fromEntries(
|
|
(terminal.nodeExecutions || []).map((item) => [item.nodeId, item.status]),
|
|
),
|
|
interestingArtifacts: interesting,
|
|
lastLogs: logs.slice(-20),
|
|
};
|
|
console.log(JSON.stringify(summary, null, 2));
|
|
if (terminal.status !== "completed") {
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
await main();
|