droboticflow/tests/frontend/scan-workflows.test.mjs

917 lines
24 KiB
JavaScript

import test from "node:test";
import assert from "node:assert/strict";
import {
buildWorkflowEditorPath,
buildScheduledWorkflowSchedule,
DEFAULT_SCAN_WORKFLOW_DESCRIPTION,
applyScanConfigPatchToWorkflow,
buildDraftScanWorkflowPayload,
buildScanConfigPayloadFromWorkflow,
buildScanWorkflowPayloadFromDraft,
effectiveWorkflowScheduleMode,
ensureScanWorkflowShell,
getScanWorkflowConfigIssue,
getScanWorkflowDraftIssue,
isWorkflowScheduleEnabled,
isScanWorkflowConfigReady,
normalizePositiveInteger,
normalizeWorkflowSchedule,
stripScanWorkflowShell,
validateScanWorkflowEndpoint,
} from "../../src/lib/scan-workflows.js";
test("workflow editor path stays project-based after removing dedicated scan workflow pages", () => {
assert.equal(buildWorkflowEditorPath("workflow-1"), "/workflow/workflow-1");
assert.equal(buildWorkflowEditorPath(""), "/projects");
});
test("workflow editor path can carry one-shot focus instructions", () => {
assert.equal(
buildWorkflowEditorPath("workflow-1", {
projectId: "project-1",
focusNodeId: "scan-source",
openConfigPanel: true,
}),
"/workflow/workflow-1?projectId=project-1&focusNodeId=scan-source&openConfigPanel=true",
);
assert.equal(buildWorkflowEditorPath("workflow-1"), "/workflow/workflow-1");
});
test("workflow editor path carries run context without turning stale running metadata into readonly mode", () => {
const path = buildWorkflowEditorPath("workflow-1", {
projectId: "project-1",
runId: "run-last-known",
});
assert.equal(path, "/workflow/workflow-1?projectId=project-1&runId=run-last-known");
assert.equal(path.includes("readOnly="), false);
});
test("scan workflow config payload is derived from workflow-level settings", () => {
const payload = buildScanConfigPayloadFromWorkflow({
name: "BOS 扫描",
description: "scan",
schedule: {
interval: 300,
sourcePath: "bos://datastorage/algo/X-platform-test",
},
scanConfig: {
dataSource: "BOS",
sourcePath: "bos://datastorage/algo/X-platform-test",
bucket: "datastorage",
prefix: "algo/X-platform-test",
endpoint: "https://bos.example.test",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
pattern: "*.mp4",
enabled: true,
},
});
assert.deepEqual(payload, {
name: "BOS 扫描",
description: "scan",
dataSource: "BOS",
connectionId: "",
sourcePath: "",
bucket: "datastorage",
prefix: "algo/X-platform-test",
endpoint: "https://bos.example.test",
accessKeyId: "",
secretAccessKey: "",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
pattern: "*.mp4",
interval: 300,
enabled: true,
});
});
test("scan workflow config payload leaves omitted BOS endpoint for the global startup value", () => {
const payload = buildScanConfigPayloadFromWorkflow({
name: "BOS 扫描",
description: "scan",
schedule: {
interval: 300,
sourcePath: "bos://datastorage/algo/X-platform-test",
},
scanConfig: {
dataSource: "BOS",
sourcePath: "bos://datastorage/algo/X-platform-test",
bucket: "datastorage",
prefix: "algo/X-platform-test",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
enabled: true,
},
});
assert.equal(payload.endpoint, "");
});
test("scan workflow endpoint validation rejects inline non-http endpoints", () => {
assert.equal(
validateScanWorkflowEndpoint({
dataSource: "BOS",
endpoint: "https://bos.example.test",
}),
"",
);
assert.match(
validateScanWorkflowEndpoint({
dataSource: "BOS",
endpoint: "ftp://bos.example.test",
}),
/absolute http\(s\) URL/,
);
assert.match(
validateScanWorkflowEndpoint({
dataSource: "BOS",
endpoint: "https://bos.example.test/bad path",
}),
/without whitespace/,
);
});
test("scan workflow endpoint validation ignores endpoints behind connection references", () => {
assert.equal(
validateScanWorkflowEndpoint({
dataSource: "BOS",
connectionId: "conn-bos-main",
endpoint: "ftp://ignored.example.test",
}),
"",
);
});
test("scan workflow draft validation reports invalid enum values", () => {
assert.match(
getScanWorkflowDraftIssue({
scanConfig: {
dataSource: "BOS",
scanMode: "APPEND",
},
}),
/scanMode must be FULL or INCREMENTAL/,
);
assert.match(
getScanWorkflowDraftIssue({
scanConfig: {
dataSource: "BOS",
incrementalUnit: "FILE",
},
}),
/incrementalUnit must be OBJECT or DIRECTORY/,
);
});
test("scan workflow config payload falls back to incremental source node when workflow shell is missing", () => {
const payload = buildScanConfigPayloadFromWorkflow({
name: "增量全链路",
description: "generic dag",
schedule: {
interval: 120,
},
nodes: [
{
id: "source",
data: {
label: "BOS 增量扫描",
bucket: "wjzh-data-migration",
prefix: "wjzh-0330-Data-migration",
pattern: "*.mp4",
hyperparams: [
{ key: "DATA_SOURCE", value: "BOS" },
{
key: "SOURCE_PATH",
value: "bos://wjzh-data-migration/wjzh-0330-Data-migration",
},
{ key: "BOS_ENDPOINT", value: "https://bos.example.test" },
{ key: "BOS_ACCESS_KEY_ID", value: "ak-node" },
{ key: "BOS_SECRET_ACCESS_KEY", value: "sk-node" },
{ key: "SCAN_MODE", value: "INCREMENTAL" },
{ key: "INCREMENTAL_UNIT", value: "DIRECTORY" },
{ key: "INCREMENTAL_DIRECTORY_DEPTH", value: "2" },
],
},
},
],
});
assert.deepEqual(payload, {
name: "增量全链路",
description: "generic dag",
dataSource: "BOS",
connectionId: "",
sourcePath: "",
bucket: "wjzh-data-migration",
prefix: "wjzh-0330-Data-migration",
endpoint: "https://bos.example.test",
accessKeyId: "",
secretAccessKey: "",
scanMode: "INCREMENTAL",
incrementalUnit: "DIRECTORY",
directoryDepth: 2,
pattern: "*.mp4",
interval: 120,
enabled: true,
});
});
test("scan workflow config rejects lossy incremental directory depth", () => {
assert.throws(
() =>
buildScanConfigPayloadFromWorkflow({
name: "增量全链路",
description: "generic dag",
schedule: {
interval: 120,
},
nodes: [
{
id: "source",
data: {
label: "BOS 增量扫描",
bucket: "wjzh-data-migration",
prefix: "wjzh-0330-Data-migration",
hyperparams: [
{ key: "DATA_SOURCE", value: "BOS" },
{
key: "SOURCE_PATH",
value: "bos://wjzh-data-migration/wjzh-0330-Data-migration",
},
{ key: "INCREMENTAL_DIRECTORY_DEPTH", value: "2abc" },
],
},
},
],
}),
/INCREMENTAL_DIRECTORY_DEPTH must be a positive integer/,
);
assert.throws(
() =>
applyScanConfigPatchToWorkflow(
{
name: "扫描",
scanConfig: {
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test",
directoryDepth: 1,
},
nodes: [{ id: "scan-source", data: { label: "BOS 增量扫描", hyperparams: [] } }],
},
{ directoryDepth: "0" },
),
/INCREMENTAL_DIRECTORY_DEPTH must be a positive integer/,
);
});
test("scan workflow config rejects invalid incremental unit", () => {
assert.throws(
() =>
buildScanConfigPayloadFromWorkflow({
name: "增量全链路",
description: "generic dag",
schedule: {
interval: 120,
},
nodes: [
{
id: "source",
data: {
label: "BOS 增量扫描",
bucket: "wjzh-data-migration",
prefix: "wjzh-0330-Data-migration",
hyperparams: [
{ key: "DATA_SOURCE", value: "BOS" },
{
key: "SOURCE_PATH",
value: "bos://wjzh-data-migration/wjzh-0330-Data-migration",
},
{ key: "INCREMENTAL_UNIT", value: "ROW" },
],
},
},
],
}),
/incrementalUnit must be OBJECT or DIRECTORY/,
);
});
test("scan workflow config rejects invalid scan mode", () => {
assert.throws(
() =>
buildScanConfigPayloadFromWorkflow({
name: "增量全链路",
description: "generic dag",
nodes: [
{
id: "source",
data: {
label: "BOS 增量扫描",
bucket: "wjzh-data-migration",
prefix: "wjzh-0330-Data-migration",
hyperparams: [
{ key: "DATA_SOURCE", value: "BOS" },
{ key: "SCAN_MODE", value: "ROW" },
],
},
},
],
}),
/scanMode must be FULL or INCREMENTAL/,
);
});
test("scan workflow config rejects invalid data source", () => {
assert.throws(
() =>
buildScanConfigPayloadFromWorkflow({
name: "增量全链路",
description: "generic dag",
scanConfig: {
dataSource: "ROW",
bucket: "wjzh-data-migration",
prefix: "wjzh-0330-Data-migration",
},
}),
/dataSource must be BOS for scan workflows/,
);
});
test("workflow schedule helpers support hourly and daily modes", () => {
assert.equal(effectiveWorkflowScheduleMode({ mode: "every_hours", intervalHours: 4 }), "every_hours");
assert.equal(effectiveWorkflowScheduleMode({ dayInterval: 2, timeOfDay: "18:30" }), "every_days_at_time");
assert.equal(isWorkflowScheduleEnabled({ mode: "every_hours", intervalHours: 4 }), true);
assert.equal(
isWorkflowScheduleEnabled({
mode: "every_days_at_time",
dayInterval: 0,
timeOfDay: "18:30",
}),
false,
);
assert.deepEqual(buildScheduledWorkflowSchedule({}, "every_hours"), {
interval: 0,
intervalHours: 1,
dayInterval: 0,
timeOfDay: "",
anchorAt: "",
timezone: "",
cron: "",
sourcePath: "",
mode: "every_hours",
});
assert.deepEqual(
normalizeWorkflowSchedule({
mode: "every_days_at_time",
dayInterval: 2,
timeOfDay: "18:30",
}),
{
interval: 0,
intervalHours: 0,
dayInterval: 2,
timeOfDay: "18:30",
anchorAt: "",
timezone: "Asia/Shanghai",
cron: "",
sourcePath: "",
mode: "every_days_at_time",
},
);
});
test("workflow schedule helpers reject lossy numeric schedule values", () => {
assert.equal(normalizePositiveInteger(" 2 ", "schedule.interval"), 2);
assert.throws(
() => normalizePositiveInteger("0", "schedule.interval"),
/schedule.interval must be a positive integer/,
);
assert.throws(
() => normalizeWorkflowSchedule({ interval: "60s" }),
/schedule.interval must be a non-negative integer/,
);
assert.throws(
() => normalizeWorkflowSchedule({ intervalHours: "1.5" }),
/schedule.intervalHours must be a non-negative integer/,
);
assert.throws(
() =>
buildScanWorkflowPayloadFromDraft({
name: "扫描",
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test",
enabled: true,
interval: "900s",
}),
/schedule.interval must be a non-negative integer/,
);
assert.throws(
() =>
applyScanConfigPatchToWorkflow(
{
name: "扫描",
scanConfig: {
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test",
directoryDepth: 1,
},
nodes: [{ id: "scan-source", data: { label: "BOS 增量扫描", hyperparams: [] } }],
},
{ enabled: true, dayInterval: "2days" },
),
/schedule.dayInterval must be a non-negative integer/,
);
});
test("scan config patch keeps workflow config, schedule, type, and source node data aligned", () => {
const updated = applyScanConfigPatchToWorkflow(
{
id: "workflow-1",
name: "扫描",
description: "",
kind: "scan",
type: "batch",
schedule: { interval: 0, sourcePath: "" },
scanConfig: {
dataSource: "BOS",
sourcePath: "",
bucket: "",
prefix: "",
endpoint: "https://bos.example.test",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
pattern: "",
enabled: false,
},
nodes: [
{
id: "scan-source",
data: {
label: "BOS 增量扫描",
hyperparams: [
{ key: "DATA_SOURCE", value: "BOS" },
{ key: "SOURCE_PATH", value: "" },
{ key: "BOS_ENDPOINT", value: "https://bos.example.test" },
],
bucket: "",
prefix: "",
pattern: "",
},
},
],
},
{
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test",
endpoint: "https://bos.example.test",
sourcePath: "",
scanMode: "FULL",
incrementalUnit: "DIRECTORY",
directoryDepth: 2,
pattern: "*.json",
enabled: true,
interval: 600,
},
);
assert.equal(updated.type, "streaming");
assert.equal(updated.schedule.interval, 600);
assert.equal(updated.schedule.sourcePath, "bos://datastorage/algo/X-platform-test");
assert.equal(updated.scanConfig.sourcePath, "bos://datastorage/algo/X-platform-test");
assert.equal(updated.scanConfig.bucket, "datastorage");
assert.equal(updated.scanConfig.scanMode, "FULL");
assert.equal(updated.scanConfig.incrementalUnit, "DIRECTORY");
assert.equal(updated.scanConfig.directoryDepth, 2);
assert.equal(updated.nodes[0].data.bucket, "datastorage");
assert.equal(updated.nodes[0].data.prefix, "algo/X-platform-test");
assert.equal(updated.nodes[0].data.label, "BOS 增量扫描");
assert.equal(updated.nodes[0].data.hyperparams.find((item) => item.key === "DATA_SOURCE")?.value, "BOS");
assert.equal(
updated.nodes[0].data.hyperparams.find((item) => item.key === "SOURCE_PATH")?.value,
"bos://datastorage/algo/X-platform-test",
);
assert.equal(updated.nodes[0].data.hyperparams.find((item) => item.key === "SCAN_MODE")?.value, "FULL");
assert.equal(updated.nodes[0].data.hyperparams.find((item) => item.key === "INCREMENTAL_UNIT")?.value, "DIRECTORY");
assert.equal(
updated.nodes[0].data.hyperparams.find((item) => item.key === "INCREMENTAL_DIRECTORY_DEPTH")?.value,
"2",
);
});
test("scan config patch preserves non-second schedule modes", () => {
const updated = applyScanConfigPatchToWorkflow(
{
id: "workflow-1",
name: "扫描",
description: "",
kind: "scan",
type: "batch",
schedule: {
mode: "every_days_at_time",
dayInterval: 2,
timeOfDay: "18:30",
timezone: "Asia/Shanghai",
sourcePath: "",
},
scanConfig: {
dataSource: "BOS",
sourcePath: "",
bucket: "",
prefix: "",
endpoint: "https://bos.example.test",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
pattern: "",
enabled: false,
},
nodes: [
{
id: "scan-source",
data: {
label: "BOS 增量扫描",
hyperparams: [],
bucket: "",
prefix: "",
pattern: "",
},
},
],
},
{
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test",
enabled: true,
mode: "every_days_at_time",
dayInterval: 2,
timeOfDay: "18:30",
timezone: "Asia/Shanghai",
},
);
assert.equal(updated.type, "streaming");
assert.equal(updated.schedule.mode, "every_days_at_time");
assert.equal(updated.schedule.dayInterval, 2);
assert.equal(updated.schedule.timeOfDay, "18:30");
assert.equal(updated.scanConfig.enabled, true);
});
test("scan workflow draft builds a native workflow payload for the generic workflow API", () => {
const payload = buildScanWorkflowPayloadFromDraft({
name: "X-platform BOS 扫描",
description: "增量扫描 BOS 目录",
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test/",
endpoint: "https://bos.example.test",
accessKeyId: "ak",
secretAccessKey: "sk",
scanMode: "INCREMENTAL",
incrementalUnit: "DIRECTORY",
directoryDepth: 3,
pattern: "*.mp4",
interval: 900,
enabled: true,
});
assert.deepEqual(payload, {
name: "X-platform BOS 扫描",
description: "增量扫描 BOS 目录",
maxConcurrentRuns: 1,
type: "streaming",
kind: "scan",
schedule: {
interval: 900,
intervalHours: 0,
dayInterval: 0,
timeOfDay: "",
anchorAt: "",
timezone: "",
cron: "",
sourcePath: "bos://datastorage/algo/X-platform-test",
mode: "interval_seconds",
},
scanConfig: {
dataSource: "BOS",
sourcePath: "bos://datastorage/algo/X-platform-test",
bucket: "datastorage",
prefix: "algo/X-platform-test",
endpoint: "https://bos.example.test",
accessKeyId: "ak",
secretAccessKey: "sk",
scanMode: "INCREMENTAL",
incrementalUnit: "DIRECTORY",
directoryDepth: 3,
pattern: "*.mp4",
enabled: true,
},
nodes: [],
edges: [],
});
});
test("scan workflow shell draft creates an incomplete BOS scan workflow for later node configuration", () => {
const payload = buildDraftScanWorkflowPayload({
name: "扫描草稿",
});
assert.deepEqual(payload, {
name: "扫描草稿",
description: DEFAULT_SCAN_WORKFLOW_DESCRIPTION,
maxConcurrentRuns: 1,
type: "batch",
kind: "scan",
schedule: {
interval: 0,
intervalHours: 0,
dayInterval: 0,
timeOfDay: "",
anchorAt: "",
timezone: "",
cron: "",
sourcePath: "",
mode: "interval_seconds",
},
scanConfig: {
dataSource: "BOS",
sourcePath: "",
connectionId: "",
bucket: "",
prefix: "",
endpoint: "",
accessKeyId: "",
secretAccessKey: "",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
pattern: "",
enabled: false,
},
nodes: [],
edges: [],
});
});
test("scan workflow shell draft allows explicit descriptions when provided", () => {
const payload = buildDraftScanWorkflowPayload({
name: "扫描草稿",
description: "保留自定义说明",
});
assert.equal(payload.description, "保留自定义说明");
});
test("scan workflow readiness requires concrete source target and credentials", () => {
assert.equal(
isScanWorkflowConfigReady({
scanConfig: {
dataSource: "BOS",
bucket: "",
prefix: "",
hasAccessKeyId: false,
hasSecretAccessKey: false,
},
}),
false,
);
assert.equal(
isScanWorkflowConfigReady({
scanConfig: {
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test",
hasAccessKeyId: true,
hasSecretAccessKey: true,
},
}),
true,
);
assert.equal(
isScanWorkflowConfigReady({
scanConfig: {
dataSource: "BOS",
connectionId: "conn-bos-main",
bucket: "datastorage",
prefix: "algo/X-platform-test",
},
}),
true,
);
assert.equal(
isScanWorkflowConfigReady({
scanConfig: {
dataSource: "BOS",
endpoint: "ftp://bos.example.test",
accessKeyId: "ak",
secretAccessKey: "sk",
bucket: "datastorage",
prefix: "algo/X-platform-test",
},
}),
false,
);
assert.match(
getScanWorkflowConfigIssue({
scanConfig: {
dataSource: "BOS",
endpoint: "ftp://bos.example.test",
accessKeyId: "ak",
secretAccessKey: "sk",
bucket: "datastorage",
prefix: "algo/X-platform-test",
},
}),
/absolute http\(s\) URL/,
);
assert.match(
getScanWorkflowConfigIssue({
scanConfig: {
dataSource: "S3",
accessKeyId: "ak",
secretAccessKey: "sk",
bucket: "datastorage",
prefix: "algo/X-platform-test",
},
}),
/dataSource must be BOS/,
);
assert.equal(
isScanWorkflowConfigReady({
scanConfig: {
dataSource: "BOS",
bucket: "",
prefix: "",
},
}),
false,
);
assert.equal(
isScanWorkflowConfigReady({
nodes: [
{
id: "source",
data: {
label: "BOS 增量扫描",
bucket: "wjzh-data-migration",
prefix: "wjzh-0330-Data-migration",
hyperparams: [
{ key: "DATA_SOURCE", value: "BOS" },
{
key: "SOURCE_PATH",
value: "bos://wjzh-data-migration/wjzh-0330-Data-migration",
},
{ key: "BOS_ENDPOINT", value: "https://bos.example.test" },
{ key: "BOS_ACCESS_KEY_ID", value: "ak-node" },
{ key: "BOS_SECRET_ACCESS_KEY", value: "sk-node" },
],
},
},
],
schedule: {
interval: 60,
},
}),
true,
);
});
test("scan config patch switches BOS auth to connection reference", () => {
const updated = applyScanConfigPatchToWorkflow(
{
kind: "scan",
type: "batch",
schedule: { interval: 0, sourcePath: "" },
scanConfig: {
dataSource: "BOS",
bucket: "datastorage",
prefix: "algo/X-platform-test",
endpoint: "https://bos.example.test",
accessKeyId: "ak-inline",
secretAccessKey: "sk-inline",
incrementalUnit: "OBJECT",
directoryDepth: 1,
enabled: false,
},
nodes: [
{
id: "scan-source",
data: {
label: "BOS 增量扫描",
hyperparams: [],
},
},
],
},
{
connectionId: "conn-bos-main",
},
);
assert.equal(updated.scanConfig.connectionId, "conn-bos-main");
assert.equal(updated.scanConfig.accessKeyId, "");
assert.equal(updated.scanConfig.secretAccessKey, "");
assert.equal(updated.scanConfig.endpoint, "");
assert.equal(updated.scanConfig.bucket, "datastorage");
assert.equal(updated.scanConfig.prefix, "algo/X-platform-test");
});
test("adding a BOS scan source can promote a normal workflow into a scan-capable workflow shell", () => {
const updated = ensureScanWorkflowShell({
id: "workflow-1",
name: "普通工作流",
description: "workflow",
type: "batch",
schedule: {
interval: 0,
sourcePath: "",
},
nodes: [],
edges: [],
});
assert.equal(updated.kind, "scan");
assert.equal(updated.maxConcurrentRuns, 1);
assert.equal(updated.type, "batch");
assert.deepEqual(updated.schedule, {
interval: 0,
intervalHours: 0,
dayInterval: 0,
timeOfDay: "",
anchorAt: "",
timezone: "",
cron: "",
sourcePath: "",
mode: "",
});
assert.deepEqual(updated.scanConfig, {
dataSource: "BOS",
sourcePath: "",
connectionId: "",
bucket: "",
prefix: "",
endpoint: "",
accessKeyId: "",
secretAccessKey: "",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
pattern: "",
enabled: false,
});
});
test("removing the last dedicated scan source drops scan-only workflow shell state", () => {
const updated = stripScanWorkflowShell({
id: "workflow-1",
name: "扫描链路",
description: "workflow",
type: "streaming",
kind: "scan",
schedule: {
interval: 300,
sourcePath: "bos://datastorage/algo/X-platform-test",
},
scanConfig: {
dataSource: "BOS",
sourcePath: "bos://datastorage/algo/X-platform-test",
bucket: "datastorage",
prefix: "algo/X-platform-test",
endpoint: "https://bos.example.test",
accessKeyId: "",
secretAccessKey: "",
scanMode: "INCREMENTAL",
incrementalUnit: "OBJECT",
directoryDepth: 1,
pattern: "*.mp4",
enabled: true,
},
});
assert.equal(updated.kind, "standard");
assert.equal(updated.scanConfig, undefined);
assert.deepEqual(updated.schedule, {
interval: 300,
sourcePath: "",
});
});