614 lines
16 KiB
JavaScript
614 lines
16 KiB
JavaScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
|
|
import {
|
|
buildWorkflowEditorPath,
|
|
buildScheduledWorkflowSchedule,
|
|
DEFAULT_SCAN_WORKFLOW_DESCRIPTION,
|
|
applyScanConfigPatchToWorkflow,
|
|
buildDraftScanWorkflowPayload,
|
|
buildScanConfigPayloadFromWorkflow,
|
|
buildScanWorkflowPayloadFromDraft,
|
|
effectiveWorkflowScheduleMode,
|
|
ensureScanWorkflowShell,
|
|
isWorkflowScheduleEnabled,
|
|
isScanWorkflowConfigReady,
|
|
normalizeWorkflowSchedule,
|
|
stripScanWorkflowShell,
|
|
} from "../../src/lib/scan-workflows.js";
|
|
|
|
test("workflow editor path stays project-based after removing dedicated scan workflow pages", () => {
|
|
assert.equal(buildWorkflowEditorPath("workflow-1"), "/workflow/workflow-1");
|
|
assert.equal(buildWorkflowEditorPath(""), "/projects");
|
|
});
|
|
|
|
test("workflow editor path can carry one-shot focus instructions", () => {
|
|
assert.equal(
|
|
buildWorkflowEditorPath("workflow-1", {
|
|
projectId: "project-1",
|
|
focusNodeId: "scan-source",
|
|
openConfigPanel: true,
|
|
}),
|
|
"/workflow/workflow-1?projectId=project-1&focusNodeId=scan-source&openConfigPanel=true",
|
|
);
|
|
assert.equal(buildWorkflowEditorPath("workflow-1"), "/workflow/workflow-1");
|
|
});
|
|
|
|
test("scan workflow config payload is derived from workflow-level settings", () => {
|
|
const payload = buildScanConfigPayloadFromWorkflow({
|
|
name: "BOS 扫描",
|
|
description: "scan",
|
|
schedule: {
|
|
interval: 300,
|
|
sourcePath: "bos://datastorage/algo/X-platform-test",
|
|
},
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
sourcePath: "bos://datastorage/algo/X-platform-test",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
endpoint: "https://bj.bcebos.com",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
pattern: "*.mp4",
|
|
enabled: true,
|
|
},
|
|
});
|
|
|
|
assert.deepEqual(payload, {
|
|
name: "BOS 扫描",
|
|
description: "scan",
|
|
dataSource: "BOS",
|
|
connectionId: "",
|
|
sourcePath: "",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "",
|
|
secretAccessKey: "",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
pattern: "*.mp4",
|
|
interval: 300,
|
|
enabled: true,
|
|
});
|
|
});
|
|
|
|
test("scan workflow config payload falls back to incremental source node when workflow shell is missing", () => {
|
|
const payload = buildScanConfigPayloadFromWorkflow({
|
|
name: "增量全链路",
|
|
description: "generic dag",
|
|
schedule: {
|
|
interval: 120,
|
|
},
|
|
nodes: [
|
|
{
|
|
id: "source",
|
|
data: {
|
|
label: "BOS 增量扫描",
|
|
bucket: "wjzh-data-migration",
|
|
prefix: "wjzh-0330-Data-migration",
|
|
pattern: "*.mp4",
|
|
hyperparams: [
|
|
{ key: "DATA_SOURCE", value: "BOS" },
|
|
{
|
|
key: "SOURCE_PATH",
|
|
value: "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
},
|
|
{ key: "BOS_ENDPOINT", value: "https://bj.bcebos.com" },
|
|
{ key: "BOS_ACCESS_KEY_ID", value: "ak-node" },
|
|
{ key: "BOS_SECRET_ACCESS_KEY", value: "sk-node" },
|
|
{ key: "SCAN_MODE", value: "INCREMENTAL" },
|
|
{ key: "INCREMENTAL_UNIT", value: "DIRECTORY" },
|
|
{ key: "INCREMENTAL_DIRECTORY_DEPTH", value: "2" },
|
|
],
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
assert.deepEqual(payload, {
|
|
name: "增量全链路",
|
|
description: "generic dag",
|
|
dataSource: "BOS",
|
|
connectionId: "",
|
|
sourcePath: "",
|
|
bucket: "wjzh-data-migration",
|
|
prefix: "wjzh-0330-Data-migration",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "",
|
|
secretAccessKey: "",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "DIRECTORY",
|
|
directoryDepth: 2,
|
|
pattern: "*.mp4",
|
|
interval: 120,
|
|
enabled: true,
|
|
});
|
|
});
|
|
|
|
test("workflow schedule helpers support hourly and daily modes", () => {
|
|
assert.equal(effectiveWorkflowScheduleMode({ mode: "every_hours", intervalHours: 4 }), "every_hours");
|
|
assert.equal(effectiveWorkflowScheduleMode({ dayInterval: 2, timeOfDay: "18:30" }), "every_days_at_time");
|
|
assert.equal(isWorkflowScheduleEnabled({ mode: "every_hours", intervalHours: 4 }), true);
|
|
assert.equal(
|
|
isWorkflowScheduleEnabled({
|
|
mode: "every_days_at_time",
|
|
dayInterval: 0,
|
|
timeOfDay: "18:30",
|
|
}),
|
|
false,
|
|
);
|
|
assert.deepEqual(buildScheduledWorkflowSchedule({}, "every_hours"), {
|
|
interval: 0,
|
|
intervalHours: 1,
|
|
dayInterval: 0,
|
|
timeOfDay: "",
|
|
anchorAt: "",
|
|
timezone: "",
|
|
cron: "",
|
|
sourcePath: "",
|
|
mode: "every_hours",
|
|
});
|
|
assert.deepEqual(
|
|
normalizeWorkflowSchedule({
|
|
mode: "every_days_at_time",
|
|
dayInterval: 2,
|
|
timeOfDay: "18:30",
|
|
}),
|
|
{
|
|
interval: 0,
|
|
intervalHours: 0,
|
|
dayInterval: 2,
|
|
timeOfDay: "18:30",
|
|
anchorAt: "",
|
|
timezone: "Asia/Shanghai",
|
|
cron: "",
|
|
sourcePath: "",
|
|
mode: "every_days_at_time",
|
|
},
|
|
);
|
|
});
|
|
|
|
test("scan config patch keeps workflow config, schedule, type, and source node data aligned", () => {
|
|
const updated = applyScanConfigPatchToWorkflow(
|
|
{
|
|
id: "workflow-1",
|
|
name: "扫描",
|
|
description: "",
|
|
kind: "scan",
|
|
type: "batch",
|
|
schedule: { interval: 0, sourcePath: "" },
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
sourcePath: "",
|
|
bucket: "",
|
|
prefix: "",
|
|
endpoint: "https://bj.bcebos.com",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
pattern: "",
|
|
enabled: false,
|
|
},
|
|
nodes: [
|
|
{
|
|
id: "scan-source",
|
|
data: {
|
|
label: "BOS 增量扫描",
|
|
hyperparams: [
|
|
{ key: "DATA_SOURCE", value: "BOS" },
|
|
{ key: "SOURCE_PATH", value: "" },
|
|
{ key: "BOS_ENDPOINT", value: "https://bj.bcebos.com" },
|
|
],
|
|
bucket: "",
|
|
prefix: "",
|
|
pattern: "",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
{
|
|
dataSource: "BOS",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
endpoint: "https://bj.bcebos.com",
|
|
sourcePath: "",
|
|
scanMode: "FULL",
|
|
incrementalUnit: "DIRECTORY",
|
|
directoryDepth: 2,
|
|
pattern: "*.json",
|
|
enabled: true,
|
|
interval: 600,
|
|
},
|
|
);
|
|
|
|
assert.equal(updated.type, "streaming");
|
|
assert.equal(updated.schedule.interval, 600);
|
|
assert.equal(updated.schedule.sourcePath, "bos://datastorage/algo/X-platform-test");
|
|
assert.equal(updated.scanConfig.sourcePath, "bos://datastorage/algo/X-platform-test");
|
|
assert.equal(updated.scanConfig.bucket, "datastorage");
|
|
assert.equal(updated.scanConfig.scanMode, "FULL");
|
|
assert.equal(updated.scanConfig.incrementalUnit, "DIRECTORY");
|
|
assert.equal(updated.scanConfig.directoryDepth, 2);
|
|
assert.equal(updated.nodes[0].data.bucket, "datastorage");
|
|
assert.equal(updated.nodes[0].data.prefix, "algo/X-platform-test");
|
|
assert.equal(updated.nodes[0].data.label, "BOS 增量扫描");
|
|
assert.equal(updated.nodes[0].data.hyperparams.find((item) => item.key === "DATA_SOURCE")?.value, "BOS");
|
|
assert.equal(
|
|
updated.nodes[0].data.hyperparams.find((item) => item.key === "SOURCE_PATH")?.value,
|
|
"bos://datastorage/algo/X-platform-test",
|
|
);
|
|
assert.equal(updated.nodes[0].data.hyperparams.find((item) => item.key === "SCAN_MODE")?.value, "FULL");
|
|
assert.equal(updated.nodes[0].data.hyperparams.find((item) => item.key === "INCREMENTAL_UNIT")?.value, "DIRECTORY");
|
|
assert.equal(
|
|
updated.nodes[0].data.hyperparams.find((item) => item.key === "INCREMENTAL_DIRECTORY_DEPTH")?.value,
|
|
"2",
|
|
);
|
|
});
|
|
|
|
test("scan config patch preserves non-second schedule modes", () => {
|
|
const updated = applyScanConfigPatchToWorkflow(
|
|
{
|
|
id: "workflow-1",
|
|
name: "扫描",
|
|
description: "",
|
|
kind: "scan",
|
|
type: "batch",
|
|
schedule: {
|
|
mode: "every_days_at_time",
|
|
dayInterval: 2,
|
|
timeOfDay: "18:30",
|
|
timezone: "Asia/Shanghai",
|
|
sourcePath: "",
|
|
},
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
sourcePath: "",
|
|
bucket: "",
|
|
prefix: "",
|
|
endpoint: "https://bj.bcebos.com",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
pattern: "",
|
|
enabled: false,
|
|
},
|
|
nodes: [
|
|
{
|
|
id: "scan-source",
|
|
data: {
|
|
label: "BOS 增量扫描",
|
|
hyperparams: [],
|
|
bucket: "",
|
|
prefix: "",
|
|
pattern: "",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
{
|
|
dataSource: "BOS",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
enabled: true,
|
|
mode: "every_days_at_time",
|
|
dayInterval: 2,
|
|
timeOfDay: "18:30",
|
|
timezone: "Asia/Shanghai",
|
|
},
|
|
);
|
|
|
|
assert.equal(updated.type, "streaming");
|
|
assert.equal(updated.schedule.mode, "every_days_at_time");
|
|
assert.equal(updated.schedule.dayInterval, 2);
|
|
assert.equal(updated.schedule.timeOfDay, "18:30");
|
|
assert.equal(updated.scanConfig.enabled, true);
|
|
});
|
|
|
|
test("scan workflow draft builds a native workflow payload for the generic workflow API", () => {
|
|
const payload = buildScanWorkflowPayloadFromDraft({
|
|
name: "X-platform BOS 扫描",
|
|
description: "增量扫描 BOS 目录",
|
|
dataSource: "BOS",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test/",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "ak",
|
|
secretAccessKey: "sk",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "DIRECTORY",
|
|
directoryDepth: 3,
|
|
pattern: "*.mp4",
|
|
interval: 900,
|
|
enabled: true,
|
|
});
|
|
|
|
assert.deepEqual(payload, {
|
|
name: "X-platform BOS 扫描",
|
|
description: "增量扫描 BOS 目录",
|
|
maxConcurrentRuns: 1,
|
|
type: "streaming",
|
|
kind: "scan",
|
|
schedule: {
|
|
interval: 900,
|
|
intervalHours: 0,
|
|
dayInterval: 0,
|
|
timeOfDay: "",
|
|
anchorAt: "",
|
|
timezone: "",
|
|
cron: "",
|
|
sourcePath: "bos://datastorage/algo/X-platform-test",
|
|
mode: "interval_seconds",
|
|
},
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
sourcePath: "bos://datastorage/algo/X-platform-test",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "ak",
|
|
secretAccessKey: "sk",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "DIRECTORY",
|
|
directoryDepth: 3,
|
|
pattern: "*.mp4",
|
|
enabled: true,
|
|
},
|
|
nodes: [],
|
|
edges: [],
|
|
});
|
|
});
|
|
|
|
test("scan workflow shell draft creates an incomplete BOS scan workflow for later node configuration", () => {
|
|
const payload = buildDraftScanWorkflowPayload({
|
|
name: "扫描草稿",
|
|
});
|
|
|
|
assert.deepEqual(payload, {
|
|
name: "扫描草稿",
|
|
description: DEFAULT_SCAN_WORKFLOW_DESCRIPTION,
|
|
maxConcurrentRuns: 1,
|
|
type: "batch",
|
|
kind: "scan",
|
|
schedule: {
|
|
interval: 0,
|
|
intervalHours: 0,
|
|
dayInterval: 0,
|
|
timeOfDay: "",
|
|
anchorAt: "",
|
|
timezone: "",
|
|
cron: "",
|
|
sourcePath: "",
|
|
mode: "interval_seconds",
|
|
},
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
sourcePath: "",
|
|
connectionId: "",
|
|
bucket: "",
|
|
prefix: "",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "",
|
|
secretAccessKey: "",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
pattern: "",
|
|
enabled: false,
|
|
},
|
|
nodes: [],
|
|
edges: [],
|
|
});
|
|
});
|
|
|
|
test("scan workflow shell draft allows explicit descriptions when provided", () => {
|
|
const payload = buildDraftScanWorkflowPayload({
|
|
name: "扫描草稿",
|
|
description: "保留自定义说明",
|
|
});
|
|
|
|
assert.equal(payload.description, "保留自定义说明");
|
|
});
|
|
|
|
test("scan workflow readiness requires concrete source target and credentials", () => {
|
|
assert.equal(
|
|
isScanWorkflowConfigReady({
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
bucket: "",
|
|
prefix: "",
|
|
hasAccessKeyId: false,
|
|
hasSecretAccessKey: false,
|
|
},
|
|
}),
|
|
false,
|
|
);
|
|
|
|
assert.equal(
|
|
isScanWorkflowConfigReady({
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
hasAccessKeyId: true,
|
|
hasSecretAccessKey: true,
|
|
},
|
|
}),
|
|
true,
|
|
);
|
|
|
|
assert.equal(
|
|
isScanWorkflowConfigReady({
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
connectionId: "conn-bos-main",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
},
|
|
}),
|
|
true,
|
|
);
|
|
|
|
assert.equal(
|
|
isScanWorkflowConfigReady({
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
bucket: "",
|
|
prefix: "",
|
|
},
|
|
}),
|
|
false,
|
|
);
|
|
|
|
assert.equal(
|
|
isScanWorkflowConfigReady({
|
|
nodes: [
|
|
{
|
|
id: "source",
|
|
data: {
|
|
label: "BOS 增量扫描",
|
|
bucket: "wjzh-data-migration",
|
|
prefix: "wjzh-0330-Data-migration",
|
|
hyperparams: [
|
|
{ key: "DATA_SOURCE", value: "BOS" },
|
|
{
|
|
key: "SOURCE_PATH",
|
|
value: "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
},
|
|
{ key: "BOS_ENDPOINT", value: "https://bj.bcebos.com" },
|
|
{ key: "BOS_ACCESS_KEY_ID", value: "ak-node" },
|
|
{ key: "BOS_SECRET_ACCESS_KEY", value: "sk-node" },
|
|
],
|
|
},
|
|
},
|
|
],
|
|
schedule: {
|
|
interval: 60,
|
|
},
|
|
}),
|
|
true,
|
|
);
|
|
});
|
|
|
|
test("scan config patch switches BOS auth to connection reference", () => {
|
|
const updated = applyScanConfigPatchToWorkflow(
|
|
{
|
|
kind: "scan",
|
|
type: "batch",
|
|
schedule: { interval: 0, sourcePath: "" },
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "ak-inline",
|
|
secretAccessKey: "sk-inline",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
enabled: false,
|
|
},
|
|
nodes: [
|
|
{
|
|
id: "scan-source",
|
|
data: {
|
|
label: "BOS 增量扫描",
|
|
hyperparams: [],
|
|
},
|
|
},
|
|
],
|
|
},
|
|
{
|
|
connectionId: "conn-bos-main",
|
|
},
|
|
);
|
|
|
|
assert.equal(updated.scanConfig.connectionId, "conn-bos-main");
|
|
assert.equal(updated.scanConfig.accessKeyId, "");
|
|
assert.equal(updated.scanConfig.secretAccessKey, "");
|
|
assert.equal(updated.scanConfig.bucket, "datastorage");
|
|
assert.equal(updated.scanConfig.prefix, "algo/X-platform-test");
|
|
});
|
|
|
|
test("adding a BOS scan source can promote a normal workflow into a scan-capable workflow shell", () => {
|
|
const updated = ensureScanWorkflowShell({
|
|
id: "workflow-1",
|
|
name: "普通工作流",
|
|
description: "workflow",
|
|
type: "batch",
|
|
schedule: {
|
|
interval: 0,
|
|
sourcePath: "",
|
|
},
|
|
nodes: [],
|
|
edges: [],
|
|
});
|
|
|
|
assert.equal(updated.kind, "scan");
|
|
assert.equal(updated.maxConcurrentRuns, 1);
|
|
assert.equal(updated.type, "batch");
|
|
assert.deepEqual(updated.schedule, {
|
|
interval: 0,
|
|
intervalHours: 0,
|
|
dayInterval: 0,
|
|
timeOfDay: "",
|
|
anchorAt: "",
|
|
timezone: "",
|
|
cron: "",
|
|
sourcePath: "",
|
|
mode: "",
|
|
});
|
|
assert.deepEqual(updated.scanConfig, {
|
|
dataSource: "BOS",
|
|
sourcePath: "",
|
|
connectionId: "",
|
|
bucket: "",
|
|
prefix: "",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "",
|
|
secretAccessKey: "",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
pattern: "",
|
|
enabled: false,
|
|
});
|
|
});
|
|
|
|
test("removing the last dedicated scan source drops scan-only workflow shell state", () => {
|
|
const updated = stripScanWorkflowShell({
|
|
id: "workflow-1",
|
|
name: "扫描链路",
|
|
description: "workflow",
|
|
type: "streaming",
|
|
kind: "scan",
|
|
schedule: {
|
|
interval: 300,
|
|
sourcePath: "bos://datastorage/algo/X-platform-test",
|
|
},
|
|
scanConfig: {
|
|
dataSource: "BOS",
|
|
sourcePath: "bos://datastorage/algo/X-platform-test",
|
|
bucket: "datastorage",
|
|
prefix: "algo/X-platform-test",
|
|
endpoint: "https://bj.bcebos.com",
|
|
accessKeyId: "",
|
|
secretAccessKey: "",
|
|
scanMode: "INCREMENTAL",
|
|
incrementalUnit: "OBJECT",
|
|
directoryDepth: 1,
|
|
pattern: "*.mp4",
|
|
enabled: true,
|
|
},
|
|
});
|
|
|
|
assert.equal(updated.kind, "standard");
|
|
assert.equal(updated.scanConfig, undefined);
|
|
assert.deepEqual(updated.schedule, {
|
|
interval: 300,
|
|
sourcePath: "",
|
|
});
|
|
});
|