2026-06-23 21:23:22 +08:00

2887 lines
134 KiB
Python

import contextlib
import hashlib
import http.server
import importlib.util
import io
import json
import os
import pathlib
import sqlite3
import threading
import tempfile
import unittest
from unittest import mock
MODULE_PATH = pathlib.Path(__file__).with_name("main.py")
SPEC = importlib.util.spec_from_file_location("operator_runner_main", MODULE_PATH)
assert SPEC is not None and SPEC.loader is not None
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
RESOURCE_MONITOR_PATH = pathlib.Path(__file__).with_name("resource_monitor.py")
RESOURCE_MONITOR_SPEC = importlib.util.spec_from_file_location("operator_runner_resource_monitor", RESOURCE_MONITOR_PATH)
assert RESOURCE_MONITOR_SPEC is not None and RESOURCE_MONITOR_SPEC.loader is not None
RESOURCE_MONITOR = importlib.util.module_from_spec(RESOURCE_MONITOR_SPEC)
RESOURCE_MONITOR_SPEC.loader.exec_module(RESOURCE_MONITOR)
class _FakeHTTPResponse:
def __init__(self, payload: bytes):
self._payload = payload
self._offset = 0
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def read(self, *_args, **_kwargs):
if self._offset >= len(self._payload):
return b""
chunk = self._payload[self._offset:]
self._offset = len(self._payload)
return chunk
class _FakeBosClient:
def __init__(self):
self.uploads = []
self.downloads = []
def put_object_from_file(self, bucket, key, file_name, **kwargs):
self.uploads.append(
{
"bucket": bucket,
"key": key,
"file_name": file_name,
"content_type": kwargs.get("content_type"),
}
)
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}"
def get_object_to_file(self, bucket, key, file_name, **_kwargs):
self.downloads.append(
{
"bucket": bucket,
"key": key,
"file_name": file_name,
}
)
pathlib.Path(file_name).write_bytes(f"{bucket}/{key}".encode("utf-8"))
class _FakeBosClientBytesURL(_FakeBosClient):
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}".encode("utf-8")
class _FakeBosListClient(_FakeBosClient):
def __init__(self, objects, object_payloads):
super().__init__()
self._objects = objects
self._object_payloads = object_payloads
self.downloaded_payload_keys = []
def list_all_objects(self, bucket, prefix=None):
return list(self._objects)
def get_object_as_string(self, bucket, key):
self.downloaded_payload_keys.append((bucket, key))
return self._object_payloads[key]
class CommandPolicyTests(unittest.TestCase):
def test_command_policy_rejects_commands_by_default(self):
with mock.patch.dict(os.environ, {}, clear=True):
with self.assertRaisesRegex(RuntimeError, "disabled"):
MODULE.validate_command_policy("printf 'ok'")
def test_command_policy_allows_commands_when_explicitly_enabled(self):
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "true"}, clear=True):
MODULE.validate_command_policy("printf 'ok'")
def test_command_policy_can_disable_shell_commands(self):
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "false"}, clear=True):
with self.assertRaisesRegex(RuntimeError, "disabled"):
MODULE.validate_command_policy("printf 'ok'")
def test_command_policy_rejects_denylisted_command(self):
with mock.patch.dict(
os.environ,
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_DENYLIST_REGEX": r"(^|\s)rm\s+-rf"},
clear=True,
):
with self.assertRaisesRegex(RuntimeError, "DENYLIST"):
MODULE.validate_command_policy("rm -rf /workspace")
def test_command_policy_requires_allowlist_match_when_configured(self):
with mock.patch.dict(
os.environ,
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_ALLOWLIST_REGEX": r"^python3?\s"},
clear=True,
):
MODULE.validate_command_policy("python3 script.py")
with self.assertRaisesRegex(RuntimeError, "ALLOWLIST"):
MODULE.validate_command_policy("bash script.sh")
class ResourceMonitorTests(unittest.TestCase):
def test_parse_gpu_snapshot_output_summarizes_real_nvidia_smi_columns(self):
snapshot = RESOURCE_MONITOR.parse_gpu_snapshot_output(
"\n".join(
[
"12, 4096, 24576, NVIDIA L4",
"83, 8192, 24576, NVIDIA L4",
"not-a-number, 1, 1, broken",
]
)
)
self.assertEqual(snapshot["gpu"], 83)
self.assertEqual(snapshot["gpuMemoryUsedMiB"], 12288)
self.assertEqual(snapshot["gpuMemoryTotalMiB"], 49152)
self.assertEqual(snapshot["gpuName"], "NVIDIA L4")
self.assertEqual(snapshot["gpuCount"], 2)
def test_parse_gpu_snapshot_output_defaults_when_nvidia_smi_has_no_valid_rows(self):
self.assertEqual(RESOURCE_MONITOR.parse_gpu_snapshot_output(""), {"gpu": 0})
class SourcePathFromItemTests(unittest.TestCase):
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "node-output"
workspace_root = pathlib.Path(temp_dir) / "workspace"
output_dir.mkdir(parents=True)
workspace_root.mkdir(parents=True)
item = {
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D.MP4",
"previewURL": "https://example.com/should-not-download.mp4",
}
}
fake_client = _FakeBosClient()
with mock.patch.dict(
os.environ,
{
"WORKSPACE_ROOT": str(workspace_root),
"BOS_ENDPOINT": "https://bos.example.test",
"BOS_ACCESS_KEY_ID": "ak",
"BOS_SECRET_ACCESS_KEY": "sk",
},
clear=False,
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")) as build_client, mock.patch.object(
MODULE.request,
"urlopen",
) as urlopen:
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(first.exists())
self.assertEqual(first, second)
self.assertIn(str(workspace_root), str(first))
self.assertEqual(len(fake_client.downloads), 1)
build_client.assert_called()
urlopen.assert_not_called()
def test_downloads_http_preview_url_to_local_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
item = {
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/demo.mp4",
"previewURL": "https://example.com/demo.mp4?signature=abc",
}
}
with mock.patch.object(
MODULE.request,
"urlopen",
return_value=_FakeHTTPResponse(b"real-video-bytes"),
) as urlopen:
resolved = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(resolved.exists())
self.assertEqual(resolved.read_bytes(), b"real-video-bytes")
urlopen.assert_called_once()
def test_prefers_workspace_cache_path_for_preview_url_when_bos_credentials_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "node-output"
workspace_root = pathlib.Path(temp_dir) / "workspace"
output_dir.mkdir(parents=True)
workspace_root.mkdir(parents=True)
item = {
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0000/DJI_001/demo.mp4",
"previewURL": "https://example.com/demo.mp4?signature=abc",
}
}
with mock.patch.dict(
os.environ,
{
"WORKSPACE_ROOT": str(workspace_root),
"BOS_ENDPOINT": "",
"BOS_ACCESS_KEY_ID": "",
"BOS_SECRET_ACCESS_KEY": "",
},
clear=False,
), mock.patch.object(
MODULE.request,
"urlopen",
return_value=_FakeHTTPResponse(b"real-video-bytes"),
) as urlopen:
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(first.exists())
self.assertEqual(first, second)
self.assertEqual(first.read_bytes(), b"real-video-bytes")
self.assertIn(str(workspace_root / "_bos_cache"), str(first))
urlopen.assert_called_once()
def test_remote_video_url_for_source_signs_bos_path_when_preview_missing(self):
source = {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_0001_D.MP4",
"previewURL": "",
}
with mock.patch.object(MODULE, "signed_bos_source_url", return_value="https://signed.example.com/video.mp4") as signer:
resolved = MODULE.remote_video_url_for_source(source)
self.assertEqual(resolved, "https://signed.example.com/video.mp4")
signer.assert_called_once_with(source["path"])
class RuntimeFileContractTests(unittest.TestCase):
def test_select_handler_prefers_specific_operator_routes(self):
self.assertEqual(MODULE.select_handler("BOS 增量扫描"), "bos_incremental_scan")
self.assertEqual(MODULE.select_handler("人工审核"), "human_review")
def test_operator_helpers_do_not_expose_legacy_builtin_handlers(self):
legacy_keys = sorted(key for key in MODULE.OPERATOR_HELPERS if key.startswith("builtin_"))
self.assertEqual(legacy_keys, [])
def test_dispatch_runs_standalone_csv_to_json_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "input.csv"
source_path.write_text("id,name,score\n1,alpha,98\n2,beta,87\n", encoding="utf-8")
output_dir = root / "out"
output_dir.mkdir()
node_data = {
"label": "CSV 转 JSON",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(source_path)}],
"outputs": [{"id": "out-1", "name": "table", "type": "json"}],
}
artifacts = MODULE.BUILTIN_HANDLERS["csv_to_json"]("run-1", "csv", output_dir, node_data, [])
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["columns"], ["id", "name", "score"])
self.assertEqual(payload["rowCount"], 2)
self.assertEqual(payload["items"][1]["name"], "beta")
def test_dispatch_runs_standalone_json_filter_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 过滤",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "EXPECTED", "value": "ready"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["json_filter"](
"run-1",
"filter",
output_dir,
node_data,
[{"data": {"items": [{"status": "ready"}, {"status": "blocked"}]}}],
)
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["count"], 1)
self.assertEqual(payload["items"], [{"status": "ready"}])
def test_dispatch_runs_named_human_review_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "人工审核",
"hyperparams": [
{"key": "REVIEW_TYPE", "value": "task"},
{"key": "REVIEWER_ROLE", "value": "qa_reviewer"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["builtin_human_review"](
"run-1",
"review",
output_dir,
node_data,
[{"data": {"items": [{"id": "clip-1", "score": 0.9}]}}],
)
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["datasetType"], "human-review-tasks")
self.assertEqual(payload["reviewerRole"], "qa_reviewer")
self.assertEqual(payload["taskCount"], 1)
self.assertEqual(payload["items"][0]["id"], "run-1-clip-1")
def test_load_input_artifacts_prefers_manifest_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
manifest_path = pathlib.Path(temp_dir) / "input-artifacts.json"
manifest_path.write_text('[{"id":"artifact-from-file"}]', encoding="utf-8")
with mock.patch.dict(
os.environ,
{
"INPUT_ARTIFACTS_PATH": str(manifest_path),
"INPUT_ARTIFACTS_JSON": '[{"id":"artifact-inline"}]',
},
clear=False,
):
loaded = MODULE.load_input_artifacts()
self.assertEqual(len(loaded), 1)
self.assertEqual(loaded[0]["id"], "artifact-from-file")
def test_main_writes_result_and_stage_manifest_files(self):
with tempfile.TemporaryDirectory() as temp_dir:
node_root = pathlib.Path(temp_dir)
output_dir = node_root / "outputs"
output_dir.mkdir(parents=True)
input_artifacts_path = node_root / "input-artifacts.json"
input_artifacts_path.write_text("[]", encoding="utf-8")
result_path = node_root / "result.json"
stage_manifest_path = node_root / "stage-manifest.json"
artifacts = [
{
"id": "artifact-1",
"runId": "run-1",
"nodeId": "node-1",
"name": "report.json",
"type": "json",
"size": "12 B",
"path": str(output_dir / "report.json"),
}
]
with mock.patch.dict(
os.environ,
{
"RUN_ID": "run-1",
"WORKFLOW_ID": "wf-1",
"NODE_ID": "node-1",
"NODE_LABEL": "通用处理",
"OUTPUT_DIR": str(output_dir),
"NODE_DATA_JSON": '{"label":"通用处理","hyperparams":[]}',
"INPUT_ARTIFACTS_PATH": str(input_artifacts_path),
"RESULT_OUTPUT_PATH": str(result_path),
"STAGE_MANIFEST_PATH": str(stage_manifest_path),
},
clear=False,
), mock.patch.object(MODULE, "builtin_generic", return_value=artifacts), mock.patch.object(
MODULE,
"resource_series",
return_value=[],
):
stdout = io.StringIO()
with contextlib.redirect_stdout(stdout):
exit_code = MODULE.main()
self.assertEqual(exit_code, 0)
self.assertIn(MODULE.RESULT_PATH_PREFIX + str(result_path), stdout.getvalue())
result_payload = json.loads(result_path.read_text(encoding="utf-8"))
self.assertEqual(result_payload["artifacts"][0]["id"], "artifact-1")
stage_manifest = json.loads(stage_manifest_path.read_text(encoding="utf-8"))
self.assertEqual(stage_manifest["artifactCount"], 1)
self.assertEqual(stage_manifest["nodeLabel"], "通用处理")
def test_write_json_artifact_externalizes_dataset_payload_to_data_ref(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"dataset-index.json",
{
"datasetType": "x-platform-index",
"collectionCount": 1,
"collections": [{"id": "collection-1"}],
"items": [{"id": "item-1"}, {"id": "item-2"}],
},
)
data_ref = artifact["data"]["dataRef"]
self.assertEqual(data_ref["kind"], "dataset")
self.assertTrue(pathlib.Path(data_ref["manifestPath"]).exists())
self.assertTrue(pathlib.Path(data_ref["payloadPath"]).exists())
manifest = json.loads(pathlib.Path(data_ref["manifestPath"]).read_text(encoding="utf-8"))
self.assertEqual(manifest["datasetType"], "x-platform-index")
self.assertIn("items", manifest["partitionFiles"])
self.assertIn("collections", manifest["partitionFiles"])
def test_load_artifact_payload_reads_externalized_payload_from_data_ref(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"filtered-videos.json",
{
"datasetType": "x-platform-filtered",
"count": 2,
"items": [{"id": "v1"}, {"id": "v2"}],
},
)
loaded = MODULE.load_artifact_payload(artifact)
self.assertEqual(loaded["datasetType"], "x-platform-filtered")
self.assertEqual(len(loaded["items"]), 2)
def test_load_payload_from_file_retries_transient_json_decode_error(self):
with tempfile.TemporaryDirectory() as temp_dir:
payload_path = pathlib.Path(temp_dir) / "payload.json"
payload_path.write_text('{"items":[{"id":"v1"}]}', encoding="utf-8")
decode_error = json.JSONDecodeError("transient", "", 0)
with mock.patch.object(
MODULE.json,
"load",
side_effect=[decode_error, {"datasetType": "x-platform", "items": [{"id": "v1"}]}],
) as json_load, mock.patch.object(MODULE.time, "sleep") as sleep_mock:
loaded = MODULE.load_payload_from_file(payload_path)
self.assertEqual(loaded["datasetType"], "x-platform")
self.assertEqual(len(loaded["items"]), 1)
self.assertEqual(json_load.call_count, 2)
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
def test_load_payload_from_file_retries_until_payload_path_exists(self):
with tempfile.TemporaryDirectory() as temp_dir:
payload_path = pathlib.Path(temp_dir) / "payload.json"
payload_path.write_text('{"datasetType":"x-platform","items":[{"id":"v1"}]}', encoding="utf-8")
original_exists = pathlib.Path.exists
seen = {"count": 0}
def fake_exists(path_obj: pathlib.Path):
if path_obj == payload_path and seen["count"] == 0:
seen["count"] += 1
return False
return original_exists(path_obj)
with mock.patch.object(pathlib.Path, "exists", autospec=True, side_effect=fake_exists), mock.patch.object(
MODULE.time,
"sleep",
) as sleep_mock:
loaded = MODULE.load_payload_from_file(payload_path)
self.assertEqual(loaded["datasetType"], "x-platform")
self.assertEqual(len(loaded["items"]), 1)
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
def test_stage_manifest_includes_data_refs(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"packages-manifest.json",
{
"datasetType": "x-platform-packages",
"packageCount": 1,
"items": [{"packageName": "pkg-1"}],
},
)
manifest = MODULE.stage_manifest_payload("run-1", "wf-1", "node-1", "成品打包", output_dir, [artifact])
self.assertEqual(len(manifest["dataRefs"]), 1)
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
class XPlatformIndexingTests(unittest.TestCase):
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "outputs"
output_dir.mkdir(parents=True)
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
manifest_path.write_text("{}", encoding="utf-8")
payload = MODULE.load_xplatform_index_payload(
{},
[
{
"path": str(manifest_path),
"data": {
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
"entries": [
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "工厂",
"place": "绿控传动",
"date": "20260330",
},
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260327132547_0003_D.MP4",
"codec": "hevc",
"resolution": "3840x2880",
"frameRate": 29.97,
"duration": 0.6,
}
],
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
},
},
],
},
}
],
output_dir,
)
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["videoCount"], 1)
self.assertEqual(payload["collections"][0]["meta"]["place"], "绿控传动")
def test_load_index_payload_raises_when_source_manifest_summary_has_no_entries(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "outputs"
output_dir.mkdir(parents=True)
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
manifest_path.write_text("{}", encoding="utf-8")
with self.assertRaisesRegex(RuntimeError, "source-manifest"):
MODULE.load_xplatform_index_payload(
{},
[
{
"name": "source-manifest.json",
"path": str(manifest_path),
"data": {
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
"entriesCount": 59,
"dataRef": {"payloadPath": str(manifest_path)},
},
}
],
output_dir,
)
def test_manifest_entries_support_video_metadata_alias(self):
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "工厂",
"place": "绿控传动",
"date": "20260330",
},
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260327132547_0003_D.MP4",
"codec": "hevc",
"resolution": "3840x2880",
"frameRate": 29.97,
"duration": 0.6,
}
],
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
},
},
],
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertEqual(payload["items"][0]["codec"], "hevc")
def test_manifest_entries_load_local_meta_when_inline_data_is_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "algo" / "X-platform-test"
collection = root / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
video_dir = collection / "DJI_001"
video_dir.mkdir(parents=True)
meta_path = collection / "meta.json"
video_meta_path = collection / "video_meta.json"
video_path = video_dir / "DJI_0001_D.mp4"
meta_path.write_text('{"type":"家庭","place":"蓟门里小区","date":"20260319"}', encoding="utf-8")
video_meta_path.write_text(
'[{"filename":"DJI_0001_D.mp4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
encoding="utf-8",
)
video_path.write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
"sizeBytes": meta_path.stat().st_size,
"mimeType": "application/json",
"source": {"path": str(meta_path)},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
"sizeBytes": video_meta_path.stat().st_size,
"mimeType": "application/json",
"source": {"path": str(video_meta_path)},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
"sizeBytes": video_path.stat().st_size,
"mimeType": "video/mp4",
"source": {"path": str(video_path)},
},
],
str(root),
)
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
self.assertEqual(payload["items"][0]["codec"], "hevc")
def test_manifest_entries_support_root_level_meta_and_video_meta(self):
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "家庭",
"place": "蓟门里小区",
"date": "20260319",
},
},
{
"path": "video_meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"duration": 41.8,
}
],
},
{
"path": "DJI_001/DJI_20260318112834_0001_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/video.mp4",
},
},
],
"bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(collection["relativePath"], "")
self.assertEqual(collection["videoCount"], 1)
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertEqual(collection["meta"]["place"], "蓟门里小区")
item = payload["items"][0]
self.assertEqual(item["collectionName"], collection["name"])
self.assertEqual(item["collectionRelativePath"], "")
self.assertEqual(item["videoFolder"], "DJI_001")
self.assertEqual(item["codec"], "hevc")
self.assertEqual(item["meta"]["type"], "家庭")
def test_local_dataset_root_with_meta_is_treated_as_collection_root(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
video_dir = root / "DJI_001"
video_dir.mkdir(parents=True)
(root / "meta.json").write_text('{"type":"家庭","place":"蓟门里小区"}', encoding="utf-8")
(root / "video_meta.json").write_text(
'[{"filename":"DJI_20260318112834_0001_D.MP4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
encoding="utf-8",
)
(video_dir / "DJI_20260318112834_0001_D.MP4").write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_dataset(root)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["name"], root.name)
self.assertEqual(collection["relativePath"], "")
self.assertEqual(collection["missingRequiredFiles"], [])
item = payload["items"][0]
self.assertEqual(item["collectionName"], root.name)
self.assertEqual(item["collectionRelativePath"], "")
self.assertEqual(item["codec"], "hevc")
def test_local_dataset_root_supports_video_metadata_alias(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "SZ_000_0005_OsmoNano_2026-03-30_16-15-11"
video_dir = root / "DJI_001"
video_dir.mkdir(parents=True)
(root / "meta.json").write_text('{"type":"工厂","place":"绿控传动"}', encoding="utf-8")
(root / "video_metadata.json").write_text(
'[{"filename":"DJI_20260327132547_0003_D.MP4","codec":"hevc","resolution":"3840x2880","frameRate":29.97,"duration":0.6}]',
encoding="utf-8",
)
(video_dir / "DJI_20260327132547_0003_D.MP4").write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_dataset(root)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(payload["items"][0]["codec"], "hevc")
class SourceOperatorTests(unittest.TestCase):
def test_builtin_source_rejects_missing_input_configuration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {"label": "真实数据源", "hyperparams": []}
with self.assertRaisesRegex(RuntimeError, "requires SOURCE_TEXT"):
MODULE.builtin_source("run-test", "source-empty", output_dir, node_data, [])
def test_builtin_source_infers_local_source_path_without_data_source(self):
with tempfile.TemporaryDirectory() as temp_dir:
source_root = pathlib.Path(temp_dir) / "source"
source_root.mkdir()
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
output_dir = pathlib.Path(temp_dir) / "output"
output_dir.mkdir()
node_data = {
"label": "真实数据源-本地路径",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(source_root)},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-local", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["dataSource"], "LOCAL")
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["path"], "payload.json")
def test_builtin_source_rejects_missing_local_source_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
missing_path = pathlib.Path(temp_dir) / "missing"
node_data = {
"label": "真实数据源-缺失本地路径",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(missing_path)},
],
}
with self.assertRaisesRegex(RuntimeError, "SOURCE_PATH does not exist"):
MODULE.builtin_source("run-test", "source-missing", output_dir, node_data, [])
def test_builtin_source_defaults_to_inline_when_source_json_is_present(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-默认内联",
"hyperparams": [
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-inline-default", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["dataSource"], "INLINE")
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
def test_source_handler_can_emit_inline_json_payload_on_named_output(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-内联JSON",
"outputs": [
{"id": "manifest", "name": "manifest", "type": "json"},
{"id": "payload", "name": "payload", "type": "json"},
],
"hyperparams": [
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "real"}]}, ensure_ascii=False)},
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
{"key": "OUTPUT_NAME", "value": "payload.json"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-json-payload", output_dir, node_data, [])
payload_artifact = next(item for item in artifacts if item["name"] == "payload.json")
payload = json.loads(pathlib.Path(payload_artifact["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["items"][0]["kind"], "real")
self.assertEqual(payload_artifact["portId"], "payload")
def test_source_handler_can_emit_inline_text_payload_with_tabular_preview(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-内联CSV",
"outputs": [
{"id": "manifest", "name": "manifest", "type": "json"},
{"id": "csv", "name": "csv", "type": "file"},
],
"hyperparams": [
{"key": "SOURCE_TEXT", "value": "id,label\nclip-a,keep\n"},
{"key": "PAYLOAD_OUTPUT_ID", "value": "csv"},
{"key": "OUTPUT_NAME", "value": "report.csv"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-csv-payload", output_dir, node_data, [])
csv_artifact = next(item for item in artifacts if item["name"] == "report.csv")
self.assertEqual(csv_artifact["portId"], "csv")
self.assertEqual(csv_artifact["data"]["preview"]["kind"], "table")
self.assertEqual(csv_artifact["data"]["preview"]["rowCount"], 1)
def test_source_handler_rejects_unsafe_inline_payload_name(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-不安全输出名",
"outputs": [
{"id": "manifest", "name": "manifest", "type": "json"},
{"id": "payload", "name": "payload", "type": "json"},
],
"hyperparams": [
{"key": "SOURCE_JSON", "value": "{}"},
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
{"key": "OUTPUT_NAME", "value": "../payload.json"},
],
}
with self.assertRaisesRegex(RuntimeError, "OUTPUT_NAME"):
MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-unsafe-payload", output_dir, node_data, [])
def test_builtin_source_prefers_source_json_over_bos_env(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-JSON",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-json", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
def test_builtin_source_prefers_source_text_over_bos_env(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-文本",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_TEXT", "value": "alpha\nbeta\ngamma"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-text", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["text"], "alpha\nbeta\ngamma")
def test_directory_incremental_source_filter_emits_full_new_directory(self):
entries = [
{"path": "batch-a/meta.json", "sizeBytes": 10, "sha256": "sha-a-meta"},
{"path": "batch-a/DJI_001/clip-a.mp4", "sizeBytes": 20, "sha256": "sha-a-clip"},
{"path": "batch-b/meta.json", "sizeBytes": 30, "sha256": "sha-b-meta"},
{"path": "batch-b/DJI_001/clip-b.mp4", "sizeBytes": 40, "sha256": "sha-b-clip"},
]
emitted, checkpoint = MODULE.apply_incremental_source_filter(
entries,
"BOS",
"wjzh-data-migration",
{
"SOURCE_STATE_JSON": json.dumps(
{
"seenFingerprints": ["batch-a"],
"incrementalUnit": "DIRECTORY",
"directoryDepth": 1,
}
),
"SOURCE_FINGERPRINT": "fp-directory",
"INCREMENTAL_UNIT": "DIRECTORY",
"INCREMENTAL_DIRECTORY_DEPTH": "1",
},
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
"wjzh-0330-Data-migration",
"",
)
self.assertEqual([item["path"] for item in emitted], ["batch-b/meta.json", "batch-b/DJI_001/clip-b.mp4"])
self.assertEqual([item["incrementalKey"] for item in emitted], ["batch-b", "batch-b"])
self.assertEqual(checkpoint["incrementalUnit"], "DIRECTORY")
self.assertEqual(checkpoint["directoryDepth"], 1)
self.assertEqual(checkpoint["newCount"], 2)
self.assertEqual(sorted(checkpoint["seenFingerprints"]), ["batch-a", "batch-b"])
def test_incremental_source_state_rejects_invalid_json(self):
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be valid JSON"):
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "{"})
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be a JSON object"):
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "[]"})
def test_incremental_source_state_rejects_invalid_shape(self):
cases = [
({"seenFingerprints": "fp-1"}, "seenFingerprints must be an array"),
({"seenFingerprints": [42]}, r"seenFingerprints\[0\] must be a string"),
({"incrementalUnit": "ROW"}, "incrementalUnit must be OBJECT or DIRECTORY"),
({"directoryDepth": 1.5}, "directoryDepth must be an integer"),
({"directoryDepth": 0}, "directoryDepth must be greater than 0"),
]
for payload, expected_error in cases:
with self.subTest(payload=payload), self.assertRaisesRegex(RuntimeError, expected_error):
MODULE.incremental_source_state({"SOURCE_STATE_JSON": json.dumps(payload)})
def test_incremental_directory_depth_rejects_lossy_values(self):
for raw in ["0", "01", "1.5", "2days"]:
with self.subTest(raw=raw), self.assertRaises(RuntimeError):
MODULE.incremental_directory_depth({"INCREMENTAL_DIRECTORY_DEPTH": raw})
def test_incremental_unit_rejects_invalid_values(self):
self.assertEqual(MODULE.incremental_unit({}), "OBJECT")
self.assertEqual(MODULE.incremental_unit({"INCREMENTAL_UNIT": " directory "}), "DIRECTORY")
with self.assertRaisesRegex(RuntimeError, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY"):
MODULE.incremental_unit({"INCREMENTAL_UNIT": "ROW"})
def test_builtin_source_rejects_invalid_scan_mode(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "BOS 增量扫描",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "INLINE"},
{"key": "SOURCE_TEXT", "value": "alpha"},
{"key": "SCAN_MODE", "value": "ROW"},
],
}
with self.assertRaisesRegex(RuntimeError, "SCAN_MODE must be FULL or INCREMENTAL"):
MODULE.builtin_source("run-test", "source-invalid-scan-mode", output_dir, node_data, [])
def test_builtin_source_rejects_invalid_data_source(self):
with tempfile.TemporaryDirectory() as temp_dir:
source_root = pathlib.Path(temp_dir) / "source"
source_root.mkdir()
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
output_dir = pathlib.Path(temp_dir) / "output"
output_dir.mkdir()
node_data = {
"label": "真实数据源-非法数据源",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "ROW"},
{"key": "SOURCE_PATH", "value": str(source_root)},
],
}
with self.assertRaisesRegex(RuntimeError, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM"):
MODULE.builtin_source("run-test", "source-invalid-data-source", output_dir, node_data, [])
def test_list_bos_entries_skips_large_non_metadata_json_payloads(self):
object_meta = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/meta.json", "size": 128, "e_tag": "meta"})()
object_imu = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json", "size": MODULE.PREVIEW_INLINE_BINARY_LIMIT + 1, "e_tag": "imu"})()
fake_client = _FakeBosListClient(
[object_meta, object_imu],
{
"wjzh-0330-Data-migration/SZ_000_0005/meta.json": b'{"type":"\xe5\xb7\xa5\xe5\x8e\x82"}',
"wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json": b'{"data_points":[1,2,3]}',
},
)
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
entries = MODULE.list_bos_entries(
"wjzh-data-migration",
"wjzh-0330-Data-migration",
"",
"https://bos.example.test",
"ak",
"sk",
)
self.assertEqual(len(entries), 2)
meta_entry = next(item for item in entries if item["path"].endswith("meta.json"))
imu_entry = next(item for item in entries if item["path"].endswith("_imu.json"))
self.assertIn("data", meta_entry)
self.assertNotIn("data", imu_entry)
self.assertEqual(
fake_client.downloaded_payload_keys,
[("wjzh-data-migration", "wjzh-0330-Data-migration/SZ_000_0005/meta.json")],
)
def test_bos_source_parses_bucket_and_prefix_from_source_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "BOS 增量扫描",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://wjzh-data-migration/wjzh-0330-Data-migration"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
{"key": "SCAN_MODE", "value": "FULL"},
],
}
fake_entries = [
{
"path": "SZ_000_0005/meta.json",
"storagePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {"type": "工厂"},
"source": {"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json"},
}
]
with mock.patch.object(MODULE, "list_bos_entries", return_value=fake_entries) as list_bos_entries, mock.patch.object(
MODULE,
"source_entry_artifact",
return_value={
"id": "artifact-1",
"runId": "run-1",
"nodeId": "source",
"name": "meta.json",
"type": "json",
"path": str(output_dir / "meta.json"),
},
):
artifacts = MODULE.builtin_source("run-1", "source", output_dir, node_data, [])
self.assertGreaterEqual(len(artifacts), 1)
list_bos_entries.assert_called_once_with(
"wjzh-data-migration",
"wjzh-0330-Data-migration",
"",
"https://bos.example.test",
"ak",
"sk",
)
class QualityMetricHelperTests(unittest.TestCase):
def test_build_skin_mask_detects_skin_tone_patch(self):
_, np = MODULE.require_cv2()
frame = np.zeros((80, 80, 3), dtype=np.uint8)
frame[20:60, 20:60] = (90, 130, 190)
mask = MODULE.build_skin_mask(frame)
self.assertGreater(int(mask.sum()), 0)
def test_build_skin_mask_rejects_neutral_background(self):
_, np = MODULE.require_cv2()
frame = np.full((80, 80, 3), 128, dtype=np.uint8)
mask = MODULE.build_skin_mask(frame)
self.assertEqual(int(mask.sum()), 0)
def test_motion_jitter_ratio_stays_low_for_smooth_motion(self):
smooth = [(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)]
jitter = MODULE.motion_jitter_ratio(smooth, 100.0)
self.assertLess(jitter, 0.01)
def test_motion_jitter_ratio_increases_for_erratic_motion(self):
smooth = MODULE.motion_jitter_ratio([(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)], 100.0)
erratic = MODULE.motion_jitter_ratio([(0.0, 0.0), (12.0, -4.0), (-10.0, 8.0), (15.0, -6.0)], 100.0)
self.assertGreater(erratic, smooth)
self.assertGreater(erratic, 0.05)
def test_largest_skin_component_ratio_ignores_small_noise(self):
_, np = MODULE.require_cv2()
mask = np.zeros((100, 100), dtype=np.uint8)
mask[10:13, 10:13] = 255
ratio = MODULE.largest_skin_component_ratio(mask)
self.assertEqual(ratio, 0.0)
def test_largest_skin_component_ratio_returns_large_component_area(self):
_, np = MODULE.require_cv2()
mask = np.zeros((100, 100), dtype=np.uint8)
mask[20:50, 30:70] = 255
ratio = MODULE.largest_skin_component_ratio(mask)
self.assertGreater(ratio, 0.1)
self.assertLess(ratio, 0.13)
class QualityOperatorTests(unittest.TestCase):
def test_video_quality_accepts_zero_hand_occlusion(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "MIN_CLARITY", "value": "0.8"},
{"key": "MAX_SHAKE", "value": "0.1"},
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
],
}
inputs = [
{
"data": {
"items": [
{
"filename": "demo.mp4",
"source": {"path": "/tmp/demo.mp4"},
}
]
}
}
]
with mock.patch.object(MODULE, "source_path_from_item", return_value=pathlib.Path("/tmp/demo.mp4")), mock.patch.object(
MODULE,
"analyze_video_quality_metrics",
return_value={
"clarity": 1.0,
"shake": 0.0,
"handOcclusion": 0.0,
"qualityScore": 1.0,
},
):
artifacts = MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["passedCount"], 1)
self.assertTrue(payload["items"][0]["quality"]["passed"])
def test_video_quality_rejects_out_of_range_thresholds(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "MIN_CLARITY", "value": "1.1"},
],
}
with self.assertRaisesRegex(RuntimeError, "MIN_CLARITY must be in the range"):
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
def test_video_quality_rejects_non_finite_thresholds(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "MAX_SHAKE", "value": "nan"},
],
}
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be finite"):
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
def test_video_quality_rejects_invalid_clarity_scale(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "CLARITY_SCALE", "value": "0"},
],
}
with self.assertRaisesRegex(RuntimeError, "CLARITY_SCALE must be greater than 0"):
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
def test_dataset_filter_keeps_zero_shake_and_hand_occlusion(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "数据过滤",
"hyperparams": [
{"key": "DATE_START", "value": "2026-03-01"},
{"key": "DATE_END", "value": "2026-03-31"},
{"key": "PLACE", "value": "蓟门里小区"},
{"key": "TYPE", "value": "家庭"},
{"key": "MIN_CLARITY", "value": "0.8"},
{"key": "MAX_SHAKE", "value": "0.1"},
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
],
}
inputs = [
{
"data": {
"items": [
{
"meta": {
"date": "20260319",
"place": "蓟门里小区",
"type": "家庭",
},
"quality": {
"clarity": 0.95,
"shake": 0.0,
"handOcclusion": 0.0,
"passed": True,
},
}
]
}
}
]
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["count"], 1)
self.assertEqual(len(payload["items"]), 1)
def test_dataset_filter_missing_thresholds_use_catalog_defaults(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "数据过滤",
"hyperparams": [],
}
inputs = [
{
"data": {
"items": [
{
"quality": {
"clarity": 0.5,
"shake": 0.0,
"handOcclusion": 0.0,
"passed": True,
},
},
{
"quality": {
"clarity": 0.95,
"shake": 0.0,
"handOcclusion": 0.0,
"passed": True,
},
},
]
}
}
]
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["criteria"]["minClarity"], MODULE.DEFAULT_MIN_CLARITY)
self.assertEqual(payload["criteria"]["maxShake"], MODULE.DEFAULT_MAX_SHAKE)
self.assertEqual(payload["criteria"]["maxHandOcclusion"], MODULE.DEFAULT_MAX_HAND_OCCLUSION)
self.assertEqual(payload["count"], 1)
self.assertEqual(payload["items"][0]["quality"]["clarity"], 0.95)
def test_dataset_filter_rejects_invalid_threshold_values(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "数据过滤",
"hyperparams": [
{"key": "MAX_SHAKE", "value": "not-a-number"},
],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be numeric"):
MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
class OperatorHyperparamParsingTests(unittest.TestCase):
def test_collection_validate_rejects_invalid_boolean_hyperparam(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "采集目录校验",
"hyperparams": [{"key": "REQUIRE_INTRINSICS", "value": "sometimes"}],
}
with self.assertRaisesRegex(RuntimeError, "REQUIRE_INTRINSICS must be a boolean"):
MODULE.builtin_collection_validate("run-1", "validate", output_dir, node_data, [{"data": {"collections": [], "items": []}}])
def test_json_filter_rejects_invalid_expected_json(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 过滤",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "EXPECTED_JSON", "value": '{"status"'},
],
}
with self.assertRaisesRegex(RuntimeError, "EXPECTED_JSON must be valid JSON"):
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
def test_json_filter_rejects_unknown_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 过滤",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "OPERATOR", "value": "roughly"},
{"key": "EXPECTED", "value": "ready"},
],
}
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
def test_json_map_rejects_non_object_field_map(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": "[]"}],
}
with self.assertRaisesRegex(RuntimeError, "FIELD_MAP_JSON must be a JSON object"):
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
def test_json_map_renames_fields_without_overwriting(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new","status":"state"}'}],
}
artifacts = MODULE.builtin_json_map(
"run-1",
"map",
output_dir,
node_data,
[{"data": {"items": [{"old": "value", "status": "ready", "unchanged": 1}]}}],
)
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["fieldMap"], {"old": "new", "status": "state"})
self.assertEqual(payload["items"], [{"new": "value", "state": "ready", "unchanged": 1}])
def test_json_map_rejects_invalid_field_map_entries(self):
cases = [
('{" old":"new"}', "must not have leading or trailing whitespace"),
('{"old":" new"}', "destination field must not have leading or trailing whitespace"),
('{"old":""}', "destination field must be non-empty"),
('{"old":42}', "destination field must be a string"),
('{"first":"name","last":"name"}', "maps multiple source fields to destination field"),
]
for field_map_json, expected_error in cases:
with self.subTest(field_map_json=field_map_json), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": field_map_json}],
}
with self.assertRaisesRegex(RuntimeError, expected_error):
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
def test_json_map_rejects_output_field_collisions(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new"}'}],
}
with self.assertRaisesRegex(RuntimeError, "same output field 'new'"):
MODULE.builtin_json_map(
"run-1",
"map",
output_dir,
node_data,
[{"data": {"items": [{"old": "mapped", "new": "existing"}]}}],
)
def test_audit_sampling_rejects_non_array_rules(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [{"key": "RULES_JSON", "value": "{}"}],
}
with self.assertRaisesRegex(RuntimeError, "RULES_JSON must be a JSON array"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_rejects_incomplete_rule_shape(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [
{"key": "RULES_JSON", "value": '[{"id":"missing-percent","field":"quality.clarity","operator":"gte","value":0.8}]'}
],
}
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent is required"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_rejects_duplicate_rule_ids(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
rules_json = json.dumps(
[
{"id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.8, "samplePercent": 100},
{"id": "clarity", "field": "quality.shake", "operator": "lte", "value": 0.1, "samplePercent": 50},
],
ensure_ascii=False,
)
node_data = {
"label": "抽检采样",
"hyperparams": [{"key": "RULES_JSON", "value": rules_json}],
}
with self.assertRaisesRegex(RuntimeError, "RULES_JSON contains duplicate id: clarity"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_rejects_invalid_sample_percent(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [
{
"key": "RULES_JSON",
"value": '[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":0}]',
}
],
}
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be in the range"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
node_data["hyperparams"][0]["value"] = (
'[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":true}]'
)
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be numeric"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_allows_exists_rule_without_value(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [
{
"key": "RULES_JSON",
"value": '[{"id":"has-collection","field":"collectionId","operator":"exists","samplePercent":100}]',
}
],
}
artifacts = MODULE.builtin_audit_sampling(
"run-1",
"sample",
output_dir,
node_data,
[
{
"data": {
"packageCount": 2,
"items": [
{"packageName": "clip-a", "collectionId": "collection-a"},
{"packageName": "clip-b"},
],
}
}
],
)
self.assertEqual(len(artifacts), 1)
payload = json.loads((output_dir / "audit-samples.json").read_text(encoding="utf-8"))
self.assertEqual(payload["rules"][0]["matchedCount"], 1)
self.assertEqual(payload["sampledCount"], 1)
def test_condition_router_rejects_unknown_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "条件分流",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "OPERATOR", "value": "roughly"},
{"key": "EXPECTED", "value": "ready"},
],
}
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
MODULE.builtin_condition_router("run-1", "route", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
class UploadOperatorTests(unittest.TestCase):
def test_build_artifact_skips_sha256_for_large_files(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
clip_path = root / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
MODULE,
"sha256_file",
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
):
artifact = MODULE.build_artifact("run-1", "slice", clip_path, "video")
metadata = artifact.get("metadata") or {}
self.assertEqual(metadata.get("sha256", ""), "")
self.assertTrue(metadata.get("sha256Skipped"))
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
def test_uploads_nested_package_outputs_to_bos(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
package_root = root / "package"
package_dir = package_root / "packages" / "clip001_package"
package_dir.mkdir(parents=True)
manifest_path = package_root / "packages-manifest.json"
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
clip_path = package_dir / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
annotation_path = package_dir / "clip001_annotation.json"
annotation_path.write_text('{"label":"walking"}', encoding="utf-8")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
MODULE.build_artifact("run-1", "package", clip_path, "video"),
MODULE.build_artifact("run-1", "package", annotation_path, "json", data={"label": "walking"}),
]
fake_client = _FakeBosClient()
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
uploaded_names = {item["name"] for item in artifacts}
uploaded_keys = {item["key"] for item in fake_client.uploads}
self.assertEqual(len(fake_client.uploads), 3)
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
self.assertEqual(uploaded_summary_payload["uploadedCount"], 3)
self.assertEqual(uploaded_summary_payload["bucket"], "datastorage")
self.assertEqual(uploaded_summary_payload["prefix"], "factory/output/demo")
def test_upload_skips_sha256_for_large_remote_artifacts(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
clip_path = root / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [MODULE.build_artifact("run-1", "slice", clip_path, "video")]
fake_client = _FakeBosClient()
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
MODULE,
"sha256_file",
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
remote_video = next(item for item in artifacts if item.get("type") == "video" and str(item.get("path", "")).startswith("bos://"))
metadata = remote_video.get("metadata") or {}
self.assertEqual(metadata.get("sha256", ""), "")
self.assertTrue(metadata.get("sha256Skipped"))
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
class _FakeInsertManyResult:
def __init__(self, inserted_ids):
self.inserted_ids = inserted_ids
class _FakeUpdateResult:
def __init__(self, matched_count=0, modified_count=0, upserted_id=None):
self.matched_count = matched_count
self.modified_count = modified_count
self.upserted_id = upserted_id
class _FakeMongoCollection:
def __init__(self):
self.insert_many_calls = []
self.replace_one_calls = []
def insert_many(self, documents, ordered=False):
self.insert_many_calls.append({"documents": documents, "ordered": ordered})
return _FakeInsertManyResult([f"id-{index + 1}" for index in range(len(documents))])
def replace_one(self, filter_document, replacement, upsert=False):
self.replace_one_calls.append(
{
"filter": filter_document,
"replacement": replacement,
"upsert": upsert,
}
)
return _FakeUpdateResult(matched_count=1, modified_count=1, upserted_id=None)
class _FakeMongoDatabase:
def __init__(self, collection):
self._collection = collection
self.requested_collections = []
def __getitem__(self, name):
self.requested_collections.append(name)
return self._collection
class _FakeMongoClient:
def __init__(self, database):
self._database = database
self.requested_databases = []
self.closed = False
def __getitem__(self, name):
self.requested_databases.append(name)
return self._database
def close(self):
self.closed = True
class MongoWriteOperatorTests(unittest.TestCase):
def test_inserts_documents_into_default_business_database(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
{"key": "MONGO_MODE", "value": "insert_many"},
],
}
inputs = [
{
"data": {
"items": [
{"clipId": "clip-001", "qualityScore": 0.91},
{"clipId": "clip-002", "qualityScore": 0.87},
]
}
}
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-1", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(database.requested_collections, ["qualityReports"])
self.assertEqual(client.requested_databases, ["business-data"])
self.assertEqual(len(collection.insert_many_calls), 1)
self.assertEqual(collection.insert_many_calls[0]["documents"][0]["clipId"], "clip-001")
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["collection"], "qualityReports")
self.assertEqual(summary["data"]["mode"], "insert_many")
self.assertEqual(summary["data"]["target"], "business")
self.assertEqual(summary["data"]["insertedCount"], 2)
self.assertTrue(client.closed)
def test_upserts_documents_by_key(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "annotations"},
{"key": "MONGO_MODE", "value": "upsert"},
{"key": "UPSERT_KEY", "value": "clipId"},
],
}
inputs = [
{"data": {"clipId": "clip-001", "label": "walking"}},
{"data": {"clipId": "clip-002", "label": "standing"}},
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-2", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(len(collection.replace_one_calls), 2)
self.assertEqual(collection.replace_one_calls[0]["filter"], {"clipId": "clip-001"})
self.assertTrue(collection.replace_one_calls[0]["upsert"])
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["mode"], "upsert")
self.assertEqual(summary["data"]["upsertKey"], "clipId")
self.assertEqual(summary["data"]["target"], "business")
self.assertEqual(summary["data"]["upsertedCount"], 2)
def test_unwraps_single_value_dict_documents_for_business_writes(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "businessWriteValidation"},
{"key": "MONGO_MODE", "value": "insert_many"},
],
}
inputs = [
{
"data": {
"items": [
{
"value": {
"runLabel": "business-mongo-test",
"source": "workflow",
"createdAt": "2026-04-14T00:00:00Z",
"value": 1,
}
}
]
}
}
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
MODULE.builtin_mongodb_write("run-unwrap", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(len(collection.insert_many_calls), 1)
inserted = collection.insert_many_calls[0]["documents"][0]
self.assertEqual(inserted["runLabel"], "business-mongo-test")
self.assertEqual(inserted["source"], "workflow")
self.assertEqual(inserted["value"], 1)
self.assertNotEqual(set(inserted.keys()), {"value"})
def test_can_target_platform_database_explicitly(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
{"key": "MONGO_TARGET", "value": "platform"},
],
}
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"PLATFORM_MONGO_URI": "mongodb://platform-mongo:27017", "PLATFORM_MONGO_DB": "droboticflow"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-3", "mongo-node", pathlib.Path(temp_dir), node_data, [{"data": {"clipId": "clip-001"}}])
self.assertEqual(client.requested_databases, ["droboticflow"])
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["target"], "platform")
def test_rebuilds_package_files_from_manifest_when_temp_paths_are_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
package_manifest = {
"packageCount": 1,
"items": [
{
"packageName": "clip001_package",
"clipId": "clip001",
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"startSecond": 0,
"endSecond": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
"quality": {"clarity": 0.9, "passed": True},
"annotation": {"objects": ["人"], "minConfidence": 0.92},
}
],
}
inputs = [
{"name": "packages-manifest.json", "type": "json", "data": package_manifest},
{
"name": "clip001.mp4",
"type": "video",
"path": "/tmp/non-existent-clip001.mp4",
"data": {"preview": {"kind": "video", "format": "mp4"}},
},
{
"name": "clip001_annotation.json",
"type": "json",
"path": "/tmp/non-existent-clip001_annotation.json",
"data": {"objects": ["人"], "minConfidence": 0.92},
},
{
"name": "clip001_quality.json",
"type": "json",
"path": "/tmp/non-existent-clip001_quality.json",
"data": {"clarity": 0.9, "passed": True},
},
{
"name": "clip001_config.json",
"type": "json",
"path": "/tmp/non-existent-clip001_config.json",
"data": {"collectionId": "collection-1"},
},
]
fake_client = _FakeBosClient()
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
uploaded_keys = {item["key"] for item in fake_client.uploads}
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_quality.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_config.json", uploaded_keys)
self.assertFalse(
(output_dir / "uploaded" / "_materialized_upload_inputs" / "packages" / "clip001_package" / "clip001.mp4").exists()
)
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
self.assertEqual(uploaded_summary_payload["uploadedCount"], 5)
def test_upload_artifacts_are_json_serializable_when_preview_url_is_bytes(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
package_root = root / "package"
package_dir = package_root / "packages" / "clip001_package"
package_dir.mkdir(parents=True)
manifest_path = package_root / "packages-manifest.json"
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
clip_path = package_dir / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
MODULE.build_artifact("run-1", "package", clip_path, "video"),
]
with mock.patch.object(MODULE, "build_bos_client", return_value=(_FakeBosClientBytesURL(), "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
json.dumps({"artifacts": artifacts}, ensure_ascii=False)
class XPlatformClipHandoffTests(unittest.TestCase):
def test_summarize_sqlite_payload_does_not_create_wal_sidecars(self):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = pathlib.Path(temp_dir) / "talker.db3"
connection = sqlite3.connect(db_path)
try:
connection.execute("pragma journal_mode=wal")
connection.execute("create table topics (id integer primary key, name text, type text, serialization_format text)")
connection.execute("create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)")
connection.execute(
"insert into topics (id, name, type, serialization_format) values (1, '/chatter', 'std_msgs/msg/String', 'cdr')"
)
connection.execute("insert into messages (topic_id, timestamp, data) values (1, 1, X'00')")
connection.commit()
finally:
connection.close()
for sidecar in db_path.parent.glob("talker.db3-*"):
sidecar.unlink()
payload = MODULE.summarize_sqlite_payload(db_path)
self.assertEqual(payload["preview"]["format"], "rosbag2")
self.assertEqual(payload["preview"]["messageCount"], 1)
self.assertEqual(list(db_path.parent.glob("talker.db3-*")), [])
def test_video_slicing_rejects_invalid_segment_duration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "0"}],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be greater than 0"):
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
def test_video_slicing_rejects_lossy_segment_duration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "01"}],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be an exact positive integer"):
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
def test_video_slicing_rejects_unsafe_segment_duration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": str(MODULE.MAX_SAFE_INTEGER + 1)}],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be a safe integer"):
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
def test_video_slicing_manifest_preserves_source_reference(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 41.8,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
clip = manifest_payload["items"][0]
self.assertEqual(clip["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
self.assertEqual(clip["clipName"], "DJI_20260318112834_0001_D_clip001.mp4")
def test_video_slicing_uses_single_pass_writer_for_multi_clip_video(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 365.0,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_write_video_segments(_source_path, segments):
outputs = []
for segment in segments:
destination = pathlib.Path(segment["destination"])
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(f"clip-{segment['clipIndex']}".encode("utf-8"))
outputs.append(
{
**segment,
"duration": round(float(segment["endSecond"]) - float(segment["startSecond"]), 3),
"frameRate": 29.97,
"resolution": "3840x2160",
}
)
return outputs
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(
MODULE,
"write_video_segments",
side_effect=fake_write_video_segments,
) as multi_writer, mock.patch.object(
MODULE,
"write_video_segment",
) as single_writer, mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["clipCount"], 2)
self.assertEqual([item["clipName"] for item in manifest_payload["items"]], [
"DJI_20260318112834_0001_D_clip001.mp4",
"DJI_20260318112834_0001_D_clip002.mp4",
])
multi_writer.assert_called_once()
single_writer.assert_not_called()
def test_write_video_segment_with_ffmpeg_prefers_stream_copy_and_uses_inspected_metrics(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
destination = root / "clips" / "clip001.mp4"
def fake_run(args, stdout=None, stderr=None, text=None):
self.assertEqual(args[0], "ffmpeg")
self.assertIn(str(source_path), args)
self.assertEqual(args[-1], str(destination))
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-video")
return mock.Mock(returncode=0, stdout="", stderr="")
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE.subprocess,
"run",
side_effect=fake_run,
) as run_mock, mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
):
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
self.assertEqual(metrics["duration"], 21.5)
self.assertEqual(metrics["frameRate"], 29.97)
self.assertEqual(metrics["resolution"], "3840x2160")
run_args = run_mock.call_args.args[0]
self.assertIn("-ss", run_args)
self.assertIn("-t", run_args)
self.assertIn("-c", run_args)
self.assertIn("copy", run_args)
self.assertNotIn("-movflags", run_args)
self.assertNotIn("+faststart", run_args)
def test_write_video_segment_with_ffmpeg_falls_back_to_reencode_when_copy_fails(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
destination = root / "clips" / "clip001.mp4"
calls = []
def fake_run(args, stdout=None, stderr=None, text=None):
calls.append(list(args))
if len(calls) == 1:
return mock.Mock(returncode=1, stdout="", stderr="copy failed")
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-video")
return mock.Mock(returncode=0, stdout="", stderr="")
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE.subprocess,
"run",
side_effect=fake_run,
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
):
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
self.assertEqual(metrics["duration"], 21.5)
self.assertEqual(len(calls), 2)
self.assertIn("copy", calls[0])
self.assertIn("libx264", calls[1])
self.assertIn("aac", calls[1])
def test_video_slicing_prefers_ffmpeg_when_available(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 365.0,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_ffmpeg_writer(_source_path, destination, start_second, end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(f"{start_second}-{end_second}".encode("utf-8"))
return {
"duration": round(float(end_second) - float(start_second), 3),
"frameRate": 29.97,
"resolution": "3840x2160",
}
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(
MODULE,
"write_video_segment_with_ffmpeg",
side_effect=fake_ffmpeg_writer,
) as ffmpeg_writer, mock.patch.object(
MODULE,
"write_video_segments",
) as multi_writer, mock.patch.object(
MODULE,
"write_video_segment",
) as single_writer, mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["clipCount"], 2)
self.assertEqual(ffmpeg_writer.call_count, 2)
multi_writer.assert_not_called()
single_writer.assert_not_called()
def test_video_annotation_rejects_invalid_frame_interval(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频标注",
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "not-a-number"}],
}
inputs = [{"data": {"clipCount": 0, "items": []}}]
with self.assertRaisesRegex(RuntimeError, "FRAME_INTERVAL_SECONDS must be numeric"):
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
def test_video_annotation_uses_explicit_positive_frame_interval_without_clamping(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
clip_path = output_dir / "clip001.mp4"
clip_path.write_bytes(b"clip-bytes")
node_data = {
"label": "视频标注",
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "0.05"}],
}
inputs = [
{
"data": {
"clipCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"clipPath": str(clip_path),
"sourceVideo": "source.mp4",
}
],
}
}
]
with mock.patch.object(MODULE, "annotate_video_clip", return_value={"objects": [], "frameCount": 1}) as annotator:
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
annotator.assert_called_once()
self.assertEqual(annotator.call_args.args[1], 0.05)
def test_video_annotation_reuses_shared_clip_path_without_copying_clip_artifact(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
shared_clip_path = output_dir / "shared" / "clip001.mp4"
shared_clip_path.parent.mkdir(parents=True, exist_ok=True)
shared_clip_path.write_bytes(b"clip-bytes")
inputs = [
{
"name": "clips-manifest.json",
"type": "json",
"data": {
"clipCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"clipPath": str(shared_clip_path),
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"duration": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频标注",
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "1.0"}],
}
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
MODULE,
"annotate_video_clip",
return_value={
"duration": 41.8,
"actions": [{"label": "walking", "confidence": 0.92, "start": 0, "end": 41.8}],
"objects": ["人"],
"minConfidence": 0.92,
},
), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "annotations-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["annotationCount"], 1)
self.assertEqual(manifest_payload["items"][0]["clipPath"], str(shared_clip_path))
self.assertFalse(any(item["name"] == "clip001.mp4" for item in artifacts))
def test_materialize_clip_from_source_prefers_remote_segment_generation(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
item = {
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
},
}
def fake_remote_segment(_source_url, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8}
with mock.patch.object(MODULE, "remote_video_url_for_source", return_value="https://signed.example.com/video.mp4"), mock.patch.object(
MODULE,
"write_video_segment_from_remote_url",
side_effect=fake_remote_segment,
) as remote_writer, mock.patch.object(MODULE, "source_path_from_item") as local_materializer:
resolved = MODULE.materialize_clip_from_source(item, output_dir)
self.assertIsNotNone(resolved)
self.assertTrue(resolved.exists())
self.assertEqual(resolved.read_bytes(), b"clip-bytes")
remote_writer.assert_called_once()
local_materializer.assert_not_called()
def test_result_package_rebuilds_clip_from_source_when_clip_artifact_is_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "annotations-manifest.json",
"type": "json",
"data": {
"annotationCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"duration": 41.8,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"clarity": 0.9, "passed": True},
"annotation": {"actions": [], "objects": ["人"], "minConfidence": 0.92},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "成品打包",
"hyperparams": [],
}
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
MODULE,
"write_video_segment",
side_effect=fake_write_video_segment,
), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
package_clip = next(item for item in artifacts if item["name"] == "clip001.mp4")
self.assertEqual(manifest_payload["packageCount"], 1)
manifest_item = manifest_payload["items"][0]
self.assertEqual(manifest_item["clipName"], "clip001.mp4")
self.assertEqual(manifest_item["collectionId"], "collection-1")
self.assertEqual(manifest_item["startSecond"], 0)
self.assertEqual(manifest_item["endSecond"], 41.8)
self.assertEqual(manifest_item["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
self.assertEqual(package_clip["name"], "clip001.mp4")
class ModelInferenceOperatorTests(unittest.TestCase):
def test_builtin_model_inference_runs_linear_model(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
model = {
"type": "linear",
"name": "quality-gate",
"features": ["quality.clarity", "quality.shake"],
"weights": [1.0, -2.0],
"bias": 0.0,
"threshold": 0.5,
"positiveLabel": "pass",
"negativeLabel": "fail",
}
node_data = {
"label": "模型推理",
"hyperparams": [{"key": "MODEL_JSON", "value": json.dumps(model)}],
}
inputs = [
{
"name": "dataset.json",
"type": "json",
"data": {
"items": [
{"id": "clip-1", "quality": {"clarity": 0.9, "shake": 0.1}},
{"id": "clip-2", "quality": {"clarity": 0.2, "shake": 0.2}},
]
},
}
]
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, inputs)
self.assertEqual([item["name"] for item in artifacts], ["inference-result.json"])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["modelType"], "linear")
self.assertEqual(payload["modelName"], "quality-gate")
expected_digest = hashlib.sha256(
json.dumps(model, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
self.assertEqual(payload["modelSource"], "MODEL_JSON")
self.assertEqual(payload["modelSha256"], expected_digest)
self.assertEqual(payload["inputCount"], 2)
self.assertEqual([item["prediction"]["label"] for item in payload["predictions"]], ["pass", "fail"])
self.assertEqual(payload["predictions"][0]["input"]["id"], "clip-1")
def test_builtin_model_inference_runs_nearest_centroid_from_inline_input(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
model = {
"type": "nearest_centroid",
"name": "motion-cluster",
"features": ["speed", "turnRate"],
"labels": ["slow-straight", "fast-turn"],
"centroids": [[0.1, 0.0], [1.0, 0.8]],
}
node_data = {
"label": "模型推理",
"hyperparams": [
{"key": "MODEL_JSON", "value": json.dumps(model)},
{"key": "INPUT_JSON", "value": json.dumps([{"id": "clip-1", "speed": 0.95, "turnRate": 0.7}])},
],
}
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["inputCount"], 1)
self.assertEqual(payload["predictions"][0]["prediction"]["label"], "fast-turn")
self.assertGreater(payload["predictions"][0]["prediction"]["score"], 0)
def test_builtin_model_inference_rejects_invalid_rule_comparison(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "模型推理",
"hyperparams": [
{
"key": "MODEL_JSON",
"value": json.dumps(
{
"type": "rule",
"rules": [{"field": "scene", "operator": "gt", "value": 1, "label": "bad"}],
}
),
},
{"key": "INPUT_JSON", "value": json.dumps({"scene": "kitchen"})},
],
}
with self.assertRaisesRegex(RuntimeError, "requires numeric values"):
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
def test_builtin_model_inference_rejects_constant_model(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "模型推理",
"hyperparams": [
{"key": "MODEL_JSON", "value": json.dumps({"type": "constant", "label": "pass"})},
{"key": "INPUT_JSON", "value": json.dumps({"id": "clip-1"})},
],
}
with self.assertRaisesRegex(RuntimeError, "constant model type is not accepted"):
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
class ShellCommandOperatorTests(unittest.TestCase):
def test_builtin_shell_command_requires_command(self):
with tempfile.TemporaryDirectory() as temp_dir:
with self.assertRaisesRegex(RuntimeError, "需要 COMMAND 或 SCRIPT"):
MODULE.builtin_shell_command(
"run-1",
"shell",
pathlib.Path(temp_dir),
{"label": "Shell 命令执行", "hyperparams": []},
[],
)
def test_builtin_shell_command_executes_command_and_collects_output(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "Shell 命令执行",
"hyperparams": [
{
"key": "COMMAND",
"value": "python3 -c 'import json; json.dump({\"ok\": True}, open(\"result.json\", \"w\"))'",
}
],
"outputs": [{"name": "result", "type": "json"}],
}
previous_policy = os.environ.get("OPERATOR_ALLOW_SHELL_COMMANDS")
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = "true"
try:
artifacts = MODULE.builtin_shell_command("run-1", "shell", output_dir, node_data, [])
finally:
if previous_policy is None:
os.environ.pop("OPERATOR_ALLOW_SHELL_COMMANDS", None)
else:
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = previous_policy
self.assertEqual([item["name"] for item in artifacts], ["result.json"])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload, {"ok": True})
class _DCloudAPIHandler(http.server.BaseHTTPRequestHandler):
requests = []
def do_POST(self):
length = int(self.headers.get("Content-Length", "0") or "0")
body = self.rfile.read(length).decode("utf-8")
payload = json.loads(body) if body else {}
if self.path == "/api/signin":
self.__class__.requests.append(
{
"method": "POST",
"path": self.path,
"authorization": self.headers.get("Authorization", ""),
"cookie": self.headers.get("Cookie", ""),
"d_user_id": self.headers.get("d-user-id", ""),
"org_symbol": self.headers.get("org-symbol", ""),
"sourceapp": self.headers.get("sourceapp", ""),
"body": {"hasUsername": bool(payload.get("Username")), "hasPassword": bool(payload.get("Password"))},
}
)
response_bytes = json.dumps({"status": 0, "data": {"token": "signin.header.payload"}}).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_bytes)))
self.end_headers()
self.wfile.write(response_bytes)
return
self.__class__.requests.append(
{
"method": "POST",
"path": self.path,
"authorization": self.headers.get("Authorization", ""),
"cookie": self.headers.get("Cookie", ""),
"d_user_id": self.headers.get("d-user-id", ""),
"org_symbol": self.headers.get("org-symbol", ""),
"sourceapp": self.headers.get("sourceapp", ""),
"body": payload,
}
)
response = {
"taskId": "task-" + str(len(self.__class__.requests)),
"previewURL": f"http://{self.server.server_address[0]}:{self.server.server_address[1]}/preview/{payload.get('name', 'item')}.json",
"payloadEcho": payload,
}
response_bytes = json.dumps(response).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_bytes)))
self.end_headers()
self.wfile.write(response_bytes)
def do_GET(self):
self.__class__.requests.append(
{
"method": "GET",
"path": self.path,
"authorization": self.headers.get("Authorization", ""),
"cookie": self.headers.get("Cookie", ""),
"d_user_id": self.headers.get("d-user-id", ""),
"org_symbol": self.headers.get("org-symbol", ""),
"sourceapp": self.headers.get("sourceapp", ""),
"body": None,
}
)
response = {"jobId": "job-preview", "previewUrl": f"http://{self.server.server_address[0]}:{self.server.server_address[1]}/asset.png"}
response_bytes = json.dumps(response).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_bytes)))
self.end_headers()
self.wfile.write(response_bytes)
def log_message(self, *_args):
return
@contextlib.contextmanager
def dcloud_api_server():
_DCloudAPIHandler.requests = []
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _DCloudAPIHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
previous_no_proxy = os.environ.get("NO_PROXY")
previous_no_proxy_lower = os.environ.get("no_proxy")
os.environ["NO_PROXY"] = "127.0.0.1,localhost"
os.environ["no_proxy"] = "127.0.0.1,localhost"
thread.start()
try:
yield server, _DCloudAPIHandler.requests
finally:
server.shutdown()
thread.join(timeout=5)
server.server_close()
if previous_no_proxy is None:
os.environ.pop("NO_PROXY", None)
else:
os.environ["NO_PROXY"] = previous_no_proxy
if previous_no_proxy_lower is None:
os.environ.pop("no_proxy", None)
else:
os.environ["no_proxy"] = previous_no_proxy_lower
class DCloudFATAPIOperatorTests(unittest.TestCase):
def test_dcloud_embodied_annotation_operator_calls_real_http_endpoint(self):
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "D-Cloud 具身标注 API",
"hyperparams": [
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
{"key": "ENDPOINT", "value": "/api/annoMidwareApi/v1/annotate/detection"},
{"key": "METHOD", "value": "POST"},
{"key": "REQUEST_JSON", "value": json.dumps({"name": "clip-1", "videoUrl": "http://example.test/clip.mp4"})},
{"key": "AUTH_TOKEN", "value": "real-token"},
{"key": "COOKIE", "value": "dcloud_session=real-session"},
{"key": "D_USER_ID", "value": "user-1"},
{"key": "ORG_SYMBOL", "value": "org-1"},
{"key": "SOURCEAPP", "value": "dataCopilot"},
],
}
artifacts = MODULE.builtin_dcloud_embodied_annotation("run-1", "anno", output_dir, node_data, [])
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0]["path"], "/api/annoMidwareApi/v1/annotate/detection")
self.assertEqual(requests[0]["authorization"], "Bearer real-token")
self.assertEqual(requests[0]["cookie"], "dcloud_session=real-session")
self.assertEqual(requests[0]["d_user_id"], "user-1")
self.assertEqual(requests[0]["org_symbol"], "org-1")
self.assertEqual(requests[0]["sourceapp"], "dataCopilot")
self.assertEqual(requests[0]["body"]["name"], "clip-1")
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["response"]["status"], 200)
self.assertEqual(payload["response"]["body"]["payloadEcho"]["videoUrl"], "http://example.test/clip.mp4")
self.assertTrue(payload["preview"]["urls"])
self.assertEqual(payload["sourceRepo"], "https://gitea.d-robotics.cc/d-cloud/robo_annotation_backend")
def test_dcloud_data_generation_operator_supports_preview_query(self):
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "D-Cloud 具身数据生成 API",
"hyperparams": [
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
{"key": "ENDPOINT", "value": "/api/generationApi/tasks/preview"},
{"key": "METHOD", "value": "GET"},
{"key": "QUERY_JSON", "value": json.dumps({"taskId": "task-1"})},
],
}
with mock.patch.dict(
os.environ,
{
"DCLOUD_D_USER_ID": "user-2",
"DCLOUD_ORG_SYMBOL": "org-2",
"DCLOUD_SOURCEAPP": "dataCopilot",
},
clear=False,
):
artifacts = MODULE.builtin_dcloud_data_generation("run-1", "gen", output_dir, node_data, [])
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0]["method"], "GET")
self.assertEqual(requests[0]["path"], "/api/generationApi/tasks/preview?taskId=task-1")
self.assertEqual(requests[0]["d_user_id"], "user-2")
self.assertEqual(requests[0]["org_symbol"], "org-2")
self.assertEqual(requests[0]["sourceapp"], "dataCopilot")
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["response"]["status"], 200)
self.assertEqual(payload["preview"]["ids"], ["job-preview"])
self.assertEqual(payload["sourceRepo"], "https://gitea.d-robotics.cc/d-cloud/data_generation")
def test_dcloud_operator_can_sign_in_for_real_fat_token(self):
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "D-Cloud 具身数据生成 API",
"hyperparams": [
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
{"key": "ENDPOINT", "value": "/api/generationApi/imgGeneration/tasks"},
{"key": "METHOD", "value": "POST"},
{"key": "REQUEST_JSON", "value": json.dumps({"currentPage": 1, "pageSize": 1})},
],
}
with mock.patch.dict(
os.environ,
{
"DCLOUD_FAT_USERNAME": "fat-user",
"DCLOUD_FAT_PASSWORD": "fat-password",
"DCLOUD_ORG_SYMBOL": "org-3",
"DCLOUD_SOURCEAPP": "dataCopilot",
},
clear=False,
):
artifacts = MODULE.builtin_dcloud_data_generation("run-1", "gen", output_dir, node_data, [])
self.assertEqual(len(requests), 2)
self.assertEqual(requests[0]["path"], "/api/signin")
self.assertEqual(requests[0]["body"], {"hasUsername": True, "hasPassword": True})
self.assertEqual(requests[1]["path"], "/api/generationApi/imgGeneration/tasks")
self.assertEqual(requests[1]["authorization"], "Bearer signin.header.payload")
self.assertEqual(requests[1]["org_symbol"], "org-3")
self.assertEqual(requests[1]["sourceapp"], "dataCopilot")
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["response"]["status"], 200)
if __name__ == "__main__":
unittest.main()