forked from longtao.wu/droboticflow
2887 lines
134 KiB
Python
2887 lines
134 KiB
Python
import contextlib
|
|
import hashlib
|
|
import http.server
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import sqlite3
|
|
import threading
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
MODULE_PATH = pathlib.Path(__file__).with_name("main.py")
|
|
SPEC = importlib.util.spec_from_file_location("operator_runner_main", MODULE_PATH)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
MODULE = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(MODULE)
|
|
|
|
RESOURCE_MONITOR_PATH = pathlib.Path(__file__).with_name("resource_monitor.py")
|
|
RESOURCE_MONITOR_SPEC = importlib.util.spec_from_file_location("operator_runner_resource_monitor", RESOURCE_MONITOR_PATH)
|
|
assert RESOURCE_MONITOR_SPEC is not None and RESOURCE_MONITOR_SPEC.loader is not None
|
|
RESOURCE_MONITOR = importlib.util.module_from_spec(RESOURCE_MONITOR_SPEC)
|
|
RESOURCE_MONITOR_SPEC.loader.exec_module(RESOURCE_MONITOR)
|
|
|
|
|
|
class _FakeHTTPResponse:
|
|
def __init__(self, payload: bytes):
|
|
self._payload = payload
|
|
self._offset = 0
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def read(self, *_args, **_kwargs):
|
|
if self._offset >= len(self._payload):
|
|
return b""
|
|
chunk = self._payload[self._offset:]
|
|
self._offset = len(self._payload)
|
|
return chunk
|
|
|
|
|
|
class _FakeBosClient:
|
|
def __init__(self):
|
|
self.uploads = []
|
|
self.downloads = []
|
|
|
|
def put_object_from_file(self, bucket, key, file_name, **kwargs):
|
|
self.uploads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"key": key,
|
|
"file_name": file_name,
|
|
"content_type": kwargs.get("content_type"),
|
|
}
|
|
)
|
|
|
|
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
|
|
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}"
|
|
|
|
def get_object_to_file(self, bucket, key, file_name, **_kwargs):
|
|
self.downloads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"key": key,
|
|
"file_name": file_name,
|
|
}
|
|
)
|
|
pathlib.Path(file_name).write_bytes(f"{bucket}/{key}".encode("utf-8"))
|
|
|
|
|
|
class _FakeBosClientBytesURL(_FakeBosClient):
|
|
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
|
|
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}".encode("utf-8")
|
|
|
|
|
|
class _FakeBosListClient(_FakeBosClient):
|
|
def __init__(self, objects, object_payloads):
|
|
super().__init__()
|
|
self._objects = objects
|
|
self._object_payloads = object_payloads
|
|
self.downloaded_payload_keys = []
|
|
|
|
def list_all_objects(self, bucket, prefix=None):
|
|
return list(self._objects)
|
|
|
|
def get_object_as_string(self, bucket, key):
|
|
self.downloaded_payload_keys.append((bucket, key))
|
|
return self._object_payloads[key]
|
|
|
|
|
|
class CommandPolicyTests(unittest.TestCase):
|
|
def test_command_policy_rejects_commands_by_default(self):
|
|
with mock.patch.dict(os.environ, {}, clear=True):
|
|
with self.assertRaisesRegex(RuntimeError, "disabled"):
|
|
MODULE.validate_command_policy("printf 'ok'")
|
|
|
|
def test_command_policy_allows_commands_when_explicitly_enabled(self):
|
|
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "true"}, clear=True):
|
|
MODULE.validate_command_policy("printf 'ok'")
|
|
|
|
def test_command_policy_can_disable_shell_commands(self):
|
|
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "false"}, clear=True):
|
|
with self.assertRaisesRegex(RuntimeError, "disabled"):
|
|
MODULE.validate_command_policy("printf 'ok'")
|
|
|
|
def test_command_policy_rejects_denylisted_command(self):
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_DENYLIST_REGEX": r"(^|\s)rm\s+-rf"},
|
|
clear=True,
|
|
):
|
|
with self.assertRaisesRegex(RuntimeError, "DENYLIST"):
|
|
MODULE.validate_command_policy("rm -rf /workspace")
|
|
|
|
def test_command_policy_requires_allowlist_match_when_configured(self):
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_ALLOWLIST_REGEX": r"^python3?\s"},
|
|
clear=True,
|
|
):
|
|
MODULE.validate_command_policy("python3 script.py")
|
|
with self.assertRaisesRegex(RuntimeError, "ALLOWLIST"):
|
|
MODULE.validate_command_policy("bash script.sh")
|
|
|
|
|
|
class ResourceMonitorTests(unittest.TestCase):
|
|
def test_parse_gpu_snapshot_output_summarizes_real_nvidia_smi_columns(self):
|
|
snapshot = RESOURCE_MONITOR.parse_gpu_snapshot_output(
|
|
"\n".join(
|
|
[
|
|
"12, 4096, 24576, NVIDIA L4",
|
|
"83, 8192, 24576, NVIDIA L4",
|
|
"not-a-number, 1, 1, broken",
|
|
]
|
|
)
|
|
)
|
|
|
|
self.assertEqual(snapshot["gpu"], 83)
|
|
self.assertEqual(snapshot["gpuMemoryUsedMiB"], 12288)
|
|
self.assertEqual(snapshot["gpuMemoryTotalMiB"], 49152)
|
|
self.assertEqual(snapshot["gpuName"], "NVIDIA L4")
|
|
self.assertEqual(snapshot["gpuCount"], 2)
|
|
|
|
def test_parse_gpu_snapshot_output_defaults_when_nvidia_smi_has_no_valid_rows(self):
|
|
self.assertEqual(RESOURCE_MONITOR.parse_gpu_snapshot_output(""), {"gpu": 0})
|
|
|
|
|
|
class SourcePathFromItemTests(unittest.TestCase):
|
|
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "node-output"
|
|
workspace_root = pathlib.Path(temp_dir) / "workspace"
|
|
output_dir.mkdir(parents=True)
|
|
workspace_root.mkdir(parents=True)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"previewURL": "https://example.com/should-not-download.mp4",
|
|
}
|
|
}
|
|
fake_client = _FakeBosClient()
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"WORKSPACE_ROOT": str(workspace_root),
|
|
"BOS_ENDPOINT": "https://bos.example.test",
|
|
"BOS_ACCESS_KEY_ID": "ak",
|
|
"BOS_SECRET_ACCESS_KEY": "sk",
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")) as build_client, mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
) as urlopen:
|
|
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
|
|
self.assertTrue(first.exists())
|
|
self.assertEqual(first, second)
|
|
self.assertIn(str(workspace_root), str(first))
|
|
self.assertEqual(len(fake_client.downloads), 1)
|
|
build_client.assert_called()
|
|
urlopen.assert_not_called()
|
|
|
|
def test_downloads_http_preview_url_to_local_file(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/demo.mp4",
|
|
"previewURL": "https://example.com/demo.mp4?signature=abc",
|
|
}
|
|
}
|
|
with mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
return_value=_FakeHTTPResponse(b"real-video-bytes"),
|
|
) as urlopen:
|
|
resolved = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
self.assertTrue(resolved.exists())
|
|
self.assertEqual(resolved.read_bytes(), b"real-video-bytes")
|
|
urlopen.assert_called_once()
|
|
|
|
def test_prefers_workspace_cache_path_for_preview_url_when_bos_credentials_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "node-output"
|
|
workspace_root = pathlib.Path(temp_dir) / "workspace"
|
|
output_dir.mkdir(parents=True)
|
|
workspace_root.mkdir(parents=True)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0000/DJI_001/demo.mp4",
|
|
"previewURL": "https://example.com/demo.mp4?signature=abc",
|
|
}
|
|
}
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"WORKSPACE_ROOT": str(workspace_root),
|
|
"BOS_ENDPOINT": "",
|
|
"BOS_ACCESS_KEY_ID": "",
|
|
"BOS_SECRET_ACCESS_KEY": "",
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
return_value=_FakeHTTPResponse(b"real-video-bytes"),
|
|
) as urlopen:
|
|
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
self.assertTrue(first.exists())
|
|
self.assertEqual(first, second)
|
|
self.assertEqual(first.read_bytes(), b"real-video-bytes")
|
|
self.assertIn(str(workspace_root / "_bos_cache"), str(first))
|
|
urlopen.assert_called_once()
|
|
|
|
def test_remote_video_url_for_source_signs_bos_path_when_preview_missing(self):
|
|
source = {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_0001_D.MP4",
|
|
"previewURL": "",
|
|
}
|
|
with mock.patch.object(MODULE, "signed_bos_source_url", return_value="https://signed.example.com/video.mp4") as signer:
|
|
resolved = MODULE.remote_video_url_for_source(source)
|
|
self.assertEqual(resolved, "https://signed.example.com/video.mp4")
|
|
signer.assert_called_once_with(source["path"])
|
|
|
|
|
|
class RuntimeFileContractTests(unittest.TestCase):
|
|
def test_select_handler_prefers_specific_operator_routes(self):
|
|
self.assertEqual(MODULE.select_handler("BOS 增量扫描"), "bos_incremental_scan")
|
|
self.assertEqual(MODULE.select_handler("人工审核"), "human_review")
|
|
|
|
def test_operator_helpers_do_not_expose_legacy_builtin_handlers(self):
|
|
legacy_keys = sorted(key for key in MODULE.OPERATOR_HELPERS if key.startswith("builtin_"))
|
|
self.assertEqual(legacy_keys, [])
|
|
|
|
def test_dispatch_runs_standalone_csv_to_json_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "input.csv"
|
|
source_path.write_text("id,name,score\n1,alpha,98\n2,beta,87\n", encoding="utf-8")
|
|
output_dir = root / "out"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "CSV 转 JSON",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(source_path)}],
|
|
"outputs": [{"id": "out-1", "name": "table", "type": "json"}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["csv_to_json"]("run-1", "csv", output_dir, node_data, [])
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["columns"], ["id", "name", "score"])
|
|
self.assertEqual(payload["rowCount"], 2)
|
|
self.assertEqual(payload["items"][1]["name"], "beta")
|
|
|
|
def test_dispatch_runs_standalone_json_filter_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 过滤",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "EXPECTED", "value": "ready"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["json_filter"](
|
|
"run-1",
|
|
"filter",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"status": "ready"}, {"status": "blocked"}]}}],
|
|
)
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(payload["items"], [{"status": "ready"}])
|
|
|
|
def test_dispatch_runs_named_human_review_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "人工审核",
|
|
"hyperparams": [
|
|
{"key": "REVIEW_TYPE", "value": "task"},
|
|
{"key": "REVIEWER_ROLE", "value": "qa_reviewer"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_human_review"](
|
|
"run-1",
|
|
"review",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"id": "clip-1", "score": 0.9}]}}],
|
|
)
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["datasetType"], "human-review-tasks")
|
|
self.assertEqual(payload["reviewerRole"], "qa_reviewer")
|
|
self.assertEqual(payload["taskCount"], 1)
|
|
self.assertEqual(payload["items"][0]["id"], "run-1-clip-1")
|
|
|
|
def test_load_input_artifacts_prefers_manifest_file(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
manifest_path = pathlib.Path(temp_dir) / "input-artifacts.json"
|
|
manifest_path.write_text('[{"id":"artifact-from-file"}]', encoding="utf-8")
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"INPUT_ARTIFACTS_PATH": str(manifest_path),
|
|
"INPUT_ARTIFACTS_JSON": '[{"id":"artifact-inline"}]',
|
|
},
|
|
clear=False,
|
|
):
|
|
loaded = MODULE.load_input_artifacts()
|
|
|
|
self.assertEqual(len(loaded), 1)
|
|
self.assertEqual(loaded[0]["id"], "artifact-from-file")
|
|
|
|
def test_main_writes_result_and_stage_manifest_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
node_root = pathlib.Path(temp_dir)
|
|
output_dir = node_root / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
input_artifacts_path = node_root / "input-artifacts.json"
|
|
input_artifacts_path.write_text("[]", encoding="utf-8")
|
|
result_path = node_root / "result.json"
|
|
stage_manifest_path = node_root / "stage-manifest.json"
|
|
artifacts = [
|
|
{
|
|
"id": "artifact-1",
|
|
"runId": "run-1",
|
|
"nodeId": "node-1",
|
|
"name": "report.json",
|
|
"type": "json",
|
|
"size": "12 B",
|
|
"path": str(output_dir / "report.json"),
|
|
}
|
|
]
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"RUN_ID": "run-1",
|
|
"WORKFLOW_ID": "wf-1",
|
|
"NODE_ID": "node-1",
|
|
"NODE_LABEL": "通用处理",
|
|
"OUTPUT_DIR": str(output_dir),
|
|
"NODE_DATA_JSON": '{"label":"通用处理","hyperparams":[]}',
|
|
"INPUT_ARTIFACTS_PATH": str(input_artifacts_path),
|
|
"RESULT_OUTPUT_PATH": str(result_path),
|
|
"STAGE_MANIFEST_PATH": str(stage_manifest_path),
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(MODULE, "builtin_generic", return_value=artifacts), mock.patch.object(
|
|
MODULE,
|
|
"resource_series",
|
|
return_value=[],
|
|
):
|
|
stdout = io.StringIO()
|
|
with contextlib.redirect_stdout(stdout):
|
|
exit_code = MODULE.main()
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertIn(MODULE.RESULT_PATH_PREFIX + str(result_path), stdout.getvalue())
|
|
result_payload = json.loads(result_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(result_payload["artifacts"][0]["id"], "artifact-1")
|
|
stage_manifest = json.loads(stage_manifest_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(stage_manifest["artifactCount"], 1)
|
|
self.assertEqual(stage_manifest["nodeLabel"], "通用处理")
|
|
|
|
def test_write_json_artifact_externalizes_dataset_payload_to_data_ref(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"dataset-index.json",
|
|
{
|
|
"datasetType": "x-platform-index",
|
|
"collectionCount": 1,
|
|
"collections": [{"id": "collection-1"}],
|
|
"items": [{"id": "item-1"}, {"id": "item-2"}],
|
|
},
|
|
)
|
|
|
|
data_ref = artifact["data"]["dataRef"]
|
|
self.assertEqual(data_ref["kind"], "dataset")
|
|
self.assertTrue(pathlib.Path(data_ref["manifestPath"]).exists())
|
|
self.assertTrue(pathlib.Path(data_ref["payloadPath"]).exists())
|
|
manifest = json.loads(pathlib.Path(data_ref["manifestPath"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["datasetType"], "x-platform-index")
|
|
self.assertIn("items", manifest["partitionFiles"])
|
|
self.assertIn("collections", manifest["partitionFiles"])
|
|
|
|
def test_load_artifact_payload_reads_externalized_payload_from_data_ref(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"filtered-videos.json",
|
|
{
|
|
"datasetType": "x-platform-filtered",
|
|
"count": 2,
|
|
"items": [{"id": "v1"}, {"id": "v2"}],
|
|
},
|
|
)
|
|
|
|
loaded = MODULE.load_artifact_payload(artifact)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform-filtered")
|
|
self.assertEqual(len(loaded["items"]), 2)
|
|
|
|
def test_load_payload_from_file_retries_transient_json_decode_error(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
payload_path = pathlib.Path(temp_dir) / "payload.json"
|
|
payload_path.write_text('{"items":[{"id":"v1"}]}', encoding="utf-8")
|
|
decode_error = json.JSONDecodeError("transient", "", 0)
|
|
|
|
with mock.patch.object(
|
|
MODULE.json,
|
|
"load",
|
|
side_effect=[decode_error, {"datasetType": "x-platform", "items": [{"id": "v1"}]}],
|
|
) as json_load, mock.patch.object(MODULE.time, "sleep") as sleep_mock:
|
|
loaded = MODULE.load_payload_from_file(payload_path)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform")
|
|
self.assertEqual(len(loaded["items"]), 1)
|
|
self.assertEqual(json_load.call_count, 2)
|
|
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
|
|
|
|
def test_load_payload_from_file_retries_until_payload_path_exists(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
payload_path = pathlib.Path(temp_dir) / "payload.json"
|
|
payload_path.write_text('{"datasetType":"x-platform","items":[{"id":"v1"}]}', encoding="utf-8")
|
|
original_exists = pathlib.Path.exists
|
|
seen = {"count": 0}
|
|
|
|
def fake_exists(path_obj: pathlib.Path):
|
|
if path_obj == payload_path and seen["count"] == 0:
|
|
seen["count"] += 1
|
|
return False
|
|
return original_exists(path_obj)
|
|
|
|
with mock.patch.object(pathlib.Path, "exists", autospec=True, side_effect=fake_exists), mock.patch.object(
|
|
MODULE.time,
|
|
"sleep",
|
|
) as sleep_mock:
|
|
loaded = MODULE.load_payload_from_file(payload_path)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform")
|
|
self.assertEqual(len(loaded["items"]), 1)
|
|
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
|
|
|
|
def test_stage_manifest_includes_data_refs(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"packages-manifest.json",
|
|
{
|
|
"datasetType": "x-platform-packages",
|
|
"packageCount": 1,
|
|
"items": [{"packageName": "pkg-1"}],
|
|
},
|
|
)
|
|
|
|
manifest = MODULE.stage_manifest_payload("run-1", "wf-1", "node-1", "成品打包", output_dir, [artifact])
|
|
|
|
self.assertEqual(len(manifest["dataRefs"]), 1)
|
|
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
|
|
|
|
|
|
class XPlatformIndexingTests(unittest.TestCase):
|
|
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
|
|
manifest_path.write_text("{}", encoding="utf-8")
|
|
|
|
payload = MODULE.load_xplatform_index_payload(
|
|
{},
|
|
[
|
|
{
|
|
"path": str(manifest_path),
|
|
"data": {
|
|
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"entries": [
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "工厂",
|
|
"place": "绿控传动",
|
|
"date": "20260330",
|
|
},
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260327132547_0003_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2880",
|
|
"frameRate": 29.97,
|
|
"duration": 0.6,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
}
|
|
],
|
|
output_dir,
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["videoCount"], 1)
|
|
self.assertEqual(payload["collections"][0]["meta"]["place"], "绿控传动")
|
|
|
|
def test_load_index_payload_raises_when_source_manifest_summary_has_no_entries(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
|
|
manifest_path.write_text("{}", encoding="utf-8")
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "source-manifest"):
|
|
MODULE.load_xplatform_index_payload(
|
|
{},
|
|
[
|
|
{
|
|
"name": "source-manifest.json",
|
|
"path": str(manifest_path),
|
|
"data": {
|
|
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"entriesCount": 59,
|
|
"dataRef": {"payloadPath": str(manifest_path)},
|
|
},
|
|
}
|
|
],
|
|
output_dir,
|
|
)
|
|
|
|
def test_manifest_entries_support_video_metadata_alias(self):
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "工厂",
|
|
"place": "绿控传动",
|
|
"date": "20260330",
|
|
},
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260327132547_0003_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2880",
|
|
"frameRate": 29.97,
|
|
"duration": 0.6,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
},
|
|
},
|
|
],
|
|
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
def test_manifest_entries_load_local_meta_when_inline_data_is_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "algo" / "X-platform-test"
|
|
collection = root / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
|
|
video_dir = collection / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
meta_path = collection / "meta.json"
|
|
video_meta_path = collection / "video_meta.json"
|
|
video_path = video_dir / "DJI_0001_D.mp4"
|
|
meta_path.write_text('{"type":"家庭","place":"蓟门里小区","date":"20260319"}', encoding="utf-8")
|
|
video_meta_path.write_text(
|
|
'[{"filename":"DJI_0001_D.mp4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
|
|
encoding="utf-8",
|
|
)
|
|
video_path.write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
|
|
"sizeBytes": meta_path.stat().st_size,
|
|
"mimeType": "application/json",
|
|
"source": {"path": str(meta_path)},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
|
|
"sizeBytes": video_meta_path.stat().st_size,
|
|
"mimeType": "application/json",
|
|
"source": {"path": str(video_meta_path)},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": video_path.stat().st_size,
|
|
"mimeType": "video/mp4",
|
|
"source": {"path": str(video_path)},
|
|
},
|
|
],
|
|
str(root),
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
|
|
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
def test_manifest_entries_support_root_level_meta_and_video_meta(self):
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "家庭",
|
|
"place": "蓟门里小区",
|
|
"date": "20260319",
|
|
},
|
|
},
|
|
{
|
|
"path": "video_meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2160",
|
|
"frameRate": 29.97,
|
|
"duration": 41.8,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/video.mp4",
|
|
},
|
|
},
|
|
],
|
|
"bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(collection["relativePath"], "")
|
|
self.assertEqual(collection["videoCount"], 1)
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertEqual(collection["meta"]["place"], "蓟门里小区")
|
|
|
|
item = payload["items"][0]
|
|
self.assertEqual(item["collectionName"], collection["name"])
|
|
self.assertEqual(item["collectionRelativePath"], "")
|
|
self.assertEqual(item["videoFolder"], "DJI_001")
|
|
self.assertEqual(item["codec"], "hevc")
|
|
self.assertEqual(item["meta"]["type"], "家庭")
|
|
|
|
def test_local_dataset_root_with_meta_is_treated_as_collection_root(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
|
|
video_dir = root / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
(root / "meta.json").write_text('{"type":"家庭","place":"蓟门里小区"}', encoding="utf-8")
|
|
(root / "video_meta.json").write_text(
|
|
'[{"filename":"DJI_20260318112834_0001_D.MP4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
|
|
encoding="utf-8",
|
|
)
|
|
(video_dir / "DJI_20260318112834_0001_D.MP4").write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_dataset(root)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["name"], root.name)
|
|
self.assertEqual(collection["relativePath"], "")
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
item = payload["items"][0]
|
|
self.assertEqual(item["collectionName"], root.name)
|
|
self.assertEqual(item["collectionRelativePath"], "")
|
|
self.assertEqual(item["codec"], "hevc")
|
|
|
|
def test_local_dataset_root_supports_video_metadata_alias(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "SZ_000_0005_OsmoNano_2026-03-30_16-15-11"
|
|
video_dir = root / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
(root / "meta.json").write_text('{"type":"工厂","place":"绿控传动"}', encoding="utf-8")
|
|
(root / "video_metadata.json").write_text(
|
|
'[{"filename":"DJI_20260327132547_0003_D.MP4","codec":"hevc","resolution":"3840x2880","frameRate":29.97,"duration":0.6}]',
|
|
encoding="utf-8",
|
|
)
|
|
(video_dir / "DJI_20260327132547_0003_D.MP4").write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_dataset(root)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
|
|
class SourceOperatorTests(unittest.TestCase):
|
|
def test_builtin_source_rejects_missing_input_configuration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {"label": "真实数据源", "hyperparams": []}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "requires SOURCE_TEXT"):
|
|
MODULE.builtin_source("run-test", "source-empty", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_infers_local_source_path_without_data_source(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
source_root = pathlib.Path(temp_dir) / "source"
|
|
source_root.mkdir()
|
|
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
|
|
output_dir = pathlib.Path(temp_dir) / "output"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "真实数据源-本地路径",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(source_root)},
|
|
],
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-local", output_dir, node_data, [])
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["dataSource"], "LOCAL")
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["path"], "payload.json")
|
|
|
|
def test_builtin_source_rejects_missing_local_source_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
missing_path = pathlib.Path(temp_dir) / "missing"
|
|
node_data = {
|
|
"label": "真实数据源-缺失本地路径",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(missing_path)},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SOURCE_PATH does not exist"):
|
|
MODULE.builtin_source("run-test", "source-missing", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_defaults_to_inline_when_source_json_is_present(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-默认内联",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-inline-default", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["dataSource"], "INLINE")
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
|
|
|
|
def test_source_handler_can_emit_inline_json_payload_on_named_output(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-内联JSON",
|
|
"outputs": [
|
|
{"id": "manifest", "name": "manifest", "type": "json"},
|
|
{"id": "payload", "name": "payload", "type": "json"},
|
|
],
|
|
"hyperparams": [
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "real"}]}, ensure_ascii=False)},
|
|
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
|
|
{"key": "OUTPUT_NAME", "value": "payload.json"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-json-payload", output_dir, node_data, [])
|
|
|
|
payload_artifact = next(item for item in artifacts if item["name"] == "payload.json")
|
|
payload = json.loads(pathlib.Path(payload_artifact["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["items"][0]["kind"], "real")
|
|
self.assertEqual(payload_artifact["portId"], "payload")
|
|
|
|
def test_source_handler_can_emit_inline_text_payload_with_tabular_preview(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-内联CSV",
|
|
"outputs": [
|
|
{"id": "manifest", "name": "manifest", "type": "json"},
|
|
{"id": "csv", "name": "csv", "type": "file"},
|
|
],
|
|
"hyperparams": [
|
|
{"key": "SOURCE_TEXT", "value": "id,label\nclip-a,keep\n"},
|
|
{"key": "PAYLOAD_OUTPUT_ID", "value": "csv"},
|
|
{"key": "OUTPUT_NAME", "value": "report.csv"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-csv-payload", output_dir, node_data, [])
|
|
|
|
csv_artifact = next(item for item in artifacts if item["name"] == "report.csv")
|
|
self.assertEqual(csv_artifact["portId"], "csv")
|
|
self.assertEqual(csv_artifact["data"]["preview"]["kind"], "table")
|
|
self.assertEqual(csv_artifact["data"]["preview"]["rowCount"], 1)
|
|
|
|
def test_source_handler_rejects_unsafe_inline_payload_name(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-不安全输出名",
|
|
"outputs": [
|
|
{"id": "manifest", "name": "manifest", "type": "json"},
|
|
{"id": "payload", "name": "payload", "type": "json"},
|
|
],
|
|
"hyperparams": [
|
|
{"key": "SOURCE_JSON", "value": "{}"},
|
|
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
|
|
{"key": "OUTPUT_NAME", "value": "../payload.json"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "OUTPUT_NAME"):
|
|
MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-unsafe-payload", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_prefers_source_json_over_bos_env(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-JSON",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-json", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
|
|
|
|
def test_builtin_source_prefers_source_text_over_bos_env(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-文本",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_TEXT", "value": "alpha\nbeta\ngamma"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-text", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["text"], "alpha\nbeta\ngamma")
|
|
|
|
def test_directory_incremental_source_filter_emits_full_new_directory(self):
|
|
entries = [
|
|
{"path": "batch-a/meta.json", "sizeBytes": 10, "sha256": "sha-a-meta"},
|
|
{"path": "batch-a/DJI_001/clip-a.mp4", "sizeBytes": 20, "sha256": "sha-a-clip"},
|
|
{"path": "batch-b/meta.json", "sizeBytes": 30, "sha256": "sha-b-meta"},
|
|
{"path": "batch-b/DJI_001/clip-b.mp4", "sizeBytes": 40, "sha256": "sha-b-clip"},
|
|
]
|
|
emitted, checkpoint = MODULE.apply_incremental_source_filter(
|
|
entries,
|
|
"BOS",
|
|
"wjzh-data-migration",
|
|
{
|
|
"SOURCE_STATE_JSON": json.dumps(
|
|
{
|
|
"seenFingerprints": ["batch-a"],
|
|
"incrementalUnit": "DIRECTORY",
|
|
"directoryDepth": 1,
|
|
}
|
|
),
|
|
"SOURCE_FINGERPRINT": "fp-directory",
|
|
"INCREMENTAL_UNIT": "DIRECTORY",
|
|
"INCREMENTAL_DIRECTORY_DEPTH": "1",
|
|
},
|
|
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
)
|
|
|
|
self.assertEqual([item["path"] for item in emitted], ["batch-b/meta.json", "batch-b/DJI_001/clip-b.mp4"])
|
|
self.assertEqual([item["incrementalKey"] for item in emitted], ["batch-b", "batch-b"])
|
|
self.assertEqual(checkpoint["incrementalUnit"], "DIRECTORY")
|
|
self.assertEqual(checkpoint["directoryDepth"], 1)
|
|
self.assertEqual(checkpoint["newCount"], 2)
|
|
self.assertEqual(sorted(checkpoint["seenFingerprints"]), ["batch-a", "batch-b"])
|
|
|
|
def test_incremental_source_state_rejects_invalid_json(self):
|
|
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be valid JSON"):
|
|
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "{"})
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be a JSON object"):
|
|
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "[]"})
|
|
|
|
def test_incremental_source_state_rejects_invalid_shape(self):
|
|
cases = [
|
|
({"seenFingerprints": "fp-1"}, "seenFingerprints must be an array"),
|
|
({"seenFingerprints": [42]}, r"seenFingerprints\[0\] must be a string"),
|
|
({"incrementalUnit": "ROW"}, "incrementalUnit must be OBJECT or DIRECTORY"),
|
|
({"directoryDepth": 1.5}, "directoryDepth must be an integer"),
|
|
({"directoryDepth": 0}, "directoryDepth must be greater than 0"),
|
|
]
|
|
for payload, expected_error in cases:
|
|
with self.subTest(payload=payload), self.assertRaisesRegex(RuntimeError, expected_error):
|
|
MODULE.incremental_source_state({"SOURCE_STATE_JSON": json.dumps(payload)})
|
|
|
|
def test_incremental_directory_depth_rejects_lossy_values(self):
|
|
for raw in ["0", "01", "1.5", "2days"]:
|
|
with self.subTest(raw=raw), self.assertRaises(RuntimeError):
|
|
MODULE.incremental_directory_depth({"INCREMENTAL_DIRECTORY_DEPTH": raw})
|
|
|
|
def test_incremental_unit_rejects_invalid_values(self):
|
|
self.assertEqual(MODULE.incremental_unit({}), "OBJECT")
|
|
self.assertEqual(MODULE.incremental_unit({"INCREMENTAL_UNIT": " directory "}), "DIRECTORY")
|
|
with self.assertRaisesRegex(RuntimeError, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY"):
|
|
MODULE.incremental_unit({"INCREMENTAL_UNIT": "ROW"})
|
|
|
|
def test_builtin_source_rejects_invalid_scan_mode(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "BOS 增量扫描",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "INLINE"},
|
|
{"key": "SOURCE_TEXT", "value": "alpha"},
|
|
{"key": "SCAN_MODE", "value": "ROW"},
|
|
],
|
|
}
|
|
with self.assertRaisesRegex(RuntimeError, "SCAN_MODE must be FULL or INCREMENTAL"):
|
|
MODULE.builtin_source("run-test", "source-invalid-scan-mode", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_rejects_invalid_data_source(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
source_root = pathlib.Path(temp_dir) / "source"
|
|
source_root.mkdir()
|
|
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
|
|
output_dir = pathlib.Path(temp_dir) / "output"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "真实数据源-非法数据源",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "ROW"},
|
|
{"key": "SOURCE_PATH", "value": str(source_root)},
|
|
],
|
|
}
|
|
with self.assertRaisesRegex(RuntimeError, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM"):
|
|
MODULE.builtin_source("run-test", "source-invalid-data-source", output_dir, node_data, [])
|
|
|
|
def test_list_bos_entries_skips_large_non_metadata_json_payloads(self):
|
|
object_meta = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/meta.json", "size": 128, "e_tag": "meta"})()
|
|
object_imu = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json", "size": MODULE.PREVIEW_INLINE_BINARY_LIMIT + 1, "e_tag": "imu"})()
|
|
fake_client = _FakeBosListClient(
|
|
[object_meta, object_imu],
|
|
{
|
|
"wjzh-0330-Data-migration/SZ_000_0005/meta.json": b'{"type":"\xe5\xb7\xa5\xe5\x8e\x82"}',
|
|
"wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json": b'{"data_points":[1,2,3]}',
|
|
},
|
|
)
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
entries = MODULE.list_bos_entries(
|
|
"wjzh-data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
"https://bos.example.test",
|
|
"ak",
|
|
"sk",
|
|
)
|
|
|
|
self.assertEqual(len(entries), 2)
|
|
meta_entry = next(item for item in entries if item["path"].endswith("meta.json"))
|
|
imu_entry = next(item for item in entries if item["path"].endswith("_imu.json"))
|
|
self.assertIn("data", meta_entry)
|
|
self.assertNotIn("data", imu_entry)
|
|
self.assertEqual(
|
|
fake_client.downloaded_payload_keys,
|
|
[("wjzh-data-migration", "wjzh-0330-Data-migration/SZ_000_0005/meta.json")],
|
|
)
|
|
|
|
def test_bos_source_parses_bucket_and_prefix_from_source_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "BOS 增量扫描",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://wjzh-data-migration/wjzh-0330-Data-migration"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
{"key": "SCAN_MODE", "value": "FULL"},
|
|
],
|
|
}
|
|
fake_entries = [
|
|
{
|
|
"path": "SZ_000_0005/meta.json",
|
|
"storagePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {"type": "工厂"},
|
|
"source": {"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json"},
|
|
}
|
|
]
|
|
with mock.patch.object(MODULE, "list_bos_entries", return_value=fake_entries) as list_bos_entries, mock.patch.object(
|
|
MODULE,
|
|
"source_entry_artifact",
|
|
return_value={
|
|
"id": "artifact-1",
|
|
"runId": "run-1",
|
|
"nodeId": "source",
|
|
"name": "meta.json",
|
|
"type": "json",
|
|
"path": str(output_dir / "meta.json"),
|
|
},
|
|
):
|
|
artifacts = MODULE.builtin_source("run-1", "source", output_dir, node_data, [])
|
|
|
|
self.assertGreaterEqual(len(artifacts), 1)
|
|
list_bos_entries.assert_called_once_with(
|
|
"wjzh-data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
"https://bos.example.test",
|
|
"ak",
|
|
"sk",
|
|
)
|
|
|
|
|
|
class QualityMetricHelperTests(unittest.TestCase):
|
|
def test_build_skin_mask_detects_skin_tone_patch(self):
|
|
_, np = MODULE.require_cv2()
|
|
frame = np.zeros((80, 80, 3), dtype=np.uint8)
|
|
frame[20:60, 20:60] = (90, 130, 190)
|
|
mask = MODULE.build_skin_mask(frame)
|
|
self.assertGreater(int(mask.sum()), 0)
|
|
|
|
def test_build_skin_mask_rejects_neutral_background(self):
|
|
_, np = MODULE.require_cv2()
|
|
frame = np.full((80, 80, 3), 128, dtype=np.uint8)
|
|
mask = MODULE.build_skin_mask(frame)
|
|
self.assertEqual(int(mask.sum()), 0)
|
|
|
|
def test_motion_jitter_ratio_stays_low_for_smooth_motion(self):
|
|
smooth = [(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)]
|
|
jitter = MODULE.motion_jitter_ratio(smooth, 100.0)
|
|
self.assertLess(jitter, 0.01)
|
|
|
|
def test_motion_jitter_ratio_increases_for_erratic_motion(self):
|
|
smooth = MODULE.motion_jitter_ratio([(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)], 100.0)
|
|
erratic = MODULE.motion_jitter_ratio([(0.0, 0.0), (12.0, -4.0), (-10.0, 8.0), (15.0, -6.0)], 100.0)
|
|
self.assertGreater(erratic, smooth)
|
|
self.assertGreater(erratic, 0.05)
|
|
|
|
def test_largest_skin_component_ratio_ignores_small_noise(self):
|
|
_, np = MODULE.require_cv2()
|
|
mask = np.zeros((100, 100), dtype=np.uint8)
|
|
mask[10:13, 10:13] = 255
|
|
ratio = MODULE.largest_skin_component_ratio(mask)
|
|
self.assertEqual(ratio, 0.0)
|
|
|
|
def test_largest_skin_component_ratio_returns_large_component_area(self):
|
|
_, np = MODULE.require_cv2()
|
|
mask = np.zeros((100, 100), dtype=np.uint8)
|
|
mask[20:50, 30:70] = 255
|
|
ratio = MODULE.largest_skin_component_ratio(mask)
|
|
self.assertGreater(ratio, 0.1)
|
|
self.assertLess(ratio, 0.13)
|
|
|
|
|
|
class QualityOperatorTests(unittest.TestCase):
|
|
def test_video_quality_accepts_zero_hand_occlusion(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "MIN_CLARITY", "value": "0.8"},
|
|
{"key": "MAX_SHAKE", "value": "0.1"},
|
|
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"filename": "demo.mp4",
|
|
"source": {"path": "/tmp/demo.mp4"},
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=pathlib.Path("/tmp/demo.mp4")), mock.patch.object(
|
|
MODULE,
|
|
"analyze_video_quality_metrics",
|
|
return_value={
|
|
"clarity": 1.0,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"qualityScore": 1.0,
|
|
},
|
|
):
|
|
artifacts = MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["passedCount"], 1)
|
|
self.assertTrue(payload["items"][0]["quality"]["passed"])
|
|
|
|
def test_video_quality_rejects_out_of_range_thresholds(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "MIN_CLARITY", "value": "1.1"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "MIN_CLARITY must be in the range"):
|
|
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
|
|
|
|
def test_video_quality_rejects_non_finite_thresholds(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "MAX_SHAKE", "value": "nan"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be finite"):
|
|
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
|
|
|
|
def test_video_quality_rejects_invalid_clarity_scale(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "CLARITY_SCALE", "value": "0"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "CLARITY_SCALE must be greater than 0"):
|
|
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
|
|
|
|
def test_dataset_filter_keeps_zero_shake_and_hand_occlusion(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "数据过滤",
|
|
"hyperparams": [
|
|
{"key": "DATE_START", "value": "2026-03-01"},
|
|
{"key": "DATE_END", "value": "2026-03-31"},
|
|
{"key": "PLACE", "value": "蓟门里小区"},
|
|
{"key": "TYPE", "value": "家庭"},
|
|
{"key": "MIN_CLARITY", "value": "0.8"},
|
|
{"key": "MAX_SHAKE", "value": "0.1"},
|
|
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"meta": {
|
|
"date": "20260319",
|
|
"place": "蓟门里小区",
|
|
"type": "家庭",
|
|
},
|
|
"quality": {
|
|
"clarity": 0.95,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"passed": True,
|
|
},
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(len(payload["items"]), 1)
|
|
|
|
def test_dataset_filter_missing_thresholds_use_catalog_defaults(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "数据过滤",
|
|
"hyperparams": [],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"quality": {
|
|
"clarity": 0.5,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"passed": True,
|
|
},
|
|
},
|
|
{
|
|
"quality": {
|
|
"clarity": 0.95,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"passed": True,
|
|
},
|
|
},
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["criteria"]["minClarity"], MODULE.DEFAULT_MIN_CLARITY)
|
|
self.assertEqual(payload["criteria"]["maxShake"], MODULE.DEFAULT_MAX_SHAKE)
|
|
self.assertEqual(payload["criteria"]["maxHandOcclusion"], MODULE.DEFAULT_MAX_HAND_OCCLUSION)
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(payload["items"][0]["quality"]["clarity"], 0.95)
|
|
|
|
def test_dataset_filter_rejects_invalid_threshold_values(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "数据过滤",
|
|
"hyperparams": [
|
|
{"key": "MAX_SHAKE", "value": "not-a-number"},
|
|
],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be numeric"):
|
|
MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
|
|
|
|
|
|
class OperatorHyperparamParsingTests(unittest.TestCase):
|
|
def test_collection_validate_rejects_invalid_boolean_hyperparam(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "采集目录校验",
|
|
"hyperparams": [{"key": "REQUIRE_INTRINSICS", "value": "sometimes"}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "REQUIRE_INTRINSICS must be a boolean"):
|
|
MODULE.builtin_collection_validate("run-1", "validate", output_dir, node_data, [{"data": {"collections": [], "items": []}}])
|
|
|
|
def test_json_filter_rejects_invalid_expected_json(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 过滤",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "EXPECTED_JSON", "value": '{"status"'},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "EXPECTED_JSON must be valid JSON"):
|
|
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
|
|
|
|
def test_json_filter_rejects_unknown_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 过滤",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "OPERATOR", "value": "roughly"},
|
|
{"key": "EXPECTED", "value": "ready"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
|
|
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
|
|
|
|
def test_json_map_rejects_non_object_field_map(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": "[]"}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "FIELD_MAP_JSON must be a JSON object"):
|
|
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
|
|
|
|
def test_json_map_renames_fields_without_overwriting(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new","status":"state"}'}],
|
|
}
|
|
|
|
artifacts = MODULE.builtin_json_map(
|
|
"run-1",
|
|
"map",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"old": "value", "status": "ready", "unchanged": 1}]}}],
|
|
)
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["fieldMap"], {"old": "new", "status": "state"})
|
|
self.assertEqual(payload["items"], [{"new": "value", "state": "ready", "unchanged": 1}])
|
|
|
|
def test_json_map_rejects_invalid_field_map_entries(self):
|
|
cases = [
|
|
('{" old":"new"}', "must not have leading or trailing whitespace"),
|
|
('{"old":" new"}', "destination field must not have leading or trailing whitespace"),
|
|
('{"old":""}', "destination field must be non-empty"),
|
|
('{"old":42}', "destination field must be a string"),
|
|
('{"first":"name","last":"name"}', "maps multiple source fields to destination field"),
|
|
]
|
|
for field_map_json, expected_error in cases:
|
|
with self.subTest(field_map_json=field_map_json), tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": field_map_json}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, expected_error):
|
|
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
|
|
|
|
def test_json_map_rejects_output_field_collisions(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new"}'}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "same output field 'new'"):
|
|
MODULE.builtin_json_map(
|
|
"run-1",
|
|
"map",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"old": "mapped", "new": "existing"}]}}],
|
|
)
|
|
|
|
def test_audit_sampling_rejects_non_array_rules(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [{"key": "RULES_JSON", "value": "{}"}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "RULES_JSON must be a JSON array"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_rejects_incomplete_rule_shape(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [
|
|
{"key": "RULES_JSON", "value": '[{"id":"missing-percent","field":"quality.clarity","operator":"gte","value":0.8}]'}
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent is required"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_rejects_duplicate_rule_ids(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
rules_json = json.dumps(
|
|
[
|
|
{"id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.8, "samplePercent": 100},
|
|
{"id": "clarity", "field": "quality.shake", "operator": "lte", "value": 0.1, "samplePercent": 50},
|
|
],
|
|
ensure_ascii=False,
|
|
)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [{"key": "RULES_JSON", "value": rules_json}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "RULES_JSON contains duplicate id: clarity"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_rejects_invalid_sample_percent(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [
|
|
{
|
|
"key": "RULES_JSON",
|
|
"value": '[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":0}]',
|
|
}
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be in the range"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
node_data["hyperparams"][0]["value"] = (
|
|
'[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":true}]'
|
|
)
|
|
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be numeric"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_allows_exists_rule_without_value(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [
|
|
{
|
|
"key": "RULES_JSON",
|
|
"value": '[{"id":"has-collection","field":"collectionId","operator":"exists","samplePercent":100}]',
|
|
}
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.builtin_audit_sampling(
|
|
"run-1",
|
|
"sample",
|
|
output_dir,
|
|
node_data,
|
|
[
|
|
{
|
|
"data": {
|
|
"packageCount": 2,
|
|
"items": [
|
|
{"packageName": "clip-a", "collectionId": "collection-a"},
|
|
{"packageName": "clip-b"},
|
|
],
|
|
}
|
|
}
|
|
],
|
|
)
|
|
|
|
self.assertEqual(len(artifacts), 1)
|
|
payload = json.loads((output_dir / "audit-samples.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["rules"][0]["matchedCount"], 1)
|
|
self.assertEqual(payload["sampledCount"], 1)
|
|
|
|
def test_condition_router_rejects_unknown_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "条件分流",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "OPERATOR", "value": "roughly"},
|
|
{"key": "EXPECTED", "value": "ready"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
|
|
MODULE.builtin_condition_router("run-1", "route", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
|
|
|
|
|
|
class UploadOperatorTests(unittest.TestCase):
|
|
def test_build_artifact_skips_sha256_for_large_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
clip_path = root / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
|
|
MODULE,
|
|
"sha256_file",
|
|
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
|
|
):
|
|
artifact = MODULE.build_artifact("run-1", "slice", clip_path, "video")
|
|
|
|
metadata = artifact.get("metadata") or {}
|
|
self.assertEqual(metadata.get("sha256", ""), "")
|
|
self.assertTrue(metadata.get("sha256Skipped"))
|
|
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
|
|
|
|
def test_uploads_nested_package_outputs_to_bos(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
package_root = root / "package"
|
|
package_dir = package_root / "packages" / "clip001_package"
|
|
package_dir.mkdir(parents=True)
|
|
|
|
manifest_path = package_root / "packages-manifest.json"
|
|
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
|
|
clip_path = package_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
annotation_path = package_dir / "clip001_annotation.json"
|
|
annotation_path.write_text('{"label":"walking"}', encoding="utf-8")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [
|
|
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
|
|
MODULE.build_artifact("run-1", "package", clip_path, "video"),
|
|
MODULE.build_artifact("run-1", "package", annotation_path, "json", data={"label": "walking"}),
|
|
]
|
|
fake_client = _FakeBosClient()
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
|
|
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
|
|
uploaded_names = {item["name"] for item in artifacts}
|
|
uploaded_keys = {item["key"] for item in fake_client.uploads}
|
|
|
|
self.assertEqual(len(fake_client.uploads), 3)
|
|
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
|
|
self.assertEqual(uploaded_summary_payload["uploadedCount"], 3)
|
|
self.assertEqual(uploaded_summary_payload["bucket"], "datastorage")
|
|
self.assertEqual(uploaded_summary_payload["prefix"], "factory/output/demo")
|
|
|
|
def test_upload_skips_sha256_for_large_remote_artifacts(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
clip_path = root / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [MODULE.build_artifact("run-1", "slice", clip_path, "video")]
|
|
fake_client = _FakeBosClient()
|
|
|
|
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
|
|
MODULE,
|
|
"sha256_file",
|
|
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
|
|
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
remote_video = next(item for item in artifacts if item.get("type") == "video" and str(item.get("path", "")).startswith("bos://"))
|
|
metadata = remote_video.get("metadata") or {}
|
|
self.assertEqual(metadata.get("sha256", ""), "")
|
|
self.assertTrue(metadata.get("sha256Skipped"))
|
|
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
|
|
|
|
|
|
class _FakeInsertManyResult:
|
|
def __init__(self, inserted_ids):
|
|
self.inserted_ids = inserted_ids
|
|
|
|
|
|
class _FakeUpdateResult:
|
|
def __init__(self, matched_count=0, modified_count=0, upserted_id=None):
|
|
self.matched_count = matched_count
|
|
self.modified_count = modified_count
|
|
self.upserted_id = upserted_id
|
|
|
|
|
|
class _FakeMongoCollection:
|
|
def __init__(self):
|
|
self.insert_many_calls = []
|
|
self.replace_one_calls = []
|
|
|
|
def insert_many(self, documents, ordered=False):
|
|
self.insert_many_calls.append({"documents": documents, "ordered": ordered})
|
|
return _FakeInsertManyResult([f"id-{index + 1}" for index in range(len(documents))])
|
|
|
|
def replace_one(self, filter_document, replacement, upsert=False):
|
|
self.replace_one_calls.append(
|
|
{
|
|
"filter": filter_document,
|
|
"replacement": replacement,
|
|
"upsert": upsert,
|
|
}
|
|
)
|
|
return _FakeUpdateResult(matched_count=1, modified_count=1, upserted_id=None)
|
|
|
|
|
|
class _FakeMongoDatabase:
|
|
def __init__(self, collection):
|
|
self._collection = collection
|
|
self.requested_collections = []
|
|
|
|
def __getitem__(self, name):
|
|
self.requested_collections.append(name)
|
|
return self._collection
|
|
|
|
|
|
class _FakeMongoClient:
|
|
def __init__(self, database):
|
|
self._database = database
|
|
self.requested_databases = []
|
|
self.closed = False
|
|
|
|
def __getitem__(self, name):
|
|
self.requested_databases.append(name)
|
|
return self._database
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class MongoWriteOperatorTests(unittest.TestCase):
|
|
def test_inserts_documents_into_default_business_database(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
|
|
{"key": "MONGO_MODE", "value": "insert_many"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{"clipId": "clip-001", "qualityScore": 0.91},
|
|
{"clipId": "clip-002", "qualityScore": 0.87},
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-1", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(database.requested_collections, ["qualityReports"])
|
|
self.assertEqual(client.requested_databases, ["business-data"])
|
|
self.assertEqual(len(collection.insert_many_calls), 1)
|
|
self.assertEqual(collection.insert_many_calls[0]["documents"][0]["clipId"], "clip-001")
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["collection"], "qualityReports")
|
|
self.assertEqual(summary["data"]["mode"], "insert_many")
|
|
self.assertEqual(summary["data"]["target"], "business")
|
|
self.assertEqual(summary["data"]["insertedCount"], 2)
|
|
self.assertTrue(client.closed)
|
|
|
|
def test_upserts_documents_by_key(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "annotations"},
|
|
{"key": "MONGO_MODE", "value": "upsert"},
|
|
{"key": "UPSERT_KEY", "value": "clipId"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{"data": {"clipId": "clip-001", "label": "walking"}},
|
|
{"data": {"clipId": "clip-002", "label": "standing"}},
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-2", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(len(collection.replace_one_calls), 2)
|
|
self.assertEqual(collection.replace_one_calls[0]["filter"], {"clipId": "clip-001"})
|
|
self.assertTrue(collection.replace_one_calls[0]["upsert"])
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["mode"], "upsert")
|
|
self.assertEqual(summary["data"]["upsertKey"], "clipId")
|
|
self.assertEqual(summary["data"]["target"], "business")
|
|
self.assertEqual(summary["data"]["upsertedCount"], 2)
|
|
|
|
def test_unwraps_single_value_dict_documents_for_business_writes(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "businessWriteValidation"},
|
|
{"key": "MONGO_MODE", "value": "insert_many"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"value": {
|
|
"runLabel": "business-mongo-test",
|
|
"source": "workflow",
|
|
"createdAt": "2026-04-14T00:00:00Z",
|
|
"value": 1,
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
MODULE.builtin_mongodb_write("run-unwrap", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(len(collection.insert_many_calls), 1)
|
|
inserted = collection.insert_many_calls[0]["documents"][0]
|
|
self.assertEqual(inserted["runLabel"], "business-mongo-test")
|
|
self.assertEqual(inserted["source"], "workflow")
|
|
self.assertEqual(inserted["value"], 1)
|
|
self.assertNotEqual(set(inserted.keys()), {"value"})
|
|
|
|
def test_can_target_platform_database_explicitly(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
|
|
{"key": "MONGO_TARGET", "value": "platform"},
|
|
],
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"PLATFORM_MONGO_URI": "mongodb://platform-mongo:27017", "PLATFORM_MONGO_DB": "droboticflow"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-3", "mongo-node", pathlib.Path(temp_dir), node_data, [{"data": {"clipId": "clip-001"}}])
|
|
|
|
self.assertEqual(client.requested_databases, ["droboticflow"])
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["target"], "platform")
|
|
|
|
def test_rebuilds_package_files_from_manifest_when_temp_paths_are_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
package_manifest = {
|
|
"packageCount": 1,
|
|
"items": [
|
|
{
|
|
"packageName": "clip001_package",
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
"quality": {"clarity": 0.9, "passed": True},
|
|
"annotation": {"objects": ["人"], "minConfidence": 0.92},
|
|
}
|
|
],
|
|
}
|
|
inputs = [
|
|
{"name": "packages-manifest.json", "type": "json", "data": package_manifest},
|
|
{
|
|
"name": "clip001.mp4",
|
|
"type": "video",
|
|
"path": "/tmp/non-existent-clip001.mp4",
|
|
"data": {"preview": {"kind": "video", "format": "mp4"}},
|
|
},
|
|
{
|
|
"name": "clip001_annotation.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_annotation.json",
|
|
"data": {"objects": ["人"], "minConfidence": 0.92},
|
|
},
|
|
{
|
|
"name": "clip001_quality.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_quality.json",
|
|
"data": {"clarity": 0.9, "passed": True},
|
|
},
|
|
{
|
|
"name": "clip001_config.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_config.json",
|
|
"data": {"collectionId": "collection-1"},
|
|
},
|
|
]
|
|
fake_client = _FakeBosClient()
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
uploaded_keys = {item["key"] for item in fake_client.uploads}
|
|
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_quality.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_config.json", uploaded_keys)
|
|
self.assertFalse(
|
|
(output_dir / "uploaded" / "_materialized_upload_inputs" / "packages" / "clip001_package" / "clip001.mp4").exists()
|
|
)
|
|
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
|
|
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
|
|
self.assertEqual(uploaded_summary_payload["uploadedCount"], 5)
|
|
|
|
def test_upload_artifacts_are_json_serializable_when_preview_url_is_bytes(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
package_root = root / "package"
|
|
package_dir = package_root / "packages" / "clip001_package"
|
|
package_dir.mkdir(parents=True)
|
|
|
|
manifest_path = package_root / "packages-manifest.json"
|
|
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
|
|
clip_path = package_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [
|
|
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
|
|
MODULE.build_artifact("run-1", "package", clip_path, "video"),
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(_FakeBosClientBytesURL(), "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
json.dumps({"artifacts": artifacts}, ensure_ascii=False)
|
|
|
|
|
|
class XPlatformClipHandoffTests(unittest.TestCase):
|
|
def test_summarize_sqlite_payload_does_not_create_wal_sidecars(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
db_path = pathlib.Path(temp_dir) / "talker.db3"
|
|
connection = sqlite3.connect(db_path)
|
|
try:
|
|
connection.execute("pragma journal_mode=wal")
|
|
connection.execute("create table topics (id integer primary key, name text, type text, serialization_format text)")
|
|
connection.execute("create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)")
|
|
connection.execute(
|
|
"insert into topics (id, name, type, serialization_format) values (1, '/chatter', 'std_msgs/msg/String', 'cdr')"
|
|
)
|
|
connection.execute("insert into messages (topic_id, timestamp, data) values (1, 1, X'00')")
|
|
connection.commit()
|
|
finally:
|
|
connection.close()
|
|
for sidecar in db_path.parent.glob("talker.db3-*"):
|
|
sidecar.unlink()
|
|
|
|
payload = MODULE.summarize_sqlite_payload(db_path)
|
|
|
|
self.assertEqual(payload["preview"]["format"], "rosbag2")
|
|
self.assertEqual(payload["preview"]["messageCount"], 1)
|
|
self.assertEqual(list(db_path.parent.glob("talker.db3-*")), [])
|
|
|
|
def test_video_slicing_rejects_invalid_segment_duration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "0"}],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be greater than 0"):
|
|
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
def test_video_slicing_rejects_lossy_segment_duration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "01"}],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be an exact positive integer"):
|
|
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
def test_video_slicing_rejects_unsafe_segment_duration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": str(MODULE.MAX_SAFE_INTEGER + 1)}],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be a safe integer"):
|
|
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
def test_video_slicing_manifest_preserves_source_reference(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 41.8,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
clip = manifest_payload["items"][0]
|
|
self.assertEqual(clip["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
|
|
self.assertEqual(clip["clipName"], "DJI_20260318112834_0001_D_clip001.mp4")
|
|
|
|
def test_video_slicing_uses_single_pass_writer_for_multi_clip_video(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 365.0,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_write_video_segments(_source_path, segments):
|
|
outputs = []
|
|
for segment in segments:
|
|
destination = pathlib.Path(segment["destination"])
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(f"clip-{segment['clipIndex']}".encode("utf-8"))
|
|
outputs.append(
|
|
{
|
|
**segment,
|
|
"duration": round(float(segment["endSecond"]) - float(segment["startSecond"]), 3),
|
|
"frameRate": 29.97,
|
|
"resolution": "3840x2160",
|
|
}
|
|
)
|
|
return outputs
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segments",
|
|
side_effect=fake_write_video_segments,
|
|
) as multi_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
) as single_writer, mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["clipCount"], 2)
|
|
self.assertEqual([item["clipName"] for item in manifest_payload["items"]], [
|
|
"DJI_20260318112834_0001_D_clip001.mp4",
|
|
"DJI_20260318112834_0001_D_clip002.mp4",
|
|
])
|
|
multi_writer.assert_called_once()
|
|
single_writer.assert_not_called()
|
|
|
|
def test_write_video_segment_with_ffmpeg_prefers_stream_copy_and_uses_inspected_metrics(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
destination = root / "clips" / "clip001.mp4"
|
|
|
|
def fake_run(args, stdout=None, stderr=None, text=None):
|
|
self.assertEqual(args[0], "ffmpeg")
|
|
self.assertIn(str(source_path), args)
|
|
self.assertEqual(args[-1], str(destination))
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-video")
|
|
return mock.Mock(returncode=0, stdout="", stderr="")
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE.subprocess,
|
|
"run",
|
|
side_effect=fake_run,
|
|
) as run_mock, mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
|
|
):
|
|
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
|
|
|
|
self.assertEqual(metrics["duration"], 21.5)
|
|
self.assertEqual(metrics["frameRate"], 29.97)
|
|
self.assertEqual(metrics["resolution"], "3840x2160")
|
|
run_args = run_mock.call_args.args[0]
|
|
self.assertIn("-ss", run_args)
|
|
self.assertIn("-t", run_args)
|
|
self.assertIn("-c", run_args)
|
|
self.assertIn("copy", run_args)
|
|
self.assertNotIn("-movflags", run_args)
|
|
self.assertNotIn("+faststart", run_args)
|
|
|
|
def test_write_video_segment_with_ffmpeg_falls_back_to_reencode_when_copy_fails(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
destination = root / "clips" / "clip001.mp4"
|
|
|
|
calls = []
|
|
|
|
def fake_run(args, stdout=None, stderr=None, text=None):
|
|
calls.append(list(args))
|
|
if len(calls) == 1:
|
|
return mock.Mock(returncode=1, stdout="", stderr="copy failed")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-video")
|
|
return mock.Mock(returncode=0, stdout="", stderr="")
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE.subprocess,
|
|
"run",
|
|
side_effect=fake_run,
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
|
|
):
|
|
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
|
|
|
|
self.assertEqual(metrics["duration"], 21.5)
|
|
self.assertEqual(len(calls), 2)
|
|
self.assertIn("copy", calls[0])
|
|
self.assertIn("libx264", calls[1])
|
|
self.assertIn("aac", calls[1])
|
|
|
|
def test_video_slicing_prefers_ffmpeg_when_available(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 365.0,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_ffmpeg_writer(_source_path, destination, start_second, end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(f"{start_second}-{end_second}".encode("utf-8"))
|
|
return {
|
|
"duration": round(float(end_second) - float(start_second), 3),
|
|
"frameRate": 29.97,
|
|
"resolution": "3840x2160",
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment_with_ffmpeg",
|
|
side_effect=fake_ffmpeg_writer,
|
|
) as ffmpeg_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segments",
|
|
) as multi_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
) as single_writer, mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["clipCount"], 2)
|
|
self.assertEqual(ffmpeg_writer.call_count, 2)
|
|
multi_writer.assert_not_called()
|
|
single_writer.assert_not_called()
|
|
|
|
def test_video_annotation_rejects_invalid_frame_interval(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频标注",
|
|
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "not-a-number"}],
|
|
}
|
|
inputs = [{"data": {"clipCount": 0, "items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "FRAME_INTERVAL_SECONDS must be numeric"):
|
|
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
|
|
|
|
def test_video_annotation_uses_explicit_positive_frame_interval_without_clamping(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
clip_path = output_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"clip-bytes")
|
|
node_data = {
|
|
"label": "视频标注",
|
|
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "0.05"}],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"clipCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"clipPath": str(clip_path),
|
|
"sourceVideo": "source.mp4",
|
|
}
|
|
],
|
|
}
|
|
}
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "annotate_video_clip", return_value={"objects": [], "frameCount": 1}) as annotator:
|
|
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
|
|
|
|
annotator.assert_called_once()
|
|
self.assertEqual(annotator.call_args.args[1], 0.05)
|
|
|
|
def test_video_annotation_reuses_shared_clip_path_without_copying_clip_artifact(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
shared_clip_path = output_dir / "shared" / "clip001.mp4"
|
|
shared_clip_path.parent.mkdir(parents=True, exist_ok=True)
|
|
shared_clip_path.write_bytes(b"clip-bytes")
|
|
inputs = [
|
|
{
|
|
"name": "clips-manifest.json",
|
|
"type": "json",
|
|
"data": {
|
|
"clipCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"clipPath": str(shared_clip_path),
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"duration": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频标注",
|
|
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "1.0"}],
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
|
|
MODULE,
|
|
"annotate_video_clip",
|
|
return_value={
|
|
"duration": 41.8,
|
|
"actions": [{"label": "walking", "confidence": 0.92, "start": 0, "end": 41.8}],
|
|
"objects": ["人"],
|
|
"minConfidence": 0.92,
|
|
},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "annotations-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["annotationCount"], 1)
|
|
self.assertEqual(manifest_payload["items"][0]["clipPath"], str(shared_clip_path))
|
|
self.assertFalse(any(item["name"] == "clip001.mp4" for item in artifacts))
|
|
|
|
def test_materialize_clip_from_source_prefers_remote_segment_generation(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
item = {
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
},
|
|
}
|
|
|
|
def fake_remote_segment(_source_url, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8}
|
|
|
|
with mock.patch.object(MODULE, "remote_video_url_for_source", return_value="https://signed.example.com/video.mp4"), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment_from_remote_url",
|
|
side_effect=fake_remote_segment,
|
|
) as remote_writer, mock.patch.object(MODULE, "source_path_from_item") as local_materializer:
|
|
resolved = MODULE.materialize_clip_from_source(item, output_dir)
|
|
|
|
self.assertIsNotNone(resolved)
|
|
self.assertTrue(resolved.exists())
|
|
self.assertEqual(resolved.read_bytes(), b"clip-bytes")
|
|
remote_writer.assert_called_once()
|
|
local_materializer.assert_not_called()
|
|
|
|
def test_result_package_rebuilds_clip_from_source_when_clip_artifact_is_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "annotations-manifest.json",
|
|
"type": "json",
|
|
"data": {
|
|
"annotationCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"duration": 41.8,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"clarity": 0.9, "passed": True},
|
|
"annotation": {"actions": [], "objects": ["人"], "minConfidence": 0.92},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "成品打包",
|
|
"hyperparams": [],
|
|
}
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
side_effect=fake_write_video_segment,
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
package_clip = next(item for item in artifacts if item["name"] == "clip001.mp4")
|
|
self.assertEqual(manifest_payload["packageCount"], 1)
|
|
manifest_item = manifest_payload["items"][0]
|
|
self.assertEqual(manifest_item["clipName"], "clip001.mp4")
|
|
self.assertEqual(manifest_item["collectionId"], "collection-1")
|
|
self.assertEqual(manifest_item["startSecond"], 0)
|
|
self.assertEqual(manifest_item["endSecond"], 41.8)
|
|
self.assertEqual(manifest_item["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
|
|
self.assertEqual(package_clip["name"], "clip001.mp4")
|
|
|
|
|
|
class ModelInferenceOperatorTests(unittest.TestCase):
|
|
def test_builtin_model_inference_runs_linear_model(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
model = {
|
|
"type": "linear",
|
|
"name": "quality-gate",
|
|
"features": ["quality.clarity", "quality.shake"],
|
|
"weights": [1.0, -2.0],
|
|
"bias": 0.0,
|
|
"threshold": 0.5,
|
|
"positiveLabel": "pass",
|
|
"negativeLabel": "fail",
|
|
}
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [{"key": "MODEL_JSON", "value": json.dumps(model)}],
|
|
}
|
|
inputs = [
|
|
{
|
|
"name": "dataset.json",
|
|
"type": "json",
|
|
"data": {
|
|
"items": [
|
|
{"id": "clip-1", "quality": {"clarity": 0.9, "shake": 0.1}},
|
|
{"id": "clip-2", "quality": {"clarity": 0.2, "shake": 0.2}},
|
|
]
|
|
},
|
|
}
|
|
]
|
|
|
|
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, inputs)
|
|
|
|
self.assertEqual([item["name"] for item in artifacts], ["inference-result.json"])
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["modelType"], "linear")
|
|
self.assertEqual(payload["modelName"], "quality-gate")
|
|
expected_digest = hashlib.sha256(
|
|
json.dumps(model, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
self.assertEqual(payload["modelSource"], "MODEL_JSON")
|
|
self.assertEqual(payload["modelSha256"], expected_digest)
|
|
self.assertEqual(payload["inputCount"], 2)
|
|
self.assertEqual([item["prediction"]["label"] for item in payload["predictions"]], ["pass", "fail"])
|
|
self.assertEqual(payload["predictions"][0]["input"]["id"], "clip-1")
|
|
|
|
def test_builtin_model_inference_runs_nearest_centroid_from_inline_input(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
model = {
|
|
"type": "nearest_centroid",
|
|
"name": "motion-cluster",
|
|
"features": ["speed", "turnRate"],
|
|
"labels": ["slow-straight", "fast-turn"],
|
|
"centroids": [[0.1, 0.0], [1.0, 0.8]],
|
|
}
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [
|
|
{"key": "MODEL_JSON", "value": json.dumps(model)},
|
|
{"key": "INPUT_JSON", "value": json.dumps([{"id": "clip-1", "speed": 0.95, "turnRate": 0.7}])},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["inputCount"], 1)
|
|
self.assertEqual(payload["predictions"][0]["prediction"]["label"], "fast-turn")
|
|
self.assertGreater(payload["predictions"][0]["prediction"]["score"], 0)
|
|
|
|
def test_builtin_model_inference_rejects_invalid_rule_comparison(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [
|
|
{
|
|
"key": "MODEL_JSON",
|
|
"value": json.dumps(
|
|
{
|
|
"type": "rule",
|
|
"rules": [{"field": "scene", "operator": "gt", "value": 1, "label": "bad"}],
|
|
}
|
|
),
|
|
},
|
|
{"key": "INPUT_JSON", "value": json.dumps({"scene": "kitchen"})},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "requires numeric values"):
|
|
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
|
|
|
|
def test_builtin_model_inference_rejects_constant_model(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [
|
|
{"key": "MODEL_JSON", "value": json.dumps({"type": "constant", "label": "pass"})},
|
|
{"key": "INPUT_JSON", "value": json.dumps({"id": "clip-1"})},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "constant model type is not accepted"):
|
|
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
|
|
|
|
|
|
class ShellCommandOperatorTests(unittest.TestCase):
|
|
def test_builtin_shell_command_requires_command(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with self.assertRaisesRegex(RuntimeError, "需要 COMMAND 或 SCRIPT"):
|
|
MODULE.builtin_shell_command(
|
|
"run-1",
|
|
"shell",
|
|
pathlib.Path(temp_dir),
|
|
{"label": "Shell 命令执行", "hyperparams": []},
|
|
[],
|
|
)
|
|
|
|
def test_builtin_shell_command_executes_command_and_collects_output(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "Shell 命令执行",
|
|
"hyperparams": [
|
|
{
|
|
"key": "COMMAND",
|
|
"value": "python3 -c 'import json; json.dump({\"ok\": True}, open(\"result.json\", \"w\"))'",
|
|
}
|
|
],
|
|
"outputs": [{"name": "result", "type": "json"}],
|
|
}
|
|
previous_policy = os.environ.get("OPERATOR_ALLOW_SHELL_COMMANDS")
|
|
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = "true"
|
|
try:
|
|
artifacts = MODULE.builtin_shell_command("run-1", "shell", output_dir, node_data, [])
|
|
finally:
|
|
if previous_policy is None:
|
|
os.environ.pop("OPERATOR_ALLOW_SHELL_COMMANDS", None)
|
|
else:
|
|
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = previous_policy
|
|
|
|
self.assertEqual([item["name"] for item in artifacts], ["result.json"])
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload, {"ok": True})
|
|
|
|
|
|
class _DCloudAPIHandler(http.server.BaseHTTPRequestHandler):
|
|
requests = []
|
|
|
|
def do_POST(self):
|
|
length = int(self.headers.get("Content-Length", "0") or "0")
|
|
body = self.rfile.read(length).decode("utf-8")
|
|
payload = json.loads(body) if body else {}
|
|
if self.path == "/api/signin":
|
|
self.__class__.requests.append(
|
|
{
|
|
"method": "POST",
|
|
"path": self.path,
|
|
"authorization": self.headers.get("Authorization", ""),
|
|
"cookie": self.headers.get("Cookie", ""),
|
|
"d_user_id": self.headers.get("d-user-id", ""),
|
|
"org_symbol": self.headers.get("org-symbol", ""),
|
|
"sourceapp": self.headers.get("sourceapp", ""),
|
|
"body": {"hasUsername": bool(payload.get("Username")), "hasPassword": bool(payload.get("Password"))},
|
|
}
|
|
)
|
|
response_bytes = json.dumps({"status": 0, "data": {"token": "signin.header.payload"}}).encode("utf-8")
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(response_bytes)))
|
|
self.end_headers()
|
|
self.wfile.write(response_bytes)
|
|
return
|
|
self.__class__.requests.append(
|
|
{
|
|
"method": "POST",
|
|
"path": self.path,
|
|
"authorization": self.headers.get("Authorization", ""),
|
|
"cookie": self.headers.get("Cookie", ""),
|
|
"d_user_id": self.headers.get("d-user-id", ""),
|
|
"org_symbol": self.headers.get("org-symbol", ""),
|
|
"sourceapp": self.headers.get("sourceapp", ""),
|
|
"body": payload,
|
|
}
|
|
)
|
|
response = {
|
|
"taskId": "task-" + str(len(self.__class__.requests)),
|
|
"previewURL": f"http://{self.server.server_address[0]}:{self.server.server_address[1]}/preview/{payload.get('name', 'item')}.json",
|
|
"payloadEcho": payload,
|
|
}
|
|
response_bytes = json.dumps(response).encode("utf-8")
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(response_bytes)))
|
|
self.end_headers()
|
|
self.wfile.write(response_bytes)
|
|
|
|
def do_GET(self):
|
|
self.__class__.requests.append(
|
|
{
|
|
"method": "GET",
|
|
"path": self.path,
|
|
"authorization": self.headers.get("Authorization", ""),
|
|
"cookie": self.headers.get("Cookie", ""),
|
|
"d_user_id": self.headers.get("d-user-id", ""),
|
|
"org_symbol": self.headers.get("org-symbol", ""),
|
|
"sourceapp": self.headers.get("sourceapp", ""),
|
|
"body": None,
|
|
}
|
|
)
|
|
response = {"jobId": "job-preview", "previewUrl": f"http://{self.server.server_address[0]}:{self.server.server_address[1]}/asset.png"}
|
|
response_bytes = json.dumps(response).encode("utf-8")
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(response_bytes)))
|
|
self.end_headers()
|
|
self.wfile.write(response_bytes)
|
|
|
|
def log_message(self, *_args):
|
|
return
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def dcloud_api_server():
|
|
_DCloudAPIHandler.requests = []
|
|
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _DCloudAPIHandler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
previous_no_proxy = os.environ.get("NO_PROXY")
|
|
previous_no_proxy_lower = os.environ.get("no_proxy")
|
|
os.environ["NO_PROXY"] = "127.0.0.1,localhost"
|
|
os.environ["no_proxy"] = "127.0.0.1,localhost"
|
|
thread.start()
|
|
try:
|
|
yield server, _DCloudAPIHandler.requests
|
|
finally:
|
|
server.shutdown()
|
|
thread.join(timeout=5)
|
|
server.server_close()
|
|
if previous_no_proxy is None:
|
|
os.environ.pop("NO_PROXY", None)
|
|
else:
|
|
os.environ["NO_PROXY"] = previous_no_proxy
|
|
if previous_no_proxy_lower is None:
|
|
os.environ.pop("no_proxy", None)
|
|
else:
|
|
os.environ["no_proxy"] = previous_no_proxy_lower
|
|
|
|
|
|
class DCloudFATAPIOperatorTests(unittest.TestCase):
|
|
def test_dcloud_embodied_annotation_operator_calls_real_http_endpoint(self):
|
|
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "D-Cloud 具身标注 API",
|
|
"hyperparams": [
|
|
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
|
|
{"key": "ENDPOINT", "value": "/api/annoMidwareApi/v1/annotate/detection"},
|
|
{"key": "METHOD", "value": "POST"},
|
|
{"key": "REQUEST_JSON", "value": json.dumps({"name": "clip-1", "videoUrl": "http://example.test/clip.mp4"})},
|
|
{"key": "AUTH_TOKEN", "value": "real-token"},
|
|
{"key": "COOKIE", "value": "dcloud_session=real-session"},
|
|
{"key": "D_USER_ID", "value": "user-1"},
|
|
{"key": "ORG_SYMBOL", "value": "org-1"},
|
|
{"key": "SOURCEAPP", "value": "dataCopilot"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.builtin_dcloud_embodied_annotation("run-1", "anno", output_dir, node_data, [])
|
|
|
|
self.assertEqual(len(requests), 1)
|
|
self.assertEqual(requests[0]["path"], "/api/annoMidwareApi/v1/annotate/detection")
|
|
self.assertEqual(requests[0]["authorization"], "Bearer real-token")
|
|
self.assertEqual(requests[0]["cookie"], "dcloud_session=real-session")
|
|
self.assertEqual(requests[0]["d_user_id"], "user-1")
|
|
self.assertEqual(requests[0]["org_symbol"], "org-1")
|
|
self.assertEqual(requests[0]["sourceapp"], "dataCopilot")
|
|
self.assertEqual(requests[0]["body"]["name"], "clip-1")
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["response"]["status"], 200)
|
|
self.assertEqual(payload["response"]["body"]["payloadEcho"]["videoUrl"], "http://example.test/clip.mp4")
|
|
self.assertTrue(payload["preview"]["urls"])
|
|
self.assertEqual(payload["sourceRepo"], "https://gitea.d-robotics.cc/d-cloud/robo_annotation_backend")
|
|
|
|
def test_dcloud_data_generation_operator_supports_preview_query(self):
|
|
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "D-Cloud 具身数据生成 API",
|
|
"hyperparams": [
|
|
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
|
|
{"key": "ENDPOINT", "value": "/api/generationApi/tasks/preview"},
|
|
{"key": "METHOD", "value": "GET"},
|
|
{"key": "QUERY_JSON", "value": json.dumps({"taskId": "task-1"})},
|
|
],
|
|
}
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"DCLOUD_D_USER_ID": "user-2",
|
|
"DCLOUD_ORG_SYMBOL": "org-2",
|
|
"DCLOUD_SOURCEAPP": "dataCopilot",
|
|
},
|
|
clear=False,
|
|
):
|
|
artifacts = MODULE.builtin_dcloud_data_generation("run-1", "gen", output_dir, node_data, [])
|
|
|
|
self.assertEqual(len(requests), 1)
|
|
self.assertEqual(requests[0]["method"], "GET")
|
|
self.assertEqual(requests[0]["path"], "/api/generationApi/tasks/preview?taskId=task-1")
|
|
self.assertEqual(requests[0]["d_user_id"], "user-2")
|
|
self.assertEqual(requests[0]["org_symbol"], "org-2")
|
|
self.assertEqual(requests[0]["sourceapp"], "dataCopilot")
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["response"]["status"], 200)
|
|
self.assertEqual(payload["preview"]["ids"], ["job-preview"])
|
|
self.assertEqual(payload["sourceRepo"], "https://gitea.d-robotics.cc/d-cloud/data_generation")
|
|
|
|
def test_dcloud_operator_can_sign_in_for_real_fat_token(self):
|
|
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "D-Cloud 具身数据生成 API",
|
|
"hyperparams": [
|
|
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
|
|
{"key": "ENDPOINT", "value": "/api/generationApi/imgGeneration/tasks"},
|
|
{"key": "METHOD", "value": "POST"},
|
|
{"key": "REQUEST_JSON", "value": json.dumps({"currentPage": 1, "pageSize": 1})},
|
|
],
|
|
}
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"DCLOUD_FAT_USERNAME": "fat-user",
|
|
"DCLOUD_FAT_PASSWORD": "fat-password",
|
|
"DCLOUD_ORG_SYMBOL": "org-3",
|
|
"DCLOUD_SOURCEAPP": "dataCopilot",
|
|
},
|
|
clear=False,
|
|
):
|
|
artifacts = MODULE.builtin_dcloud_data_generation("run-1", "gen", output_dir, node_data, [])
|
|
|
|
self.assertEqual(len(requests), 2)
|
|
self.assertEqual(requests[0]["path"], "/api/signin")
|
|
self.assertEqual(requests[0]["body"], {"hasUsername": True, "hasPassword": True})
|
|
self.assertEqual(requests[1]["path"], "/api/generationApi/imgGeneration/tasks")
|
|
self.assertEqual(requests[1]["authorization"], "Bearer signin.header.payload")
|
|
self.assertEqual(requests[1]["org_symbol"], "org-3")
|
|
self.assertEqual(requests[1]["sourceapp"], "dataCopilot")
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["response"]["status"], 200)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|