2026-06-24 20:35:59 +08:00

5540 lines
271 KiB
Python

import contextlib
import hashlib
import http.server
import importlib.util
import io
import json
import os
import pathlib
import pickle
import sqlite3
import struct
import tarfile
import threading
import tempfile
import unittest
from unittest import mock
MODULE_PATH = pathlib.Path(__file__).with_name("main.py")
SPEC = importlib.util.spec_from_file_location("operator_runner_main", MODULE_PATH)
assert SPEC is not None and SPEC.loader is not None
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
RESOURCE_MONITOR_PATH = pathlib.Path(__file__).with_name("resource_monitor.py")
RESOURCE_MONITOR_SPEC = importlib.util.spec_from_file_location("operator_runner_resource_monitor", RESOURCE_MONITOR_PATH)
assert RESOURCE_MONITOR_SPEC is not None and RESOURCE_MONITOR_SPEC.loader is not None
RESOURCE_MONITOR = importlib.util.module_from_spec(RESOURCE_MONITOR_SPEC)
RESOURCE_MONITOR_SPEC.loader.exec_module(RESOURCE_MONITOR)
class _FakeHTTPResponse:
def __init__(self, payload: bytes):
self._payload = payload
self._offset = 0
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def read(self, *_args, **_kwargs):
if self._offset >= len(self._payload):
return b""
chunk = self._payload[self._offset:]
self._offset = len(self._payload)
return chunk
class _FakeBosClient:
def __init__(self):
self.uploads = []
self.downloads = []
def put_object_from_file(self, bucket, key, file_name, **kwargs):
self.uploads.append(
{
"bucket": bucket,
"key": key,
"file_name": file_name,
"content_type": kwargs.get("content_type"),
}
)
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}"
def get_object_to_file(self, bucket, key, file_name, **_kwargs):
self.downloads.append(
{
"bucket": bucket,
"key": key,
"file_name": file_name,
}
)
pathlib.Path(file_name).write_bytes(f"{bucket}/{key}".encode("utf-8"))
class _FakeBosClientBytesURL(_FakeBosClient):
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}".encode("utf-8")
class _FakeBosListClient(_FakeBosClient):
def __init__(self, objects, object_payloads):
super().__init__()
self._objects = objects
self._object_payloads = object_payloads
self.downloaded_payload_keys = []
def list_all_objects(self, bucket, prefix=None):
return list(self._objects)
def get_object_as_string(self, bucket, key):
self.downloaded_payload_keys.append((bucket, key))
return self._object_payloads[key]
class CommandPolicyTests(unittest.TestCase):
def test_command_policy_rejects_commands_by_default(self):
with mock.patch.dict(os.environ, {}, clear=True):
with self.assertRaisesRegex(RuntimeError, "disabled"):
MODULE.validate_command_policy("printf 'ok'")
def test_command_policy_allows_commands_when_explicitly_enabled(self):
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "true"}, clear=True):
MODULE.validate_command_policy("printf 'ok'")
def test_command_policy_can_disable_shell_commands(self):
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "false"}, clear=True):
with self.assertRaisesRegex(RuntimeError, "disabled"):
MODULE.validate_command_policy("printf 'ok'")
def test_command_policy_rejects_denylisted_command(self):
with mock.patch.dict(
os.environ,
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_DENYLIST_REGEX": r"(^|\s)rm\s+-rf"},
clear=True,
):
with self.assertRaisesRegex(RuntimeError, "DENYLIST"):
MODULE.validate_command_policy("rm -rf /workspace")
def test_command_policy_requires_allowlist_match_when_configured(self):
with mock.patch.dict(
os.environ,
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_ALLOWLIST_REGEX": r"^python3?\s"},
clear=True,
):
MODULE.validate_command_policy("python3 script.py")
with self.assertRaisesRegex(RuntimeError, "ALLOWLIST"):
MODULE.validate_command_policy("bash script.sh")
class ResourceMonitorTests(unittest.TestCase):
def test_parse_gpu_snapshot_output_summarizes_real_nvidia_smi_columns(self):
snapshot = RESOURCE_MONITOR.parse_gpu_snapshot_output(
"\n".join(
[
"12, 4096, 24576, NVIDIA L4",
"83, 8192, 24576, NVIDIA L4",
"not-a-number, 1, 1, broken",
]
)
)
self.assertEqual(snapshot["gpu"], 83)
self.assertEqual(snapshot["gpuMemoryUsedMiB"], 12288)
self.assertEqual(snapshot["gpuMemoryTotalMiB"], 49152)
self.assertEqual(snapshot["gpuName"], "NVIDIA L4")
self.assertEqual(snapshot["gpuCount"], 2)
def test_parse_gpu_snapshot_output_defaults_when_nvidia_smi_has_no_valid_rows(self):
self.assertEqual(RESOURCE_MONITOR.parse_gpu_snapshot_output(""), {"gpu": 0})
class SourcePathFromItemTests(unittest.TestCase):
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "node-output"
workspace_root = pathlib.Path(temp_dir) / "workspace"
output_dir.mkdir(parents=True)
workspace_root.mkdir(parents=True)
item = {
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D.MP4",
"previewURL": "https://example.com/should-not-download.mp4",
}
}
fake_client = _FakeBosClient()
with mock.patch.dict(
os.environ,
{
"WORKSPACE_ROOT": str(workspace_root),
"BOS_ENDPOINT": "https://bos.example.test",
"BOS_ACCESS_KEY_ID": "ak",
"BOS_SECRET_ACCESS_KEY": "sk",
},
clear=False,
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")) as build_client, mock.patch.object(
MODULE.request,
"urlopen",
) as urlopen:
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(first.exists())
self.assertEqual(first, second)
self.assertIn(str(workspace_root), str(first))
self.assertEqual(len(fake_client.downloads), 1)
build_client.assert_called()
urlopen.assert_not_called()
def test_downloads_http_preview_url_to_local_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
item = {
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/demo.mp4",
"previewURL": "https://example.com/demo.mp4?signature=abc",
}
}
with mock.patch.object(
MODULE.request,
"urlopen",
return_value=_FakeHTTPResponse(b"real-video-bytes"),
) as urlopen:
resolved = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(resolved.exists())
self.assertEqual(resolved.read_bytes(), b"real-video-bytes")
urlopen.assert_called_once()
def test_prefers_workspace_cache_path_for_preview_url_when_bos_credentials_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "node-output"
workspace_root = pathlib.Path(temp_dir) / "workspace"
output_dir.mkdir(parents=True)
workspace_root.mkdir(parents=True)
item = {
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0000/DJI_001/demo.mp4",
"previewURL": "https://example.com/demo.mp4?signature=abc",
}
}
with mock.patch.dict(
os.environ,
{
"WORKSPACE_ROOT": str(workspace_root),
"BOS_ENDPOINT": "",
"BOS_ACCESS_KEY_ID": "",
"BOS_SECRET_ACCESS_KEY": "",
},
clear=False,
), mock.patch.object(
MODULE.request,
"urlopen",
return_value=_FakeHTTPResponse(b"real-video-bytes"),
) as urlopen:
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(first.exists())
self.assertEqual(first, second)
self.assertEqual(first.read_bytes(), b"real-video-bytes")
self.assertIn(str(workspace_root / "_bos_cache"), str(first))
urlopen.assert_called_once()
def test_remote_video_url_for_source_signs_bos_path_when_preview_missing(self):
source = {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_0001_D.MP4",
"previewURL": "",
}
with mock.patch.object(MODULE, "signed_bos_source_url", return_value="https://signed.example.com/video.mp4") as signer:
resolved = MODULE.remote_video_url_for_source(source)
self.assertEqual(resolved, "https://signed.example.com/video.mp4")
signer.assert_called_once_with(source["path"])
class RuntimeFileContractTests(unittest.TestCase):
def test_select_handler_prefers_specific_operator_routes(self):
self.assertEqual(MODULE.select_handler("BOS 增量扫描"), "bos_incremental_scan")
self.assertEqual(MODULE.select_handler("人工审核"), "human_review")
self.assertEqual(MODULE.select_handler("ALOHA HDF5 示教校验"), "aloha_hdf5_validate")
self.assertEqual(MODULE.select_handler("robomimic HDF5 示教校验"), "robomimic_hdf5_validate")
self.assertEqual(MODULE.select_handler("VLA 样本清单校验"), "vla_sample_manifest_validate")
self.assertEqual(MODULE.select_handler("URDF 关节限位校验"), "joint_limit_validate")
self.assertEqual(MODULE.select_handler("MJCF 执行器校验"), "mjcf_actuator_validate")
self.assertEqual(MODULE.select_handler("TF 坐标树校验"), "tf_transform_tree_validate")
self.assertEqual(MODULE.select_handler("LeRobot 视频对齐校验"), "lerobot_video_alignment_validate")
self.assertEqual(MODULE.select_handler("ROS JointState 校验"), "ros_joint_state_validate")
self.assertEqual(MODULE.select_handler("DexYCB 数据集校验"), "dexycb_dataset_validate")
self.assertEqual(MODULE.select_handler("HO3D/H2O3D 数据集校验"), "ho3d_dataset_validate")
self.assertEqual(MODULE.select_handler("手-物体姿态标注校验"), "hand_object_pose_validate")
self.assertEqual(MODULE.select_handler("RLBench 数据集校验"), "rlbench_dataset_validate")
self.assertEqual(MODULE.select_handler("ManiSkill 轨迹校验"), "maniskill_trajectory_validate")
self.assertEqual(MODULE.select_handler("CALVIN Episode 校验"), "calvin_episode_validate")
self.assertEqual(MODULE.select_handler("RT-X动作空间Schema校验"), "rtx_action_schema_validate")
self.assertEqual(MODULE.select_handler("多机器人Embodiment元信息校验"), "embodiment_metadata_validate")
self.assertEqual(MODULE.select_handler("力扭矩传感器数据校验"), "force_torque_sensor_validate")
self.assertEqual(MODULE.select_handler("AgiBotWorld2026 数据集校验"), "agibot_world2026_validate")
self.assertEqual(MODULE.select_handler("BridgeData TFDS-RLDS清单校验"), "bridgedata_tfds_manifest_validate")
self.assertEqual(MODULE.select_handler("DROID Raw Episode校验"), "droid_raw_episode_validate")
self.assertEqual(MODULE.select_handler("LIBERO HDF5任务校验"), "libero_hdf5_task_validate")
self.assertEqual(MODULE.select_handler("RoboCasa HDF5示教校验"), "robocasa_hdf5_demonstration_validate")
self.assertEqual(MODULE.select_handler("FurnitureBench PKL示教校验"), "furniturebench_pkl_demonstration_validate")
self.assertEqual(MODULE.select_handler("RoboTurk Real HDF5校验"), "roboturk_real_hdf5_validate")
self.assertEqual(MODULE.select_handler("RoboSet HDF5轨迹校验"), "roboset_hdf5_validate")
self.assertEqual(MODULE.select_handler("RH20T 场景结构校验"), "rh20t_scene_validate")
self.assertEqual(MODULE.select_handler("RoboMIND HDF5质量校验"), "robomind_hdf5_quality_validate")
self.assertEqual(MODULE.select_handler("UMI Zarr数据集校验"), "umi_zarr_dataset_validate")
self.assertEqual(MODULE.select_handler("LeRobot v3 Episode索引校验"), "lerobot_v3_episode_index_validate")
self.assertEqual(MODULE.select_handler("数据集切分泄漏校验"), "dataset_split_leakage_validate")
self.assertEqual(MODULE.select_handler("媒体引用可读性校验"), "media_reference_validate")
self.assertEqual(MODULE.select_handler("数据集Schema合约校验"), "dataset_schema_contract_validate")
def test_operator_helpers_do_not_expose_legacy_builtin_handlers(self):
legacy_keys = sorted(key for key in MODULE.OPERATOR_HELPERS if key.startswith("builtin_"))
self.assertEqual(legacy_keys, [])
def test_dispatch_runs_standalone_csv_to_json_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "input.csv"
source_path.write_text("id,name,score\n1,alpha,98\n2,beta,87\n", encoding="utf-8")
output_dir = root / "out"
output_dir.mkdir()
node_data = {
"label": "CSV 转 JSON",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(source_path)}],
"outputs": [{"id": "out-1", "name": "table", "type": "json"}],
}
artifacts = MODULE.BUILTIN_HANDLERS["csv_to_json"]("run-1", "csv", output_dir, node_data, [])
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["columns"], ["id", "name", "score"])
self.assertEqual(payload["rowCount"], 2)
self.assertEqual(payload["items"][1]["name"], "beta")
def test_dispatch_runs_standalone_json_filter_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 过滤",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "EXPECTED", "value": "ready"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["json_filter"](
"run-1",
"filter",
output_dir,
node_data,
[{"data": {"items": [{"status": "ready"}, {"status": "blocked"}]}}],
)
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["count"], 1)
self.assertEqual(payload["items"], [{"status": "ready"}])
def test_dispatch_runs_named_human_review_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "人工审核",
"hyperparams": [
{"key": "REVIEW_TYPE", "value": "task"},
{"key": "REVIEWER_ROLE", "value": "qa_reviewer"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["builtin_human_review"](
"run-1",
"review",
output_dir,
node_data,
[{"data": {"items": [{"id": "clip-1", "score": 0.9}]}}],
)
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["datasetType"], "human-review-tasks")
self.assertEqual(payload["reviewerRole"], "qa_reviewer")
self.assertEqual(payload["taskCount"], 1)
self.assertEqual(payload["items"][0]["id"], "run-1-clip-1")
def test_load_input_artifacts_prefers_manifest_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
manifest_path = pathlib.Path(temp_dir) / "input-artifacts.json"
manifest_path.write_text('[{"id":"artifact-from-file"}]', encoding="utf-8")
with mock.patch.dict(
os.environ,
{
"INPUT_ARTIFACTS_PATH": str(manifest_path),
"INPUT_ARTIFACTS_JSON": '[{"id":"artifact-inline"}]',
},
clear=False,
):
loaded = MODULE.load_input_artifacts()
self.assertEqual(len(loaded), 1)
self.assertEqual(loaded[0]["id"], "artifact-from-file")
def test_main_writes_result_and_stage_manifest_files(self):
with tempfile.TemporaryDirectory() as temp_dir:
node_root = pathlib.Path(temp_dir)
output_dir = node_root / "outputs"
output_dir.mkdir(parents=True)
input_artifacts_path = node_root / "input-artifacts.json"
input_artifacts_path.write_text("[]", encoding="utf-8")
result_path = node_root / "result.json"
stage_manifest_path = node_root / "stage-manifest.json"
artifacts = [
{
"id": "artifact-1",
"runId": "run-1",
"nodeId": "node-1",
"name": "report.json",
"type": "json",
"size": "12 B",
"path": str(output_dir / "report.json"),
}
]
with mock.patch.dict(
os.environ,
{
"RUN_ID": "run-1",
"WORKFLOW_ID": "wf-1",
"NODE_ID": "node-1",
"NODE_LABEL": "通用处理",
"OUTPUT_DIR": str(output_dir),
"NODE_DATA_JSON": '{"label":"通用处理","hyperparams":[]}',
"INPUT_ARTIFACTS_PATH": str(input_artifacts_path),
"RESULT_OUTPUT_PATH": str(result_path),
"STAGE_MANIFEST_PATH": str(stage_manifest_path),
},
clear=False,
), mock.patch.object(MODULE, "builtin_generic", return_value=artifacts), mock.patch.object(
MODULE,
"resource_series",
return_value=[],
):
stdout = io.StringIO()
with contextlib.redirect_stdout(stdout):
exit_code = MODULE.main()
self.assertEqual(exit_code, 0)
self.assertIn(MODULE.RESULT_PATH_PREFIX + str(result_path), stdout.getvalue())
result_payload = json.loads(result_path.read_text(encoding="utf-8"))
self.assertEqual(result_payload["artifacts"][0]["id"], "artifact-1")
stage_manifest = json.loads(stage_manifest_path.read_text(encoding="utf-8"))
self.assertEqual(stage_manifest["artifactCount"], 1)
self.assertEqual(stage_manifest["nodeLabel"], "通用处理")
def test_write_json_artifact_externalizes_dataset_payload_to_data_ref(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"dataset-index.json",
{
"datasetType": "x-platform-index",
"collectionCount": 1,
"collections": [{"id": "collection-1"}],
"items": [{"id": "item-1"}, {"id": "item-2"}],
},
)
data_ref = artifact["data"]["dataRef"]
self.assertEqual(data_ref["kind"], "dataset")
self.assertTrue(pathlib.Path(data_ref["manifestPath"]).exists())
self.assertTrue(pathlib.Path(data_ref["payloadPath"]).exists())
manifest = json.loads(pathlib.Path(data_ref["manifestPath"]).read_text(encoding="utf-8"))
self.assertEqual(manifest["datasetType"], "x-platform-index")
self.assertIn("items", manifest["partitionFiles"])
self.assertIn("collections", manifest["partitionFiles"])
def test_load_artifact_payload_reads_externalized_payload_from_data_ref(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"filtered-videos.json",
{
"datasetType": "x-platform-filtered",
"count": 2,
"items": [{"id": "v1"}, {"id": "v2"}],
},
)
loaded = MODULE.load_artifact_payload(artifact)
self.assertEqual(loaded["datasetType"], "x-platform-filtered")
self.assertEqual(len(loaded["items"]), 2)
def test_load_payload_from_file_retries_transient_json_decode_error(self):
with tempfile.TemporaryDirectory() as temp_dir:
payload_path = pathlib.Path(temp_dir) / "payload.json"
payload_path.write_text('{"items":[{"id":"v1"}]}', encoding="utf-8")
decode_error = json.JSONDecodeError("transient", "", 0)
with mock.patch.object(
MODULE.json,
"load",
side_effect=[decode_error, {"datasetType": "x-platform", "items": [{"id": "v1"}]}],
) as json_load, mock.patch.object(MODULE.time, "sleep") as sleep_mock:
loaded = MODULE.load_payload_from_file(payload_path)
self.assertEqual(loaded["datasetType"], "x-platform")
self.assertEqual(len(loaded["items"]), 1)
self.assertEqual(json_load.call_count, 2)
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
def test_load_payload_from_file_retries_until_payload_path_exists(self):
with tempfile.TemporaryDirectory() as temp_dir:
payload_path = pathlib.Path(temp_dir) / "payload.json"
payload_path.write_text('{"datasetType":"x-platform","items":[{"id":"v1"}]}', encoding="utf-8")
original_exists = pathlib.Path.exists
seen = {"count": 0}
def fake_exists(path_obj: pathlib.Path):
if path_obj == payload_path and seen["count"] == 0:
seen["count"] += 1
return False
return original_exists(path_obj)
with mock.patch.object(pathlib.Path, "exists", autospec=True, side_effect=fake_exists), mock.patch.object(
MODULE.time,
"sleep",
) as sleep_mock:
loaded = MODULE.load_payload_from_file(payload_path)
self.assertEqual(loaded["datasetType"], "x-platform")
self.assertEqual(len(loaded["items"]), 1)
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
def test_stage_manifest_includes_data_refs(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"packages-manifest.json",
{
"datasetType": "x-platform-packages",
"packageCount": 1,
"items": [{"packageName": "pkg-1"}],
},
)
manifest = MODULE.stage_manifest_payload("run-1", "wf-1", "node-1", "成品打包", output_dir, [artifact])
self.assertEqual(len(manifest["dataRefs"]), 1)
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
class XPlatformIndexingTests(unittest.TestCase):
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "outputs"
output_dir.mkdir(parents=True)
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
manifest_path.write_text("{}", encoding="utf-8")
payload = MODULE.load_xplatform_index_payload(
{},
[
{
"path": str(manifest_path),
"data": {
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
"entries": [
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "工厂",
"place": "绿控传动",
"date": "20260330",
},
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260327132547_0003_D.MP4",
"codec": "hevc",
"resolution": "3840x2880",
"frameRate": 29.97,
"duration": 0.6,
}
],
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
},
},
],
},
}
],
output_dir,
)
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["videoCount"], 1)
self.assertEqual(payload["collections"][0]["meta"]["place"], "绿控传动")
def test_load_index_payload_raises_when_source_manifest_summary_has_no_entries(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "outputs"
output_dir.mkdir(parents=True)
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
manifest_path.write_text("{}", encoding="utf-8")
with self.assertRaisesRegex(RuntimeError, "source-manifest"):
MODULE.load_xplatform_index_payload(
{},
[
{
"name": "source-manifest.json",
"path": str(manifest_path),
"data": {
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
"entriesCount": 59,
"dataRef": {"payloadPath": str(manifest_path)},
},
}
],
output_dir,
)
def test_manifest_entries_support_video_metadata_alias(self):
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "工厂",
"place": "绿控传动",
"date": "20260330",
},
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260327132547_0003_D.MP4",
"codec": "hevc",
"resolution": "3840x2880",
"frameRate": 29.97,
"duration": 0.6,
}
],
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
},
},
],
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertEqual(payload["items"][0]["codec"], "hevc")
def test_manifest_entries_load_local_meta_when_inline_data_is_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "algo" / "X-platform-test"
collection = root / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
video_dir = collection / "DJI_001"
video_dir.mkdir(parents=True)
meta_path = collection / "meta.json"
video_meta_path = collection / "video_meta.json"
video_path = video_dir / "DJI_0001_D.mp4"
meta_path.write_text('{"type":"家庭","place":"蓟门里小区","date":"20260319"}', encoding="utf-8")
video_meta_path.write_text(
'[{"filename":"DJI_0001_D.mp4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
encoding="utf-8",
)
video_path.write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
"sizeBytes": meta_path.stat().st_size,
"mimeType": "application/json",
"source": {"path": str(meta_path)},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
"sizeBytes": video_meta_path.stat().st_size,
"mimeType": "application/json",
"source": {"path": str(video_meta_path)},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
"sizeBytes": video_path.stat().st_size,
"mimeType": "video/mp4",
"source": {"path": str(video_path)},
},
],
str(root),
)
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
self.assertEqual(payload["items"][0]["codec"], "hevc")
def test_manifest_entries_support_root_level_meta_and_video_meta(self):
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "家庭",
"place": "蓟门里小区",
"date": "20260319",
},
},
{
"path": "video_meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"duration": 41.8,
}
],
},
{
"path": "DJI_001/DJI_20260318112834_0001_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/video.mp4",
},
},
],
"bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(collection["relativePath"], "")
self.assertEqual(collection["videoCount"], 1)
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertEqual(collection["meta"]["place"], "蓟门里小区")
item = payload["items"][0]
self.assertEqual(item["collectionName"], collection["name"])
self.assertEqual(item["collectionRelativePath"], "")
self.assertEqual(item["videoFolder"], "DJI_001")
self.assertEqual(item["codec"], "hevc")
self.assertEqual(item["meta"]["type"], "家庭")
def test_local_dataset_root_with_meta_is_treated_as_collection_root(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
video_dir = root / "DJI_001"
video_dir.mkdir(parents=True)
(root / "meta.json").write_text('{"type":"家庭","place":"蓟门里小区"}', encoding="utf-8")
(root / "video_meta.json").write_text(
'[{"filename":"DJI_20260318112834_0001_D.MP4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
encoding="utf-8",
)
(video_dir / "DJI_20260318112834_0001_D.MP4").write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_dataset(root)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["name"], root.name)
self.assertEqual(collection["relativePath"], "")
self.assertEqual(collection["missingRequiredFiles"], [])
item = payload["items"][0]
self.assertEqual(item["collectionName"], root.name)
self.assertEqual(item["collectionRelativePath"], "")
self.assertEqual(item["codec"], "hevc")
def test_local_dataset_root_supports_video_metadata_alias(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "SZ_000_0005_OsmoNano_2026-03-30_16-15-11"
video_dir = root / "DJI_001"
video_dir.mkdir(parents=True)
(root / "meta.json").write_text('{"type":"工厂","place":"绿控传动"}', encoding="utf-8")
(root / "video_metadata.json").write_text(
'[{"filename":"DJI_20260327132547_0003_D.MP4","codec":"hevc","resolution":"3840x2880","frameRate":29.97,"duration":0.6}]',
encoding="utf-8",
)
(video_dir / "DJI_20260327132547_0003_D.MP4").write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_dataset(root)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(payload["items"][0]["codec"], "hevc")
class SourceOperatorTests(unittest.TestCase):
def test_builtin_source_rejects_missing_input_configuration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {"label": "真实数据源", "hyperparams": []}
with self.assertRaisesRegex(RuntimeError, "requires SOURCE_TEXT"):
MODULE.builtin_source("run-test", "source-empty", output_dir, node_data, [])
def test_builtin_source_infers_local_source_path_without_data_source(self):
with tempfile.TemporaryDirectory() as temp_dir:
source_root = pathlib.Path(temp_dir) / "source"
source_root.mkdir()
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
output_dir = pathlib.Path(temp_dir) / "output"
output_dir.mkdir()
node_data = {
"label": "真实数据源-本地路径",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(source_root)},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-local", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["dataSource"], "LOCAL")
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["path"], "payload.json")
def test_builtin_source_rejects_missing_local_source_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
missing_path = pathlib.Path(temp_dir) / "missing"
node_data = {
"label": "真实数据源-缺失本地路径",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(missing_path)},
],
}
with self.assertRaisesRegex(RuntimeError, "SOURCE_PATH does not exist"):
MODULE.builtin_source("run-test", "source-missing", output_dir, node_data, [])
def test_builtin_source_defaults_to_inline_when_source_json_is_present(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-默认内联",
"hyperparams": [
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-inline-default", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["dataSource"], "INLINE")
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
def test_source_handler_can_emit_inline_json_payload_on_named_output(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-内联JSON",
"outputs": [
{"id": "manifest", "name": "manifest", "type": "json"},
{"id": "payload", "name": "payload", "type": "json"},
],
"hyperparams": [
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "real"}]}, ensure_ascii=False)},
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
{"key": "OUTPUT_NAME", "value": "payload.json"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-json-payload", output_dir, node_data, [])
payload_artifact = next(item for item in artifacts if item["name"] == "payload.json")
payload = json.loads(pathlib.Path(payload_artifact["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["items"][0]["kind"], "real")
self.assertEqual(payload_artifact["portId"], "payload")
def test_source_handler_can_emit_inline_text_payload_with_tabular_preview(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-内联CSV",
"outputs": [
{"id": "manifest", "name": "manifest", "type": "json"},
{"id": "csv", "name": "csv", "type": "file"},
],
"hyperparams": [
{"key": "SOURCE_TEXT", "value": "id,label\nclip-a,keep\n"},
{"key": "PAYLOAD_OUTPUT_ID", "value": "csv"},
{"key": "OUTPUT_NAME", "value": "report.csv"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-csv-payload", output_dir, node_data, [])
csv_artifact = next(item for item in artifacts if item["name"] == "report.csv")
self.assertEqual(csv_artifact["portId"], "csv")
self.assertEqual(csv_artifact["data"]["preview"]["kind"], "table")
self.assertEqual(csv_artifact["data"]["preview"]["rowCount"], 1)
def test_source_handler_rejects_unsafe_inline_payload_name(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-不安全输出名",
"outputs": [
{"id": "manifest", "name": "manifest", "type": "json"},
{"id": "payload", "name": "payload", "type": "json"},
],
"hyperparams": [
{"key": "SOURCE_JSON", "value": "{}"},
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
{"key": "OUTPUT_NAME", "value": "../payload.json"},
],
}
with self.assertRaisesRegex(RuntimeError, "OUTPUT_NAME"):
MODULE.BUILTIN_HANDLERS["source"]("run-test", "source-unsafe-payload", output_dir, node_data, [])
def test_builtin_source_prefers_source_json_over_bos_env(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-JSON",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-json", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
def test_builtin_source_prefers_source_text_over_bos_env(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-文本",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_TEXT", "value": "alpha\nbeta\ngamma"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-text", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["text"], "alpha\nbeta\ngamma")
def test_directory_incremental_source_filter_emits_full_new_directory(self):
entries = [
{"path": "batch-a/meta.json", "sizeBytes": 10, "sha256": "sha-a-meta"},
{"path": "batch-a/DJI_001/clip-a.mp4", "sizeBytes": 20, "sha256": "sha-a-clip"},
{"path": "batch-b/meta.json", "sizeBytes": 30, "sha256": "sha-b-meta"},
{"path": "batch-b/DJI_001/clip-b.mp4", "sizeBytes": 40, "sha256": "sha-b-clip"},
]
emitted, checkpoint = MODULE.apply_incremental_source_filter(
entries,
"BOS",
"wjzh-data-migration",
{
"SOURCE_STATE_JSON": json.dumps(
{
"seenFingerprints": ["batch-a"],
"incrementalUnit": "DIRECTORY",
"directoryDepth": 1,
}
),
"SOURCE_FINGERPRINT": "fp-directory",
"INCREMENTAL_UNIT": "DIRECTORY",
"INCREMENTAL_DIRECTORY_DEPTH": "1",
},
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
"wjzh-0330-Data-migration",
"",
)
self.assertEqual([item["path"] for item in emitted], ["batch-b/meta.json", "batch-b/DJI_001/clip-b.mp4"])
self.assertEqual([item["incrementalKey"] for item in emitted], ["batch-b", "batch-b"])
self.assertEqual(checkpoint["incrementalUnit"], "DIRECTORY")
self.assertEqual(checkpoint["directoryDepth"], 1)
self.assertEqual(checkpoint["newCount"], 2)
self.assertEqual(sorted(checkpoint["seenFingerprints"]), ["batch-a", "batch-b"])
def test_incremental_source_state_rejects_invalid_json(self):
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be valid JSON"):
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "{"})
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be a JSON object"):
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "[]"})
def test_incremental_source_state_rejects_invalid_shape(self):
cases = [
({"seenFingerprints": "fp-1"}, "seenFingerprints must be an array"),
({"seenFingerprints": [42]}, r"seenFingerprints\[0\] must be a string"),
({"incrementalUnit": "ROW"}, "incrementalUnit must be OBJECT or DIRECTORY"),
({"directoryDepth": 1.5}, "directoryDepth must be an integer"),
({"directoryDepth": 0}, "directoryDepth must be greater than 0"),
]
for payload, expected_error in cases:
with self.subTest(payload=payload), self.assertRaisesRegex(RuntimeError, expected_error):
MODULE.incremental_source_state({"SOURCE_STATE_JSON": json.dumps(payload)})
def test_incremental_directory_depth_rejects_lossy_values(self):
for raw in ["0", "01", "1.5", "2days"]:
with self.subTest(raw=raw), self.assertRaises(RuntimeError):
MODULE.incremental_directory_depth({"INCREMENTAL_DIRECTORY_DEPTH": raw})
def test_incremental_unit_rejects_invalid_values(self):
self.assertEqual(MODULE.incremental_unit({}), "OBJECT")
self.assertEqual(MODULE.incremental_unit({"INCREMENTAL_UNIT": " directory "}), "DIRECTORY")
with self.assertRaisesRegex(RuntimeError, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY"):
MODULE.incremental_unit({"INCREMENTAL_UNIT": "ROW"})
def test_builtin_source_rejects_invalid_scan_mode(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "BOS 增量扫描",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "INLINE"},
{"key": "SOURCE_TEXT", "value": "alpha"},
{"key": "SCAN_MODE", "value": "ROW"},
],
}
with self.assertRaisesRegex(RuntimeError, "SCAN_MODE must be FULL or INCREMENTAL"):
MODULE.builtin_source("run-test", "source-invalid-scan-mode", output_dir, node_data, [])
def test_builtin_source_rejects_invalid_data_source(self):
with tempfile.TemporaryDirectory() as temp_dir:
source_root = pathlib.Path(temp_dir) / "source"
source_root.mkdir()
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
output_dir = pathlib.Path(temp_dir) / "output"
output_dir.mkdir()
node_data = {
"label": "真实数据源-非法数据源",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "ROW"},
{"key": "SOURCE_PATH", "value": str(source_root)},
],
}
with self.assertRaisesRegex(RuntimeError, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM"):
MODULE.builtin_source("run-test", "source-invalid-data-source", output_dir, node_data, [])
def test_list_bos_entries_skips_large_non_metadata_json_payloads(self):
object_meta = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/meta.json", "size": 128, "e_tag": "meta"})()
object_imu = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json", "size": MODULE.PREVIEW_INLINE_BINARY_LIMIT + 1, "e_tag": "imu"})()
fake_client = _FakeBosListClient(
[object_meta, object_imu],
{
"wjzh-0330-Data-migration/SZ_000_0005/meta.json": b'{"type":"\xe5\xb7\xa5\xe5\x8e\x82"}',
"wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json": b'{"data_points":[1,2,3]}',
},
)
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
entries = MODULE.list_bos_entries(
"wjzh-data-migration",
"wjzh-0330-Data-migration",
"",
"https://bos.example.test",
"ak",
"sk",
)
self.assertEqual(len(entries), 2)
meta_entry = next(item for item in entries if item["path"].endswith("meta.json"))
imu_entry = next(item for item in entries if item["path"].endswith("_imu.json"))
self.assertIn("data", meta_entry)
self.assertNotIn("data", imu_entry)
self.assertEqual(
fake_client.downloaded_payload_keys,
[("wjzh-data-migration", "wjzh-0330-Data-migration/SZ_000_0005/meta.json")],
)
def test_bos_source_parses_bucket_and_prefix_from_source_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "BOS 增量扫描",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://wjzh-data-migration/wjzh-0330-Data-migration"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
{"key": "SCAN_MODE", "value": "FULL"},
],
}
fake_entries = [
{
"path": "SZ_000_0005/meta.json",
"storagePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {"type": "工厂"},
"source": {"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json"},
}
]
with mock.patch.object(MODULE, "list_bos_entries", return_value=fake_entries) as list_bos_entries, mock.patch.object(
MODULE,
"source_entry_artifact",
return_value={
"id": "artifact-1",
"runId": "run-1",
"nodeId": "source",
"name": "meta.json",
"type": "json",
"path": str(output_dir / "meta.json"),
},
):
artifacts = MODULE.builtin_source("run-1", "source", output_dir, node_data, [])
self.assertGreaterEqual(len(artifacts), 1)
list_bos_entries.assert_called_once_with(
"wjzh-data-migration",
"wjzh-0330-Data-migration",
"",
"https://bos.example.test",
"ak",
"sk",
)
class QualityMetricHelperTests(unittest.TestCase):
def test_build_skin_mask_detects_skin_tone_patch(self):
_, np = MODULE.require_cv2()
frame = np.zeros((80, 80, 3), dtype=np.uint8)
frame[20:60, 20:60] = (90, 130, 190)
mask = MODULE.build_skin_mask(frame)
self.assertGreater(int(mask.sum()), 0)
def test_build_skin_mask_rejects_neutral_background(self):
_, np = MODULE.require_cv2()
frame = np.full((80, 80, 3), 128, dtype=np.uint8)
mask = MODULE.build_skin_mask(frame)
self.assertEqual(int(mask.sum()), 0)
def test_motion_jitter_ratio_stays_low_for_smooth_motion(self):
smooth = [(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)]
jitter = MODULE.motion_jitter_ratio(smooth, 100.0)
self.assertLess(jitter, 0.01)
def test_motion_jitter_ratio_increases_for_erratic_motion(self):
smooth = MODULE.motion_jitter_ratio([(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)], 100.0)
erratic = MODULE.motion_jitter_ratio([(0.0, 0.0), (12.0, -4.0), (-10.0, 8.0), (15.0, -6.0)], 100.0)
self.assertGreater(erratic, smooth)
self.assertGreater(erratic, 0.05)
def test_largest_skin_component_ratio_ignores_small_noise(self):
_, np = MODULE.require_cv2()
mask = np.zeros((100, 100), dtype=np.uint8)
mask[10:13, 10:13] = 255
ratio = MODULE.largest_skin_component_ratio(mask)
self.assertEqual(ratio, 0.0)
def test_largest_skin_component_ratio_returns_large_component_area(self):
_, np = MODULE.require_cv2()
mask = np.zeros((100, 100), dtype=np.uint8)
mask[20:50, 30:70] = 255
ratio = MODULE.largest_skin_component_ratio(mask)
self.assertGreater(ratio, 0.1)
self.assertLess(ratio, 0.13)
class QualityOperatorTests(unittest.TestCase):
def test_video_quality_accepts_zero_hand_occlusion(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "MIN_CLARITY", "value": "0.8"},
{"key": "MAX_SHAKE", "value": "0.1"},
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
],
}
inputs = [
{
"data": {
"items": [
{
"filename": "demo.mp4",
"source": {"path": "/tmp/demo.mp4"},
}
]
}
}
]
with mock.patch.object(MODULE, "source_path_from_item", return_value=pathlib.Path("/tmp/demo.mp4")), mock.patch.object(
MODULE,
"analyze_video_quality_metrics",
return_value={
"clarity": 1.0,
"shake": 0.0,
"handOcclusion": 0.0,
"qualityScore": 1.0,
},
):
artifacts = MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["passedCount"], 1)
self.assertTrue(payload["items"][0]["quality"]["passed"])
def test_video_quality_rejects_out_of_range_thresholds(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "MIN_CLARITY", "value": "1.1"},
],
}
with self.assertRaisesRegex(RuntimeError, "MIN_CLARITY must be in the range"):
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
def test_video_quality_rejects_non_finite_thresholds(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "MAX_SHAKE", "value": "nan"},
],
}
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be finite"):
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
def test_video_quality_rejects_invalid_clarity_scale(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "CLARITY_SCALE", "value": "0"},
],
}
with self.assertRaisesRegex(RuntimeError, "CLARITY_SCALE must be greater than 0"):
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
def test_dataset_filter_keeps_zero_shake_and_hand_occlusion(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "数据过滤",
"hyperparams": [
{"key": "DATE_START", "value": "2026-03-01"},
{"key": "DATE_END", "value": "2026-03-31"},
{"key": "PLACE", "value": "蓟门里小区"},
{"key": "TYPE", "value": "家庭"},
{"key": "MIN_CLARITY", "value": "0.8"},
{"key": "MAX_SHAKE", "value": "0.1"},
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
],
}
inputs = [
{
"data": {
"items": [
{
"meta": {
"date": "20260319",
"place": "蓟门里小区",
"type": "家庭",
},
"quality": {
"clarity": 0.95,
"shake": 0.0,
"handOcclusion": 0.0,
"passed": True,
},
}
]
}
}
]
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["count"], 1)
self.assertEqual(len(payload["items"]), 1)
def test_dataset_filter_missing_thresholds_use_catalog_defaults(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "数据过滤",
"hyperparams": [],
}
inputs = [
{
"data": {
"items": [
{
"quality": {
"clarity": 0.5,
"shake": 0.0,
"handOcclusion": 0.0,
"passed": True,
},
},
{
"quality": {
"clarity": 0.95,
"shake": 0.0,
"handOcclusion": 0.0,
"passed": True,
},
},
]
}
}
]
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["criteria"]["minClarity"], MODULE.DEFAULT_MIN_CLARITY)
self.assertEqual(payload["criteria"]["maxShake"], MODULE.DEFAULT_MAX_SHAKE)
self.assertEqual(payload["criteria"]["maxHandOcclusion"], MODULE.DEFAULT_MAX_HAND_OCCLUSION)
self.assertEqual(payload["count"], 1)
self.assertEqual(payload["items"][0]["quality"]["clarity"], 0.95)
def test_dataset_filter_rejects_invalid_threshold_values(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "数据过滤",
"hyperparams": [
{"key": "MAX_SHAKE", "value": "not-a-number"},
],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be numeric"):
MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
class OperatorHyperparamParsingTests(unittest.TestCase):
def test_collection_validate_rejects_invalid_boolean_hyperparam(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "采集目录校验",
"hyperparams": [{"key": "REQUIRE_INTRINSICS", "value": "sometimes"}],
}
with self.assertRaisesRegex(RuntimeError, "REQUIRE_INTRINSICS must be a boolean"):
MODULE.builtin_collection_validate("run-1", "validate", output_dir, node_data, [{"data": {"collections": [], "items": []}}])
def test_json_filter_rejects_invalid_expected_json(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 过滤",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "EXPECTED_JSON", "value": '{"status"'},
],
}
with self.assertRaisesRegex(RuntimeError, "EXPECTED_JSON must be valid JSON"):
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
def test_json_filter_rejects_unknown_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 过滤",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "OPERATOR", "value": "roughly"},
{"key": "EXPECTED", "value": "ready"},
],
}
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
def test_json_map_rejects_non_object_field_map(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": "[]"}],
}
with self.assertRaisesRegex(RuntimeError, "FIELD_MAP_JSON must be a JSON object"):
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
def test_json_map_renames_fields_without_overwriting(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new","status":"state"}'}],
}
artifacts = MODULE.builtin_json_map(
"run-1",
"map",
output_dir,
node_data,
[{"data": {"items": [{"old": "value", "status": "ready", "unchanged": 1}]}}],
)
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["fieldMap"], {"old": "new", "status": "state"})
self.assertEqual(payload["items"], [{"new": "value", "state": "ready", "unchanged": 1}])
def test_json_map_rejects_invalid_field_map_entries(self):
cases = [
('{" old":"new"}', "must not have leading or trailing whitespace"),
('{"old":" new"}', "destination field must not have leading or trailing whitespace"),
('{"old":""}', "destination field must be non-empty"),
('{"old":42}', "destination field must be a string"),
('{"first":"name","last":"name"}', "maps multiple source fields to destination field"),
]
for field_map_json, expected_error in cases:
with self.subTest(field_map_json=field_map_json), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": field_map_json}],
}
with self.assertRaisesRegex(RuntimeError, expected_error):
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
def test_json_map_rejects_output_field_collisions(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "JSON 映射",
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new"}'}],
}
with self.assertRaisesRegex(RuntimeError, "same output field 'new'"):
MODULE.builtin_json_map(
"run-1",
"map",
output_dir,
node_data,
[{"data": {"items": [{"old": "mapped", "new": "existing"}]}}],
)
def test_audit_sampling_rejects_non_array_rules(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [{"key": "RULES_JSON", "value": "{}"}],
}
with self.assertRaisesRegex(RuntimeError, "RULES_JSON must be a JSON array"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_rejects_incomplete_rule_shape(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [
{"key": "RULES_JSON", "value": '[{"id":"missing-percent","field":"quality.clarity","operator":"gte","value":0.8}]'}
],
}
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent is required"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_rejects_duplicate_rule_ids(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
rules_json = json.dumps(
[
{"id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.8, "samplePercent": 100},
{"id": "clarity", "field": "quality.shake", "operator": "lte", "value": 0.1, "samplePercent": 50},
],
ensure_ascii=False,
)
node_data = {
"label": "抽检采样",
"hyperparams": [{"key": "RULES_JSON", "value": rules_json}],
}
with self.assertRaisesRegex(RuntimeError, "RULES_JSON contains duplicate id: clarity"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_rejects_invalid_sample_percent(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [
{
"key": "RULES_JSON",
"value": '[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":0}]',
}
],
}
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be in the range"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
node_data["hyperparams"][0]["value"] = (
'[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":true}]'
)
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be numeric"):
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
def test_audit_sampling_allows_exists_rule_without_value(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "抽检采样",
"hyperparams": [
{
"key": "RULES_JSON",
"value": '[{"id":"has-collection","field":"collectionId","operator":"exists","samplePercent":100}]',
}
],
}
artifacts = MODULE.builtin_audit_sampling(
"run-1",
"sample",
output_dir,
node_data,
[
{
"data": {
"packageCount": 2,
"items": [
{"packageName": "clip-a", "collectionId": "collection-a"},
{"packageName": "clip-b"},
],
}
}
],
)
self.assertEqual(len(artifacts), 1)
payload = json.loads((output_dir / "audit-samples.json").read_text(encoding="utf-8"))
self.assertEqual(payload["rules"][0]["matchedCount"], 1)
self.assertEqual(payload["sampledCount"], 1)
def test_condition_router_rejects_unknown_operator(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "条件分流",
"hyperparams": [
{"key": "FIELD_PATH", "value": "status"},
{"key": "OPERATOR", "value": "roughly"},
{"key": "EXPECTED", "value": "ready"},
],
}
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
MODULE.builtin_condition_router("run-1", "route", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
class UploadOperatorTests(unittest.TestCase):
def test_build_artifact_skips_sha256_for_large_files(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
clip_path = root / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
MODULE,
"sha256_file",
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
):
artifact = MODULE.build_artifact("run-1", "slice", clip_path, "video")
metadata = artifact.get("metadata") or {}
self.assertEqual(metadata.get("sha256", ""), "")
self.assertTrue(metadata.get("sha256Skipped"))
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
def test_uploads_nested_package_outputs_to_bos(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
package_root = root / "package"
package_dir = package_root / "packages" / "clip001_package"
package_dir.mkdir(parents=True)
manifest_path = package_root / "packages-manifest.json"
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
clip_path = package_dir / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
annotation_path = package_dir / "clip001_annotation.json"
annotation_path.write_text('{"label":"walking"}', encoding="utf-8")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
MODULE.build_artifact("run-1", "package", clip_path, "video"),
MODULE.build_artifact("run-1", "package", annotation_path, "json", data={"label": "walking"}),
]
fake_client = _FakeBosClient()
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
uploaded_names = {item["name"] for item in artifacts}
uploaded_keys = {item["key"] for item in fake_client.uploads}
self.assertEqual(len(fake_client.uploads), 3)
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
self.assertEqual(uploaded_summary_payload["uploadedCount"], 3)
self.assertEqual(uploaded_summary_payload["bucket"], "datastorage")
self.assertEqual(uploaded_summary_payload["prefix"], "factory/output/demo")
def test_upload_skips_sha256_for_large_remote_artifacts(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
clip_path = root / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [MODULE.build_artifact("run-1", "slice", clip_path, "video")]
fake_client = _FakeBosClient()
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
MODULE,
"sha256_file",
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
remote_video = next(item for item in artifacts if item.get("type") == "video" and str(item.get("path", "")).startswith("bos://"))
metadata = remote_video.get("metadata") or {}
self.assertEqual(metadata.get("sha256", ""), "")
self.assertTrue(metadata.get("sha256Skipped"))
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
class _FakeInsertManyResult:
def __init__(self, inserted_ids):
self.inserted_ids = inserted_ids
class _FakeUpdateResult:
def __init__(self, matched_count=0, modified_count=0, upserted_id=None):
self.matched_count = matched_count
self.modified_count = modified_count
self.upserted_id = upserted_id
class _FakeMongoCollection:
def __init__(self):
self.insert_many_calls = []
self.replace_one_calls = []
def insert_many(self, documents, ordered=False):
self.insert_many_calls.append({"documents": documents, "ordered": ordered})
return _FakeInsertManyResult([f"id-{index + 1}" for index in range(len(documents))])
def replace_one(self, filter_document, replacement, upsert=False):
self.replace_one_calls.append(
{
"filter": filter_document,
"replacement": replacement,
"upsert": upsert,
}
)
return _FakeUpdateResult(matched_count=1, modified_count=1, upserted_id=None)
class _FakeMongoDatabase:
def __init__(self, collection):
self._collection = collection
self.requested_collections = []
def __getitem__(self, name):
self.requested_collections.append(name)
return self._collection
class _FakeMongoClient:
def __init__(self, database):
self._database = database
self.requested_databases = []
self.closed = False
def __getitem__(self, name):
self.requested_databases.append(name)
return self._database
def close(self):
self.closed = True
class MongoWriteOperatorTests(unittest.TestCase):
def test_inserts_documents_into_default_business_database(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
{"key": "MONGO_MODE", "value": "insert_many"},
],
}
inputs = [
{
"data": {
"items": [
{"clipId": "clip-001", "qualityScore": 0.91},
{"clipId": "clip-002", "qualityScore": 0.87},
]
}
}
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-1", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(database.requested_collections, ["qualityReports"])
self.assertEqual(client.requested_databases, ["business-data"])
self.assertEqual(len(collection.insert_many_calls), 1)
self.assertEqual(collection.insert_many_calls[0]["documents"][0]["clipId"], "clip-001")
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["collection"], "qualityReports")
self.assertEqual(summary["data"]["mode"], "insert_many")
self.assertEqual(summary["data"]["target"], "business")
self.assertEqual(summary["data"]["insertedCount"], 2)
self.assertTrue(client.closed)
def test_upserts_documents_by_key(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "annotations"},
{"key": "MONGO_MODE", "value": "upsert"},
{"key": "UPSERT_KEY", "value": "clipId"},
],
}
inputs = [
{"data": {"clipId": "clip-001", "label": "walking"}},
{"data": {"clipId": "clip-002", "label": "standing"}},
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-2", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(len(collection.replace_one_calls), 2)
self.assertEqual(collection.replace_one_calls[0]["filter"], {"clipId": "clip-001"})
self.assertTrue(collection.replace_one_calls[0]["upsert"])
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["mode"], "upsert")
self.assertEqual(summary["data"]["upsertKey"], "clipId")
self.assertEqual(summary["data"]["target"], "business")
self.assertEqual(summary["data"]["upsertedCount"], 2)
def test_unwraps_single_value_dict_documents_for_business_writes(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "businessWriteValidation"},
{"key": "MONGO_MODE", "value": "insert_many"},
],
}
inputs = [
{
"data": {
"items": [
{
"value": {
"runLabel": "business-mongo-test",
"source": "workflow",
"createdAt": "2026-04-14T00:00:00Z",
"value": 1,
}
}
]
}
}
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
MODULE.builtin_mongodb_write("run-unwrap", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(len(collection.insert_many_calls), 1)
inserted = collection.insert_many_calls[0]["documents"][0]
self.assertEqual(inserted["runLabel"], "business-mongo-test")
self.assertEqual(inserted["source"], "workflow")
self.assertEqual(inserted["value"], 1)
self.assertNotEqual(set(inserted.keys()), {"value"})
def test_can_target_platform_database_explicitly(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
{"key": "MONGO_TARGET", "value": "platform"},
],
}
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"PLATFORM_MONGO_URI": "mongodb://platform-mongo:27017", "PLATFORM_MONGO_DB": "droboticflow"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-3", "mongo-node", pathlib.Path(temp_dir), node_data, [{"data": {"clipId": "clip-001"}}])
self.assertEqual(client.requested_databases, ["droboticflow"])
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["target"], "platform")
def test_rebuilds_package_files_from_manifest_when_temp_paths_are_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
package_manifest = {
"packageCount": 1,
"items": [
{
"packageName": "clip001_package",
"clipId": "clip001",
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"startSecond": 0,
"endSecond": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
"quality": {"clarity": 0.9, "passed": True},
"annotation": {"objects": ["人"], "minConfidence": 0.92},
}
],
}
inputs = [
{"name": "packages-manifest.json", "type": "json", "data": package_manifest},
{
"name": "clip001.mp4",
"type": "video",
"path": "/tmp/non-existent-clip001.mp4",
"data": {"preview": {"kind": "video", "format": "mp4"}},
},
{
"name": "clip001_annotation.json",
"type": "json",
"path": "/tmp/non-existent-clip001_annotation.json",
"data": {"objects": ["人"], "minConfidence": 0.92},
},
{
"name": "clip001_quality.json",
"type": "json",
"path": "/tmp/non-existent-clip001_quality.json",
"data": {"clarity": 0.9, "passed": True},
},
{
"name": "clip001_config.json",
"type": "json",
"path": "/tmp/non-existent-clip001_config.json",
"data": {"collectionId": "collection-1"},
},
]
fake_client = _FakeBosClient()
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
uploaded_keys = {item["key"] for item in fake_client.uploads}
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_quality.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_config.json", uploaded_keys)
self.assertFalse(
(output_dir / "uploaded" / "_materialized_upload_inputs" / "packages" / "clip001_package" / "clip001.mp4").exists()
)
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
self.assertEqual(uploaded_summary_payload["uploadedCount"], 5)
def test_upload_artifacts_are_json_serializable_when_preview_url_is_bytes(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
package_root = root / "package"
package_dir = package_root / "packages" / "clip001_package"
package_dir.mkdir(parents=True)
manifest_path = package_root / "packages-manifest.json"
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
clip_path = package_dir / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
MODULE.build_artifact("run-1", "package", clip_path, "video"),
]
with mock.patch.object(MODULE, "build_bos_client", return_value=(_FakeBosClientBytesURL(), "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
json.dumps({"artifacts": artifacts}, ensure_ascii=False)
class XPlatformClipHandoffTests(unittest.TestCase):
def test_summarize_sqlite_payload_does_not_create_wal_sidecars(self):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = pathlib.Path(temp_dir) / "talker.db3"
connection = sqlite3.connect(db_path)
try:
connection.execute("pragma journal_mode=wal")
connection.execute("create table topics (id integer primary key, name text, type text, serialization_format text)")
connection.execute("create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)")
connection.execute(
"insert into topics (id, name, type, serialization_format) values (1, '/chatter', 'std_msgs/msg/String', 'cdr')"
)
connection.execute("insert into messages (topic_id, timestamp, data) values (1, 1, X'00')")
connection.commit()
finally:
connection.close()
for sidecar in db_path.parent.glob("talker.db3-*"):
sidecar.unlink()
payload = MODULE.summarize_sqlite_payload(db_path)
self.assertEqual(payload["preview"]["format"], "rosbag2")
self.assertEqual(payload["preview"]["messageCount"], 1)
self.assertEqual(list(db_path.parent.glob("talker.db3-*")), [])
def test_video_slicing_rejects_invalid_segment_duration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "0"}],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be greater than 0"):
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
def test_video_slicing_rejects_lossy_segment_duration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "01"}],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be an exact positive integer"):
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
def test_video_slicing_rejects_unsafe_segment_duration(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": str(MODULE.MAX_SAFE_INTEGER + 1)}],
}
inputs = [{"data": {"items": []}}]
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be a safe integer"):
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
def test_video_slicing_manifest_preserves_source_reference(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 41.8,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
clip = manifest_payload["items"][0]
self.assertEqual(clip["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
self.assertEqual(clip["clipName"], "DJI_20260318112834_0001_D_clip001.mp4")
def test_video_slicing_uses_single_pass_writer_for_multi_clip_video(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 365.0,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_write_video_segments(_source_path, segments):
outputs = []
for segment in segments:
destination = pathlib.Path(segment["destination"])
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(f"clip-{segment['clipIndex']}".encode("utf-8"))
outputs.append(
{
**segment,
"duration": round(float(segment["endSecond"]) - float(segment["startSecond"]), 3),
"frameRate": 29.97,
"resolution": "3840x2160",
}
)
return outputs
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(
MODULE,
"write_video_segments",
side_effect=fake_write_video_segments,
) as multi_writer, mock.patch.object(
MODULE,
"write_video_segment",
) as single_writer, mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["clipCount"], 2)
self.assertEqual([item["clipName"] for item in manifest_payload["items"]], [
"DJI_20260318112834_0001_D_clip001.mp4",
"DJI_20260318112834_0001_D_clip002.mp4",
])
multi_writer.assert_called_once()
single_writer.assert_not_called()
def test_write_video_segment_with_ffmpeg_prefers_stream_copy_and_uses_inspected_metrics(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
destination = root / "clips" / "clip001.mp4"
def fake_run(args, stdout=None, stderr=None, text=None):
self.assertEqual(args[0], "ffmpeg")
self.assertIn(str(source_path), args)
self.assertEqual(args[-1], str(destination))
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-video")
return mock.Mock(returncode=0, stdout="", stderr="")
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE.subprocess,
"run",
side_effect=fake_run,
) as run_mock, mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
):
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
self.assertEqual(metrics["duration"], 21.5)
self.assertEqual(metrics["frameRate"], 29.97)
self.assertEqual(metrics["resolution"], "3840x2160")
run_args = run_mock.call_args.args[0]
self.assertIn("-ss", run_args)
self.assertIn("-t", run_args)
self.assertIn("-c", run_args)
self.assertIn("copy", run_args)
self.assertNotIn("-movflags", run_args)
self.assertNotIn("+faststart", run_args)
def test_write_video_segment_with_ffmpeg_falls_back_to_reencode_when_copy_fails(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
destination = root / "clips" / "clip001.mp4"
calls = []
def fake_run(args, stdout=None, stderr=None, text=None):
calls.append(list(args))
if len(calls) == 1:
return mock.Mock(returncode=1, stdout="", stderr="copy failed")
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-video")
return mock.Mock(returncode=0, stdout="", stderr="")
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE.subprocess,
"run",
side_effect=fake_run,
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
):
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
self.assertEqual(metrics["duration"], 21.5)
self.assertEqual(len(calls), 2)
self.assertIn("copy", calls[0])
self.assertIn("libx264", calls[1])
self.assertIn("aac", calls[1])
def test_video_slicing_prefers_ffmpeg_when_available(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 365.0,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_ffmpeg_writer(_source_path, destination, start_second, end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(f"{start_second}-{end_second}".encode("utf-8"))
return {
"duration": round(float(end_second) - float(start_second), 3),
"frameRate": 29.97,
"resolution": "3840x2160",
}
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(
MODULE,
"write_video_segment_with_ffmpeg",
side_effect=fake_ffmpeg_writer,
) as ffmpeg_writer, mock.patch.object(
MODULE,
"write_video_segments",
) as multi_writer, mock.patch.object(
MODULE,
"write_video_segment",
) as single_writer, mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["clipCount"], 2)
self.assertEqual(ffmpeg_writer.call_count, 2)
multi_writer.assert_not_called()
single_writer.assert_not_called()
def test_video_annotation_rejects_invalid_frame_interval(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频标注",
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "not-a-number"}],
}
inputs = [{"data": {"clipCount": 0, "items": []}}]
with self.assertRaisesRegex(RuntimeError, "FRAME_INTERVAL_SECONDS must be numeric"):
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
def test_video_annotation_uses_explicit_positive_frame_interval_without_clamping(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
clip_path = output_dir / "clip001.mp4"
clip_path.write_bytes(b"clip-bytes")
node_data = {
"label": "视频标注",
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "0.05"}],
}
inputs = [
{
"data": {
"clipCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"clipPath": str(clip_path),
"sourceVideo": "source.mp4",
}
],
}
}
]
with mock.patch.object(MODULE, "annotate_video_clip", return_value={"objects": [], "frameCount": 1}) as annotator:
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
annotator.assert_called_once()
self.assertEqual(annotator.call_args.args[1], 0.05)
def test_video_annotation_reuses_shared_clip_path_without_copying_clip_artifact(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
shared_clip_path = output_dir / "shared" / "clip001.mp4"
shared_clip_path.parent.mkdir(parents=True, exist_ok=True)
shared_clip_path.write_bytes(b"clip-bytes")
inputs = [
{
"name": "clips-manifest.json",
"type": "json",
"data": {
"clipCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"clipPath": str(shared_clip_path),
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"duration": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频标注",
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "1.0"}],
}
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
MODULE,
"annotate_video_clip",
return_value={
"duration": 41.8,
"actions": [{"label": "walking", "confidence": 0.92, "start": 0, "end": 41.8}],
"objects": ["人"],
"minConfidence": 0.92,
},
), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "annotations-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["annotationCount"], 1)
self.assertEqual(manifest_payload["items"][0]["clipPath"], str(shared_clip_path))
self.assertFalse(any(item["name"] == "clip001.mp4" for item in artifacts))
def test_materialize_clip_from_source_prefers_remote_segment_generation(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
item = {
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
},
}
def fake_remote_segment(_source_url, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8}
with mock.patch.object(MODULE, "remote_video_url_for_source", return_value="https://signed.example.com/video.mp4"), mock.patch.object(
MODULE,
"write_video_segment_from_remote_url",
side_effect=fake_remote_segment,
) as remote_writer, mock.patch.object(MODULE, "source_path_from_item") as local_materializer:
resolved = MODULE.materialize_clip_from_source(item, output_dir)
self.assertIsNotNone(resolved)
self.assertTrue(resolved.exists())
self.assertEqual(resolved.read_bytes(), b"clip-bytes")
remote_writer.assert_called_once()
local_materializer.assert_not_called()
def test_result_package_rebuilds_clip_from_source_when_clip_artifact_is_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "annotations-manifest.json",
"type": "json",
"data": {
"annotationCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"duration": 41.8,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"clarity": 0.9, "passed": True},
"annotation": {"actions": [], "objects": ["人"], "minConfidence": 0.92},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "成品打包",
"hyperparams": [],
}
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
MODULE,
"write_video_segment",
side_effect=fake_write_video_segment,
), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
package_clip = next(item for item in artifacts if item["name"] == "clip001.mp4")
self.assertEqual(manifest_payload["packageCount"], 1)
manifest_item = manifest_payload["items"][0]
self.assertEqual(manifest_item["clipName"], "clip001.mp4")
self.assertEqual(manifest_item["collectionId"], "collection-1")
self.assertEqual(manifest_item["startSecond"], 0)
self.assertEqual(manifest_item["endSecond"], 41.8)
self.assertEqual(manifest_item["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
self.assertEqual(package_clip["name"], "clip001.mp4")
class ModelInferenceOperatorTests(unittest.TestCase):
def test_builtin_model_inference_runs_linear_model(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
model = {
"type": "linear",
"name": "quality-gate",
"features": ["quality.clarity", "quality.shake"],
"weights": [1.0, -2.0],
"bias": 0.0,
"threshold": 0.5,
"positiveLabel": "pass",
"negativeLabel": "fail",
}
node_data = {
"label": "模型推理",
"hyperparams": [{"key": "MODEL_JSON", "value": json.dumps(model)}],
}
inputs = [
{
"name": "dataset.json",
"type": "json",
"data": {
"items": [
{"id": "clip-1", "quality": {"clarity": 0.9, "shake": 0.1}},
{"id": "clip-2", "quality": {"clarity": 0.2, "shake": 0.2}},
]
},
}
]
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, inputs)
self.assertEqual([item["name"] for item in artifacts], ["inference-result.json"])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["modelType"], "linear")
self.assertEqual(payload["modelName"], "quality-gate")
expected_digest = hashlib.sha256(
json.dumps(model, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
self.assertEqual(payload["modelSource"], "MODEL_JSON")
self.assertEqual(payload["modelSha256"], expected_digest)
self.assertEqual(payload["inputCount"], 2)
self.assertEqual([item["prediction"]["label"] for item in payload["predictions"]], ["pass", "fail"])
self.assertEqual(payload["predictions"][0]["input"]["id"], "clip-1")
def test_builtin_model_inference_runs_nearest_centroid_from_inline_input(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
model = {
"type": "nearest_centroid",
"name": "motion-cluster",
"features": ["speed", "turnRate"],
"labels": ["slow-straight", "fast-turn"],
"centroids": [[0.1, 0.0], [1.0, 0.8]],
}
node_data = {
"label": "模型推理",
"hyperparams": [
{"key": "MODEL_JSON", "value": json.dumps(model)},
{"key": "INPUT_JSON", "value": json.dumps([{"id": "clip-1", "speed": 0.95, "turnRate": 0.7}])},
],
}
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["inputCount"], 1)
self.assertEqual(payload["predictions"][0]["prediction"]["label"], "fast-turn")
self.assertGreater(payload["predictions"][0]["prediction"]["score"], 0)
def test_builtin_model_inference_rejects_invalid_rule_comparison(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "模型推理",
"hyperparams": [
{
"key": "MODEL_JSON",
"value": json.dumps(
{
"type": "rule",
"rules": [{"field": "scene", "operator": "gt", "value": 1, "label": "bad"}],
}
),
},
{"key": "INPUT_JSON", "value": json.dumps({"scene": "kitchen"})},
],
}
with self.assertRaisesRegex(RuntimeError, "requires numeric values"):
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
def test_builtin_model_inference_rejects_constant_model(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "模型推理",
"hyperparams": [
{"key": "MODEL_JSON", "value": json.dumps({"type": "constant", "label": "pass"})},
{"key": "INPUT_JSON", "value": json.dumps({"id": "clip-1"})},
],
}
with self.assertRaisesRegex(RuntimeError, "constant model type is not accepted"):
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
class ShellCommandOperatorTests(unittest.TestCase):
def test_builtin_shell_command_requires_command(self):
with tempfile.TemporaryDirectory() as temp_dir:
with self.assertRaisesRegex(RuntimeError, "需要 COMMAND 或 SCRIPT"):
MODULE.builtin_shell_command(
"run-1",
"shell",
pathlib.Path(temp_dir),
{"label": "Shell 命令执行", "hyperparams": []},
[],
)
def test_builtin_shell_command_executes_command_and_collects_output(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "Shell 命令执行",
"hyperparams": [
{
"key": "COMMAND",
"value": "python3 -c 'import json; json.dump({\"ok\": True}, open(\"result.json\", \"w\"))'",
}
],
"outputs": [{"name": "result", "type": "json"}],
}
previous_policy = os.environ.get("OPERATOR_ALLOW_SHELL_COMMANDS")
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = "true"
try:
artifacts = MODULE.builtin_shell_command("run-1", "shell", output_dir, node_data, [])
finally:
if previous_policy is None:
os.environ.pop("OPERATOR_ALLOW_SHELL_COMMANDS", None)
else:
os.environ["OPERATOR_ALLOW_SHELL_COMMANDS"] = previous_policy
self.assertEqual([item["name"] for item in artifacts], ["result.json"])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload, {"ok": True})
class _HFDatasetViewerHandler(http.server.BaseHTTPRequestHandler):
requests = []
def do_GET(self):
parsed = MODULE.urlparse(self.path)
query = dict(item.split("=", 1) if "=" in item else (item, "") for item in parsed.query.split("&") if item)
self.__class__.requests.append({"path": parsed.path, "query": query})
if parsed.path == "/is-valid":
response = {"valid": True}
elif parsed.path == "/splits":
response = {"splits": [{"dataset": "lerobot/pusht", "config": "default", "split": "train", "num_examples": 2}]}
elif parsed.path == "/size":
response = {"size": {"dataset": {"num_rows": 2, "num_bytes": 128}}}
elif parsed.path == "/parquet":
response = {
"parquet_files": [
{
"dataset": "lerobot/pusht",
"config": "default",
"split": "train",
"filename": "0000.parquet",
"size": 128,
}
]
}
elif parsed.path == "/rows":
response = {
"rows": [
{"row_idx": 0, "row": {"episode_index": 0, "frame_index": 145, "action": [120.0, 373.0]}},
{"row_idx": 1, "row": {"episode_index": 0, "frame_index": 146, "action": [124.0, 371.0]}},
],
"num_rows_total": 2,
}
else:
self.send_error(404)
return
response_bytes = json.dumps(response).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_bytes)))
self.end_headers()
self.wfile.write(response_bytes)
def log_message(self, *_args):
return
@contextlib.contextmanager
def hf_dataset_viewer_server():
_HFDatasetViewerHandler.requests = []
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _HFDatasetViewerHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
previous_no_proxy = os.environ.get("NO_PROXY")
previous_no_proxy_lower = os.environ.get("no_proxy")
os.environ["NO_PROXY"] = "127.0.0.1,localhost"
os.environ["no_proxy"] = "127.0.0.1,localhost"
thread.start()
try:
yield server, _HFDatasetViewerHandler.requests
finally:
server.shutdown()
thread.join(timeout=5)
server.server_close()
if previous_no_proxy is None:
os.environ.pop("NO_PROXY", None)
else:
os.environ["NO_PROXY"] = previous_no_proxy
if previous_no_proxy_lower is None:
os.environ.pop("no_proxy", None)
else:
os.environ["no_proxy"] = previous_no_proxy_lower
class EmbodiedDatasetOperatorTests(unittest.TestCase):
def test_hf_dataset_probe_calls_dataset_viewer_api(self):
with hf_dataset_viewer_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "HF 数据集探测",
"hyperparams": [
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
{"key": "DATASET", "value": "lerobot/pusht"},
{"key": "ROW_LENGTH", "value": "2"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["hf_dataset_probe"]("run-1", "hf", output_dir, node_data, [])
self.assertEqual([item["path"] for item in requests], ["/is-valid", "/splits", "/size", "/parquet", "/rows"])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "huggingface-dataset")
self.assertEqual(payload["dataset"], "lerobot/pusht")
self.assertEqual(payload["selectedConfig"], "default")
self.assertEqual(payload["selectedSplit"], "train")
self.assertEqual(payload["rowCount"], 2)
self.assertEqual(payload["parquetFileCount"], 1)
def test_rlds_episode_summary_validates_real_jsonl_episodes(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "rlds"
root.mkdir()
(root / "dataset_info.json").write_text(json.dumps({"name": "bridge_v2"}), encoding="utf-8")
episodes_path = root / "episodes.jsonl"
episodes = [
{
"episode_id": "ep-1",
"steps": [
{
"is_first": True,
"is_last": False,
"is_terminal": False,
"action": [0.1, 0.2, 0.3],
"reward": 0.0,
"observation": {"image": "frame-1.jpg", "state": [1.0, 2.0]},
"language_instruction": "pick up the block",
},
{
"is_first": False,
"is_last": True,
"is_terminal": True,
"action": [0.3, 0.2, 0.1],
"reward": 1.0,
"observation": {"image": "frame-2.jpg", "state": [1.5, 2.5]},
},
],
},
{
"episode_id": "ep-2",
"steps": [
{
"is_first": True,
"is_last": True,
"is_terminal": True,
"action": [0.0, 0.0, 0.1],
"reward": 0.5,
"observation": {"image": "frame-3.jpg", "state": [0.0, 1.0]},
}
],
},
]
episodes_path.write_text("\n".join(json.dumps(item) for item in episodes), encoding="utf-8")
node_data = {"label": "RLDS 轨迹概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["rlds_episode_summary"]("run-1", "rlds", pathlib.Path(temp_dir) / "out", node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "rlds")
self.assertEqual(payload["episodeCount"], 2)
self.assertEqual(payload["stepCount"], 3)
self.assertEqual(payload["invalidEpisodeCount"], 0)
self.assertEqual(payload["episodes"][0]["actionDimension"], 3)
self.assertIn("image", payload["episodes"][0]["observationKeys"])
self.assertIn("dataset_info.json", payload["metadata"])
def test_agibot_world2026_validate_reads_real_parquet_video_layout(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "agibot"
meta = root / "meta"
data = root / "data" / "chunk-000"
video = root / "videos" / "observation.images.head" / "chunk-000"
meta.mkdir(parents=True)
data.mkdir(parents=True)
video.mkdir(parents=True)
(meta / "info.json").write_text(
json.dumps(
{
"dataset_name": "agibot-world/AgiBotWorld2026",
"data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
"video_path": "videos/{video_key}/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.mp4",
"features": {
"observation.images.head": {"dtype": "video"},
"action": {"dtype": "float32", "shape": [7]},
},
}
),
encoding="utf-8",
)
episodes = [
{"episode_index": 0, "length": 2, "tasks": [0], "data_path": "data/chunk-000/episode_000000.parquet"},
{"episode_index": 1, "length": 1, "tasks": [1], "data_path": "data/chunk-000/episode_000001.parquet"},
]
(meta / "episodes.jsonl").write_text("\n".join(json.dumps(item) for item in episodes), encoding="utf-8")
tasks = [
{"task_index": 0, "task": "pick up cup"},
{"task_index": 1, "task": "place cup"},
]
(meta / "tasks.jsonl").write_text("\n".join(json.dumps(item) for item in tasks), encoding="utf-8")
pq.write_table(
pa.table(
{
"episode_index": [0, 0],
"frame_index": [0, 1],
"timestamp": [0.0, 0.033],
"action": [[0.1] * 7, [0.2] * 7],
}
),
data / "episode_000000.parquet",
)
pq.write_table(
pa.table(
{
"episode_index": [1],
"frame_index": [0],
"timestamp": [0.0],
"action": [[0.3] * 7],
}
),
data / "episode_000001.parquet",
)
(video / "episode_000000.mp4").write_bytes(b"real-mp4-bytes-0")
(video / "episode_000001.mp4").write_bytes(b"real-mp4-bytes-1")
node_data = {"label": "AgiBotWorld2026 数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["agibot_world2026_validate"](
"run-1", "agibot", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "agibot-world2026-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["episodeMetadataCount"], 2)
self.assertEqual(payload["taskCount"], 2)
self.assertEqual(payload["dataParquetCount"], 2)
self.assertEqual(payload["dataRowCount"], 3)
self.assertEqual(payload["videoFileCount"], 2)
self.assertEqual(payload["validEpisodeCount"], 2)
self.assertEqual(payload["videoKeys"], ["observation.images.head"])
def test_droid_raw_episode_validate_reads_real_hdf5_metadata_and_video(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "droid-raw"
episode = root / "episode_000001"
recordings = episode / "recordings" / "MP4"
recordings.mkdir(parents=True)
with h5py.File(episode / "trajectory.h5", "w") as handle:
action = handle.create_group("action")
action.create_dataset(
"cartesian_position",
data=np.array([[0.1, 0.2, 0.3, 0.0, 0.0, 0.0, 1.0]] * 3, dtype="float32"),
)
observation = handle.create_group("observation")
observation.create_dataset("cartesian_position", data=np.ones((3, 6), dtype="float32"))
observation.create_dataset("gripper_position", data=np.array([[0.1], [0.2], [0.3]], dtype="float32"))
(episode / "metadata_camera.json").write_text(
json.dumps({"uuid": "episode-000001", "task": "pick the cup", "robot_serial": "franka-1"}),
encoding="utf-8",
)
(episode / "metadata_recording.json").write_text(
json.dumps({"episode_id": "episode_000001", "time": "2026-06-24T15:00:00+08:00"}),
encoding="utf-8",
)
(recordings / "camera_0.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42real-droid-video")
node_data = {"label": "DROID Raw Episode校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["droid_raw_episode_validate"](
"run-1", "droid", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "droid-raw-episode-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["episodeCount"], 1)
self.assertEqual(payload["trajectoryFileCount"], 1)
self.assertEqual(payload["metadataFileCount"], 2)
self.assertEqual(payload["videoFileCount"], 1)
self.assertEqual(payload["validEpisodeCount"], 1)
self.assertEqual(payload["episodes"][0]["frameCount"], 3)
self.assertIn("/action/cartesian_position", payload["episodes"][0]["actionDatasetPaths"])
self.assertIn("/observation/gripper_position", payload["episodes"][0]["stateDatasetPaths"])
def test_libero_hdf5_task_validate_reads_real_hdf5_and_bddl(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "libero_spatial"
demos = root / "demos"
bddl = root / "bddl"
demos.mkdir(parents=True)
bddl.mkdir()
(bddl / "pick_up_the_black_bowl.bddl").write_text(
"(define (problem pick_up_the_black_bowl) (:language \"pick up the black bowl\"))\n",
encoding="utf-8",
)
path = demos / "pick_up_the_black_bowl_demo.hdf5"
with h5py.File(path, "w") as handle:
data = handle.create_group("data")
data.attrs["total"] = 3
data.attrs["env_args"] = json.dumps({"env_name": "LIBERO_SPATIAL", "bddl_file_name": "pick_up_the_black_bowl.bddl"})
demo = data.create_group("demo_0")
demo.attrs["language_instruction"] = "pick up the black bowl"
demo.create_dataset("actions", data=np.ones((3, 7), dtype="float32"))
demo.create_dataset("states", data=np.ones((3, 12), dtype="float32"))
demo.create_dataset("rewards", data=np.array([0.0, 0.0, 1.0], dtype="float32"))
demo.create_dataset("dones", data=np.array([False, False, True], dtype="bool"))
obs = demo.create_group("obs")
obs.create_dataset("agentview_rgb", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
obs.create_dataset("eye_in_hand_rgb", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
obs.create_dataset("gripper_states", data=np.ones((3, 2), dtype="float32"))
obs.create_dataset("joint_states", data=np.ones((3, 7), dtype="float32"))
obs.create_dataset("ee_states", data=np.ones((3, 6), dtype="float32"))
next_obs = demo.create_group("next_obs")
next_obs.create_dataset("agentview_rgb", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
mask = handle.create_group("mask")
mask.create_dataset("train", data=np.array([b"demo_0"]))
node_data = {
"label": "LIBERO HDF5任务校验",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "BDDL_PATH", "value": str(bddl)},
{"key": "REQUIRE_BDDL", "value": "true"},
{"key": "REQUIRE_LANGUAGE", "value": "true"},
{"key": "REQUIRE_TERMINAL_DONE", "value": "true"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["libero_hdf5_task_validate"](
"run-1", "libero", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "libero-hdf5-task-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["bddlFileCount"], 1)
self.assertEqual(payload["taskCount"], 1)
self.assertEqual(payload["demoCount"], 1)
self.assertEqual(payload["validDemoCount"], 1)
self.assertEqual(payload["frameCount"], 3)
self.assertEqual(payload["taskSuites"], ["libero_spatial"])
self.assertEqual(payload["files"][0]["bddlPath"], "pick_up_the_black_bowl.bddl")
self.assertIn("agentview_rgb", payload["files"][0]["demos"][0]["imageObservationKeys"])
self.assertIn("joint_states", payload["files"][0]["demos"][0]["proprioObservationKeys"])
self.assertEqual(payload["files"][0]["demos"][0]["languageFields"], ["pick up the black bowl"])
def test_robocasa_hdf5_demonstration_validate_reads_real_demo_metadata_and_action_dict(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "robocasa"
root.mkdir()
path = root / "demo.hdf5"
with h5py.File(path, "w") as handle:
data = handle.create_group("data")
data.attrs["total"] = 3
data.attrs["env_args"] = json.dumps({"env_name": "KitchenPnPCounterToCab", "type": "robocasa"})
demo = data.create_group("demo_0")
demo.attrs["model_file"] = "<mujoco><worldbody /></mujoco>"
demo.attrs["ep_meta"] = json.dumps({"lang": "pick up the mug and place it in the cabinet", "layout_id": 1, "style_id": 2})
demo.create_dataset("actions", data=np.ones((3, 7), dtype="float32"))
action_dict = demo.create_group("action_dict")
action_dict.create_dataset("abs_pos", data=np.ones((3, 3), dtype="float32"))
action_dict.create_dataset("abs_rot_6d", data=np.ones((3, 6), dtype="float32"))
action_dict.create_dataset("gripper", data=np.ones((3, 1), dtype="float32"))
demo.create_dataset("states", data=np.ones((4, 10), dtype="float32"))
obs = demo.create_group("obs")
obs.create_dataset("robot0_agentview_left_image", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
obs.create_dataset("robot0_eye_in_hand_image", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
obs.create_dataset("robot0_joint_pos", data=np.ones((3, 7), dtype="float32"))
obs.create_dataset("robot0_gripper_qpos", data=np.ones((3, 2), dtype="float32"))
next_obs = demo.create_group("next_obs")
next_obs.create_dataset("robot0_agentview_left_image", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
mask = handle.create_group("mask")
mask.create_dataset("train", data=np.array([b"demo_0"]))
node_data = {"label": "RoboCasa HDF5示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["robocasa_hdf5_demonstration_validate"](
"run-1", "robocasa", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "robocasa-hdf5-demonstration-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["demoCount"], 1)
self.assertEqual(payload["validDemoCount"], 1)
self.assertEqual(payload["frameCount"], 3)
self.assertEqual(payload["languageInstructions"], ["pick up the mug and place it in the cabinet"])
demo_payload = payload["files"][0]["demos"][0]
self.assertEqual(demo_payload["actionShape"], [3, 7])
self.assertEqual(demo_payload["stateShape"], [4, 10])
self.assertIn("robot0_agentview_left_image", demo_payload["imageObservationKeys"])
self.assertIn("robot0_joint_pos", demo_payload["proprioObservationKeys"])
self.assertEqual(sorted(demo_payload["actionDictKeys"]), ["abs_pos", "abs_rot_6d", "gripper"])
self.assertTrue(demo_payload["modelFilePresent"])
self.assertTrue(demo_payload["epMetaPresent"])
def test_furniturebench_pkl_demonstration_validate_reads_real_pkl_file(self):
import pickle
try:
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "furniturebench" / "low" / "lamp"
root.mkdir(parents=True)
observations = []
for index in range(2):
observations.append(
{
"color_image1": np.full((224, 224, 3), 20 + index, dtype=np.uint8),
"color_image2": np.full((224, 224, 3), 30 + index, dtype=np.uint8),
"robot_state": {
"ee_pos": np.array([0.1, 0.2, 0.3], dtype=np.float32),
"ee_quat": np.array([1.0, 0.0, 0.0, 0.0], dtype=np.float32),
"ee_pos_vel": np.zeros((3,), dtype=np.float32),
"ee_ori_vel": np.zeros((3,), dtype=np.float32),
"joint_positions": np.ones((7,), dtype=np.float32) * index,
"joint_velocities": np.zeros((7,), dtype=np.float32),
"joint_torques": np.zeros((7,), dtype=np.float32),
"gripper_width": np.array([0.04], dtype=np.float32),
},
}
)
demo = {
"furniture": "lamp",
"observations": observations,
"actions": [np.ones((8,), dtype=np.float32), np.ones((8,), dtype=np.float32) * 2],
"rewards": [0, 1],
"skills": [0, 1],
}
path = root / "0.pkl"
with path.open("wb") as handle:
pickle.dump(demo, handle)
node_data = {"label": "FurnitureBench PKL示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path.parents[2])}]}
artifacts = MODULE.BUILTIN_HANDLERS["furniturebench_pkl_demonstration_validate"](
"run-1", "furniturebench", path.parents[2] / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "furniturebench-pkl-demonstration-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["validFileCount"], 1)
self.assertEqual(payload["totalFrames"], 2)
self.assertEqual(payload["furnitureCounts"], {"lamp": 1})
self.assertEqual(payload["randomnessCounts"], {"low": 1})
demo_payload = payload["demos"][0]
self.assertEqual(demo_payload["actionDimensions"], [8])
self.assertEqual(demo_payload["imageKeyCounts"], {"color_image1": 2, "color_image2": 2})
self.assertEqual(demo_payload["robotStateKeyCounts"]["joint_positions"], 2)
self.assertEqual(demo_payload["rewardValues"], [0.0, 1.0])
self.assertEqual(demo_payload["skillValues"], [0, 1])
def test_roboturk_real_hdf5_validate_reads_aligned_hdf5_file(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "roboturk"
root.mkdir()
path = root / "SawyerLaundryLayout_aligned_dataset.hdf5"
with h5py.File(path, "w") as handle:
data = handle.create_group("data")
data.attrs["total_demos"] = 1
data.attrs["total_time"] = 3.0
user = data.create_group("user_001")
demo = user.create_group("demo_1")
demo.attrs["clock_time"] = "2021-02-24T00:00:00Z"
demo.attrs["session_id"] = "session-001"
demo.attrs["demo_time"] = 3.0
demo.attrs["date"] = "2021-02-24"
demo.attrs["task"] = "SawyerLaundryLayout"
demo.attrs["top_depth_video_file"] = "SawyerLaundryLayout/roboturk_day1/user_001/SawyerLaundryLayout/t/demo_1/kinect_depth_aligned.mp4"
demo.attrs["top_rgb_video_file"] = "SawyerLaundryLayout/roboturk_day1/user_001/SawyerLaundryLayout/t/demo_1/kinect_rgb_aligned.mp4"
demo.attrs["front_rgb_video_file"] = "SawyerLaundryLayout/roboturk_day1/user_001/SawyerLaundryLayout/t/demo_1/usb_aligned.mp4"
robot_observation = demo.create_group("robot_observation")
robot_observation.create_dataset("eef_poses", data=np.ones((3, 7), dtype=np.float32))
robot_observation.create_dataset("eef_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
robot_observation.create_dataset("joint_states_arm", data=np.ones((3, 27), dtype=np.float32))
robot_observation.create_dataset("joint_states_arm_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
robot_observation.create_dataset("joint_states_gripper", data=np.ones((3, 3), dtype=np.float32))
robot_observation.create_dataset("joint_states_gripper_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
user_control = demo.create_group("user_control")
user_control.create_dataset("user_control", data=np.ones((3, 22), dtype=np.float32))
user_control.create_dataset("user_control_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
node_data = {"label": "RoboTurk Real HDF5校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["roboturk_real_hdf5_validate"](
"run-1", "roboturk-real", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "roboturk-real-hdf5-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["issueCount"], 0)
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["validFileCount"], 1)
self.assertEqual(payload["totalUsers"], 1)
self.assertEqual(payload["totalDemos"], 1)
self.assertEqual(payload["returnedDemoCount"], 1)
self.assertEqual(payload["taskCounts"], {"SawyerLaundryLayout": 1})
file_payload = payload["files"][0]
self.assertEqual(file_payload["dataAttrs"]["total_demos"], 1)
demo_payload = file_payload["demos"][0]
self.assertEqual(demo_payload["userId"], "user_001")
self.assertEqual(demo_payload["demoId"], "demo_1")
self.assertEqual(demo_payload["datasetShapes"]["robot_observation/eef_poses"], [3, 7])
self.assertEqual(demo_payload["datasetShapes"]["robot_observation/joint_states_arm"], [3, 27])
self.assertEqual(demo_payload["datasetShapes"]["user_control/user_control"], [3, 22])
self.assertEqual(demo_payload["frameCounts"]["user_control/user_control_times"], 3)
def test_roboset_hdf5_validate_reads_trial_data_and_multicamera_rgb(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "roboset"
root.mkdir()
path = root / "FK1_KitchenHuman.h5"
with h5py.File(path, "w") as handle:
handle.attrs["dataset"] = "RoboSet"
trial = handle.create_group("Trial0")
trial.attrs["env_name"] = "FK1_Knob1OnRandom-v4"
data = trial.create_group("data")
trial.create_group("derived").create_dataset("success", data=np.array([0, 0, 1], dtype=np.int8))
config = trial.create_group("config")
config.attrs["robot"] = "FrankaKitchen"
data.create_dataset("actions", data=np.ones((3, 9), dtype=np.float32))
data.create_dataset("rewards", data=np.array([0.0, 0.1, 1.0], dtype=np.float32))
data.create_dataset("dones", data=np.array([0, 0, 1], dtype=np.int8))
observations = data.create_group("observations")
observations.create_dataset("qpos", data=np.ones((3, 30), dtype=np.float32))
observations.create_dataset("qvel", data=np.ones((3, 30), dtype=np.float32) * 2)
rgb = data.create_group("rgb")
for camera_index in range(4):
rgb.create_dataset(
f"camera_{camera_index}",
data=np.ones((3, 12, 16, 3), dtype=np.uint8) * camera_index,
)
node_data = {
"label": "RoboSet HDF5轨迹校验",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "MIN_RGB_VIEWS", "value": "4"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["roboset_hdf5_validate"](
"run-1", "roboset", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "roboset-hdf5-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["issueCount"], 0)
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["validFileCount"], 1)
self.assertEqual(payload["totalTrials"], 1)
self.assertEqual(payload["returnedTrialCount"], 1)
self.assertEqual(payload["rgbViewCounts"], {"4": 1})
self.assertEqual(payload["actionDimensionCounts"], {"9": 1})
trial_payload = payload["files"][0]["trials"][0]
self.assertEqual(trial_payload["trialId"], "Trial0")
self.assertEqual(trial_payload["trialKeys"], ["config", "data", "derived"])
self.assertEqual(trial_payload["frameCount"], 3)
self.assertEqual(trial_payload["actionDatasetPaths"], ["actions"])
self.assertEqual(trial_payload["rewardDatasetPaths"], ["rewards"])
self.assertEqual(trial_payload["doneDatasetPaths"], ["dones"])
self.assertEqual(trial_payload["rgbViewCount"], 4)
self.assertEqual(trial_payload["datasetShapes"]["actions"], [3, 9])
self.assertEqual(trial_payload["datasetShapes"]["observations/qpos"], [3, 30])
self.assertEqual(trial_payload["datasetShapes"]["rgb/camera_0"], [3, 12, 16, 3])
def test_rh20t_scene_validate_reads_real_scene_structure_and_arrays(self):
try:
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "rh20t"
scene = root / "scene_000001"
camera = scene / "cam_123456"
transformed = scene / "transformed"
high_freq = scene / "high_freq_data"
calib = root / "calib" / "20240624"
camera.mkdir(parents=True)
transformed.mkdir(parents=True)
high_freq.mkdir(parents=True)
calib.mkdir(parents=True)
(scene / "metadata.json").write_text(
json.dumps({"calib": "20240624", "task_description": "open the drawer", "operator": "robot-teacher"}),
encoding="utf-8",
)
np.save(camera / "timestamps.npy", {"color": np.array([1000, 1033, 1066], dtype=np.int64)})
(camera / "color.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42rh20t-video")
np.save(transformed / "force_torque.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 6), dtype=np.float32)})
np.save(transformed / "tcp.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 7), dtype=np.float32)})
np.save(transformed / "joint.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 7), dtype=np.float32)})
np.save(transformed / "gripper.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 1), dtype=np.float32)})
np.save(high_freq / "force_torque_tcp_joint_timestamp.npy", np.array([[1000, 0.1], [1001, 0.2]], dtype=np.float32))
np.save(calib / "intrinsics.npy", {"cam_123456": np.eye(3, dtype=np.float32)})
np.save(calib / "extrinsics.npy", {"cam_123456": np.eye(4, dtype=np.float32)})
np.save(calib / "tcp.npy", np.ones((4, 4), dtype=np.float32))
node_data = {"label": "RH20T 场景结构校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["rh20t_scene_validate"](
"run-1", "rh20t", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "rh20t-scene-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["sceneCount"], 1)
self.assertEqual(payload["validSceneCount"], 1)
self.assertEqual(payload["cameraCount"], 1)
self.assertEqual(payload["cameraTimestampCount"], 3)
scene_payload = payload["scenes"][0]
self.assertTrue(scene_payload["metadataPresent"])
self.assertEqual(scene_payload["calib"], "20240624")
self.assertEqual(scene_payload["cameras"][0]["videoFiles"], ["cam_123456/color.mp4"])
self.assertIn("force_torque.npy", scene_payload["transformed"]["files"])
self.assertTrue(scene_payload["highFreq"]["present"])
self.assertIn("intrinsics.npy", scene_payload["calibration"]["files"])
def test_robomind_hdf5_quality_validate_reads_real_hdf5_joint_position_and_end_effector(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "robomind" / "benchmark1_0"
data_dir = root / "h5_franka_1rgb" / "bread_in_basket" / "success_episodes" / "train" / "1014_144602" / "data"
data_dir.mkdir(parents=True)
path = data_dir / "trajectory.hdf5"
with h5py.File(path, "w") as handle:
puppet = handle.create_group("puppet")
puppet.create_dataset(
"joint_position",
data=np.array(
[
[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6],
[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7],
[0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8],
],
dtype=np.float32,
),
)
puppet.create_dataset(
"end_effector",
data=np.array(
[
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.1, 0.0, 0.0, 0.0, 0.0, 0.1],
[0.2, 0.0, 0.0, 0.0, 0.0, 0.2],
],
dtype=np.float32,
),
)
master = handle.create_group("master")
master.create_dataset(
"joint_position",
data=np.array(
[
[1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6],
[1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7],
[1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8],
],
dtype=np.float32,
),
)
rgb_images = handle.create_group("rgb_images")
rgb_images.create_dataset("front", data=np.zeros((3, 4, 4, 3), dtype=np.uint8))
handle.create_dataset("action", data=np.ones((3, 7), dtype=np.float32))
node_data = {
"label": "RoboMIND HDF5质量校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
}
artifacts = MODULE.BUILTIN_HANDLERS["robomind_hdf5_quality_validate"](
"run-1", "robomind", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "robomind-hdf5-quality-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["trajectoryCount"], 1)
self.assertEqual(payload["validFileCount"], 1)
self.assertEqual(payload["jointPositionFileCount"], 1)
self.assertEqual(payload["endEffectorFileCount"], 1)
self.assertEqual(payload["stuckJointPositionCount"], 0)
self.assertEqual(payload["stuckEndEffectorCount"], 0)
self.assertEqual(payload["imageDatasetCount"], 1)
file_payload = payload["files"][0]
self.assertEqual(file_payload["benchmark"], "benchmark1_0")
self.assertEqual(file_payload["robotType"], "h5_franka_1rgb")
self.assertEqual(file_payload["taskName"], "bread_in_basket")
self.assertEqual(file_payload["split"], "train")
self.assertEqual(file_payload["episodeId"], "1014_144602")
self.assertEqual(file_payload["frameCount"], 3)
self.assertEqual(file_payload["puppetJointPosition"]["key"], "puppet/joint_position")
self.assertEqual(file_payload["puppetJointPosition"]["shape"], [3, 7])
self.assertEqual(file_payload["puppetEndEffector"]["key"], "puppet/end_effector")
def test_hdf5_demonstration_summary_reads_real_hdf5_file(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
path = pathlib.Path(temp_dir) / "demo.hdf5"
with h5py.File(path, "w") as handle:
data = handle.create_group("data")
data.attrs["total"] = 3
data.attrs["env_args"] = json.dumps({"env_name": "Lift"})
demo_1 = data.create_group("demo_1")
demo_1.attrs["num_samples"] = 2
demo_1.create_dataset("actions", data=np.array([[0.1, 0.2], [0.3, 0.4]], dtype="float32"))
demo_1.create_dataset("states", data=np.array([[1.0], [2.0]], dtype="float32"))
obs = demo_1.create_group("obs")
obs.create_dataset("robot0_eef_pos", data=np.array([[0.0, 0.1, 0.2], [0.2, 0.3, 0.4]], dtype="float32"))
next_obs = demo_1.create_group("next_obs")
next_obs.create_dataset("robot0_eef_pos", data=np.array([[0.1, 0.2, 0.3], [0.3, 0.4, 0.5]], dtype="float32"))
demo_2 = data.create_group("demo_2")
demo_2.attrs["num_samples"] = 1
demo_2.create_dataset("actions", data=np.array([[0.5, 0.6]], dtype="float32"))
mask = handle.create_group("mask")
mask.create_dataset("train", data=np.array([b"demo_1"]))
node_data = {"label": "HDF5 示教数据概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
artifacts = MODULE.BUILTIN_HANDLERS["hdf5_demonstration_summary"](
"run-1", "hdf5", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "hdf5-demonstration")
self.assertEqual(payload["demoCount"], 2)
self.assertEqual(payload["totalSamples"], 3)
self.assertEqual(payload["demos"][0]["actionsShape"], [2, 2])
self.assertEqual(payload["demos"][0]["observationKeys"], ["robot0_eef_pos"])
self.assertEqual(payload["maskKeys"], ["train"])
def test_aloha_hdf5_validate_reads_real_act_layout(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
path = pathlib.Path(temp_dir) / "episode_000000.hdf5"
with h5py.File(path, "w") as handle:
handle.attrs["sim"] = False
observations = handle.create_group("observations")
observations.create_dataset("qpos", data=np.ones((3, 14), dtype="float32"))
observations.create_dataset("qvel", data=np.zeros((3, 14), dtype="float32"))
images = observations.create_group("images")
images.create_dataset("cam_high", data=np.zeros((3, 4, 5, 3), dtype="uint8"))
images.create_dataset("cam_wrist", data=np.zeros((3, 4, 5, 3), dtype="uint8"))
handle.create_dataset("action", data=np.ones((3, 14), dtype="float32"))
node_data = {"label": "ALOHA HDF5 示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
artifacts = MODULE.BUILTIN_HANDLERS["aloha_hdf5_validate"](
"run-1", "aloha", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "aloha-hdf5-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["totalFrames"], 3)
self.assertEqual(payload["cameraCount"], 2)
self.assertEqual(payload["actionDimensions"], [14])
self.assertEqual(payload["qposDimensions"], [14])
self.assertEqual(payload["files"][0]["cameraNames"], ["cam_high", "cam_wrist"])
def test_robomimic_hdf5_validate_reads_real_demo_groups(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
path = pathlib.Path(temp_dir) / "robomimic.hdf5"
with h5py.File(path, "w") as handle:
data = handle.create_group("data")
data.attrs["total"] = 3
data.attrs["env_args"] = json.dumps({"env_name": "Lift"})
demo_1 = data.create_group("demo_1")
demo_1.attrs["num_samples"] = 2
demo_1.create_dataset("actions", data=np.array([[0.1, 0.2], [0.3, 0.4]], dtype="float32"))
demo_1.create_dataset("rewards", data=np.array([0.0, 1.0], dtype="float32"))
demo_1.create_dataset("dones", data=np.array([False, True], dtype="bool"))
demo_1.create_dataset("states", data=np.array([[1.0], [2.0]], dtype="float32"))
obs = demo_1.create_group("obs")
obs.create_dataset("robot0_eef_pos", data=np.array([[0.0, 0.1, 0.2], [0.2, 0.3, 0.4]], dtype="float32"))
next_obs = demo_1.create_group("next_obs")
next_obs.create_dataset("robot0_eef_pos", data=np.array([[0.1, 0.2, 0.3], [0.3, 0.4, 0.5]], dtype="float32"))
demo_2 = data.create_group("demo_2")
demo_2.attrs["num_samples"] = 1
demo_2.create_dataset("actions", data=np.array([[0.5, 0.6]], dtype="float32"))
demo_2.create_dataset("rewards", data=np.array([0.5], dtype="float32"))
demo_2.create_dataset("dones", data=np.array([True], dtype="bool"))
obs = demo_2.create_group("obs")
obs.create_dataset("robot0_eef_pos", data=np.array([[0.4, 0.5, 0.6]], dtype="float32"))
next_obs = demo_2.create_group("next_obs")
next_obs.create_dataset("robot0_eef_pos", data=np.array([[0.5, 0.6, 0.7]], dtype="float32"))
mask = handle.create_group("mask")
mask.create_dataset("train", data=np.array([b"demo_1"]))
mask.create_dataset("valid", data=np.array([b"demo_2"]))
node_data = {"label": "robomimic HDF5 示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
artifacts = MODULE.BUILTIN_HANDLERS["robomimic_hdf5_validate"](
"run-1", "robomimic", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "robomimic-hdf5-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["demoCount"], 2)
self.assertEqual(payload["totalSamples"], 3)
self.assertEqual(payload["maskKeys"], ["train", "valid"])
self.assertEqual(payload["actionDimensions"], [2])
self.assertEqual(payload["files"][0]["observationKeyCounts"], {"robot0_eef_pos": 2})
def test_rlbench_dataset_validate_reads_real_episode_pickles_and_images(self):
import pickle
try:
import cv2 # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"cv2/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "rlbench"
episode = root / "pick_up_cup" / "variation0" / "episodes" / "episode0"
for camera in ("front_rgb", "left_shoulder_rgb"):
(episode / camera).mkdir(parents=True, exist_ok=True)
for index in range(3):
frame = np.full((6, 8, 3), 40 + index, dtype=np.uint8)
cv2.imwrite(str(episode / camera / f"{index}.png"), frame)
with (episode / "low_dim_obs.pkl").open("wb") as handle:
pickle.dump([{"gripper_open": True}, {"gripper_open": False}, {"gripper_open": True}], handle)
with (episode / "variation_descriptions.pkl").open("wb") as handle:
pickle.dump(["pick up the cup"], handle)
node_data = {"label": "RLBench 数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["rlbench_dataset_validate"](
"run-1", "rlbench", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "rlbench-dataset-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["episodeCount"], 1)
self.assertEqual(payload["validEpisodeCount"], 1)
self.assertEqual(payload["totalLowDimFrames"], 3)
self.assertEqual(payload["cameraTotals"], {"front_rgb": 3, "left_shoulder_rgb": 3})
def test_maniskill_trajectory_validate_reads_real_hdf5_and_metadata(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "maniskill"
root.mkdir()
h5_path = root / "trajectory.h5"
with h5py.File(h5_path, "w") as handle:
traj_0 = handle.create_group("traj_0")
traj_0.create_dataset("actions", data=np.array([[0.1, 0.2], [0.2, 0.3], [0.3, 0.4]], dtype="float32"))
traj_0.create_dataset("rewards", data=np.array([0.0, 0.0, 1.0], dtype="float32"))
traj_0.create_dataset("terminated", data=np.array([False, False, True], dtype="bool"))
traj_0.create_dataset("env_states", data=np.ones((4, 3), dtype="float32"))
traj_1 = handle.create_group("traj_1")
traj_1.create_dataset("actions", data=np.array([[0.4, 0.5], [0.5, 0.6]], dtype="float32"))
traj_1.create_dataset("rewards", data=np.array([0.0, 1.0], dtype="float32"))
traj_1.create_dataset("terminated", data=np.array([False, True], dtype="bool"))
(root / "trajectory.json").write_text(
json.dumps({"env_info": {"env_id": "PickCube-v1"}, "episodes": [{"episode_id": 0, "elapsed_steps": 3}, {"episode_id": 1, "elapsed_steps": 2}]}),
encoding="utf-8",
)
node_data = {"label": "ManiSkill 轨迹校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["maniskill_trajectory_validate"](
"run-1", "maniskill", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "maniskill-trajectory-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["trajectoryCount"], 2)
self.assertEqual(payload["actionStepCount"], 5)
self.assertEqual(payload["files"][0]["envId"], "PickCube-v1")
def test_calvin_episode_validate_reads_real_npz_and_language_annotations(self):
try:
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "calvin"
(root / "lang_annotations").mkdir(parents=True)
for index in range(2):
np.savez(
root / f"episode_{index:07d}.npz",
actions=np.array([0.1, 0.2, 0.3], dtype="float32"),
rel_actions=np.array([0.01, 0.02, 0.03], dtype="float32"),
robot_obs=np.ones((15,), dtype="float32"),
scene_obs=np.ones((24,), dtype="float32"),
rgb_static=np.zeros((8, 8, 3), dtype="uint8"),
rgb_gripper=np.zeros((6, 6, 3), dtype="uint8"),
)
np.save(
root / "lang_annotations" / "auto_lang_ann.npy",
{"language": {"ann": ["open the drawer", "push the block"], "task": ["open_drawer", "push_block"]}},
allow_pickle=True,
)
node_data = {"label": "CALVIN Episode 校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["calvin_episode_validate"](
"run-1", "calvin", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "calvin-episode-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["episodeCount"], 2)
self.assertEqual(payload["validEpisodeCount"], 2)
self.assertEqual(payload["languageAnnotationCount"], 2)
self.assertEqual(payload["episodes"][0]["shapes"]["actions"], [3])
def test_rtx_action_schema_validate_reads_real_parquet_component_actions(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "rtx"
root.mkdir()
pq.write_table(
pa.table(
{
"episode_id": ["episode-000", "episode-000"],
"world_vector": [[0.1, 0.0, -0.1], [0.2, 0.1, -0.2]],
"rotation_delta": [[0.01, 0.02, 0.03], [0.02, 0.03, 0.04]],
"gripper_closedness_action": [0.0, 1.0],
}
),
root / "actions.parquet",
)
node_data = {"label": "RT-X动作空间Schema校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["rtx_action_schema_validate"](
"run-1", "rtx-action", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "rtx-action-schema-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["rowCount"], 2)
self.assertEqual(payload["actionCount"], 2)
self.assertEqual(payload["expectedActionDim"], 7)
self.assertEqual(payload["fieldCounts"]["world_vector+rotation_delta+gripper_closedness_action"], 2)
def test_embodiment_metadata_validate_reads_real_jsonl_metadata(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "embodiments"
root.mkdir()
rows = [
{
"robot_type": "franka_panda",
"task": "pick cup",
"camera_names": ["front", "wrist"],
"action_dim": 7,
},
{
"robot_type": "aloha_bimanual",
"task": "fold cloth",
"camera_names": ["left_wrist", "right_wrist", "top"],
"action_dim": 14,
},
]
(root / "metadata.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
node_data = {
"label": "多机器人Embodiment元信息校验",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "MIN_ROBOT_TYPES", "value": "2"},
{"key": "REQUIRED_FIELDS", "value": "robot_type,task,camera,action_dim"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["embodiment_metadata_validate"](
"run-1", "embodiment", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "embodiment-metadata-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["robotTypeCount"], 2)
self.assertEqual(payload["taskCount"], 2)
self.assertEqual(payload["actionDimCounts"], {"7": 1, "14": 1})
def test_force_torque_sensor_validate_reads_real_hdf5_wrench(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "force"
root.mkdir()
with h5py.File(root / "force_torque.h5", "w") as handle:
sensor = handle.create_group("sensor")
sensor.create_dataset(
"wrench",
data=np.array([[1.0, 2.0, 3.0, 0.1, 0.2, 0.3], [1.5, 2.5, 3.5, 0.2, 0.3, 0.4]], dtype="float32"),
)
node_data = {"label": "力扭矩传感器数据校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["force_torque_sensor_validate"](
"run-1", "force", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "force-torque-sensor-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["rowCount"], 2)
self.assertEqual(payload["vectorCount"], 2)
self.assertEqual(payload["expectedDim"], 6)
self.assertEqual(payload["maxAbsByDimension"], [1.5, 2.5, 3.5, 0.2, 0.3, 0.4])
def test_dataset_split_leakage_validate_reads_real_split_csv(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "splits"
root.mkdir()
(root / "dataset-split-manifest.csv").write_text(
"\n".join(
[
"episodeId,split,task",
"episode-000,train,pick",
"episode-001,train,place",
"episode-002,val,pick",
"episode-003,test,place",
]
)
+ "\n",
encoding="utf-8",
)
node_data = {"label": "数据集切分泄漏校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["dataset_split_leakage_validate"](
"run-1", "split-leakage", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "dataset-split-leakage-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["episodeCount"], 4)
self.assertEqual(payload["splitCounts"], {"train": 2, "val": 1, "test": 1})
self.assertEqual(payload["leakedEpisodeCount"], 0)
def test_media_reference_validate_reads_real_manifest_and_media_files(self):
try:
import cv2 # type: ignore
import numpy as np # type: ignore
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"cv2/numpy/pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "media"
media_dir = root / "assets"
media_dir.mkdir(parents=True)
image = media_dir / "frame_000.png"
depth = media_dir / "depth_000.npy"
pointcloud = media_dir / "cloud_000.pcd"
cv2.imwrite(str(image), np.full((8, 10, 3), 120, dtype=np.uint8))
np.save(depth, np.ones((8, 10), dtype=np.float32))
pointcloud.write_text(
"VERSION .7\nFIELDS x y z\nSIZE 4 4 4\nTYPE F F F\nCOUNT 1 1 1\nWIDTH 1\nHEIGHT 1\nPOINTS 1\nDATA ascii\n0 0 1\n",
encoding="utf-8",
)
pq.write_table(
pa.table(
{
"episode_id": ["episode-000"],
"image_path": ["assets/frame_000.png"],
"depth_path": ["assets/depth_000.npy"],
"pointcloud_path": ["assets/cloud_000.pcd"],
}
),
root / "manifest.parquet",
)
node_data = {"label": "媒体引用可读性校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["media_reference_validate"](
"run-1", "media", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "media-reference-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["referenceCount"], 3)
self.assertEqual(payload["missingCount"], 0)
self.assertEqual(payload["unreadableCount"], 0)
self.assertEqual(payload["fieldCounts"]["image_path"], 1)
def test_dataset_schema_contract_validate_reads_parquet_hdf5_and_npz(self):
try:
import h5py # type: ignore
import numpy as np # type: ignore
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"h5py/numpy/pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "schema"
root.mkdir()
pq.write_table(
pa.table(
{
"episode_id": ["episode-000", "episode-001"],
"split": ["train", "val"],
"action": [[0.1, 0.2], [0.3, 0.4]],
}
),
root / "manifest.parquet",
)
with h5py.File(root / "trajectory.h5", "w") as handle:
data = handle.create_group("data")
data.create_dataset("actions", data=np.ones((2, 2), dtype="float32"))
data.create_dataset("observations", data=np.ones((2, 4), dtype="float32"))
np.savez(root / "episode_0000000.npz", actions=np.ones((2, 2), dtype="float32"), robot_obs=np.ones((2, 4), dtype="float32"))
node_data = {
"label": "数据集Schema合约校验",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "REQUIRED_COLUMNS", "value": "episode_id,split,action"},
{"key": "REQUIRED_HDF5_DATASETS", "value": "data/actions,data/observations"},
{"key": "REQUIRED_NPZ_ARRAYS", "value": "actions,robot_obs"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["dataset_schema_contract_validate"](
"run-1", "schema", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "dataset-schema-contract-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 3)
self.assertEqual(payload["validFileCount"], 3)
kinds = sorted(item["kind"] for item in payload["files"])
self.assertEqual(kinds, ["hdf5", "npz", "parquet"])
def test_lerobot_dataset_validate_reads_real_parquet_and_metadata(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "lerobot"
meta = root / "meta"
episode_meta_dir = meta / "episodes" / "chunk-000"
data_dir = root / "data" / "chunk-000"
meta.mkdir(parents=True)
episode_meta_dir.mkdir(parents=True)
data_dir.mkdir(parents=True)
(meta / "info.json").write_text(
json.dumps({"total_episodes": 2, "total_frames": 3, "total_videos": 0, "fps": 10}),
encoding="utf-8",
)
(meta / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "push cube"}) + "\n", encoding="utf-8")
pq.write_table(
pa.table(
{
"episode_index": [0, 1],
"length": [2, 1],
"task_index": [0, 0],
}
),
episode_meta_dir / "file-000.parquet",
)
pq.write_table(
pa.table(
{
"episode_index": [0, 0, 1],
"frame_index": [0, 1, 0],
"timestamp": [0.0, 0.1, 0.0],
"action": [[0.1, 0.2], [0.2, 0.3], [0.4, 0.5]],
"observation.state": [[1.0, 2.0], [1.1, 2.1], [3.0, 4.0]],
}
),
data_dir / "episode_000000.parquet",
)
node_data = {"label": "LeRobot 数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["lerobot_dataset_validate"](
"run-1", "lerobot-validate", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "lerobot-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["episodeMetadataCount"], 2)
self.assertEqual(payload["episodeMetadataFiles"], ["meta/episodes/chunk-000/file-000.parquet"])
self.assertEqual(payload["totalParquetRows"], 3)
self.assertEqual(payload["episodeRows"], {"0": 2, "1": 1})
self.assertIn("action", payload["schemaColumns"])
def test_lerobot_metadata_consistency_validate_reads_v3_meta_and_data(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "lerobot-v3"
meta = root / "meta"
episode_meta_dir = meta / "episodes" / "chunk-000"
data_dir = root / "data" / "chunk-000"
meta.mkdir(parents=True)
episode_meta_dir.mkdir(parents=True)
data_dir.mkdir(parents=True)
(meta / "info.json").write_text(
json.dumps(
{
"codebase_version": "v3.0",
"total_episodes": 2,
"total_frames": 3,
"total_tasks": 1,
"features": {
"action": {"dtype": "float32", "shape": [3]},
"observation.state": {"dtype": "float32", "shape": [4]},
},
}
),
encoding="utf-8",
)
(meta / "stats.json").write_text(
json.dumps(
{
"action": {"mean": [0.2, 0.4, 0.6]},
"observation.state": {"mean": [1.2, 2.2, 3.2, 4.2]},
}
),
encoding="utf-8",
)
(meta / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "pick cube"}) + "\n", encoding="utf-8")
pq.write_table(
pa.table({"episode_index": [0, 1], "length": [2, 1], "task_index": [0, 0]}),
episode_meta_dir / "file-000.parquet",
)
pq.write_table(
pa.table(
{
"episode_index": [0, 0, 1],
"frame_index": [0, 1, 0],
"timestamp": [0.0, 0.1, 0.0],
"action": [[0.1, 0.2, 0.3], [0.2, 0.4, 0.6], [0.3, 0.6, 0.9]],
"observation.state": [
[1.0, 2.0, 3.0, 4.0],
[1.2, 2.2, 3.2, 4.2],
[1.4, 2.4, 3.4, 4.4],
],
}
),
data_dir / "episode_000000.parquet",
)
node_data = {
"label": "LeRobot 元数据一致性校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
}
artifacts = MODULE.BUILTIN_HANDLERS["lerobot_metadata_consistency_validate"](
"run-1", "lerobot-meta", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "lerobot-metadata-consistency-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["declaredFrames"], 3)
self.assertEqual(payload["declaredEpisodes"], 2)
self.assertEqual(payload["declaredTasks"], 1)
self.assertEqual(payload["episodeMetadataCount"], 2)
self.assertEqual(payload["dataRowCount"], 3)
self.assertEqual(payload["episodeRows"], {"0": 2, "1": 1})
self.assertIn("observation.state", payload["featureNames"])
def test_lerobot_v3_episode_index_validate_reads_real_path_windows(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "lerobot-v3-index"
meta = root / "meta"
episode_meta_dir = meta / "episodes" / "chunk-000"
data_dir = root / "data" / "chunk-000"
episode_meta_dir.mkdir(parents=True)
data_dir.mkdir(parents=True)
(meta / "info.json").write_text(
json.dumps(
{
"codebase_version": "v3.0",
"chunks_size": 1000,
"data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
}
),
encoding="utf-8",
)
pq.write_table(
pa.table(
{
"episode_index": [0, 1],
"episode_chunk": [0, 0],
"length": [2, 1],
"data_path": [
"data/chunk-000/episode_000000.parquet",
"data/chunk-000/episode_000001.parquet",
],
"dataset_from_index": [0, 0],
"dataset_to_index": [2, 1],
}
),
episode_meta_dir / "file-000.parquet",
)
pq.write_table(
pa.table(
{
"episode_index": [0, 0],
"frame_index": [0, 1],
"action": [[0.1, 0.2], [0.2, 0.3]],
}
),
data_dir / "episode_000000.parquet",
)
pq.write_table(
pa.table({"episode_index": [1], "frame_index": [0], "action": [[0.4, 0.5]]}),
data_dir / "episode_000001.parquet",
)
node_data = {
"label": "LeRobot v3 Episode索引校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
}
artifacts = MODULE.BUILTIN_HANDLERS["lerobot_v3_episode_index_validate"](
"run-1", "lerobot-v3-index", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "lerobot-v3-episode-index-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["episodeMetadataCount"], 2)
self.assertEqual(payload["dataParquetCount"], 2)
self.assertEqual(payload["dataRowCount"], 3)
self.assertEqual(payload["validEpisodeCount"], 2)
self.assertEqual(payload["episodes"][0]["matchedPaths"], ["data/chunk-000/episode_000000.parquet"])
self.assertEqual(payload["episodes"][0]["indexWindow"], {"from": 0, "to": 2})
def test_lerobot_video_alignment_validate_reads_real_mp4_and_parquet(self):
try:
import cv2 # type: ignore
import numpy as np # type: ignore
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"cv2/numpy/pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "lerobot-video"
meta = root / "meta"
episode_meta_dir = meta / "episodes" / "chunk-000"
data_dir = root / "data" / "chunk-000"
video_dir = root / "videos" / "observation.images.main" / "chunk-000"
meta.mkdir(parents=True)
episode_meta_dir.mkdir(parents=True)
data_dir.mkdir(parents=True)
video_dir.mkdir(parents=True)
video_path = video_dir / "episode_000000.mp4"
writer = cv2.VideoWriter(str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), 10.0, (8, 6))
if not writer.isOpened():
self.skipTest("OpenCV cannot create mp4v video in this environment")
for index in range(3):
frame = np.full((6, 8, 3), 30 + index * 40, dtype=np.uint8)
writer.write(frame)
writer.release()
(meta / "info.json").write_text(
json.dumps(
{
"codebase_version": "v3.0",
"total_episodes": 1,
"total_frames": 3,
"total_videos": 1,
"features": {
"action": {"dtype": "float32", "shape": [2]},
"observation.state": {"dtype": "float32", "shape": [2]},
"observation.images.main": {"dtype": "video", "shape": [6, 8, 3]},
},
}
),
encoding="utf-8",
)
pq.write_table(pa.table({"episode_index": [0], "length": [3], "task_index": [0]}), episode_meta_dir / "file-000.parquet")
pq.write_table(
pa.table(
{
"episode_index": [0, 0, 0],
"frame_index": [0, 1, 2],
"timestamp": [0.0, 0.1, 0.2],
"action": [[0.1, 0.2], [0.2, 0.3], [0.3, 0.4]],
"observation.state": [[1.0, 2.0], [1.1, 2.1], [1.2, 2.2]],
"observation.images.main": [
"videos/observation.images.main/chunk-000/episode_000000.mp4",
"videos/observation.images.main/chunk-000/episode_000000.mp4",
"videos/observation.images.main/chunk-000/episode_000000.mp4",
],
}
),
data_dir / "episode_000000.parquet",
)
node_data = {"label": "LeRobot 视频对齐校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["lerobot_video_alignment_validate"](
"run-1", "lerobot-video", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "lerobot-video-alignment-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["videoCount"], 1)
self.assertEqual(payload["videoFrameCount"], 3)
self.assertEqual(payload["dataRowCount"], 3)
self.assertEqual(payload["missingVideoCount"], 0)
self.assertEqual(payload["mismatchedVideoCount"], 0)
def test_trajectory_integrity_validate_checks_modalities_and_file_refs(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "trajectory"
image_dir = root / "images"
image_dir.mkdir(parents=True)
(image_dir / "frame-000.png").write_bytes(b"real-image-bytes")
(root / "episodes.jsonl").write_text(
json.dumps(
{
"episode_id": "ep-1",
"steps": [
{
"action": [0.1, 0.2, 0.3],
"observation": {"image": "images/frame-000.png", "state": [1.0, 2.0]},
},
{
"action": [0.2, 0.3, 0.4],
"observation": {"image": "images/frame-000.png", "state": [1.2, 2.2]},
},
],
}
)
+ "\n",
encoding="utf-8",
)
node_data = {
"label": "轨迹多模态完整性校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
}
artifacts = MODULE.BUILTIN_HANDLERS["trajectory_integrity_validate"](
"run-1", "trajectory", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "trajectory-integrity-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["stepCount"], 2)
self.assertEqual(payload["actionDimension"], 3)
self.assertEqual(payload["fieldMissingCounts"]["observation.image"], 0)
def test_action_state_quality_validate_reads_real_parquet_vectors(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "action-state"
data_dir = root / "data" / "chunk-000"
meta_dir = root / "meta"
data_dir.mkdir(parents=True)
meta_dir.mkdir(parents=True)
(meta_dir / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "ignored meta row"}) + "\n", encoding="utf-8")
pq.write_table(
pa.table(
{
"episode_index": [0, 0, 1],
"frame_index": [0, 1, 0],
"action": [[0.1, 0.2, 0.3], [0.2, 0.4, 0.6], [0.3, 0.6, 0.9]],
"observation.state": [
[1.0, 2.0, 3.0, 4.0],
[1.2, 2.2, 3.2, 4.2],
[1.4, 2.4, 3.4, 4.4],
],
}
),
data_dir / "episode_000000.parquet",
)
node_data = {
"label": "动作状态数值质量校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
}
artifacts = MODULE.BUILTIN_HANDLERS["action_state_quality_validate"](
"run-1", "action-state", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "action-state-quality-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["rowCount"], 3)
self.assertEqual(payload["actionRowCount"], 3)
self.assertEqual(payload["stateRowCount"], 3)
self.assertEqual(payload["actionDimension"], 3)
self.assertEqual(payload["stateDimension"], 4)
self.assertEqual(payload["actionStats"]["max"], [0.3, 0.6, 0.9])
self.assertEqual(payload["stateStats"]["min"], [1.0, 2.0, 3.0, 4.0])
def test_episode_boundary_validate_reads_real_parquet_rows(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "episode-boundary"
data_dir = root / "data" / "chunk-000"
meta_dir = root / "meta"
data_dir.mkdir(parents=True)
meta_dir.mkdir(parents=True)
(meta_dir / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "ignored meta row"}) + "\n", encoding="utf-8")
pq.write_table(
pa.table(
{
"episode_index": [0, 0, 1],
"frame_index": [0, 1, 0],
"done": [False, True, True],
"action": [[0.1], [0.2], [0.3]],
}
),
data_dir / "episode_000000.parquet",
)
node_data = {"label": "Episode 边界校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["episode_boundary_validate"](
"run-1", "episode-boundary", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "episode-boundary-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["rowCount"], 3)
self.assertEqual(payload["episodeCount"], 2)
self.assertEqual(payload["invalidEpisodeCount"], 0)
self.assertEqual(payload["terminalFieldCounts"], {"done": 3})
self.assertEqual(payload["episodes"][0]["lastFrameIndex"], 1)
def test_vla_sample_manifest_validate_reads_real_parquet_and_images(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "vla"
image_dir = root / "images"
image_dir.mkdir(parents=True)
(image_dir / "frame-000.png").write_bytes(b"\x89PNG\r\n\x1a\nreal")
(image_dir / "frame-001.png").write_bytes(b"\x89PNG\r\n\x1a\nreal")
pq.write_table(
pa.table(
{
"image_path": ["images/frame-000.png", "images/frame-001.png"],
"language_instruction": ["pick up the cube", "place the cube"],
"action": [
[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 1.0],
[0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.0],
],
}
),
root / "manifest.parquet",
)
node_data = {"label": "VLA 样本清单校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["vla_sample_manifest_validate"](
"run-1", "vla", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "vla-sample-manifest-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["rowCount"], 2)
self.assertEqual(payload["imageReferenceCount"], 2)
self.assertEqual(payload["missingImageCount"], 0)
self.assertEqual(payload["textRowCount"], 2)
self.assertEqual(payload["actionRowCount"], 2)
self.assertEqual(payload["completeRowCount"], 2)
self.assertEqual(payload["actionDimension"], 7)
def test_dataset_split_manifest_writes_json_and_csv_from_real_episode_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "split-source"
root.mkdir()
(root / "episodes.jsonl").write_text(
"\n".join(
[
json.dumps({"episode_id": "ep-1", "steps": [{"action": [0]}], "task": "pick"}),
json.dumps({"episode_id": "ep-2", "steps": [{"action": [1]}, {"action": [2]}], "task": "place"}),
json.dumps({"episode_id": "ep-3", "steps": [], "task": "skip"}),
]
),
encoding="utf-8",
)
node_data = {
"label": "训练切分清单生成",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "TRAIN_RATIO", "value": "1"},
{"key": "VAL_RATIO", "value": "0"},
{"key": "TEST_RATIO", "value": "0"},
{"key": "MIN_STEPS", "value": "1"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["dataset_split_manifest"](
"run-1", "split", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "dataset-split-manifest")
self.assertEqual(payload["episodeCount"], 2)
self.assertEqual(payload["skippedCount"], 1)
self.assertEqual(payload["splitCounts"], {"train": 2, "val": 0, "test": 0})
csv_path = pathlib.Path(payload["csvPath"])
self.assertTrue(csv_path.exists())
self.assertEqual(artifacts[1]["name"], "dataset-split-manifest.csv")
def test_mcap_topic_summary_reads_real_mcap_messages(self):
try:
from mcap.writer import Writer # type: ignore
except Exception as exc:
self.skipTest(f"mcap not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
path = pathlib.Path(temp_dir) / "robot.mcap"
with path.open("wb") as handle:
writer = Writer(handle)
writer.start(profile="ros2")
schema_id = writer.register_schema(
name="sensor_msgs/msg/JointState",
encoding="ros2msg",
data=b"string[] name\nfloat64[] position\n",
)
channel_id = writer.register_channel(
topic="/joint_states",
message_encoding="cdr",
schema_id=schema_id,
metadata={"frame_id": "base_link"},
)
writer.add_message(channel_id=channel_id, log_time=100, publish_time=90, data=b"joint-a")
writer.add_message(channel_id=channel_id, log_time=200, publish_time=190, data=b"joint-b-longer")
writer.add_metadata("source", {"robot": "unit-test-arm"})
writer.finish()
node_data = {"label": "MCAP 主题概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
artifacts = MODULE.BUILTIN_HANDLERS["mcap_topic_summary"](
"run-1", "mcap", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "mcap-topic-summary")
self.assertEqual(payload["topicCount"], 1)
self.assertEqual(payload["messageCount"], 2)
self.assertEqual(payload["topics"][0]["topic"], "/joint_states")
self.assertEqual(payload["topics"][0]["messageCount"], 2)
self.assertEqual(payload["topics"][0]["schemaName"], "sensor_msgs/msg/JointState")
def test_tfrecord_structural_validate_reads_real_records_and_crc(self):
def record(payload: bytes) -> bytes:
length_bytes = struct.pack("<Q", len(payload))
return (
length_bytes
+ struct.pack("<I", MODULE.tfrecord_structural_validate_operator._masked_crc32c(length_bytes))
+ payload
+ struct.pack("<I", MODULE.tfrecord_structural_validate_operator._masked_crc32c(payload))
)
with tempfile.TemporaryDirectory() as temp_dir:
path = pathlib.Path(temp_dir) / "episodes.tfrecord"
path.write_bytes(
record(json.dumps({"episode_id": "ep-1", "steps": 2}).encode("utf-8"))
+ record(json.dumps({"episode_id": "ep-2", "steps": 1}).encode("utf-8"))
)
node_data = {"label": "TFRecord 结构校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
artifacts = MODULE.BUILTIN_HANDLERS["tfrecord_structural_validate"](
"run-1", "tfrecord", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "tfrecord-structural-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["recordCount"], 2)
self.assertEqual(payload["files"][0]["records"][0]["encoding"], "json")
self.assertEqual(payload["files"][0]["records"][0]["preview"]["episode_id"], "ep-1")
def test_bridgedata_tfds_manifest_validate_reads_real_tfrecord_and_features(self):
def record(payload: bytes) -> bytes:
length_bytes = struct.pack("<Q", len(payload))
return (
length_bytes
+ struct.pack("<I", MODULE.bridgedata_tfds_manifest_validate_operator._masked_crc32c(length_bytes))
+ payload
+ struct.pack("<I", MODULE.bridgedata_tfds_manifest_validate_operator._masked_crc32c(payload))
)
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "bridgedata"
train_dir = root / "train"
train_dir.mkdir(parents=True)
(root / "dataset_info.json").write_text(
json.dumps(
{
"name": "bridge_data_v2",
"splits": {"train": {"numExamples": 2}},
"features": {
"steps": {
"observation": {"image_0": {"dtype": "uint8"}},
"action": {"shape": [7]},
"language_instruction": {"dtype": "string"},
}
},
}
),
encoding="utf-8",
)
(root / "features.json").write_text(
json.dumps({"steps": {"observation": {"image_0": "image"}, "action": "float32", "language_instruction": "string"}}),
encoding="utf-8",
)
(train_dir / "episode-00000-of-00001.tfrecord").write_bytes(
record(json.dumps({"episode_id": "ep-1", "language_instruction": "open drawer"}).encode("utf-8"))
+ record(json.dumps({"episode_id": "ep-2", "language_instruction": "close drawer"}).encode("utf-8"))
)
node_data = {
"label": "BridgeData TFDS-RLDS清单校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
}
artifacts = MODULE.BUILTIN_HANDLERS["bridgedata_tfds_manifest_validate"](
"run-1", "bridge", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "bridgedata-tfds-rlds-manifest-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["recordCount"], 2)
self.assertEqual(payload["splitRecordCounts"], {"train": 2})
self.assertEqual(payload["files"][0]["records"][0]["encoding"], "json")
self.assertIn("dataset_info.features.steps.action.shape", payload["featureKeys"])
def test_dataset_provenance_audit_reads_dataset_card_license_and_citation(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "hf-dataset"
root.mkdir()
(root / "README.md").write_text(
"---\n"
"license: apache-2.0\n"
"source: https://github.com/example/robot-dataset\n"
"task_categories:\n"
" - robotics\n"
"---\n"
"# Robot Dataset\n",
encoding="utf-8",
)
(root / "LICENSE").write_text("Apache License\nVersion 2.0\n", encoding="utf-8")
(root / "CITATION.cff").write_text("cff-version: 1.2.0\ntitle: Robot Dataset\n", encoding="utf-8")
(root / "provenance.json").write_text(
json.dumps({"sources": ["s3://real-bucket/raw"], "license": "apache-2.0"}),
encoding="utf-8",
)
node_data = {"label": "数据集来源许可审计", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["dataset_provenance_audit"](
"run-1", "provenance", pathlib.Path(temp_dir) / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "dataset-provenance-audit")
self.assertTrue(payload["valid"])
self.assertEqual(payload["issueCount"], 0)
self.assertIn("apache-2.0", payload["licenses"])
self.assertIn("https://github.com/example/robot-dataset", payload["sources"])
self.assertEqual(payload["card"]["path"], "README.md")
def test_webdataset_shard_summary_reads_real_tar_samples(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
shard_path = root / "robot-000000.tar"
with tarfile.open(shard_path, "w") as archive:
for name, payload in {
"episode-000001.jpg": b"real-image-bytes",
"episode-000001.json": json.dumps({"task": "pick block"}).encode("utf-8"),
"episode-000002.png": b"real-png-bytes",
"episode-000002.txt": b"place block",
}.items():
info = tarfile.TarInfo(name)
info.size = len(payload)
archive.addfile(info, io.BytesIO(payload))
node_data = {"label": "WebDataset 分片概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(shard_path)}]}
artifacts = MODULE.BUILTIN_HANDLERS["webdataset_shard_summary"](
"run-1", "webdataset", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "webdataset-shard-summary")
self.assertTrue(payload["valid"])
self.assertEqual(payload["shardCount"], 1)
self.assertEqual(payload["sampleCount"], 2)
self.assertEqual(payload["memberFileCount"], 4)
self.assertEqual(payload["extensionCounts"]["jpg"], 1)
self.assertEqual(payload["modalityCounts"]["image"], 2)
def test_zarr_trajectory_summary_reads_real_zarr_metadata(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "trajectory.zarr"
data = root / "data"
action = data / "action"
state = data / "state"
episode_ends = root / "meta" / "episode_ends"
action.mkdir(parents=True)
state.mkdir(parents=True)
episode_ends.mkdir(parents=True)
(root / ".zgroup").write_text(json.dumps({"zarr_format": 2}), encoding="utf-8")
(root / ".zattrs").write_text(json.dumps({"episode_count": 2, "robot": "unit-test-arm"}), encoding="utf-8")
for array_path, shape, chunks, dtype in (
(action, [3, 7], [2, 7], "<f4"),
(state, [3, 14], [2, 14], "<f4"),
(episode_ends, [2], [2], "<i8"),
):
(array_path / ".zarray").write_text(
json.dumps(
{
"zarr_format": 2,
"shape": shape,
"chunks": chunks,
"dtype": dtype,
"compressor": None,
"fill_value": 0,
"order": "C",
"filters": None,
}
),
encoding="utf-8",
)
(array_path / "0").write_bytes(b"chunk-bytes")
node_data = {"label": "Zarr 轨迹数组概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["zarr_trajectory_summary"]("run-1", "zarr", root / "out", node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "zarr-trajectory-summary")
self.assertTrue(payload["valid"])
self.assertEqual(payload["arrayCount"], 3)
self.assertEqual(payload["episodeCount"], 2)
self.assertEqual(payload["stepCount"], 3)
self.assertIn("data/action", [item["path"] for item in payload["arrays"]])
def test_umi_zarr_dataset_validate_reads_real_zarr_episode_ends_and_arrays(self):
try:
import numpy as np # type: ignore
import zarr # type: ignore
except Exception as exc:
self.skipTest(f"numpy/zarr not available: {exc}")
def create_array(group, name: str, data, chunks):
if hasattr(group, "create_array"):
return group.create_array(name, data=data, chunks=chunks)
return group.create_dataset(name, data=data, chunks=chunks)
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "umi" / "dataset.zarr"
group = zarr.open_group(str(root), mode="w", zarr_format=2)
data = group.create_group("data")
create_array(data, "camera0_rgb", np.zeros((5, 8, 8, 3), dtype=np.uint8), chunks=(1, 8, 8, 3))
create_array(data, "robot0_eef_pos", np.linspace(0.1, 0.5, 15, dtype=np.float32).reshape(5, 3), chunks=(5, 3))
create_array(data, "robot0_eef_rot_axis_angle", np.ones((5, 3), dtype=np.float32), chunks=(5, 3))
create_array(data, "robot0_gripper_width", np.linspace(0.01, 0.05, 5, dtype=np.float32).reshape(5, 1), chunks=(5, 1))
meta = group.create_group("meta")
create_array(meta, "episode_ends", np.array([2, 5], dtype=np.int64), chunks=(2,))
node_data = {"label": "UMI Zarr数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root.parent)}]}
artifacts = MODULE.BUILTIN_HANDLERS["umi_zarr_dataset_validate"](
"run-1", "umi-zarr", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "umi-zarr-dataset-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["datasetPath"], str(root))
self.assertEqual(payload["dataArrayCount"], 4)
self.assertEqual(payload["episodeCount"], 2)
self.assertEqual(payload["stepCount"], 5)
self.assertEqual(payload["episodeEnds"], [2, 5])
self.assertEqual(payload["cameraArrayCount"], 1)
self.assertEqual(payload["eefArrayCount"], 2)
self.assertEqual(payload["gripperArrayCount"], 1)
self.assertTrue(payload["requiredDataKeysPresent"])
self.assertIn("robot0_eef_pos", [item["key"] for item in payload["arrays"]])
def test_camera_calibration_validate_reads_real_yaml_camera_info(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "calibration"
root.mkdir()
(root / "camera_info.yaml").write_text(
"width: 640\n"
"height: 480\n"
"K: [525.0, 0.0, 319.5, 0.0, 525.0, 239.5, 0.0, 0.0, 1.0]\n"
"D: [0.1, -0.01, 0.0, 0.0, 0.0]\n"
"R: [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]\n"
"P: [525.0, 0.0, 319.5, 0.0, 0.0, 525.0, 239.5, 0.0, 0.0, 0.0, 1.0, 0.0]\n"
"T_base_camera: [1.0, 0.0, 0.0, 0.1, 0.0, 1.0, 0.0, 0.2, 0.0, 0.0, 1.0, 0.3, 0.0, 0.0, 0.0, 1.0]\n",
encoding="utf-8",
)
node_data = {"label": "相机标定校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["camera_calibration_validate"](
"run-1", "calibration", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "camera-calibration-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["cameraCount"], 1)
self.assertEqual(payload["issueCount"], 0)
self.assertEqual(payload["cameras"][0]["width"], 640)
self.assertTrue(payload["cameras"][0]["hasExtrinsic"])
def test_robot_model_asset_validate_reads_real_urdf_and_mesh_refs(self):
with tempfile.TemporaryDirectory() as temp_dir:
package = pathlib.Path(temp_dir) / "robot_description"
mesh_dir = package / "meshes"
mesh_dir.mkdir(parents=True)
(mesh_dir / "base.stl").write_bytes(b"solid base\nendsolid base\n")
(package / "robot.urdf").write_text(
"<robot name=\"unit_arm\">"
" <link name=\"base_link\">"
" <visual><geometry><mesh filename=\"package://robot_description/meshes/base.stl\"/></geometry></visual>"
" </link>"
" <link name=\"tool_link\"/>"
" <joint name=\"tool_joint\" type=\"fixed\"><parent link=\"base_link\"/><child link=\"tool_link\"/></joint>"
"</robot>",
encoding="utf-8",
)
node_data = {"label": "机器人模型资产校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(package)}]}
artifacts = MODULE.BUILTIN_HANDLERS["robot_model_asset_validate"](
"run-1", "robot-model", package / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "robot-model-asset-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["modelCount"], 1)
self.assertEqual(payload["linkCount"], 2)
self.assertEqual(payload["jointCount"], 1)
self.assertEqual(payload["meshReferenceCount"], 1)
self.assertEqual(payload["missingMeshCount"], 0)
def test_mjcf_actuator_validate_reads_real_ctrlrange(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "mjcf"
root.mkdir()
(root / "scene.xml").write_text(
"<mujoco model=\"unit_arm\">"
" <worldbody>"
" <body name=\"arm\">"
" <joint name=\"shoulder\" type=\"hinge\"/>"
" <site name=\"tool\"/>"
" </body>"
" </worldbody>"
" <actuator>"
" <position name=\"shoulder_pos\" joint=\"shoulder\" ctrllimited=\"true\" ctrlrange=\"-1 1\" gear=\"1\"/>"
" <motor name=\"tool_motor\" site=\"tool\" gear=\"0 0 1\"/>"
" </actuator>"
"</mujoco>",
encoding="utf-8",
)
node_data = {"label": "MJCF 执行器校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["mjcf_actuator_validate"](
"run-1", "mjcf-actuator", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "mjcf-actuator-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["modelCount"], 1)
self.assertEqual(payload["actuatorCount"], 2)
self.assertEqual(payload["invalidActuatorCount"], 0)
self.assertEqual(payload["issueCount"], 0)
self.assertEqual(payload["models"][0]["targetCounts"]["joints"], 1)
self.assertEqual(payload["models"][0]["targetCounts"]["sites"], 1)
def test_tf_transform_tree_validate_reads_real_transform_jsonl(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "tf"
root.mkdir()
rows = [
{
"header": {"frame_id": "world"},
"child_frame_id": "base_link",
"transform": {
"translation": {"x": 0.0, "y": 0.0, "z": 0.0},
"rotation": {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0},
},
},
{
"header": {"frame_id": "base_link"},
"child_frame_id": "camera",
"transform": {
"translation": {"x": 0.1, "y": 0.2, "z": 0.3},
"rotation": {"x": 0.0, "y": 0.0, "z": 0.70710678, "w": 0.70710678},
},
},
]
(root / "transforms.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8")
node_data = {"label": "TF 坐标树校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["tf_transform_tree_validate"](
"run-1", "tf-tree", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "tf-transform-tree-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["transformCount"], 2)
self.assertEqual(payload["frameCount"], 3)
self.assertEqual(payload["rootFrames"], ["world"])
self.assertEqual(payload["cycleCount"], 0)
self.assertEqual(payload["issueCount"], 0)
def test_ros_joint_state_validate_reads_real_jsonl_messages(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "joint-state"
root.mkdir()
rows = [
{
"header": {"stamp": {"sec": 1, "nanosec": 0}},
"name": ["shoulder_pan", "elbow_flex"],
"position": [0.1, 0.2],
"velocity": [0.0, 0.1],
"effort": [1.0, 1.1],
},
{
"header": {"stamp": {"sec": 1, "nanosec": 100000000}},
"name": ["shoulder_pan", "elbow_flex"],
"position": [0.2, 0.3],
"velocity": [0.1, 0.2],
"effort": [1.2, 1.3],
},
]
(root / "joint_states.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8")
node_data = {"label": "ROS JointState 校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["ros_joint_state_validate"](
"run-1", "joint-state", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "ros-joint-state-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["messageCount"], 2)
self.assertEqual(payload["validMessageCount"], 2)
self.assertEqual(payload["invalidMessageCount"], 0)
self.assertEqual(payload["jointCount"], 2)
self.assertEqual(payload["jointNames"], ["shoulder_pan", "elbow_flex"])
def test_dexycb_dataset_validate_reads_label_npz_and_rgbd_pairs(self):
try:
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "dex-ycb"
camera_dir = root / "20200709-subject-01" / "20200709_141841" / "932122060861"
camera_dir.mkdir(parents=True)
(root / "calibration").mkdir()
(root / "models" / "002_master_chef_can").mkdir(parents=True)
(root / "calibration" / "932122060861.yml").write_text("fx: 613.0\n", encoding="utf-8")
(root / "models" / "002_master_chef_can" / "textured.obj").write_text("o can\n", encoding="utf-8")
frame_id = "000053"
(camera_dir / f"color_{frame_id}.jpg").write_bytes(b"real-jpeg-placeholder")
(camera_dir / f"aligned_depth_to_color_{frame_id}.png").write_bytes(b"real-png-placeholder")
seg = np.zeros((6, 8), dtype=np.uint8)
seg[1:4, 2:5] = 1
seg[4:6, 5:8] = 255
pose_y = np.ones((2, 3, 4), dtype=np.float32)
pose_m = np.ones((1, 51), dtype=np.float32)
joint_3d = np.ones((1, 21, 3), dtype=np.float32)
joint_2d = np.ones((1, 21, 2), dtype=np.float32)
np.savez(
camera_dir / f"labels_{frame_id}.npz",
seg=seg,
pose_y=pose_y,
pose_m=pose_m,
joint_3d=joint_3d,
joint_2d=joint_2d,
)
node_data = {
"label": "DexYCB 数据集校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
}
artifacts = MODULE.BUILTIN_HANDLERS["dexycb_dataset_validate"](
"run-1", "dexycb", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "dexycb-dataset-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["issueCount"], 0)
self.assertEqual(payload["sampleCount"], 1)
self.assertEqual(payload["validSampleCount"], 1)
self.assertEqual(payload["subjectCounts"], {"20200709-subject-01": 1})
self.assertEqual(payload["cameraCounts"], {"932122060861": 1})
self.assertEqual(payload["objectPoseCounts"], {"2": 1})
self.assertTrue(payload["assets"]["hasCalibrationDir"])
self.assertTrue(payload["assets"]["hasModelsDir"])
sample = payload["samples"][0]
self.assertEqual(sample["frameId"], "000053")
self.assertEqual(sample["arrayShapes"]["seg"], [6, 8])
self.assertEqual(sample["arrayShapes"]["pose_y"], [2, 3, 4])
self.assertEqual(sample["arrayShapes"]["pose_m"], [1, 51])
self.assertEqual(sample["arrayShapes"]["joint_3d"], [1, 21, 3])
self.assertEqual(sample["arrayShapes"]["joint_2d"], [1, 21, 2])
self.assertIn(255, sample["segUniqueLabels"])
self.assertEqual(sample["visibleJointCount"], 21)
def test_ho3d_dataset_validate_reads_meta_pickle_and_media_pairs(self):
try:
import cv2 # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"cv2/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "ho3d"
frame_id = "0000"
seq_dir = root / "train" / "ABF10"
for folder in ("rgb", "depth", "segr", "meta"):
(seq_dir / folder).mkdir(parents=True)
cv2.imwrite(str(seq_dir / "rgb" / f"{frame_id}.jpg"), np.full((6, 8, 3), 120, dtype=np.uint8))
cv2.imwrite(str(seq_dir / "depth" / f"{frame_id}.png"), np.full((6, 8, 3), 20, dtype=np.uint8))
cv2.imwrite(str(seq_dir / "segr" / f"{frame_id}.png"), np.full((6, 8, 3), 255, dtype=np.uint8))
meta = {
"camMat": np.array([[610.0, 0.0, 320.0], [0.0, 611.0, 240.0], [0.0, 0.0, 1.0]], dtype=np.float32),
"objRot": np.array([0.1, 0.2, 0.3], dtype=np.float32),
"objTrans": np.array([0.01, -0.02, -0.45], dtype=np.float32),
"objCorners3DRest": np.arange(24, dtype=np.float32).reshape(8, 3),
"objName": "003_cracker_box",
"handPose": np.zeros((48,), dtype=np.float32),
"handTrans": np.array([0.0, 0.0, -0.35], dtype=np.float32),
"handBeta": np.zeros((10,), dtype=np.float32),
"handJoints3D": np.ones((21, 3), dtype=np.float32),
"handBoundingBox": np.array([1.0, 1.0, 7.0, 5.0], dtype=np.float32),
"handVertContact": np.array([0.0, 1.0, 0.5], dtype=np.float32),
}
with (seq_dir / "meta" / f"{frame_id}.pkl").open("wb") as handle:
pickle.dump(meta, handle)
node_data = {
"label": "HO3D/H2O3D 数据集校验",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "CHECK_SEGMENTATION_FILES", "value": "true"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["ho3d_dataset_validate"](
"run-1", "ho3d", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "ho3d-dataset-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["issueCount"], 0)
self.assertEqual(payload["sampleCount"], 1)
self.assertEqual(payload["validSampleCount"], 1)
self.assertEqual(payload["invalidSampleCount"], 0)
self.assertEqual(payload["splitCounts"], {"train": 1})
self.assertEqual(payload["sequenceCounts"], {"ABF10": 1})
self.assertEqual(payload["objectCounts"], {"003_cracker_box": 1})
self.assertEqual(payload["handAnnotationCounts"], {"manoHand": 1, "handJoints3D": 1, "handVertContact": 1})
self.assertTrue(payload["assets"]["hasTrainDir"])
sample = payload["samples"][0]
self.assertEqual(sample["frameId"], "0000")
self.assertEqual(sample["metaPath"], "train/ABF10/meta/0000.pkl")
self.assertEqual(sample["rgbPath"], "train/ABF10/rgb/0000.jpg")
self.assertEqual(sample["depthPath"], "train/ABF10/depth/0000.png")
self.assertEqual(sample["segmentationPath"], "train/ABF10/segr/0000.png")
self.assertTrue(sample["hasObjectPose"])
self.assertTrue(sample["hasManoHand"])
self.assertTrue(sample["hasHandJoints3D"])
self.assertEqual(sample["arrayShapes"]["camMat"], [3, 3])
self.assertEqual(sample["arrayShapes"]["objCorners3DRest"], [8, 3])
self.assertEqual(sample["arrayShapes"]["handPose"], [48])
self.assertEqual(sample["arrayShapes"]["handJoints3D"], [21, 3])
def test_hand_object_pose_validate_reads_real_pose_jsonl_and_images(self):
try:
import cv2 # type: ignore
import numpy as np # type: ignore
except Exception as exc:
self.skipTest(f"cv2/numpy not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "hand-object"
(root / "images").mkdir(parents=True)
(root / "masks").mkdir(parents=True)
image = np.zeros((8, 10, 3), dtype=np.uint8)
mask = np.zeros((8, 10), dtype=np.uint8)
mask[2:6, 3:8] = 1
cv2.imwrite(str(root / "images" / "frame_0001.png"), image)
cv2.imwrite(str(root / "masks" / "frame_0001.png"), mask)
joints_3d = [[float(index), float(index + 1), float(index + 2)] for index in range(21)]
joints_2d = [[float(index), float(index + 1)] for index in range(21)]
pose_matrix = [
[1.0, 0.0, 0.0, 0.1],
[0.0, 1.0, 0.0, 0.2],
[0.0, 0.0, 1.0, 0.3],
[0.0, 0.0, 0.0, 1.0],
]
rows = [
{
"image_path": "images/frame_0001.png",
"mask_path": "masks/frame_0001.png",
"hand_joints_3d": joints_3d,
"hand_joints_2d": joints_2d,
"object_pose_matrix": pose_matrix,
"bbox": [1.0, 2.0, 5.0, 4.0],
},
{
"image_path": "images/frame_0001.png",
"mask_path": "masks/frame_0001.png",
"hand_joints_3d": joints_3d,
"object_quaternion": [0.0, 0.0, 0.0, 1.0],
"object_translation": [0.1, 0.2, 0.3],
"bbox": [2.0, 1.0, 4.0, 3.0],
},
]
(root / "annotations.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8")
node_data = {
"label": "手-物体姿态标注校验",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "CHECK_MASK_EXISTS", "value": "true"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["hand_object_pose_validate"](
"run-1", "hand-object", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "hand-object-pose-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["sampleCount"], 2)
self.assertEqual(payload["invalidSampleCount"], 0)
self.assertEqual(payload["handJoint3DCount"], 2)
self.assertEqual(payload["handJoint2DCount"], 1)
self.assertEqual(payload["objectPoseCount"], 2)
self.assertEqual(payload["missingImageCount"], 0)
self.assertEqual(payload["missingMaskCount"], 0)
def test_joint_limit_validate_reads_real_urdf_and_parquet_positions(self):
try:
import pyarrow as pa # type: ignore
import pyarrow.parquet as pq # type: ignore
except Exception as exc:
self.skipTest(f"pyarrow not available: {exc}")
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "joint-limit"
data_dir = root / "data"
data_dir.mkdir(parents=True)
(root / "robot.urdf").write_text(
"<robot name=\"unit_arm\">"
" <link name=\"base\"/>"
" <link name=\"shoulder\"/>"
" <link name=\"elbow\"/>"
" <joint name=\"shoulder_pan\" type=\"revolute\">"
" <parent link=\"base\"/><child link=\"shoulder\"/>"
" <limit lower=\"-1.0\" upper=\"1.0\" effort=\"10\" velocity=\"2\"/>"
" </joint>"
" <joint name=\"elbow_flex\" type=\"revolute\">"
" <parent link=\"shoulder\"/><child link=\"elbow\"/>"
" <limit lower=\"0.0\" upper=\"2.0\" effort=\"10\" velocity=\"2\"/>"
" </joint>"
"</robot>",
encoding="utf-8",
)
pq.write_table(
pa.table(
{
"frame_index": [0, 1],
"joint_positions": [[0.0, 1.0], [0.8, 1.5]],
}
),
data_dir / "episode_000000.parquet",
)
node_data = {"label": "URDF 关节限位校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["joint_limit_validate"](
"run-1", "joint-limit", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "joint-limit-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["jointCount"], 2)
self.assertEqual(payload["boundedJointCount"], 2)
self.assertEqual(payload["rowCount"], 2)
self.assertEqual(payload["checkedRowCount"], 2)
self.assertEqual(payload["violationCount"], 0)
def test_pointcloud_summary_reads_real_ascii_pcd_and_ply(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "pointclouds"
root.mkdir()
(root / "scan.pcd").write_text(
"# .PCD v0.7\n"
"VERSION 0.7\n"
"FIELDS x y z intensity\n"
"SIZE 4 4 4 4\n"
"TYPE F F F F\n"
"COUNT 1 1 1 1\n"
"WIDTH 3\n"
"HEIGHT 1\n"
"POINTS 3\n"
"DATA ascii\n"
"0.0 0.0 0.0 1.0\n"
"1.0 2.0 3.0 0.5\n"
"-1.0 1.0 2.0 0.2\n",
encoding="utf-8",
)
(root / "scan.ply").write_text(
"ply\n"
"format ascii 1.0\n"
"element vertex 2\n"
"property float x\n"
"property float y\n"
"property float z\n"
"end_header\n"
"0 0 0\n"
"2 3 4\n",
encoding="utf-8",
)
node_data = {"label": "点云数据概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["pointcloud_summary"]("run-1", "pointcloud", root / "out", node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "pointcloud-summary")
self.assertEqual(payload["fileCount"], 2)
self.assertEqual(payload["totalPointCount"], 5)
self.assertEqual(payload["formats"], ["pcd", "ply"])
pcd = next(item for item in payload["files"] if item["format"] == "pcd")
self.assertEqual(pcd["bounds"]["min"], [-1.0, 0.0, 0.0])
self.assertEqual(pcd["bounds"]["max"], [1.0, 2.0, 3.0])
def test_sensor_timestamp_sync_validate_reads_real_csv_streams(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "sync"
root.mkdir()
(root / "timestamps.csv").write_text(
"frame_id,stream,timestamp_ns\n"
"0,camera,100000000000\n"
"0,lidar,100010000000\n"
"0,action,100020000000\n"
"1,camera,101000000000\n"
"1,lidar,101015000000\n"
"1,action,101025000000\n",
encoding="utf-8",
)
node_data = {
"label": "多传感器时间同步校验",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "REQUIRED_STREAMS", "value": "camera,lidar,action"},
{"key": "MAX_DRIFT_MS", "value": "30"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["sensor_timestamp_sync_validate"](
"run-1", "timestamp-sync", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "sensor-timestamp-sync-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["syncGroupCount"], 2)
self.assertEqual(payload["streamCounts"], {"camera": 2, "lidar": 2, "action": 2})
self.assertEqual(payload["maxObservedDriftMs"], 25.0)
def test_parquet_table_summary_reads_real_parquet_schema_and_rows(self):
import pyarrow as pa
import pyarrow.parquet as pq
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "parquet"
root.mkdir()
table = pa.table(
{
"episode_index": [0, 0, 1],
"frame_index": [0, 1, 0],
"timestamp": [0.0, 0.1, 0.0],
"action": [[0.1, 0.2], [0.2, 0.3], [0.4, 0.5]],
}
)
pq.write_table(table, root / "episodes.parquet")
node_data = {
"label": "Parquet 表摘要",
"hyperparams": [
{"key": "SOURCE_PATH", "value": str(root)},
{"key": "REQUIRED_COLUMNS", "value": "episode_index,frame_index,timestamp,action"},
{"key": "SAMPLE_COLUMNS", "value": "episode_index,frame_index,timestamp"},
{"key": "MAX_SAMPLE_ROWS", "value": "2"},
],
}
artifacts = MODULE.BUILTIN_HANDLERS["parquet_table_summary"](
"run-1", "parquet", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "parquet-table-summary")
self.assertTrue(payload["valid"])
self.assertEqual(payload["fileCount"], 1)
self.assertEqual(payload["totalRowCount"], 3)
self.assertIn("action", payload["columnNames"])
self.assertEqual(len(payload["files"][0]["sampleRows"]), 2)
def test_coco_annotation_validate_reads_real_json_and_image_files(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "coco"
images = root / "images"
annotations = root / "annotations"
images.mkdir(parents=True)
annotations.mkdir(parents=True)
(images / "frame_0001.jpg").write_bytes(b"\xff\xd8\xff\xd9")
(annotations / "instances_train.json").write_text(
json.dumps(
{
"images": [{"id": 1, "file_name": "frame_0001.jpg", "width": 640, "height": 480}],
"categories": [{"id": 3, "name": "gripper"}],
"annotations": [
{
"id": 7,
"image_id": 1,
"category_id": 3,
"bbox": [100, 120, 80, 60],
"segmentation": [[100, 120, 180, 120, 180, 180, 100, 180]],
}
],
}
),
encoding="utf-8",
)
node_data = {"label": "COCO 标注校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["coco_annotation_validate"]("run-1", "coco", root / "out", node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "coco-annotation-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["imageCount"], 1)
self.assertEqual(payload["existingImageCount"], 1)
self.assertEqual(payload["annotationCount"], 1)
self.assertEqual(payload["classCounts"], {"gripper": 1})
def test_yolo_detection_validate_reads_real_labels_and_images(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "yolo"
(root / "images" / "train").mkdir(parents=True)
(root / "labels" / "train").mkdir(parents=True)
(root / "images" / "train" / "frame_0001.jpg").write_bytes(b"\xff\xd8\xff\xd9")
(root / "labels" / "train" / "frame_0001.txt").write_text("0 0.500 0.500 0.250 0.200\n", encoding="utf-8")
(root / "data.yaml").write_text("names: [cube]\n", encoding="utf-8")
node_data = {"label": "YOLO 检测标注校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["yolo_detection_validate"]("run-1", "yolo", root / "out", node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "yolo-detection-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["imageCount"], 1)
self.assertEqual(payload["labelFileCount"], 1)
self.assertEqual(payload["annotationCount"], 1)
self.assertEqual(payload["classCounts"], {"cube": 1})
def test_rgbd_frame_pair_validate_reads_real_rgb_depth_and_intrinsics(self):
import cv2
import numpy as np
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "rgbd"
(root / "rgb").mkdir(parents=True)
(root / "depth").mkdir(parents=True)
color = np.zeros((4, 5, 3), dtype=np.uint8)
color[:, :, 1] = 128
depth = np.array(
[
[1000, 1100, 1200, 0, 1300],
[1000, 1100, 1200, 1300, 1400],
[900, 1000, 1100, 1200, 1300],
[800, 900, 1000, 1100, 1200],
],
dtype=np.uint16,
)
cv2.imwrite(str(root / "rgb" / "frame_0001.png"), color)
cv2.imwrite(str(root / "depth" / "frame_0001.png"), depth)
(root / "camera_intrinsics.json").write_text(
json.dumps({"fx": 525.0, "fy": 526.0, "cx": 2.0, "cy": 1.5}),
encoding="utf-8",
)
node_data = {"label": "RGB-D 帧对齐校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["rgbd_frame_pair_validate"]("run-1", "rgbd", root / "out", node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "rgbd-frame-pair-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["pairCount"], 1)
self.assertEqual(payload["shapeMismatchCount"], 0)
self.assertEqual(payload["pairs"][0]["rgbShape"], [4, 5, 3])
self.assertEqual(payload["pairs"][0]["depthShape"], [4, 5])
self.assertEqual(payload["depthMinM"], 0.8)
self.assertEqual(payload["depthMaxM"], 1.4)
self.assertTrue(payload["intrinsics"]["valid"])
def test_segmentation_mask_validate_reads_real_masks_and_images(self):
import cv2
import numpy as np
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "segmentation"
(root / "images").mkdir(parents=True)
(root / "masks").mkdir(parents=True)
image = np.zeros((3, 4, 3), dtype=np.uint8)
mask = np.array([[0, 1, 1, 2], [0, 1, 2, 2], [0, 0, 2, 2]], dtype=np.uint8)
cv2.imwrite(str(root / "images" / "frame_0001.png"), image)
cv2.imwrite(str(root / "masks" / "frame_0001.png"), mask)
node_data = {
"label": "分割 Mask 校验",
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}, {"key": "CLASS_IDS", "value": "0,1,2"}],
}
artifacts = MODULE.BUILTIN_HANDLERS["segmentation_mask_validate"]("run-1", "mask", root / "out", node_data, [])
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "segmentation-mask-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["pairCount"], 1)
self.assertEqual(payload["classPixelCounts"], {"0": 4, "1": 3, "2": 5})
self.assertEqual(payload["masks"][0]["classIds"], [0, 1, 2])
def test_language_instruction_validate_reads_real_jsonl_tasks(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "language"
(root / "meta").mkdir(parents=True)
(root / "meta" / "episodes.jsonl").write_text(
json.dumps({"episode_index": 0, "language_instruction": "pick up the red cube"}) + "\n"
+ json.dumps({"episode_index": 1, "language_instruction": "place the cube in the bin"}) + "\n"
+ json.dumps({"episode_index": 2, "language_instruction": "pick up the red cube"}) + "\n",
encoding="utf-8",
)
node_data = {"label": "语言指令校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
artifacts = MODULE.BUILTIN_HANDLERS["language_instruction_validate"](
"run-1", "language", root / "out", node_data, []
)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["datasetType"], "language-instruction-validation")
self.assertTrue(payload["valid"])
self.assertEqual(payload["rowCount"], 3)
self.assertEqual(payload["instructionCount"], 3)
self.assertEqual(payload["uniqueInstructionCount"], 2)
self.assertEqual(payload["duplicateInstructionCount"], 1)
self.assertEqual(payload["groupCount"], 3)
class _DCloudAPIHandler(http.server.BaseHTTPRequestHandler):
requests = []
def do_POST(self):
length = int(self.headers.get("Content-Length", "0") or "0")
body = self.rfile.read(length).decode("utf-8")
payload = json.loads(body) if body else {}
if self.path in {"/api/signin", "/userCenterApi/signin"}:
self.__class__.requests.append(
{
"method": "POST",
"path": self.path,
"authorization": self.headers.get("Authorization", ""),
"cookie": self.headers.get("Cookie", ""),
"d_user_id": self.headers.get("d-user-id", ""),
"org_symbol": self.headers.get("org-symbol", ""),
"sourceapp": self.headers.get("sourceapp", ""),
"body": {"hasUsername": bool(payload.get("Username")), "hasPassword": bool(payload.get("Password"))},
}
)
response_bytes = json.dumps({"status": 0, "data": {"token": "signin.header.payload"}}).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_bytes)))
self.end_headers()
self.wfile.write(response_bytes)
return
self.__class__.requests.append(
{
"method": "POST",
"path": self.path,
"authorization": self.headers.get("Authorization", ""),
"cookie": self.headers.get("Cookie", ""),
"d_user_id": self.headers.get("d-user-id", ""),
"org_symbol": self.headers.get("org-symbol", ""),
"sourceapp": self.headers.get("sourceapp", ""),
"body": payload,
}
)
response = {
"taskId": "task-" + str(len(self.__class__.requests)),
"previewURL": f"http://{self.server.server_address[0]}:{self.server.server_address[1]}/preview/{payload.get('name', 'item')}.json",
"payloadEcho": payload,
}
response_bytes = json.dumps(response).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_bytes)))
self.end_headers()
self.wfile.write(response_bytes)
def do_GET(self):
self.__class__.requests.append(
{
"method": "GET",
"path": self.path,
"authorization": self.headers.get("Authorization", ""),
"cookie": self.headers.get("Cookie", ""),
"d_user_id": self.headers.get("d-user-id", ""),
"org_symbol": self.headers.get("org-symbol", ""),
"sourceapp": self.headers.get("sourceapp", ""),
"body": None,
}
)
response = {"jobId": "job-preview", "previewUrl": f"http://{self.server.server_address[0]}:{self.server.server_address[1]}/asset.png"}
response_bytes = json.dumps(response).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_bytes)))
self.end_headers()
self.wfile.write(response_bytes)
def log_message(self, *_args):
return
@contextlib.contextmanager
def dcloud_api_server():
_DCloudAPIHandler.requests = []
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _DCloudAPIHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
previous_no_proxy = os.environ.get("NO_PROXY")
previous_no_proxy_lower = os.environ.get("no_proxy")
os.environ["NO_PROXY"] = "127.0.0.1,localhost"
os.environ["no_proxy"] = "127.0.0.1,localhost"
thread.start()
try:
yield server, _DCloudAPIHandler.requests
finally:
server.shutdown()
thread.join(timeout=5)
server.server_close()
if previous_no_proxy is None:
os.environ.pop("NO_PROXY", None)
else:
os.environ["NO_PROXY"] = previous_no_proxy
if previous_no_proxy_lower is None:
os.environ.pop("no_proxy", None)
else:
os.environ["no_proxy"] = previous_no_proxy_lower
class DCloudFATAPIOperatorTests(unittest.TestCase):
def test_dcloud_embodied_annotation_operator_calls_real_http_endpoint(self):
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "D-Cloud 具身标注 API",
"hyperparams": [
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
{"key": "ENDPOINT", "value": "/api/annoMidwareApi/v1/annotate/detection"},
{"key": "METHOD", "value": "POST"},
{"key": "REQUEST_JSON", "value": json.dumps({"name": "clip-1", "videoUrl": "http://example.test/clip.mp4"})},
{"key": "AUTH_TOKEN", "value": "real-token"},
{"key": "COOKIE", "value": "dcloud_session=real-session"},
{"key": "D_USER_ID", "value": "user-1"},
{"key": "ORG_SYMBOL", "value": "org-1"},
{"key": "SOURCEAPP", "value": "dataCopilot"},
],
}
artifacts = MODULE.builtin_dcloud_embodied_annotation("run-1", "anno", output_dir, node_data, [])
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0]["path"], "/api/annoMidwareApi/v1/annotate/detection")
self.assertEqual(requests[0]["authorization"], "Bearer real-token")
self.assertEqual(requests[0]["cookie"], "dcloud_session=real-session")
self.assertEqual(requests[0]["d_user_id"], "user-1")
self.assertEqual(requests[0]["org_symbol"], "org-1")
self.assertEqual(requests[0]["sourceapp"], "dataCopilot")
self.assertEqual(requests[0]["body"]["name"], "clip-1")
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["response"]["status"], 200)
self.assertEqual(payload["response"]["body"]["payloadEcho"]["videoUrl"], "http://example.test/clip.mp4")
self.assertTrue(payload["preview"]["urls"])
self.assertEqual(payload["sourceRepo"], "https://gitea.d-robotics.cc/d-cloud/robo_annotation_backend")
def test_dcloud_data_generation_operator_supports_preview_query(self):
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "D-Cloud 具身数据生成 API",
"hyperparams": [
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
{"key": "ENDPOINT", "value": "/api/generationApi/tasks/preview"},
{"key": "METHOD", "value": "GET"},
{"key": "QUERY_JSON", "value": json.dumps({"taskId": "task-1"})},
],
}
with mock.patch.dict(
os.environ,
{
"DCLOUD_D_USER_ID": "user-2",
"DCLOUD_ORG_SYMBOL": "org-2",
"DCLOUD_SOURCEAPP": "dataCopilot",
},
clear=False,
):
artifacts = MODULE.builtin_dcloud_data_generation("run-1", "gen", output_dir, node_data, [])
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0]["method"], "GET")
self.assertEqual(requests[0]["path"], "/api/generationApi/tasks/preview?taskId=task-1")
self.assertEqual(requests[0]["d_user_id"], "user-2")
self.assertEqual(requests[0]["org_symbol"], "org-2")
self.assertEqual(requests[0]["sourceapp"], "dataCopilot")
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["response"]["status"], 200)
self.assertEqual(payload["preview"]["ids"], ["job-preview"])
self.assertEqual(payload["sourceRepo"], "https://gitea.d-robotics.cc/d-cloud/data_generation")
def test_dcloud_operator_can_sign_in_for_real_fat_token(self):
with dcloud_api_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "D-Cloud 具身数据生成 API",
"hyperparams": [
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
{"key": "ENDPOINT", "value": "/api/generationApi/imgGeneration/tasks"},
{"key": "METHOD", "value": "POST"},
{"key": "REQUEST_JSON", "value": json.dumps({"currentPage": 1, "pageSize": 1})},
],
}
with mock.patch.dict(
os.environ,
{
"DCLOUD_FAT_USERNAME": "fat-user",
"DCLOUD_FAT_PASSWORD": "fat-password",
"DCLOUD_FAT_SIGNIN_URL": f"http://127.0.0.1:{server.server_address[1]}/userCenterApi/signin",
"DCLOUD_ORG_SYMBOL": "org-3",
"DCLOUD_SOURCEAPP": "dataCopilot",
},
clear=False,
):
artifacts = MODULE.builtin_dcloud_data_generation("run-1", "gen", output_dir, node_data, [])
self.assertEqual(len(requests), 2)
self.assertEqual(requests[0]["path"], "/userCenterApi/signin")
self.assertEqual(requests[0]["body"], {"hasUsername": True, "hasPassword": True})
self.assertEqual(requests[1]["path"], "/api/generationApi/imgGeneration/tasks")
self.assertEqual(requests[1]["authorization"], "Bearer signin.header.payload")
self.assertEqual(requests[1]["org_symbol"], "org-3")
self.assertEqual(requests[1]["sourceapp"], "dataCopilot")
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["response"]["status"], 200)
if __name__ == "__main__":
unittest.main()