forked from longtao.wu/droboticflow
6798 lines
330 KiB
Python
6798 lines
330 KiB
Python
import contextlib
|
|
import hashlib
|
|
import http.client
|
|
import http.server
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import pickle
|
|
import sqlite3
|
|
import struct
|
|
import sys
|
|
import tarfile
|
|
import threading
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
MODULE_PATH = pathlib.Path(__file__).with_name("main.py")
|
|
SPEC = importlib.util.spec_from_file_location("operator_runner_main", MODULE_PATH)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
MODULE = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(MODULE)
|
|
|
|
RESOURCE_MONITOR_PATH = pathlib.Path(__file__).with_name("resource_monitor.py")
|
|
RESOURCE_MONITOR_SPEC = importlib.util.spec_from_file_location("operator_runner_resource_monitor", RESOURCE_MONITOR_PATH)
|
|
assert RESOURCE_MONITOR_SPEC is not None and RESOURCE_MONITOR_SPEC.loader is not None
|
|
RESOURCE_MONITOR = importlib.util.module_from_spec(RESOURCE_MONITOR_SPEC)
|
|
RESOURCE_MONITOR_SPEC.loader.exec_module(RESOURCE_MONITOR)
|
|
|
|
|
|
class _FakeHTTPResponse:
|
|
def __init__(self, payload: bytes):
|
|
self._payload = payload
|
|
self._offset = 0
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def read(self, *_args, **_kwargs):
|
|
if self._offset >= len(self._payload):
|
|
return b""
|
|
chunk = self._payload[self._offset:]
|
|
self._offset = len(self._payload)
|
|
return chunk
|
|
|
|
|
|
class _FakeBosClient:
|
|
def __init__(self):
|
|
self.uploads = []
|
|
self.downloads = []
|
|
|
|
def put_object_from_file(self, bucket, key, file_name, **kwargs):
|
|
self.uploads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"key": key,
|
|
"file_name": file_name,
|
|
"content_type": kwargs.get("content_type"),
|
|
}
|
|
)
|
|
|
|
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
|
|
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}"
|
|
|
|
def get_object_to_file(self, bucket, key, file_name, **_kwargs):
|
|
self.downloads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"key": key,
|
|
"file_name": file_name,
|
|
}
|
|
)
|
|
pathlib.Path(file_name).write_bytes(f"{bucket}/{key}".encode("utf-8"))
|
|
|
|
|
|
class _FakeBosClientBytesURL(_FakeBosClient):
|
|
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
|
|
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}".encode("utf-8")
|
|
|
|
|
|
class _FakeBosListClient(_FakeBosClient):
|
|
def __init__(self, objects, object_payloads):
|
|
super().__init__()
|
|
self._objects = objects
|
|
self._object_payloads = object_payloads
|
|
self.downloaded_payload_keys = []
|
|
|
|
def list_all_objects(self, bucket, prefix=None):
|
|
return list(self._objects)
|
|
|
|
def get_object_as_string(self, bucket, key):
|
|
self.downloaded_payload_keys.append((bucket, key))
|
|
return self._object_payloads[key]
|
|
|
|
|
|
class CommandPolicyTests(unittest.TestCase):
|
|
def test_command_policy_rejects_commands_by_default(self):
|
|
with mock.patch.dict(os.environ, {}, clear=True):
|
|
with self.assertRaisesRegex(RuntimeError, "disabled"):
|
|
MODULE.validate_command_policy("printf 'ok'")
|
|
|
|
def test_command_policy_allows_commands_when_explicitly_enabled(self):
|
|
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "true"}, clear=True):
|
|
MODULE.validate_command_policy("printf 'ok'")
|
|
|
|
def test_command_policy_can_disable_shell_commands(self):
|
|
with mock.patch.dict(os.environ, {"OPERATOR_ALLOW_SHELL_COMMANDS": "false"}, clear=True):
|
|
with self.assertRaisesRegex(RuntimeError, "disabled"):
|
|
MODULE.validate_command_policy("printf 'ok'")
|
|
|
|
def test_command_policy_rejects_denylisted_command(self):
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_DENYLIST_REGEX": r"(^|\s)rm\s+-rf"},
|
|
clear=True,
|
|
):
|
|
with self.assertRaisesRegex(RuntimeError, "DENYLIST"):
|
|
MODULE.validate_command_policy("rm -rf /workspace")
|
|
|
|
def test_command_policy_requires_allowlist_match_when_configured(self):
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{"OPERATOR_ALLOW_SHELL_COMMANDS": "true", "OPERATOR_COMMAND_ALLOWLIST_REGEX": r"^python3?\s"},
|
|
clear=True,
|
|
):
|
|
MODULE.validate_command_policy("python3 script.py")
|
|
with self.assertRaisesRegex(RuntimeError, "ALLOWLIST"):
|
|
MODULE.validate_command_policy("bash script.sh")
|
|
|
|
|
|
class ResourceMonitorTests(unittest.TestCase):
|
|
def test_parse_gpu_snapshot_output_summarizes_real_nvidia_smi_columns(self):
|
|
snapshot = RESOURCE_MONITOR.parse_gpu_snapshot_output(
|
|
"\n".join(
|
|
[
|
|
"12, 4096, 24576, NVIDIA L4",
|
|
"83, 8192, 24576, NVIDIA L4",
|
|
"not-a-number, 1, 1, broken",
|
|
]
|
|
)
|
|
)
|
|
|
|
self.assertEqual(snapshot["gpu"], 83)
|
|
self.assertEqual(snapshot["gpuMemoryUsedMiB"], 12288)
|
|
self.assertEqual(snapshot["gpuMemoryTotalMiB"], 49152)
|
|
self.assertEqual(snapshot["gpuName"], "NVIDIA L4")
|
|
self.assertEqual(snapshot["gpuCount"], 2)
|
|
|
|
def test_parse_gpu_snapshot_output_defaults_when_nvidia_smi_has_no_valid_rows(self):
|
|
self.assertEqual(RESOURCE_MONITOR.parse_gpu_snapshot_output(""), {"gpu": 0})
|
|
|
|
def test_gpu_snapshot_falls_back_to_nvml_when_nvidia_smi_is_missing(self):
|
|
class _Memory:
|
|
used = 3 * 1024 * 1024 * 1024
|
|
total = 80 * 1024 * 1024 * 1024
|
|
|
|
class _Utilization:
|
|
gpu = 42
|
|
|
|
class _NVML:
|
|
@staticmethod
|
|
def nvmlInit():
|
|
return None
|
|
|
|
@staticmethod
|
|
def nvmlShutdown():
|
|
return None
|
|
|
|
@staticmethod
|
|
def nvmlDeviceGetCount():
|
|
return 1
|
|
|
|
@staticmethod
|
|
def nvmlDeviceGetHandleByIndex(index):
|
|
return f"gpu-{index}"
|
|
|
|
@staticmethod
|
|
def nvmlDeviceGetMemoryInfo(handle):
|
|
return _Memory()
|
|
|
|
@staticmethod
|
|
def nvmlDeviceGetUtilizationRates(handle):
|
|
return _Utilization()
|
|
|
|
@staticmethod
|
|
def nvmlDeviceGetName(handle):
|
|
return b"NVIDIA A800-SXM4-80GB"
|
|
|
|
with mock.patch("shutil.which", return_value=None), mock.patch.dict("sys.modules", {"pynvml": _NVML}):
|
|
snapshot = RESOURCE_MONITOR.gpu_snapshot()
|
|
|
|
self.assertEqual(snapshot["gpu"], 42)
|
|
self.assertEqual(snapshot["gpuMemoryUsedMiB"], 3072)
|
|
self.assertEqual(snapshot["gpuMemoryTotalMiB"], 81920)
|
|
self.assertEqual(snapshot["gpuName"], "NVIDIA A800-SXM4-80GB")
|
|
self.assertEqual(snapshot["gpuCount"], 1)
|
|
|
|
def test_resource_sampler_keeps_execution_time_gpu_samples(self):
|
|
started_at = "2026-06-29T00:00:00Z"
|
|
ended_at = "2026-06-29T00:00:03Z"
|
|
with mock.patch.object(
|
|
RESOURCE_MONITOR,
|
|
"resource_sample",
|
|
return_value={
|
|
"timestamp": "2026-06-29T00:00:01Z",
|
|
"cpu": 7,
|
|
"memory": 512,
|
|
"gpu": 66,
|
|
"gpuMemoryUsedMiB": 2048,
|
|
"gpuMemoryTotalMiB": 81920,
|
|
"gpuName": "NVIDIA A800-SXM4-80GB",
|
|
"gpuCount": 1,
|
|
},
|
|
), mock.patch.object(RESOURCE_MONITOR, "resource_snapshot", return_value=(1.0, 640)), mock.patch.object(
|
|
RESOURCE_MONITOR, "gpu_snapshot", return_value={"gpu": 0}
|
|
):
|
|
sampler = RESOURCE_MONITOR.ResourceSampler(0.0, 256, started_at, interval_seconds=0.01)
|
|
sampler.start()
|
|
time.sleep(0.03)
|
|
series = sampler.stop(ended_at)
|
|
|
|
self.assertTrue(any(sample.get("gpu") == 66 for sample in series))
|
|
self.assertTrue(any(sample.get("gpuMemoryTotalMiB") == 81920 for sample in series))
|
|
|
|
|
|
class SourcePathFromItemTests(unittest.TestCase):
|
|
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "node-output"
|
|
workspace_root = pathlib.Path(temp_dir) / "workspace"
|
|
output_dir.mkdir(parents=True)
|
|
workspace_root.mkdir(parents=True)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"previewURL": "https://example.com/should-not-download.mp4",
|
|
}
|
|
}
|
|
fake_client = _FakeBosClient()
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"WORKSPACE_ROOT": str(workspace_root),
|
|
"BOS_ENDPOINT": "https://bos.example.test",
|
|
"BOS_ACCESS_KEY_ID": "ak",
|
|
"BOS_SECRET_ACCESS_KEY": "sk",
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")) as build_client, mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
) as urlopen:
|
|
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
|
|
self.assertTrue(first.exists())
|
|
self.assertEqual(first, second)
|
|
self.assertIn(str(workspace_root), str(first))
|
|
self.assertEqual(len(fake_client.downloads), 1)
|
|
build_client.assert_called()
|
|
urlopen.assert_not_called()
|
|
|
|
def test_downloads_http_preview_url_to_local_file(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/demo.mp4",
|
|
"previewURL": "https://example.com/demo.mp4?signature=abc",
|
|
}
|
|
}
|
|
with mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
return_value=_FakeHTTPResponse(b"real-video-bytes"),
|
|
) as urlopen:
|
|
resolved = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
self.assertTrue(resolved.exists())
|
|
self.assertEqual(resolved.read_bytes(), b"real-video-bytes")
|
|
urlopen.assert_called_once()
|
|
|
|
def test_prefers_workspace_cache_path_for_preview_url_when_bos_credentials_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "node-output"
|
|
workspace_root = pathlib.Path(temp_dir) / "workspace"
|
|
output_dir.mkdir(parents=True)
|
|
workspace_root.mkdir(parents=True)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0000/DJI_001/demo.mp4",
|
|
"previewURL": "https://example.com/demo.mp4?signature=abc",
|
|
}
|
|
}
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"WORKSPACE_ROOT": str(workspace_root),
|
|
"BOS_ENDPOINT": "",
|
|
"BOS_ACCESS_KEY_ID": "",
|
|
"BOS_SECRET_ACCESS_KEY": "",
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
return_value=_FakeHTTPResponse(b"real-video-bytes"),
|
|
) as urlopen:
|
|
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
self.assertTrue(first.exists())
|
|
self.assertEqual(first, second)
|
|
self.assertEqual(first.read_bytes(), b"real-video-bytes")
|
|
self.assertIn(str(workspace_root / "_bos_cache"), str(first))
|
|
urlopen.assert_called_once()
|
|
|
|
def test_remote_video_url_for_source_signs_bos_path_when_preview_missing(self):
|
|
source = {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_0001_D.MP4",
|
|
"previewURL": "",
|
|
}
|
|
with mock.patch.object(MODULE, "signed_bos_source_url", return_value="https://signed.example.com/video.mp4") as signer:
|
|
resolved = MODULE.remote_video_url_for_source(source)
|
|
self.assertEqual(resolved, "https://signed.example.com/video.mp4")
|
|
signer.assert_called_once_with(source["path"])
|
|
|
|
|
|
class RuntimeFileContractTests(unittest.TestCase):
|
|
def test_dispatch_requires_explicit_current_handler(self):
|
|
with mock.patch.dict(os.environ, {"DROBOTICFLOW_OPERATOR_HANDLER": ""}):
|
|
with self.assertRaisesRegex(RuntimeError, "is required"):
|
|
MODULE.runtime_handler()
|
|
with mock.patch.dict(os.environ, {"DROBOTICFLOW_OPERATOR_HANDLER": "undeclared-handler"}):
|
|
with self.assertRaisesRegex(RuntimeError, "is not registered"):
|
|
MODULE.runtime_handler()
|
|
with mock.patch.dict(os.environ, {"DROBOTICFLOW_OPERATOR_HANDLER": "builtin_source"}):
|
|
self.assertEqual(MODULE.runtime_handler(), "builtin_source")
|
|
|
|
def test_registry_uses_only_current_catalog_handler_names(self):
|
|
catalog = json.loads((MODULE.PROJECT_ROOT / "internal" / "api" / "builtin_operators_catalog.json").read_text())
|
|
for item in catalog:
|
|
if item.get("type") == "builtin":
|
|
self.assertIn(item["runtimeHandler"], MODULE.BUILTIN_HANDLERS)
|
|
self.assertNotIn("csv_to_json", MODULE.BUILTIN_HANDLERS)
|
|
self.assertNotIn("builtin_lerobot", MODULE.BUILTIN_HANDLERS)
|
|
|
|
def test_operator_helpers_do_not_expose_legacy_builtin_handlers(self):
|
|
legacy_keys = sorted(key for key in MODULE.OPERATOR_HELPERS if key.startswith("builtin_"))
|
|
self.assertEqual(legacy_keys, [])
|
|
|
|
def test_dispatch_runs_standalone_csv_to_json_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "input.csv"
|
|
source_path.write_text("id,name,score\n1,alpha,98\n2,beta,87\n", encoding="utf-8")
|
|
output_dir = root / "out"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "CSV 转 JSON",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(source_path)}],
|
|
"outputs": [{"id": "out-1", "name": "table", "type": "json"}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_csv_to_json"]("run-1", "csv", output_dir, node_data, [])
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["columns"], ["id", "name", "score"])
|
|
self.assertEqual(payload["rowCount"], 2)
|
|
self.assertEqual(payload["items"][1]["name"], "beta")
|
|
|
|
def test_dispatch_runs_standalone_json_filter_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 过滤",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "EXPECTED", "value": "ready"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_json_filter"](
|
|
"run-1",
|
|
"filter",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"status": "ready"}, {"status": "blocked"}]}}],
|
|
)
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(payload["items"], [{"status": "ready"}])
|
|
|
|
def test_dispatch_runs_named_human_review_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "人工审核",
|
|
"hyperparams": [
|
|
{"key": "REVIEW_TYPE", "value": "task"},
|
|
{"key": "REVIEWER_ROLE", "value": "qa_reviewer"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_human_review"](
|
|
"run-1",
|
|
"review",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"id": "clip-1", "score": 0.9}]}}],
|
|
)
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["datasetType"], "human-review-tasks")
|
|
self.assertEqual(payload["reviewerRole"], "qa_reviewer")
|
|
self.assertEqual(payload["taskCount"], 1)
|
|
self.assertEqual(payload["items"][0]["id"], "run-1-clip-1")
|
|
|
|
def test_load_input_artifacts_prefers_manifest_file(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
manifest_path = pathlib.Path(temp_dir) / "input-artifacts.json"
|
|
manifest_path.write_text('[{"id":"artifact-from-file"}]', encoding="utf-8")
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"INPUT_ARTIFACTS_PATH": str(manifest_path),
|
|
"INPUT_ARTIFACTS_JSON": '[{"id":"artifact-inline"}]',
|
|
},
|
|
clear=False,
|
|
):
|
|
loaded = MODULE.load_input_artifacts()
|
|
|
|
self.assertEqual(len(loaded), 1)
|
|
self.assertEqual(loaded[0]["id"], "artifact-from-file")
|
|
|
|
def test_load_input_bindings_preserves_target_port_and_payload(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
manifest_path = pathlib.Path(temp_dir) / "input-bindings.json"
|
|
manifest_path.write_text(
|
|
'[{"targetPortId":"config","workflowInputId":"config","payload":{"mode":"inline","type":"json","inline":{"ok":true}}}]',
|
|
encoding="utf-8",
|
|
)
|
|
with mock.patch.dict(os.environ, {"INPUT_BINDINGS_PATH": str(manifest_path)}, clear=False):
|
|
loaded = MODULE.load_input_bindings()
|
|
|
|
self.assertEqual(loaded[0]["targetPortId"], "config")
|
|
self.assertTrue(loaded[0]["payload"]["inline"]["ok"])
|
|
|
|
def test_main_writes_result_and_stage_manifest_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
node_root = pathlib.Path(temp_dir)
|
|
output_dir = node_root / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
input_artifacts_path = node_root / "input-artifacts.json"
|
|
input_artifacts_path.write_text("[]", encoding="utf-8")
|
|
result_path = node_root / "result.json"
|
|
stage_manifest_path = node_root / "stage-manifest.json"
|
|
artifacts = [
|
|
{
|
|
"id": "artifact-1",
|
|
"runId": "run-1",
|
|
"nodeId": "node-1",
|
|
"name": "report.json",
|
|
"type": "json",
|
|
"size": "12 B",
|
|
"path": str(output_dir / "report.json"),
|
|
}
|
|
]
|
|
|
|
class _FakeResourceSampler:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
def start(self):
|
|
pass
|
|
|
|
def stop(self, ended_at):
|
|
return []
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"RUN_ID": "run-1",
|
|
"WORKFLOW_ID": "wf-1",
|
|
"NODE_ID": "node-1",
|
|
"NODE_LABEL": "通用处理",
|
|
"DROBOTICFLOW_OPERATOR_HANDLER": "builtin_generic",
|
|
"OUTPUT_DIR": str(output_dir),
|
|
"NODE_DATA_JSON": '{"label":"通用处理","hyperparams":[]}',
|
|
"INPUT_ARTIFACTS_PATH": str(input_artifacts_path),
|
|
"RESULT_OUTPUT_PATH": str(result_path),
|
|
"STAGE_MANIFEST_PATH": str(stage_manifest_path),
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(MODULE, "builtin_generic", return_value=artifacts), mock.patch.object(MODULE, "ResourceSampler", _FakeResourceSampler):
|
|
stdout = io.StringIO()
|
|
with contextlib.redirect_stdout(stdout):
|
|
exit_code = MODULE.main()
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertIn(MODULE.RESULT_PATH_PREFIX + str(result_path), stdout.getvalue())
|
|
result_payload = json.loads(result_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(result_payload["artifacts"][0]["id"], "artifact-1")
|
|
stage_manifest = json.loads(stage_manifest_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(stage_manifest["artifactCount"], 1)
|
|
self.assertEqual(stage_manifest["nodeLabel"], "通用处理")
|
|
|
|
def test_write_json_artifact_externalizes_dataset_payload_to_data_ref(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"dataset-index.json",
|
|
{
|
|
"datasetType": "x-platform-index",
|
|
"collectionCount": 1,
|
|
"collections": [{"id": "collection-1"}],
|
|
"items": [{"id": "item-1"}, {"id": "item-2"}],
|
|
},
|
|
)
|
|
|
|
data_ref = artifact["data"]["dataRef"]
|
|
self.assertEqual(data_ref["kind"], "dataset")
|
|
self.assertTrue(pathlib.Path(data_ref["manifestPath"]).exists())
|
|
self.assertTrue(pathlib.Path(data_ref["payloadPath"]).exists())
|
|
manifest = json.loads(pathlib.Path(data_ref["manifestPath"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["datasetType"], "x-platform-index")
|
|
self.assertIn("items", manifest["partitionFiles"])
|
|
self.assertIn("collections", manifest["partitionFiles"])
|
|
|
|
def test_load_artifact_payload_reads_externalized_payload_from_data_ref(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"filtered-videos.json",
|
|
{
|
|
"datasetType": "x-platform-filtered",
|
|
"count": 2,
|
|
"items": [{"id": "v1"}, {"id": "v2"}],
|
|
},
|
|
)
|
|
|
|
loaded = MODULE.load_artifact_payload(artifact)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform-filtered")
|
|
self.assertEqual(len(loaded["items"]), 2)
|
|
|
|
def test_load_payload_from_file_retries_transient_json_decode_error(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
payload_path = pathlib.Path(temp_dir) / "payload.json"
|
|
payload_path.write_text('{"items":[{"id":"v1"}]}', encoding="utf-8")
|
|
decode_error = json.JSONDecodeError("transient", "", 0)
|
|
|
|
with mock.patch.object(
|
|
MODULE.json,
|
|
"load",
|
|
side_effect=[decode_error, {"datasetType": "x-platform", "items": [{"id": "v1"}]}],
|
|
) as json_load, mock.patch.object(MODULE.time, "sleep") as sleep_mock:
|
|
loaded = MODULE.load_payload_from_file(payload_path)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform")
|
|
self.assertEqual(len(loaded["items"]), 1)
|
|
self.assertEqual(json_load.call_count, 2)
|
|
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
|
|
|
|
def test_load_payload_from_file_retries_until_payload_path_exists(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
payload_path = pathlib.Path(temp_dir) / "payload.json"
|
|
payload_path.write_text('{"datasetType":"x-platform","items":[{"id":"v1"}]}', encoding="utf-8")
|
|
original_exists = pathlib.Path.exists
|
|
seen = {"count": 0}
|
|
|
|
def fake_exists(path_obj: pathlib.Path):
|
|
if path_obj == payload_path and seen["count"] == 0:
|
|
seen["count"] += 1
|
|
return False
|
|
return original_exists(path_obj)
|
|
|
|
with mock.patch.object(pathlib.Path, "exists", autospec=True, side_effect=fake_exists), mock.patch.object(
|
|
MODULE.time,
|
|
"sleep",
|
|
) as sleep_mock:
|
|
loaded = MODULE.load_payload_from_file(payload_path)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform")
|
|
self.assertEqual(len(loaded["items"]), 1)
|
|
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
|
|
|
|
def test_stage_manifest_includes_data_refs(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"packages-manifest.json",
|
|
{
|
|
"datasetType": "x-platform-packages",
|
|
"packageCount": 1,
|
|
"items": [{"packageName": "pkg-1"}],
|
|
},
|
|
)
|
|
|
|
manifest = MODULE.stage_manifest_payload("run-1", "wf-1", "node-1", "成品打包", output_dir, [artifact])
|
|
|
|
self.assertEqual(len(manifest["dataRefs"]), 1)
|
|
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
|
|
|
|
def test_infer_data_metrics_uses_bound_stage_and_preserves_unknowns(self):
|
|
metrics = MODULE.infer_data_metrics(
|
|
{"businessStage": "clean", "metricBoundary": True},
|
|
[{"name": "filtered.json", "data": {"count": 8, "fileCount": 16}}],
|
|
[{"name": "source.json", "data": {"count": 10, "fileCount": 20}}],
|
|
)
|
|
|
|
self.assertEqual(metrics["stage"], "clean")
|
|
self.assertEqual(metrics["input"]["trajectoryCount"], 10)
|
|
self.assertEqual(metrics["output"]["trajectoryCount"], 8)
|
|
self.assertEqual(metrics["rejected"]["trajectoryCount"], 2)
|
|
self.assertEqual(metrics["quality"], "derived")
|
|
|
|
def test_stage_manifest_includes_standard_data_metrics(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
metrics = {
|
|
"schemaVersion": 1,
|
|
"stage": "collect",
|
|
"output": {"trajectoryCount": 12},
|
|
"quality": "exact",
|
|
}
|
|
manifest = MODULE.stage_manifest_payload(
|
|
"run-1",
|
|
"wf-1",
|
|
"node-1",
|
|
"数据采集",
|
|
output_dir,
|
|
[],
|
|
metrics,
|
|
)
|
|
|
|
self.assertEqual(manifest["dataMetrics"]["stage"], "collect")
|
|
self.assertEqual(manifest["dataMetrics"]["output"]["trajectoryCount"], 12)
|
|
|
|
|
|
class XPlatformIndexingTests(unittest.TestCase):
|
|
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
|
|
manifest_path.write_text("{}", encoding="utf-8")
|
|
|
|
payload = MODULE.load_xplatform_index_payload(
|
|
{},
|
|
[
|
|
{
|
|
"path": str(manifest_path),
|
|
"data": {
|
|
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"entries": [
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "工厂",
|
|
"place": "绿控传动",
|
|
"date": "20260330",
|
|
},
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260327132547_0003_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2880",
|
|
"frameRate": 29.97,
|
|
"duration": 0.6,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
}
|
|
],
|
|
output_dir,
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["videoCount"], 1)
|
|
self.assertEqual(payload["collections"][0]["meta"]["place"], "绿控传动")
|
|
|
|
def test_load_index_payload_raises_when_source_manifest_summary_has_no_entries(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
|
|
manifest_path.write_text("{}", encoding="utf-8")
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "source-manifest"):
|
|
MODULE.load_xplatform_index_payload(
|
|
{},
|
|
[
|
|
{
|
|
"name": "source-manifest.json",
|
|
"path": str(manifest_path),
|
|
"data": {
|
|
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"entriesCount": 59,
|
|
"dataRef": {"payloadPath": str(manifest_path)},
|
|
},
|
|
}
|
|
],
|
|
output_dir,
|
|
)
|
|
|
|
def test_manifest_entries_support_video_metadata_alias(self):
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "工厂",
|
|
"place": "绿控传动",
|
|
"date": "20260330",
|
|
},
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260327132547_0003_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2880",
|
|
"frameRate": 29.97,
|
|
"duration": 0.6,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
},
|
|
},
|
|
],
|
|
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
def test_manifest_entries_load_local_meta_when_inline_data_is_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "algo" / "X-platform-test"
|
|
collection = root / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
|
|
video_dir = collection / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
meta_path = collection / "meta.json"
|
|
video_meta_path = collection / "video_meta.json"
|
|
video_path = video_dir / "DJI_0001_D.mp4"
|
|
meta_path.write_text('{"type":"家庭","place":"蓟门里小区","date":"20260319"}', encoding="utf-8")
|
|
video_meta_path.write_text(
|
|
'[{"filename":"DJI_0001_D.mp4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
|
|
encoding="utf-8",
|
|
)
|
|
video_path.write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
|
|
"sizeBytes": meta_path.stat().st_size,
|
|
"mimeType": "application/json",
|
|
"source": {"path": str(meta_path)},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
|
|
"sizeBytes": video_meta_path.stat().st_size,
|
|
"mimeType": "application/json",
|
|
"source": {"path": str(video_meta_path)},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": video_path.stat().st_size,
|
|
"mimeType": "video/mp4",
|
|
"source": {"path": str(video_path)},
|
|
},
|
|
],
|
|
str(root),
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
|
|
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
def test_manifest_entries_support_root_level_meta_and_video_meta(self):
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "家庭",
|
|
"place": "蓟门里小区",
|
|
"date": "20260319",
|
|
},
|
|
},
|
|
{
|
|
"path": "video_meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2160",
|
|
"frameRate": 29.97,
|
|
"duration": 41.8,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/video.mp4",
|
|
},
|
|
},
|
|
],
|
|
"bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(collection["relativePath"], "")
|
|
self.assertEqual(collection["videoCount"], 1)
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertEqual(collection["meta"]["place"], "蓟门里小区")
|
|
|
|
item = payload["items"][0]
|
|
self.assertEqual(item["collectionName"], collection["name"])
|
|
self.assertEqual(item["collectionRelativePath"], "")
|
|
self.assertEqual(item["videoFolder"], "DJI_001")
|
|
self.assertEqual(item["codec"], "hevc")
|
|
self.assertEqual(item["meta"]["type"], "家庭")
|
|
|
|
def test_local_dataset_root_with_meta_is_treated_as_collection_root(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
|
|
video_dir = root / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
(root / "meta.json").write_text('{"type":"家庭","place":"蓟门里小区"}', encoding="utf-8")
|
|
(root / "video_meta.json").write_text(
|
|
'[{"filename":"DJI_20260318112834_0001_D.MP4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
|
|
encoding="utf-8",
|
|
)
|
|
(video_dir / "DJI_20260318112834_0001_D.MP4").write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_dataset(root)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["name"], root.name)
|
|
self.assertEqual(collection["relativePath"], "")
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
item = payload["items"][0]
|
|
self.assertEqual(item["collectionName"], root.name)
|
|
self.assertEqual(item["collectionRelativePath"], "")
|
|
self.assertEqual(item["codec"], "hevc")
|
|
|
|
def test_local_dataset_root_supports_video_metadata_alias(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "SZ_000_0005_OsmoNano_2026-03-30_16-15-11"
|
|
video_dir = root / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
(root / "meta.json").write_text('{"type":"工厂","place":"绿控传动"}', encoding="utf-8")
|
|
(root / "video_metadata.json").write_text(
|
|
'[{"filename":"DJI_20260327132547_0003_D.MP4","codec":"hevc","resolution":"3840x2880","frameRate":29.97,"duration":0.6}]',
|
|
encoding="utf-8",
|
|
)
|
|
(video_dir / "DJI_20260327132547_0003_D.MP4").write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_dataset(root)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
|
|
class SourceOperatorTests(unittest.TestCase):
|
|
def test_builtin_source_rejects_missing_input_configuration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {"label": "真实数据源", "hyperparams": []}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "requires SOURCE_TEXT"):
|
|
MODULE.builtin_source("run-test", "source-empty", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_infers_local_source_path_without_data_source(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
source_root = pathlib.Path(temp_dir) / "source"
|
|
source_root.mkdir()
|
|
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
|
|
output_dir = pathlib.Path(temp_dir) / "output"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "真实数据源-本地路径",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(source_root)},
|
|
],
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-local", output_dir, node_data, [])
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["dataSource"], "LOCAL")
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["path"], "payload.json")
|
|
|
|
def test_builtin_source_rejects_missing_local_source_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
missing_path = pathlib.Path(temp_dir) / "missing"
|
|
node_data = {
|
|
"label": "真实数据源-缺失本地路径",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(missing_path)},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SOURCE_PATH does not exist"):
|
|
MODULE.builtin_source("run-test", "source-missing", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_defaults_to_inline_when_source_json_is_present(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-默认内联",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-inline-default", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["dataSource"], "INLINE")
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
|
|
|
|
def test_source_handler_can_emit_inline_json_payload_on_named_output(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-内联JSON",
|
|
"outputs": [
|
|
{"id": "manifest", "name": "manifest", "type": "json"},
|
|
{"id": "payload", "name": "payload", "type": "json"},
|
|
],
|
|
"hyperparams": [
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "real"}]}, ensure_ascii=False)},
|
|
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
|
|
{"key": "OUTPUT_NAME", "value": "payload.json"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_source"]("run-test", "source-json-payload", output_dir, node_data, [])
|
|
|
|
payload_artifact = next(item for item in artifacts if item["name"] == "payload.json")
|
|
payload = json.loads(pathlib.Path(payload_artifact["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["items"][0]["kind"], "real")
|
|
self.assertEqual(payload_artifact["portId"], "payload")
|
|
|
|
def test_source_handler_can_emit_top_level_inline_json_list_payload(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-内联JSON列表",
|
|
"outputs": [
|
|
{"id": "manifest", "name": "manifest", "type": "json"},
|
|
{"id": "payload", "name": "payload", "type": "json"},
|
|
],
|
|
"hyperparams": [
|
|
{
|
|
"key": "SOURCE_JSON",
|
|
"value": json.dumps(
|
|
[
|
|
{"id": "sample-1", "kind": "real"},
|
|
{"id": "sample-2", "kind": "real"},
|
|
],
|
|
ensure_ascii=False,
|
|
),
|
|
},
|
|
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
|
|
{"key": "OUTPUT_NAME", "value": "payload.json"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_source"]("run-test", "source-json-list-payload", output_dir, node_data, [])
|
|
|
|
payload_artifact = next(item for item in artifacts if item["name"] == "payload.json")
|
|
payload = json.loads(pathlib.Path(payload_artifact["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload[0]["id"], "sample-1")
|
|
self.assertEqual(payload_artifact["data"]["itemCount"], 2)
|
|
self.assertEqual(MODULE.collect_items([MODULE.load_artifact_payload(payload_artifact)])[1]["id"], "sample-2")
|
|
|
|
def test_source_handler_can_emit_inline_text_payload_with_tabular_preview(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-内联CSV",
|
|
"outputs": [
|
|
{"id": "manifest", "name": "manifest", "type": "json"},
|
|
{"id": "csv", "name": "csv", "type": "file"},
|
|
],
|
|
"hyperparams": [
|
|
{"key": "SOURCE_TEXT", "value": "id,label\nclip-a,keep\n"},
|
|
{"key": "PAYLOAD_OUTPUT_ID", "value": "csv"},
|
|
{"key": "OUTPUT_NAME", "value": "report.csv"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_source"]("run-test", "source-csv-payload", output_dir, node_data, [])
|
|
|
|
csv_artifact = next(item for item in artifacts if item["name"] == "report.csv")
|
|
self.assertEqual(csv_artifact["portId"], "csv")
|
|
self.assertEqual(csv_artifact["data"]["preview"]["kind"], "table")
|
|
self.assertEqual(csv_artifact["data"]["preview"]["rowCount"], 1)
|
|
|
|
def test_source_handler_rejects_unsafe_inline_payload_name(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-不安全输出名",
|
|
"outputs": [
|
|
{"id": "manifest", "name": "manifest", "type": "json"},
|
|
{"id": "payload", "name": "payload", "type": "json"},
|
|
],
|
|
"hyperparams": [
|
|
{"key": "SOURCE_JSON", "value": "{}"},
|
|
{"key": "PAYLOAD_OUTPUT_ID", "value": "payload"},
|
|
{"key": "OUTPUT_NAME", "value": "../payload.json"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "OUTPUT_NAME"):
|
|
MODULE.BUILTIN_HANDLERS["builtin_source"]("run-test", "source-unsafe-payload", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_prefers_source_json_over_bos_env(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-JSON",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-json", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
|
|
|
|
def test_builtin_source_prefers_source_text_over_bos_env(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-文本",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_TEXT", "value": "alpha\nbeta\ngamma"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-text", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["text"], "alpha\nbeta\ngamma")
|
|
|
|
def test_directory_incremental_source_filter_emits_full_new_directory(self):
|
|
entries = [
|
|
{"path": "batch-a/meta.json", "sizeBytes": 10, "sha256": "sha-a-meta"},
|
|
{"path": "batch-a/DJI_001/clip-a.mp4", "sizeBytes": 20, "sha256": "sha-a-clip"},
|
|
{"path": "batch-b/meta.json", "sizeBytes": 30, "sha256": "sha-b-meta"},
|
|
{"path": "batch-b/DJI_001/clip-b.mp4", "sizeBytes": 40, "sha256": "sha-b-clip"},
|
|
]
|
|
emitted, checkpoint = MODULE.apply_incremental_source_filter(
|
|
entries,
|
|
"BOS",
|
|
"wjzh-data-migration",
|
|
{
|
|
"SOURCE_STATE_JSON": json.dumps(
|
|
{
|
|
"seenFingerprints": ["batch-a"],
|
|
"incrementalUnit": "DIRECTORY",
|
|
"directoryDepth": 1,
|
|
}
|
|
),
|
|
"SOURCE_FINGERPRINT": "fp-directory",
|
|
"INCREMENTAL_UNIT": "DIRECTORY",
|
|
"INCREMENTAL_DIRECTORY_DEPTH": "1",
|
|
},
|
|
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
)
|
|
|
|
self.assertEqual([item["path"] for item in emitted], ["batch-b/meta.json", "batch-b/DJI_001/clip-b.mp4"])
|
|
self.assertEqual([item["incrementalKey"] for item in emitted], ["batch-b", "batch-b"])
|
|
self.assertEqual(checkpoint["incrementalUnit"], "DIRECTORY")
|
|
self.assertEqual(checkpoint["directoryDepth"], 1)
|
|
self.assertEqual(checkpoint["newCount"], 2)
|
|
self.assertEqual(sorted(checkpoint["seenFingerprints"]), ["batch-a", "batch-b"])
|
|
|
|
def test_incremental_source_state_rejects_invalid_json(self):
|
|
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be valid JSON"):
|
|
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "{"})
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SOURCE_STATE_JSON must be a JSON object"):
|
|
MODULE.incremental_source_state({"SOURCE_STATE_JSON": "[]"})
|
|
|
|
def test_incremental_source_state_rejects_invalid_shape(self):
|
|
cases = [
|
|
({"seenFingerprints": "fp-1"}, "seenFingerprints must be an array"),
|
|
({"seenFingerprints": [42]}, r"seenFingerprints\[0\] must be a string"),
|
|
({"incrementalUnit": "ROW"}, "incrementalUnit must be OBJECT or DIRECTORY"),
|
|
({"directoryDepth": 1.5}, "directoryDepth must be an integer"),
|
|
({"directoryDepth": 0}, "directoryDepth must be greater than 0"),
|
|
]
|
|
for payload, expected_error in cases:
|
|
with self.subTest(payload=payload), self.assertRaisesRegex(RuntimeError, expected_error):
|
|
MODULE.incremental_source_state({"SOURCE_STATE_JSON": json.dumps(payload)})
|
|
|
|
def test_incremental_directory_depth_rejects_lossy_values(self):
|
|
for raw in ["0", "01", "1.5", "2days"]:
|
|
with self.subTest(raw=raw), self.assertRaises(RuntimeError):
|
|
MODULE.incremental_directory_depth({"INCREMENTAL_DIRECTORY_DEPTH": raw})
|
|
|
|
def test_incremental_unit_rejects_invalid_values(self):
|
|
self.assertEqual(MODULE.incremental_unit({}), "OBJECT")
|
|
self.assertEqual(MODULE.incremental_unit({"INCREMENTAL_UNIT": " directory "}), "DIRECTORY")
|
|
with self.assertRaisesRegex(RuntimeError, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY"):
|
|
MODULE.incremental_unit({"INCREMENTAL_UNIT": "ROW"})
|
|
|
|
def test_builtin_source_rejects_invalid_scan_mode(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "BOS 增量扫描",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "INLINE"},
|
|
{"key": "SOURCE_TEXT", "value": "alpha"},
|
|
{"key": "SCAN_MODE", "value": "ROW"},
|
|
],
|
|
}
|
|
with self.assertRaisesRegex(RuntimeError, "SCAN_MODE must be FULL or INCREMENTAL"):
|
|
MODULE.builtin_source("run-test", "source-invalid-scan-mode", output_dir, node_data, [])
|
|
|
|
def test_builtin_source_rejects_invalid_data_source(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
source_root = pathlib.Path(temp_dir) / "source"
|
|
source_root.mkdir()
|
|
(source_root / "payload.json").write_text('{"kind":"real-record"}', encoding="utf-8")
|
|
output_dir = pathlib.Path(temp_dir) / "output"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "真实数据源-非法数据源",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "ROW"},
|
|
{"key": "SOURCE_PATH", "value": str(source_root)},
|
|
],
|
|
}
|
|
with self.assertRaisesRegex(RuntimeError, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM"):
|
|
MODULE.builtin_source("run-test", "source-invalid-data-source", output_dir, node_data, [])
|
|
|
|
def test_list_bos_entries_skips_large_non_metadata_json_payloads(self):
|
|
object_meta = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/meta.json", "size": 128, "e_tag": "meta"})()
|
|
object_imu = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json", "size": MODULE.PREVIEW_INLINE_BINARY_LIMIT + 1, "e_tag": "imu"})()
|
|
fake_client = _FakeBosListClient(
|
|
[object_meta, object_imu],
|
|
{
|
|
"wjzh-0330-Data-migration/SZ_000_0005/meta.json": b'{"type":"\xe5\xb7\xa5\xe5\x8e\x82"}',
|
|
"wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json": b'{"data_points":[1,2,3]}',
|
|
},
|
|
)
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
entries = MODULE.list_bos_entries(
|
|
"wjzh-data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
"https://bos.example.test",
|
|
"ak",
|
|
"sk",
|
|
)
|
|
|
|
self.assertEqual(len(entries), 2)
|
|
meta_entry = next(item for item in entries if item["path"].endswith("meta.json"))
|
|
imu_entry = next(item for item in entries if item["path"].endswith("_imu.json"))
|
|
self.assertIn("data", meta_entry)
|
|
self.assertNotIn("data", imu_entry)
|
|
self.assertEqual(
|
|
fake_client.downloaded_payload_keys,
|
|
[("wjzh-data-migration", "wjzh-0330-Data-migration/SZ_000_0005/meta.json")],
|
|
)
|
|
|
|
def test_bos_source_parses_bucket_and_prefix_from_source_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "BOS 增量扫描",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://wjzh-data-migration/wjzh-0330-Data-migration"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
{"key": "SCAN_MODE", "value": "FULL"},
|
|
],
|
|
}
|
|
fake_entries = [
|
|
{
|
|
"path": "SZ_000_0005/meta.json",
|
|
"storagePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {"type": "工厂"},
|
|
"source": {"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json"},
|
|
}
|
|
]
|
|
with mock.patch.object(MODULE, "list_bos_entries", return_value=fake_entries) as list_bos_entries, mock.patch.object(
|
|
MODULE,
|
|
"source_entry_artifact",
|
|
return_value={
|
|
"id": "artifact-1",
|
|
"runId": "run-1",
|
|
"nodeId": "source",
|
|
"name": "meta.json",
|
|
"type": "json",
|
|
"path": str(output_dir / "meta.json"),
|
|
},
|
|
):
|
|
artifacts = MODULE.builtin_source("run-1", "source", output_dir, node_data, [])
|
|
|
|
self.assertGreaterEqual(len(artifacts), 1)
|
|
list_bos_entries.assert_called_once_with(
|
|
"wjzh-data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
"https://bos.example.test",
|
|
"ak",
|
|
"sk",
|
|
)
|
|
|
|
|
|
class QualityMetricHelperTests(unittest.TestCase):
|
|
def test_build_skin_mask_detects_skin_tone_patch(self):
|
|
_, np = MODULE.require_cv2()
|
|
frame = np.zeros((80, 80, 3), dtype=np.uint8)
|
|
frame[20:60, 20:60] = (90, 130, 190)
|
|
mask = MODULE.build_skin_mask(frame)
|
|
self.assertGreater(int(mask.sum()), 0)
|
|
|
|
def test_build_skin_mask_rejects_neutral_background(self):
|
|
_, np = MODULE.require_cv2()
|
|
frame = np.full((80, 80, 3), 128, dtype=np.uint8)
|
|
mask = MODULE.build_skin_mask(frame)
|
|
self.assertEqual(int(mask.sum()), 0)
|
|
|
|
def test_motion_jitter_ratio_stays_low_for_smooth_motion(self):
|
|
smooth = [(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)]
|
|
jitter = MODULE.motion_jitter_ratio(smooth, 100.0)
|
|
self.assertLess(jitter, 0.01)
|
|
|
|
def test_motion_jitter_ratio_increases_for_erratic_motion(self):
|
|
smooth = MODULE.motion_jitter_ratio([(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)], 100.0)
|
|
erratic = MODULE.motion_jitter_ratio([(0.0, 0.0), (12.0, -4.0), (-10.0, 8.0), (15.0, -6.0)], 100.0)
|
|
self.assertGreater(erratic, smooth)
|
|
self.assertGreater(erratic, 0.05)
|
|
|
|
def test_largest_skin_component_ratio_ignores_small_noise(self):
|
|
_, np = MODULE.require_cv2()
|
|
mask = np.zeros((100, 100), dtype=np.uint8)
|
|
mask[10:13, 10:13] = 255
|
|
ratio = MODULE.largest_skin_component_ratio(mask)
|
|
self.assertEqual(ratio, 0.0)
|
|
|
|
def test_largest_skin_component_ratio_returns_large_component_area(self):
|
|
_, np = MODULE.require_cv2()
|
|
mask = np.zeros((100, 100), dtype=np.uint8)
|
|
mask[20:50, 30:70] = 255
|
|
ratio = MODULE.largest_skin_component_ratio(mask)
|
|
self.assertGreater(ratio, 0.1)
|
|
self.assertLess(ratio, 0.13)
|
|
|
|
|
|
class QualityOperatorTests(unittest.TestCase):
|
|
def test_video_quality_accepts_zero_hand_occlusion(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "MIN_CLARITY", "value": "0.8"},
|
|
{"key": "MAX_SHAKE", "value": "0.1"},
|
|
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"filename": "demo.mp4",
|
|
"source": {"path": "/tmp/demo.mp4"},
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=pathlib.Path("/tmp/demo.mp4")), mock.patch.object(
|
|
MODULE,
|
|
"analyze_video_quality_metrics",
|
|
return_value={
|
|
"clarity": 1.0,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"qualityScore": 1.0,
|
|
},
|
|
):
|
|
artifacts = MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["passedCount"], 1)
|
|
self.assertTrue(payload["items"][0]["quality"]["passed"])
|
|
|
|
def test_video_quality_rejects_out_of_range_thresholds(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "MIN_CLARITY", "value": "1.1"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "MIN_CLARITY must be in the range"):
|
|
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
|
|
|
|
def test_video_quality_rejects_non_finite_thresholds(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "MAX_SHAKE", "value": "nan"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be finite"):
|
|
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
|
|
|
|
def test_video_quality_rejects_invalid_clarity_scale(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "CLARITY_SCALE", "value": "0"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "CLARITY_SCALE must be greater than 0"):
|
|
MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, [])
|
|
|
|
def test_dataset_filter_keeps_zero_shake_and_hand_occlusion(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "数据过滤",
|
|
"hyperparams": [
|
|
{"key": "DATE_START", "value": "2026-03-01"},
|
|
{"key": "DATE_END", "value": "2026-03-31"},
|
|
{"key": "PLACE", "value": "蓟门里小区"},
|
|
{"key": "TYPE", "value": "家庭"},
|
|
{"key": "MIN_CLARITY", "value": "0.8"},
|
|
{"key": "MAX_SHAKE", "value": "0.1"},
|
|
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"meta": {
|
|
"date": "20260319",
|
|
"place": "蓟门里小区",
|
|
"type": "家庭",
|
|
},
|
|
"quality": {
|
|
"clarity": 0.95,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"passed": True,
|
|
},
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(len(payload["items"]), 1)
|
|
|
|
def test_dataset_filter_missing_thresholds_use_catalog_defaults(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "数据过滤",
|
|
"hyperparams": [],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"quality": {
|
|
"clarity": 0.5,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"passed": True,
|
|
},
|
|
},
|
|
{
|
|
"quality": {
|
|
"clarity": 0.95,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"passed": True,
|
|
},
|
|
},
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["criteria"]["minClarity"], MODULE.DEFAULT_MIN_CLARITY)
|
|
self.assertEqual(payload["criteria"]["maxShake"], MODULE.DEFAULT_MAX_SHAKE)
|
|
self.assertEqual(payload["criteria"]["maxHandOcclusion"], MODULE.DEFAULT_MAX_HAND_OCCLUSION)
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(payload["items"][0]["quality"]["clarity"], 0.95)
|
|
|
|
def test_dataset_filter_rejects_invalid_threshold_values(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "数据过滤",
|
|
"hyperparams": [
|
|
{"key": "MAX_SHAKE", "value": "not-a-number"},
|
|
],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "MAX_SHAKE must be numeric"):
|
|
MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
|
|
|
|
|
|
class OperatorHyperparamParsingTests(unittest.TestCase):
|
|
def test_collection_validate_rejects_invalid_boolean_hyperparam(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "采集目录校验",
|
|
"hyperparams": [{"key": "REQUIRE_INTRINSICS", "value": "sometimes"}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "REQUIRE_INTRINSICS must be a boolean"):
|
|
MODULE.builtin_collection_validate("run-1", "validate", output_dir, node_data, [{"data": {"collections": [], "items": []}}])
|
|
|
|
def test_json_filter_rejects_invalid_expected_json(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 过滤",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "EXPECTED_JSON", "value": '{"status"'},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "EXPECTED_JSON must be valid JSON"):
|
|
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
|
|
|
|
def test_json_filter_rejects_unknown_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 过滤",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "OPERATOR", "value": "roughly"},
|
|
{"key": "EXPECTED", "value": "ready"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
|
|
MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
|
|
|
|
def test_json_map_rejects_non_object_field_map(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": "[]"}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "FIELD_MAP_JSON must be a JSON object"):
|
|
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
|
|
|
|
def test_json_map_renames_fields_without_overwriting(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new","status":"state"}'}],
|
|
}
|
|
|
|
artifacts = MODULE.builtin_json_map(
|
|
"run-1",
|
|
"map",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"old": "value", "status": "ready", "unchanged": 1}]}}],
|
|
)
|
|
|
|
payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["fieldMap"], {"old": "new", "status": "state"})
|
|
self.assertEqual(payload["items"], [{"new": "value", "state": "ready", "unchanged": 1}])
|
|
|
|
def test_json_map_rejects_invalid_field_map_entries(self):
|
|
cases = [
|
|
('{" old":"new"}', "must not have leading or trailing whitespace"),
|
|
('{"old":" new"}', "destination field must not have leading or trailing whitespace"),
|
|
('{"old":""}', "destination field must be non-empty"),
|
|
('{"old":42}', "destination field must be a string"),
|
|
('{"first":"name","last":"name"}', "maps multiple source fields to destination field"),
|
|
]
|
|
for field_map_json, expected_error in cases:
|
|
with self.subTest(field_map_json=field_map_json), tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": field_map_json}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, expected_error):
|
|
MODULE.builtin_json_map("run-1", "map", output_dir, node_data, [{"data": {"items": [{"old": "value"}]}}])
|
|
|
|
def test_json_map_rejects_output_field_collisions(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "JSON 映射",
|
|
"hyperparams": [{"key": "FIELD_MAP_JSON", "value": '{"old":"new"}'}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "same output field 'new'"):
|
|
MODULE.builtin_json_map(
|
|
"run-1",
|
|
"map",
|
|
output_dir,
|
|
node_data,
|
|
[{"data": {"items": [{"old": "mapped", "new": "existing"}]}}],
|
|
)
|
|
|
|
def test_audit_sampling_rejects_non_array_rules(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [{"key": "RULES_JSON", "value": "{}"}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "RULES_JSON must be a JSON array"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_rejects_incomplete_rule_shape(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [
|
|
{"key": "RULES_JSON", "value": '[{"id":"missing-percent","field":"quality.clarity","operator":"gte","value":0.8}]'}
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent is required"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_rejects_duplicate_rule_ids(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
rules_json = json.dumps(
|
|
[
|
|
{"id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.8, "samplePercent": 100},
|
|
{"id": "clarity", "field": "quality.shake", "operator": "lte", "value": 0.1, "samplePercent": 50},
|
|
],
|
|
ensure_ascii=False,
|
|
)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [{"key": "RULES_JSON", "value": rules_json}],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "RULES_JSON contains duplicate id: clarity"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_rejects_invalid_sample_percent(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [
|
|
{
|
|
"key": "RULES_JSON",
|
|
"value": '[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":0}]',
|
|
}
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be in the range"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
node_data["hyperparams"][0]["value"] = (
|
|
'[{"id":"clarity","field":"quality.clarity","operator":"gte","value":0.8,"samplePercent":true}]'
|
|
)
|
|
with self.assertRaisesRegex(RuntimeError, r"RULES_JSON\[0\]\.samplePercent must be numeric"):
|
|
MODULE.builtin_audit_sampling("run-1", "sample", output_dir, node_data, [{"data": {"packageCount": 1, "items": []}}])
|
|
|
|
def test_audit_sampling_allows_exists_rule_without_value(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "抽检采样",
|
|
"hyperparams": [
|
|
{
|
|
"key": "RULES_JSON",
|
|
"value": '[{"id":"has-collection","field":"collectionId","operator":"exists","samplePercent":100}]',
|
|
}
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.builtin_audit_sampling(
|
|
"run-1",
|
|
"sample",
|
|
output_dir,
|
|
node_data,
|
|
[
|
|
{
|
|
"data": {
|
|
"packageCount": 2,
|
|
"items": [
|
|
{"packageName": "clip-a", "collectionId": "collection-a"},
|
|
{"packageName": "clip-b"},
|
|
],
|
|
}
|
|
}
|
|
],
|
|
)
|
|
|
|
self.assertEqual(len(artifacts), 1)
|
|
payload = json.loads((output_dir / "audit-samples.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["rules"][0]["matchedCount"], 1)
|
|
self.assertEqual(payload["sampledCount"], 1)
|
|
|
|
def test_condition_router_rejects_unknown_operator(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "条件分流",
|
|
"hyperparams": [
|
|
{"key": "FIELD_PATH", "value": "status"},
|
|
{"key": "OPERATOR", "value": "roughly"},
|
|
{"key": "EXPECTED", "value": "ready"},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"):
|
|
MODULE.builtin_condition_router("run-1", "route", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}])
|
|
|
|
|
|
class UploadOperatorTests(unittest.TestCase):
|
|
def test_build_artifact_skips_sha256_for_large_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
clip_path = root / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
|
|
MODULE,
|
|
"sha256_file",
|
|
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
|
|
):
|
|
artifact = MODULE.build_artifact("run-1", "slice", clip_path, "video")
|
|
|
|
metadata = artifact.get("metadata") or {}
|
|
self.assertEqual(metadata.get("sha256", ""), "")
|
|
self.assertTrue(metadata.get("sha256Skipped"))
|
|
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
|
|
|
|
def test_uploads_nested_package_outputs_to_bos(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
package_root = root / "package"
|
|
package_dir = package_root / "packages" / "clip001_package"
|
|
package_dir.mkdir(parents=True)
|
|
|
|
manifest_path = package_root / "packages-manifest.json"
|
|
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
|
|
clip_path = package_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
annotation_path = package_dir / "clip001_annotation.json"
|
|
annotation_path.write_text('{"label":"walking"}', encoding="utf-8")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [
|
|
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
|
|
MODULE.build_artifact("run-1", "package", clip_path, "video"),
|
|
MODULE.build_artifact("run-1", "package", annotation_path, "json", data={"label": "walking"}),
|
|
]
|
|
fake_client = _FakeBosClient()
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
|
|
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
|
|
uploaded_names = {item["name"] for item in artifacts}
|
|
uploaded_keys = {item["key"] for item in fake_client.uploads}
|
|
|
|
self.assertEqual(len(fake_client.uploads), 3)
|
|
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
|
|
self.assertEqual(uploaded_summary_payload["uploadedCount"], 3)
|
|
self.assertEqual(uploaded_summary_payload["bucket"], "datastorage")
|
|
self.assertEqual(uploaded_summary_payload["prefix"], "factory/output/demo")
|
|
|
|
def test_upload_skips_sha256_for_large_remote_artifacts(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
clip_path = root / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [MODULE.build_artifact("run-1", "slice", clip_path, "video")]
|
|
fake_client = _FakeBosClient()
|
|
|
|
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
|
|
MODULE,
|
|
"sha256_file",
|
|
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
|
|
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
remote_video = next(item for item in artifacts if item.get("type") == "video" and str(item.get("path", "")).startswith("bos://"))
|
|
metadata = remote_video.get("metadata") or {}
|
|
self.assertEqual(metadata.get("sha256", ""), "")
|
|
self.assertTrue(metadata.get("sha256Skipped"))
|
|
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
|
|
|
|
|
|
class _FakeInsertManyResult:
|
|
def __init__(self, inserted_ids):
|
|
self.inserted_ids = inserted_ids
|
|
|
|
|
|
class _FakeUpdateResult:
|
|
def __init__(self, matched_count=0, modified_count=0, upserted_id=None):
|
|
self.matched_count = matched_count
|
|
self.modified_count = modified_count
|
|
self.upserted_id = upserted_id
|
|
|
|
|
|
class _FakeMongoCollection:
|
|
def __init__(self):
|
|
self.insert_many_calls = []
|
|
self.replace_one_calls = []
|
|
|
|
def insert_many(self, documents, ordered=False):
|
|
self.insert_many_calls.append({"documents": documents, "ordered": ordered})
|
|
return _FakeInsertManyResult([f"id-{index + 1}" for index in range(len(documents))])
|
|
|
|
def replace_one(self, filter_document, replacement, upsert=False):
|
|
self.replace_one_calls.append(
|
|
{
|
|
"filter": filter_document,
|
|
"replacement": replacement,
|
|
"upsert": upsert,
|
|
}
|
|
)
|
|
return _FakeUpdateResult(matched_count=1, modified_count=1, upserted_id=None)
|
|
|
|
|
|
class _FakeMongoDatabase:
|
|
def __init__(self, collection):
|
|
self._collection = collection
|
|
self.requested_collections = []
|
|
|
|
def __getitem__(self, name):
|
|
self.requested_collections.append(name)
|
|
return self._collection
|
|
|
|
|
|
class _FakeMongoClient:
|
|
def __init__(self, database):
|
|
self._database = database
|
|
self.requested_databases = []
|
|
self.closed = False
|
|
|
|
def __getitem__(self, name):
|
|
self.requested_databases.append(name)
|
|
return self._database
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class MongoWriteOperatorTests(unittest.TestCase):
|
|
def test_inserts_documents_into_default_business_database(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
|
|
{"key": "MONGO_MODE", "value": "insert_many"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{"clipId": "clip-001", "qualityScore": 0.91},
|
|
{"clipId": "clip-002", "qualityScore": 0.87},
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-1", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(database.requested_collections, ["qualityReports"])
|
|
self.assertEqual(client.requested_databases, ["business-data"])
|
|
self.assertEqual(len(collection.insert_many_calls), 1)
|
|
self.assertEqual(collection.insert_many_calls[0]["documents"][0]["clipId"], "clip-001")
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["collection"], "qualityReports")
|
|
self.assertEqual(summary["data"]["mode"], "insert_many")
|
|
self.assertEqual(summary["data"]["target"], "business")
|
|
self.assertEqual(summary["data"]["insertedCount"], 2)
|
|
self.assertTrue(client.closed)
|
|
|
|
def test_upserts_documents_by_key(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "annotations"},
|
|
{"key": "MONGO_MODE", "value": "upsert"},
|
|
{"key": "UPSERT_KEY", "value": "clipId"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{"data": {"clipId": "clip-001", "label": "walking"}},
|
|
{"data": {"clipId": "clip-002", "label": "standing"}},
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-2", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(len(collection.replace_one_calls), 2)
|
|
self.assertEqual(collection.replace_one_calls[0]["filter"], {"clipId": "clip-001"})
|
|
self.assertTrue(collection.replace_one_calls[0]["upsert"])
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["mode"], "upsert")
|
|
self.assertEqual(summary["data"]["upsertKey"], "clipId")
|
|
self.assertEqual(summary["data"]["target"], "business")
|
|
self.assertEqual(summary["data"]["upsertedCount"], 2)
|
|
|
|
def test_unwraps_single_value_dict_documents_for_business_writes(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "businessWriteValidation"},
|
|
{"key": "MONGO_MODE", "value": "insert_many"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"value": {
|
|
"runLabel": "business-mongo-test",
|
|
"source": "workflow",
|
|
"createdAt": "2026-04-14T00:00:00Z",
|
|
"value": 1,
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
MODULE.builtin_mongodb_write("run-unwrap", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(len(collection.insert_many_calls), 1)
|
|
inserted = collection.insert_many_calls[0]["documents"][0]
|
|
self.assertEqual(inserted["runLabel"], "business-mongo-test")
|
|
self.assertEqual(inserted["source"], "workflow")
|
|
self.assertEqual(inserted["value"], 1)
|
|
self.assertNotEqual(set(inserted.keys()), {"value"})
|
|
|
|
def test_can_target_platform_database_explicitly(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
|
|
{"key": "MONGO_TARGET", "value": "platform"},
|
|
],
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"PLATFORM_MONGO_URI": "mongodb://platform-mongo:27017", "PLATFORM_MONGO_DB": "droboticflow"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-3", "mongo-node", pathlib.Path(temp_dir), node_data, [{"data": {"clipId": "clip-001"}}])
|
|
|
|
self.assertEqual(client.requested_databases, ["droboticflow"])
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["target"], "platform")
|
|
|
|
def test_rebuilds_package_files_from_manifest_when_temp_paths_are_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
package_manifest = {
|
|
"packageCount": 1,
|
|
"items": [
|
|
{
|
|
"packageName": "clip001_package",
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
"quality": {"clarity": 0.9, "passed": True},
|
|
"annotation": {"objects": ["人"], "minConfidence": 0.92},
|
|
}
|
|
],
|
|
}
|
|
inputs = [
|
|
{"name": "packages-manifest.json", "type": "json", "data": package_manifest},
|
|
{
|
|
"name": "clip001.mp4",
|
|
"type": "video",
|
|
"path": "/tmp/non-existent-clip001.mp4",
|
|
"data": {"preview": {"kind": "video", "format": "mp4"}},
|
|
},
|
|
{
|
|
"name": "clip001_annotation.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_annotation.json",
|
|
"data": {"objects": ["人"], "minConfidence": 0.92},
|
|
},
|
|
{
|
|
"name": "clip001_quality.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_quality.json",
|
|
"data": {"clarity": 0.9, "passed": True},
|
|
},
|
|
{
|
|
"name": "clip001_config.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_config.json",
|
|
"data": {"collectionId": "collection-1"},
|
|
},
|
|
]
|
|
fake_client = _FakeBosClient()
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
uploaded_keys = {item["key"] for item in fake_client.uploads}
|
|
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_quality.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_config.json", uploaded_keys)
|
|
self.assertFalse(
|
|
(output_dir / "uploaded" / "_materialized_upload_inputs" / "packages" / "clip001_package" / "clip001.mp4").exists()
|
|
)
|
|
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
|
|
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
|
|
self.assertEqual(uploaded_summary_payload["uploadedCount"], 5)
|
|
|
|
def test_upload_artifacts_are_json_serializable_when_preview_url_is_bytes(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
package_root = root / "package"
|
|
package_dir = package_root / "packages" / "clip001_package"
|
|
package_dir.mkdir(parents=True)
|
|
|
|
manifest_path = package_root / "packages-manifest.json"
|
|
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
|
|
clip_path = package_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [
|
|
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
|
|
MODULE.build_artifact("run-1", "package", clip_path, "video"),
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(_FakeBosClientBytesURL(), "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
json.dumps({"artifacts": artifacts}, ensure_ascii=False)
|
|
|
|
|
|
class XPlatformClipHandoffTests(unittest.TestCase):
|
|
def test_summarize_sqlite_payload_does_not_create_wal_sidecars(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
db_path = pathlib.Path(temp_dir) / "talker.db3"
|
|
connection = sqlite3.connect(db_path)
|
|
try:
|
|
connection.execute("pragma journal_mode=wal")
|
|
connection.execute("create table topics (id integer primary key, name text, type text, serialization_format text)")
|
|
connection.execute("create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)")
|
|
connection.execute(
|
|
"insert into topics (id, name, type, serialization_format) values (1, '/chatter', 'std_msgs/msg/String', 'cdr')"
|
|
)
|
|
connection.execute("insert into messages (topic_id, timestamp, data) values (1, 1, X'00')")
|
|
connection.commit()
|
|
finally:
|
|
connection.close()
|
|
for sidecar in db_path.parent.glob("talker.db3-*"):
|
|
sidecar.unlink()
|
|
|
|
payload = MODULE.summarize_sqlite_payload(db_path)
|
|
|
|
self.assertEqual(payload["preview"]["format"], "rosbag2")
|
|
self.assertEqual(payload["preview"]["messageCount"], 1)
|
|
self.assertEqual(list(db_path.parent.glob("talker.db3-*")), [])
|
|
|
|
def test_video_slicing_rejects_invalid_segment_duration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "0"}],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be greater than 0"):
|
|
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
def test_video_slicing_rejects_lossy_segment_duration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "01"}],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be an exact positive integer"):
|
|
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
def test_video_slicing_rejects_unsafe_segment_duration(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": str(MODULE.MAX_SAFE_INTEGER + 1)}],
|
|
}
|
|
inputs = [{"data": {"items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "SEGMENT_DURATION_SECONDS must be a safe integer"):
|
|
MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
def test_video_slicing_manifest_preserves_source_reference(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 41.8,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
clip = manifest_payload["items"][0]
|
|
self.assertEqual(clip["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
|
|
self.assertEqual(clip["clipName"], "DJI_20260318112834_0001_D_clip001.mp4")
|
|
|
|
def test_video_slicing_uses_single_pass_writer_for_multi_clip_video(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 365.0,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_write_video_segments(_source_path, segments):
|
|
outputs = []
|
|
for segment in segments:
|
|
destination = pathlib.Path(segment["destination"])
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(f"clip-{segment['clipIndex']}".encode("utf-8"))
|
|
outputs.append(
|
|
{
|
|
**segment,
|
|
"duration": round(float(segment["endSecond"]) - float(segment["startSecond"]), 3),
|
|
"frameRate": 29.97,
|
|
"resolution": "3840x2160",
|
|
}
|
|
)
|
|
return outputs
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segments",
|
|
side_effect=fake_write_video_segments,
|
|
) as multi_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
) as single_writer, mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["clipCount"], 2)
|
|
self.assertEqual([item["clipName"] for item in manifest_payload["items"]], [
|
|
"DJI_20260318112834_0001_D_clip001.mp4",
|
|
"DJI_20260318112834_0001_D_clip002.mp4",
|
|
])
|
|
multi_writer.assert_called_once()
|
|
single_writer.assert_not_called()
|
|
|
|
def test_write_video_segment_with_ffmpeg_prefers_stream_copy_and_uses_inspected_metrics(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
destination = root / "clips" / "clip001.mp4"
|
|
|
|
def fake_run(args, stdout=None, stderr=None, text=None):
|
|
self.assertEqual(args[0], "ffmpeg")
|
|
self.assertIn(str(source_path), args)
|
|
self.assertEqual(args[-1], str(destination))
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-video")
|
|
return mock.Mock(returncode=0, stdout="", stderr="")
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE.subprocess,
|
|
"run",
|
|
side_effect=fake_run,
|
|
) as run_mock, mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
|
|
):
|
|
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
|
|
|
|
self.assertEqual(metrics["duration"], 21.5)
|
|
self.assertEqual(metrics["frameRate"], 29.97)
|
|
self.assertEqual(metrics["resolution"], "3840x2160")
|
|
run_args = run_mock.call_args.args[0]
|
|
self.assertIn("-ss", run_args)
|
|
self.assertIn("-t", run_args)
|
|
self.assertIn("-c", run_args)
|
|
self.assertIn("copy", run_args)
|
|
self.assertNotIn("-movflags", run_args)
|
|
self.assertNotIn("+faststart", run_args)
|
|
|
|
def test_write_video_segment_with_ffmpeg_falls_back_to_reencode_when_copy_fails(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
destination = root / "clips" / "clip001.mp4"
|
|
|
|
calls = []
|
|
|
|
def fake_run(args, stdout=None, stderr=None, text=None):
|
|
calls.append(list(args))
|
|
if len(calls) == 1:
|
|
return mock.Mock(returncode=1, stdout="", stderr="copy failed")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-video")
|
|
return mock.Mock(returncode=0, stdout="", stderr="")
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE.subprocess,
|
|
"run",
|
|
side_effect=fake_run,
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
|
|
):
|
|
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
|
|
|
|
self.assertEqual(metrics["duration"], 21.5)
|
|
self.assertEqual(len(calls), 2)
|
|
self.assertIn("copy", calls[0])
|
|
self.assertIn("libx264", calls[1])
|
|
self.assertIn("aac", calls[1])
|
|
|
|
def test_video_slicing_prefers_ffmpeg_when_available(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 365.0,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_ffmpeg_writer(_source_path, destination, start_second, end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(f"{start_second}-{end_second}".encode("utf-8"))
|
|
return {
|
|
"duration": round(float(end_second) - float(start_second), 3),
|
|
"frameRate": 29.97,
|
|
"resolution": "3840x2160",
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment_with_ffmpeg",
|
|
side_effect=fake_ffmpeg_writer,
|
|
) as ffmpeg_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segments",
|
|
) as multi_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
) as single_writer, mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["clipCount"], 2)
|
|
self.assertEqual(ffmpeg_writer.call_count, 2)
|
|
multi_writer.assert_not_called()
|
|
single_writer.assert_not_called()
|
|
|
|
def test_video_annotation_rejects_invalid_frame_interval(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频标注",
|
|
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "not-a-number"}],
|
|
}
|
|
inputs = [{"data": {"clipCount": 0, "items": []}}]
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "FRAME_INTERVAL_SECONDS must be numeric"):
|
|
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
|
|
|
|
def test_video_annotation_uses_explicit_positive_frame_interval_without_clamping(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
clip_path = output_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"clip-bytes")
|
|
node_data = {
|
|
"label": "视频标注",
|
|
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "0.05"}],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"clipCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"clipPath": str(clip_path),
|
|
"sourceVideo": "source.mp4",
|
|
}
|
|
],
|
|
}
|
|
}
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "annotate_video_clip", return_value={"objects": [], "frameCount": 1}) as annotator:
|
|
MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
|
|
|
|
annotator.assert_called_once()
|
|
self.assertEqual(annotator.call_args.args[1], 0.05)
|
|
|
|
def test_video_annotation_reuses_shared_clip_path_without_copying_clip_artifact(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
shared_clip_path = output_dir / "shared" / "clip001.mp4"
|
|
shared_clip_path.parent.mkdir(parents=True, exist_ok=True)
|
|
shared_clip_path.write_bytes(b"clip-bytes")
|
|
inputs = [
|
|
{
|
|
"name": "clips-manifest.json",
|
|
"type": "json",
|
|
"data": {
|
|
"clipCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"clipPath": str(shared_clip_path),
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"duration": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频标注",
|
|
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "1.0"}],
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
|
|
MODULE,
|
|
"annotate_video_clip",
|
|
return_value={
|
|
"duration": 41.8,
|
|
"actions": [{"label": "walking", "confidence": 0.92, "start": 0, "end": 41.8}],
|
|
"objects": ["人"],
|
|
"minConfidence": 0.92,
|
|
},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "annotations-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["annotationCount"], 1)
|
|
self.assertEqual(manifest_payload["items"][0]["clipPath"], str(shared_clip_path))
|
|
self.assertFalse(any(item["name"] == "clip001.mp4" for item in artifacts))
|
|
|
|
def test_video_target_clip_has_explicit_handler(self):
|
|
with mock.patch.dict(os.environ, {"DROBOTICFLOW_OPERATOR_HANDLER": "builtin_yolo_video_target_clip"}):
|
|
self.assertEqual(MODULE.runtime_handler(), "builtin_yolo_video_target_clip")
|
|
|
|
def test_video_target_clip_detects_motion_and_writes_real_clip(self):
|
|
try:
|
|
cv2, np = MODULE.require_cv2()
|
|
except RuntimeError as exc:
|
|
self.skipTest(str(exc))
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "moving.mp4"
|
|
writer = cv2.VideoWriter(str(source_path), cv2.VideoWriter_fourcc(*"mp4v"), 12.0, (64, 64))
|
|
if not writer.isOpened():
|
|
self.skipTest("OpenCV cannot write mp4v video on this host")
|
|
try:
|
|
for frame_index in range(36):
|
|
frame = np.zeros((64, 64, 3), dtype=np.uint8)
|
|
if 8 <= frame_index <= 28:
|
|
x = 4 + frame_index
|
|
cv2.rectangle(frame, (x, 20), (x + 14, 36), (255, 255, 255), -1)
|
|
writer.write(frame)
|
|
finally:
|
|
writer.release()
|
|
self.assertTrue(source_path.exists())
|
|
output_dir = root / "target"
|
|
node_data = {
|
|
"label": "视频目标检测裁剪",
|
|
"hyperparams": [
|
|
{"key": "DETECTION_MODE", "value": "motion"},
|
|
{"key": "TARGET_LABELS", "value": "motion"},
|
|
{"key": "FRAME_INTERVAL_SECONDS", "value": "0.25"},
|
|
{"key": "MIN_CONFIDENCE", "value": "0.05"},
|
|
{"key": "MIN_MOTION_AREA_RATIO", "value": "0.001"},
|
|
{"key": "MERGE_GAP_SECONDS", "value": "0.5"},
|
|
{"key": "MIN_SEGMENT_SECONDS", "value": "0.25"},
|
|
{"key": "PRE_ROLL_SECONDS", "value": "0"},
|
|
{"key": "POST_ROLL_SECONDS", "value": "0"},
|
|
{"key": "MAX_CLIPS", "value": "3"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"filename": source_path.name,
|
|
"path": str(source_path),
|
|
"duration": 3.0,
|
|
"collectionId": "collection-1",
|
|
"source": {"path": str(source_path)},
|
|
}
|
|
]
|
|
},
|
|
}
|
|
]
|
|
|
|
original_ffmpeg_available = MODULE.OPERATOR_HELPERS["ffmpeg_available"]
|
|
MODULE.OPERATOR_HELPERS["ffmpeg_available"] = lambda: False
|
|
try:
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_video_target_clip"]("run-1", "target", output_dir, node_data, inputs)
|
|
finally:
|
|
MODULE.OPERATOR_HELPERS["ffmpeg_available"] = original_ffmpeg_available
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "target-clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["datasetType"], "video-target-clips")
|
|
self.assertGreater(manifest_payload["detectionCount"], 0)
|
|
self.assertGreater(manifest_payload["clipCount"], 0)
|
|
clip = manifest_payload["items"][0]
|
|
clip_path = pathlib.Path(clip["clipPath"])
|
|
self.assertTrue(clip_path.exists())
|
|
self.assertGreater(clip_path.stat().st_size, 0)
|
|
self.assertGreater(float(clip["endSecond"]), float(clip["startSecond"]))
|
|
clip_metrics = MODULE.inspect_video_file(clip_path)
|
|
self.assertGreater(float(clip_metrics.get("duration") or 0), 0)
|
|
self.assertTrue(any(item["name"].endswith(".mp4") for item in artifacts))
|
|
|
|
def test_materialize_clip_from_source_prefers_remote_segment_generation(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
item = {
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
},
|
|
}
|
|
|
|
def fake_remote_segment(_source_url, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8}
|
|
|
|
with mock.patch.object(MODULE, "remote_video_url_for_source", return_value="https://signed.example.com/video.mp4"), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment_from_remote_url",
|
|
side_effect=fake_remote_segment,
|
|
) as remote_writer, mock.patch.object(MODULE, "source_path_from_item") as local_materializer:
|
|
resolved = MODULE.materialize_clip_from_source(item, output_dir)
|
|
|
|
self.assertIsNotNone(resolved)
|
|
self.assertTrue(resolved.exists())
|
|
self.assertEqual(resolved.read_bytes(), b"clip-bytes")
|
|
remote_writer.assert_called_once()
|
|
local_materializer.assert_not_called()
|
|
|
|
def test_result_package_rebuilds_clip_from_source_when_clip_artifact_is_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "annotations-manifest.json",
|
|
"type": "json",
|
|
"data": {
|
|
"annotationCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"duration": 41.8,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"clarity": 0.9, "passed": True},
|
|
"annotation": {"actions": [], "objects": ["人"], "minConfidence": 0.92},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "成品打包",
|
|
"hyperparams": [],
|
|
}
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
side_effect=fake_write_video_segment,
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
package_clip = next(item for item in artifacts if item["name"] == "clip001.mp4")
|
|
self.assertEqual(manifest_payload["packageCount"], 1)
|
|
manifest_item = manifest_payload["items"][0]
|
|
self.assertEqual(manifest_item["clipName"], "clip001.mp4")
|
|
self.assertEqual(manifest_item["collectionId"], "collection-1")
|
|
self.assertEqual(manifest_item["startSecond"], 0)
|
|
self.assertEqual(manifest_item["endSecond"], 41.8)
|
|
self.assertEqual(manifest_item["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
|
|
self.assertEqual(package_clip["name"], "clip001.mp4")
|
|
|
|
def test_result_package_accepts_target_clip_manifest(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
clip_path = output_dir / "target001.mp4"
|
|
clip_path.write_bytes(b"target-clip")
|
|
inputs = [
|
|
{
|
|
"name": "target-clips-manifest.json",
|
|
"type": "json",
|
|
"data": {
|
|
"datasetType": "video-target-clips",
|
|
"clipCount": 1,
|
|
"detectionMode": "motion",
|
|
"targetLabels": ["motion"],
|
|
"items": [
|
|
{
|
|
"clipId": "target001",
|
|
"clipName": "target001.mp4",
|
|
"clipPath": str(clip_path),
|
|
"sourceVideo": "source.mp4",
|
|
"startSecond": 1.0,
|
|
"endSecond": 2.5,
|
|
"duration": 1.5,
|
|
"collectionId": "collection-1",
|
|
"quality": {"passed": True},
|
|
"detections": [{"label": "motion", "confidence": 0.8, "second": 1.2}],
|
|
"detectionCount": 1,
|
|
"maxConfidence": 0.8,
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {"label": "成品打包", "hyperparams": []}
|
|
|
|
with mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
package_clip = next(item for item in artifacts if item["name"] == "target001.mp4")
|
|
annotation_artifact = next(item for item in artifacts if item["name"] == "target001_annotation.json")
|
|
annotation_payload = MODULE.load_artifact_payload(annotation_artifact)
|
|
self.assertEqual(manifest_payload["packageCount"], 1)
|
|
self.assertEqual(manifest_payload["items"][0]["clipName"], "target001.mp4")
|
|
self.assertEqual(manifest_payload["items"][0]["annotation"]["type"], "target_detections")
|
|
self.assertEqual(annotation_payload["detectionCount"], 1)
|
|
self.assertEqual(package_clip["name"], "target001.mp4")
|
|
|
|
|
|
class ModelInferenceOperatorTests(unittest.TestCase):
|
|
def test_builtin_model_inference_runs_linear_model(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
model = {
|
|
"type": "linear",
|
|
"name": "quality-gate",
|
|
"features": ["quality.clarity", "quality.shake"],
|
|
"weights": [1.0, -2.0],
|
|
"bias": 0.0,
|
|
"threshold": 0.5,
|
|
"positiveLabel": "pass",
|
|
"negativeLabel": "fail",
|
|
}
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [{"key": "MODEL_JSON", "value": json.dumps(model)}],
|
|
}
|
|
inputs = [
|
|
{
|
|
"name": "dataset.json",
|
|
"type": "json",
|
|
"data": {
|
|
"items": [
|
|
{"id": "clip-1", "quality": {"clarity": 0.9, "shake": 0.1}},
|
|
{"id": "clip-2", "quality": {"clarity": 0.2, "shake": 0.2}},
|
|
]
|
|
},
|
|
}
|
|
]
|
|
|
|
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, inputs)
|
|
|
|
self.assertEqual([item["name"] for item in artifacts], ["inference-result.json"])
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["modelType"], "linear")
|
|
self.assertEqual(payload["modelName"], "quality-gate")
|
|
expected_digest = hashlib.sha256(
|
|
json.dumps(model, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
self.assertEqual(payload["modelSource"], "MODEL_JSON")
|
|
self.assertEqual(payload["modelSha256"], expected_digest)
|
|
self.assertEqual(payload["inputCount"], 2)
|
|
self.assertEqual([item["prediction"]["label"] for item in payload["predictions"]], ["pass", "fail"])
|
|
self.assertEqual(payload["predictions"][0]["input"]["id"], "clip-1")
|
|
|
|
def test_builtin_model_inference_runs_nearest_centroid_from_inline_input(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
model = {
|
|
"type": "nearest_centroid",
|
|
"name": "motion-cluster",
|
|
"features": ["speed", "turnRate"],
|
|
"labels": ["slow-straight", "fast-turn"],
|
|
"centroids": [[0.1, 0.0], [1.0, 0.8]],
|
|
}
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [
|
|
{"key": "MODEL_JSON", "value": json.dumps(model)},
|
|
{"key": "INPUT_JSON", "value": json.dumps([{"id": "clip-1", "speed": 0.95, "turnRate": 0.7}])},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["inputCount"], 1)
|
|
self.assertEqual(payload["predictions"][0]["prediction"]["label"], "fast-turn")
|
|
self.assertGreater(payload["predictions"][0]["prediction"]["score"], 0)
|
|
|
|
def test_builtin_model_inference_rejects_invalid_rule_comparison(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [
|
|
{
|
|
"key": "MODEL_JSON",
|
|
"value": json.dumps(
|
|
{
|
|
"type": "rule",
|
|
"rules": [{"field": "scene", "operator": "gt", "value": 1, "label": "bad"}],
|
|
}
|
|
),
|
|
},
|
|
{"key": "INPUT_JSON", "value": json.dumps({"scene": "kitchen"})},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "requires numeric values"):
|
|
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
|
|
|
|
def test_builtin_model_inference_rejects_constant_model(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "模型推理",
|
|
"hyperparams": [
|
|
{"key": "MODEL_JSON", "value": json.dumps({"type": "constant", "label": "pass"})},
|
|
{"key": "INPUT_JSON", "value": json.dumps({"id": "clip-1"})},
|
|
],
|
|
}
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "constant model type is not accepted"):
|
|
MODULE.builtin_model_inference("run-1", "infer", output_dir, node_data, [])
|
|
|
|
|
|
class VLAModelWorkflowOperatorTests(unittest.TestCase):
|
|
def test_vla_train_sim3d_inference_evaluate_chain_uses_real_artifacts(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
samples = [
|
|
{
|
|
"id": "sample-1",
|
|
"instruction": "pick up the folded shirt from the table",
|
|
"duration": 4.2,
|
|
"width": 1920,
|
|
"height": 1080,
|
|
"action": [0.12, 0.25, 0.04],
|
|
},
|
|
{
|
|
"id": "sample-2",
|
|
"instruction": "move the shirt to the storage box",
|
|
"duration": 5.1,
|
|
"width": 1920,
|
|
"height": 1080,
|
|
"action": [0.31, 0.42, 0.08],
|
|
},
|
|
{
|
|
"id": "sample-3",
|
|
"instruction": "release the shirt into the storage box",
|
|
"duration": 3.7,
|
|
"width": 1920,
|
|
"height": 1080,
|
|
"action": [0.28, 0.18, 0.02],
|
|
},
|
|
]
|
|
sample_path = root / "vla_samples.json"
|
|
sample_path.write_text(json.dumps({"items": samples}, ensure_ascii=False), encoding="utf-8")
|
|
train_node = {
|
|
"label": "VLA 模型训练",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(sample_path)},
|
|
{"key": "MODEL_NAME", "value": "shirt-folding-policy"},
|
|
],
|
|
}
|
|
|
|
train_artifacts = MODULE.builtin_vla_model_train("run-1", "train", root / "train", train_node, [])
|
|
model_artifact = next(item for item in train_artifacts if item["name"] == "vla-policy-model.json")
|
|
metrics_artifact = next(item for item in train_artifacts if item["name"] == "vla-training-metrics.json")
|
|
model_payload = MODULE.load_artifact_payload(model_artifact)
|
|
metrics_payload = MODULE.load_artifact_payload(metrics_artifact)
|
|
self.assertEqual(model_payload["type"], "vla_linear_policy")
|
|
self.assertEqual(model_payload["actionDimension"], 3)
|
|
self.assertEqual(metrics_payload["sampleCount"], 3)
|
|
|
|
sample_artifact = {"name": "vla_samples.json", "type": "json", "data": {"items": samples}, "path": str(sample_path)}
|
|
sim_artifacts = MODULE.builtin_sim3d_asset_generate(
|
|
"run-1",
|
|
"sim",
|
|
root / "sim",
|
|
{"label": "3D 资产仿真生成", "hyperparams": []},
|
|
[model_artifact, sample_artifact],
|
|
)
|
|
sim_manifest = next(item for item in sim_artifacts if item["name"] == "sim3d-asset-manifest.json")
|
|
sim_payload = MODULE.load_artifact_payload(sim_manifest)
|
|
self.assertEqual(sim_payload["datasetType"], "sim3d-asset-model")
|
|
self.assertTrue((root / "sim" / "sim3d-assets" / "scene.obj").is_file())
|
|
self.assertTrue((root / "sim" / "sim3d-assets" / "scene.gltf").is_file())
|
|
|
|
inference_artifacts = MODULE.builtin_vla_model_inference(
|
|
"run-1",
|
|
"infer",
|
|
root / "infer",
|
|
{"label": "VLA 模型推理", "hyperparams": []},
|
|
[sim_manifest],
|
|
)
|
|
inference_payload = MODULE.load_artifact_payload(inference_artifacts[0])
|
|
self.assertEqual(inference_payload["datasetType"], "vla-model-inference")
|
|
self.assertEqual(inference_payload["sampleCount"], 3)
|
|
self.assertEqual(len(inference_payload["predictions"][0]["action"]), 3)
|
|
|
|
evaluation_artifacts = MODULE.builtin_vla_model_evaluate(
|
|
"run-1",
|
|
"eval",
|
|
root / "eval",
|
|
{"label": "VLA 模型评测", "hyperparams": [{"key": "MAX_RMSE", "value": "1.0"}]},
|
|
[inference_artifacts[0]],
|
|
)
|
|
evaluation_payload = MODULE.load_artifact_payload(evaluation_artifacts[0])
|
|
self.assertEqual(evaluation_payload["datasetType"], "vla-model-evaluation")
|
|
self.assertEqual(evaluation_payload["sampleCount"], 3)
|
|
self.assertLessEqual(evaluation_payload["rmse"], 1.0)
|
|
|
|
|
|
class ShellCommandOperatorTests(unittest.TestCase):
|
|
def test_holobrain_run_command_tees_stdout_and_stderr(self):
|
|
from backend.operators.builtin import holobrain_operator
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
output = root / "out"
|
|
command = [
|
|
sys.executable,
|
|
"-c",
|
|
"import sys; print('train stdout line'); print('train stderr line', file=sys.stderr)",
|
|
]
|
|
stdout_buffer = io.StringIO()
|
|
stderr_buffer = io.StringIO()
|
|
with contextlib.redirect_stdout(stdout_buffer), contextlib.redirect_stderr(stderr_buffer):
|
|
command_path, stdout_path, stderr_path = holobrain_operator._run_command(
|
|
command,
|
|
root,
|
|
output,
|
|
dict(os.environ),
|
|
{"framework": "test"},
|
|
)
|
|
self.assertTrue(command_path.is_file())
|
|
self.assertEqual(stdout_path.read_text(encoding="utf-8").strip(), "train stdout line")
|
|
self.assertEqual(stderr_path.read_text(encoding="utf-8").strip(), "train stderr line")
|
|
self.assertIn("train stdout line", stdout_buffer.getvalue())
|
|
self.assertIn("train stderr line", stderr_buffer.getvalue())
|
|
|
|
def test_holobrain_data_prepare_accepts_real_local_dataset_dirs(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_root = root / "robotwin-source"
|
|
dataset_dir = source_root / "aloha_agilex_demo_clean" / "index"
|
|
dataset_dir.mkdir(parents=True)
|
|
(dataset_dir / "data.mdb").write_bytes(b"real-lmdb")
|
|
(dataset_dir / "lock.mdb").write_bytes(b"")
|
|
artifacts = MODULE.builtin_holobrain_data_prepare(
|
|
"run-1",
|
|
"data",
|
|
root / "out",
|
|
{
|
|
"label": "HoloBrain 数据准备",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "local"},
|
|
{"key": "ROBOTWIN_SOURCE_PATH", "value": str(source_root)},
|
|
{"key": "EXPECTED_LMDB_DIRS_JSON", "value": "[\"aloha_agilex_demo_clean\"]"},
|
|
],
|
|
},
|
|
[],
|
|
)
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "holobrain-data-manifest")
|
|
self.assertEqual(payload["datasetCount"], 1)
|
|
self.assertEqual(pathlib.Path(payload["robotwinDataRoot"]).resolve(), source_root.resolve())
|
|
self.assertFalse((root / "out" / "holobrain-data").exists())
|
|
|
|
def test_holobrain_data_prepare_fails_without_compatible_dataset(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_root = root / "robotwin-source"
|
|
(source_root / "unrelated_dataset").mkdir(parents=True)
|
|
with self.assertRaisesRegex(RuntimeError, "No HoloBrain-compatible RoboTwin LMDB dataset layout"):
|
|
MODULE.builtin_holobrain_data_prepare(
|
|
"run-1",
|
|
"data",
|
|
root / "out",
|
|
{
|
|
"label": "HoloBrain 数据准备",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "local"},
|
|
{"key": "ROBOTWIN_SOURCE_PATH", "value": str(source_root)},
|
|
{"key": "EXPECTED_LMDB_DIRS_JSON", "value": "[\"aloha_agilex_demo_clean\"]"},
|
|
],
|
|
},
|
|
[],
|
|
)
|
|
|
|
def test_holobrain_data_prepare_converts_raw_robotwin_zip_layout(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_root = root / "robotwin-source"
|
|
(source_root / "dataset" / "adjust_bottle").mkdir(parents=True)
|
|
(source_root / "dataset" / "adjust_bottle" / "aloha-agilex_clean_50.zip").write_bytes(b"real-zip-placeholder")
|
|
|
|
def fake_pack(local_root, output_path, expected, params):
|
|
converted_root = output_path / "holobrain-converted-raw" / "aloha_agilex_demo_clean" / "index"
|
|
converted_root.mkdir(parents=True)
|
|
(converted_root / "data.mdb").write_bytes(b"converted-lmdb")
|
|
return (
|
|
[
|
|
{
|
|
"name": "aloha_agilex_demo_clean",
|
|
"path": str(converted_root.parent),
|
|
"indexPath": str(converted_root),
|
|
"hasLmdbFiles": True,
|
|
"lmdbFiles": ["data.mdb"],
|
|
"fileCount": 1,
|
|
"sampledSizeBytes": 14,
|
|
}
|
|
],
|
|
[
|
|
{
|
|
"name": "aloha_agilex_demo_clean",
|
|
"sourceZips": [str(source_root / "dataset" / "adjust_bottle" / "aloha-agilex_clean_50.zip")],
|
|
}
|
|
],
|
|
)
|
|
|
|
with mock.patch("backend.operators.builtin.holobrain_operator._pack_robotwin_raw_to_lmdb", side_effect=fake_pack):
|
|
artifacts = MODULE.builtin_holobrain_data_prepare(
|
|
"run-1",
|
|
"data",
|
|
root / "out",
|
|
{
|
|
"label": "HoloBrain 数据准备",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "local_raw"},
|
|
{"key": "ROBOTWIN_SOURCE_PATH", "value": str(source_root)},
|
|
{"key": "EXPECTED_LMDB_DIRS_JSON", "value": "[\"aloha_agilex_demo_clean\"]"},
|
|
],
|
|
},
|
|
[],
|
|
)
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "holobrain-data-manifest")
|
|
self.assertEqual(payload["datasetCount"], 1)
|
|
self.assertEqual(payload["conversions"][0]["name"], "aloha_agilex_demo_clean")
|
|
|
|
def test_holobrain_data_prepare_converts_arrow_cache_layout(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_root = root / "robotwin-source"
|
|
source_root.mkdir(parents=True)
|
|
(source_root / "dataset_info.json").write_text("{}", encoding="utf-8")
|
|
(source_root / "data-00000-of-00001.arrow").write_bytes(b"arrow")
|
|
|
|
def fake_pack(local_root, output_path, expected, params):
|
|
converted_root = output_path / "holobrain-converted-arrow" / "robotwin2_0" / "index"
|
|
converted_root.mkdir(parents=True)
|
|
(converted_root / "data.mdb").write_bytes(b"converted-lmdb")
|
|
return (
|
|
[
|
|
{
|
|
"name": "robotwin2_0",
|
|
"path": str(converted_root.parent),
|
|
"indexPath": str(converted_root),
|
|
"hasLmdbFiles": True,
|
|
"lmdbFiles": ["data.mdb"],
|
|
"fileCount": 1,
|
|
"sampledSizeBytes": 14,
|
|
}
|
|
],
|
|
[
|
|
{
|
|
"name": "robotwin2_0",
|
|
"format": "huggingface_arrow",
|
|
"sourceJointDim": 16,
|
|
"targetJointDim": 14,
|
|
"jointIndexes": [0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 15],
|
|
}
|
|
],
|
|
)
|
|
|
|
with mock.patch("backend.operators.builtin.holobrain_operator._pack_robotwin_arrow_to_lmdb", side_effect=fake_pack):
|
|
artifacts = MODULE.builtin_holobrain_data_prepare(
|
|
"run-1",
|
|
"data",
|
|
root / "out",
|
|
{
|
|
"label": "HoloBrain 数据准备",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "local"},
|
|
{"key": "ROBOTWIN_SOURCE_PATH", "value": str(source_root)},
|
|
{"key": "EXPECTED_LMDB_DIRS_JSON", "value": "[\"robotwin2_0\"]"},
|
|
],
|
|
},
|
|
[],
|
|
)
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "holobrain-data-manifest")
|
|
self.assertEqual(payload["datasetCount"], 1)
|
|
self.assertEqual(payload["conversions"][0]["format"], "huggingface_arrow")
|
|
self.assertEqual(payload["conversions"][0]["targetJointDim"], 14)
|
|
self.assertTrue(payload["robotwinDataRoot"].endswith("holobrain-converted-arrow"))
|
|
|
|
def test_holobrain_data_prepare_maps_bos_source_to_workspace_mount(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
workspace = root / "workspace"
|
|
source_root = workspace / "datasets" / "holobrain" / "arrow"
|
|
dataset_dir = source_root / "robotwin2_0" / "index"
|
|
dataset_dir.mkdir(parents=True)
|
|
(dataset_dir / "data.mdb").write_bytes(b"real-lmdb")
|
|
(dataset_dir / "lock.mdb").write_bytes(b"")
|
|
with mock.patch.dict(os.environ, {"WORKSPACE_BOS_BUCKET": "testxyz", "WORKSPACE_MOUNT_PATH": str(workspace)}):
|
|
artifacts = MODULE.builtin_holobrain_data_prepare(
|
|
"run-1",
|
|
"data",
|
|
root / "out",
|
|
{
|
|
"label": "HoloBrain 数据准备",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "ROBOTWIN_SOURCE_PATH", "value": "bos://testxyz/datasets/holobrain/arrow"},
|
|
{"key": "EXPECTED_LMDB_DIRS_JSON", "value": "[\"robotwin2_0\"]"},
|
|
],
|
|
},
|
|
[],
|
|
)
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["dataSource"], "bos")
|
|
self.assertEqual(payload["source"]["path"], "bos://testxyz/datasets/holobrain/arrow")
|
|
self.assertEqual(payload["datasetCount"], 1)
|
|
|
|
def test_holobrain_data_prepare_rejects_runtime_hf_download(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
with self.assertRaisesRegex(RuntimeError, "Runtime dataset download is disabled"):
|
|
MODULE.builtin_holobrain_data_prepare(
|
|
"run-1",
|
|
"data",
|
|
root / "out",
|
|
{
|
|
"label": "HoloBrain 数据准备",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "hf_snapshot"},
|
|
],
|
|
},
|
|
[],
|
|
)
|
|
|
|
def test_holobrain_data_prepare_rejects_lerobot_dataset_root(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_root = root / "robotwin-source"
|
|
(source_root / "meta").mkdir(parents=True)
|
|
(source_root / "meta" / "info.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"codebase_version": "v3.0",
|
|
"features": {
|
|
"observation.images.cam_high": {"dtype": "video"},
|
|
"observation.state": {"dtype": "float32"},
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(source_root / "data" / "chunk-000").mkdir(parents=True)
|
|
(source_root / "data" / "chunk-000" / "file-000.parquet").write_bytes(b"PAR1")
|
|
with self.assertRaisesRegex(RuntimeError, "LeRobot parquet/video datasets"):
|
|
MODULE.builtin_holobrain_data_prepare(
|
|
"run-1",
|
|
"data",
|
|
root / "out",
|
|
{
|
|
"label": "HoloBrain 数据准备",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "local"},
|
|
{"key": "ROBOTWIN_SOURCE_PATH", "value": str(source_root)},
|
|
],
|
|
},
|
|
[],
|
|
)
|
|
|
|
def test_holobrain_train_dry_run_writes_training_report(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
repo = root / "RoboOrchardLab"
|
|
project = repo / "projects" / "holobrain"
|
|
(project / "scripts").mkdir(parents=True)
|
|
(project / "configs").mkdir(parents=True)
|
|
(project / "scripts" / "train.py").write_text("print('train')\n", encoding="utf-8")
|
|
(project / "configs" / "config_holobrain_qwen_common.py").write_text("config={}\n", encoding="utf-8")
|
|
data_root = root / "data" / "robotwin2.0"
|
|
(data_root / "aloha_agilex_demo_clean").mkdir(parents=True)
|
|
data_artifact = {
|
|
"name": "holobrain-data-manifest.json",
|
|
"type": "json",
|
|
"path": str(root / "holobrain-data-manifest.json"),
|
|
"data": {
|
|
"datasetType": "holobrain-data-manifest",
|
|
"robotwinDataRoot": str(data_root),
|
|
},
|
|
}
|
|
artifacts = MODULE.builtin_holobrain_train(
|
|
"run-1",
|
|
"train",
|
|
root / "train-out",
|
|
{
|
|
"label": "HoloBrain 完整训练",
|
|
"hyperparams": [
|
|
{"key": "ROBOORCHARDLAB_ROOT", "value": str(repo)},
|
|
{"key": "ENV_JSON", "value": "{\"HOLOBRAIN_OPERATOR_DRY_RUN\":\"1\"}"},
|
|
],
|
|
},
|
|
[data_artifact],
|
|
)
|
|
report = MODULE.load_artifact_payload(next(item for item in artifacts if item["name"] == "holobrain-training-report.json"))
|
|
self.assertEqual(report["datasetType"], "holobrain-training-report")
|
|
self.assertEqual(report["config"]["maxStep"], 100000)
|
|
command_payload = json.loads((root / "train-out" / "framework-command.json").read_text(encoding="utf-8"))
|
|
self.assertIn("train.py", " ".join(command_payload["command"]))
|
|
self.assertEqual(command_payload["metadata"]["model"], "HorizonRobotics/HoloBrain_v0.0_Qwen")
|
|
self.assertTrue(command_payload["metadata"]["hfRuntime"]["offline"])
|
|
self.assertEqual(command_payload["metadata"]["hfRuntime"]["hfHome"], "/opt/hf-cache")
|
|
|
|
def test_holobrain_export_eval_and_inference_reports(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
repo = root / "RoboOrchardLab"
|
|
project = repo / "projects" / "holobrain"
|
|
(project / "scripts").mkdir(parents=True)
|
|
(project / "configs").mkdir(parents=True)
|
|
(project / "scripts" / "train.py").write_text("print('train')\n", encoding="utf-8")
|
|
(project / "scripts" / "export.py").write_text(
|
|
"\n".join(
|
|
[
|
|
"import os",
|
|
"from pathlib import Path",
|
|
"",
|
|
"_REPO_ROOT = os.path.abspath(",
|
|
" os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)",
|
|
")",
|
|
"PROJECT_ROOT = Path(__file__).resolve().parents[1]",
|
|
"CONFIGS_DIR = PROJECT_ROOT / \"configs\"",
|
|
"",
|
|
"def main(args):",
|
|
" model.save_model(model_path, required_empty=False)",
|
|
" logger.info(\"Export model successfully.\")",
|
|
"",
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(project / "scripts" / "robotwin_eval.py").write_text("print('eval')\n", encoding="utf-8")
|
|
(project / "configs" / "config_holobrain_qwen_common.py").write_text("config={}\n", encoding="utf-8")
|
|
workspace = root / "workspace"
|
|
checkpoint_dir = workspace / "checkpoints" / "checkpoint_1"
|
|
checkpoint_dir.mkdir(parents=True)
|
|
train_report = {
|
|
"name": "holobrain-training-report.json",
|
|
"type": "json",
|
|
"path": str(root / "holobrain-training-report.json"),
|
|
"data": {
|
|
"datasetType": "holobrain-training-report",
|
|
"trainingWorkspace": str(workspace),
|
|
"latestCheckpoint": str(checkpoint_dir),
|
|
},
|
|
}
|
|
export_artifacts = MODULE.builtin_holobrain_export(
|
|
"run-1",
|
|
"export",
|
|
root / "export-out",
|
|
{
|
|
"label": "HoloBrain 模型导出",
|
|
"hyperparams": [
|
|
{"key": "ROBOORCHARDLAB_ROOT", "value": str(repo)},
|
|
{"key": "ENV_JSON", "value": "{\"HOLOBRAIN_OPERATOR_DRY_RUN\":\"1\"}"},
|
|
],
|
|
},
|
|
[train_report],
|
|
)
|
|
export_report = MODULE.load_artifact_payload(next(item for item in export_artifacts if item["name"] == "holobrain-export-manifest.json"))
|
|
self.assertEqual(export_report["datasetType"], "holobrain-export-manifest")
|
|
|
|
model_dir = root / "model"
|
|
model_dir.mkdir()
|
|
(model_dir / "robotwin2_0.config.json").write_text("{}", encoding="utf-8")
|
|
robotwin = root / "RoboTwin"
|
|
for relative in [
|
|
"task_config/demo_clean.yml",
|
|
"task_config/_embodiment_config.yml",
|
|
"task_config/_camera_config.yml",
|
|
"envs/beat_block_hammer.py",
|
|
"description/task_instruction/beat_block_hammer.json",
|
|
]:
|
|
path = robotwin / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("{}\n", encoding="utf-8")
|
|
(robotwin / "assets" / "embodiments" / "aloha-agilex").mkdir(parents=True)
|
|
for relative in [
|
|
"assets/objects/objaverse/list.json",
|
|
"assets/objects/same.json",
|
|
"assets/objects/020_hammer/model_data0.json",
|
|
"assets/objects/020_hammer/collision/base0.glb",
|
|
"assets/objects/020_hammer/visual/base0.glb",
|
|
]:
|
|
path = robotwin / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(b"fixture")
|
|
export_manifest = {
|
|
"name": "holobrain-export-manifest.json",
|
|
"type": "json",
|
|
"path": str(root / "holobrain-export-manifest.json"),
|
|
"data": {"datasetType": "holobrain-export-manifest", "exportedModelDir": str(model_dir)},
|
|
}
|
|
eval_artifacts = MODULE.builtin_holobrain_robotwin_eval(
|
|
"run-1",
|
|
"eval",
|
|
root / "eval-out",
|
|
{
|
|
"label": "HoloBrain RoboTwin 评测",
|
|
"hyperparams": [
|
|
{"key": "ROBOORCHARDLAB_ROOT", "value": str(repo)},
|
|
{"key": "ROBOTWIN_ROOT", "value": str(robotwin)},
|
|
{"key": "ENV_JSON", "value": "{\"HOLOBRAIN_OPERATOR_DRY_RUN\":\"1\"}"},
|
|
],
|
|
},
|
|
[export_manifest],
|
|
)
|
|
eval_report = MODULE.load_artifact_payload(next(item for item in eval_artifacts if item["name"] == "holobrain-robotwin-eval-report.json"))
|
|
self.assertEqual(eval_report["datasetType"], "holobrain-robotwin-evaluation")
|
|
|
|
inference_artifacts = MODULE.builtin_holobrain_inference_test(
|
|
"run-1",
|
|
"test",
|
|
root / "test-out",
|
|
{
|
|
"label": "HoloBrain 模型测试",
|
|
"hyperparams": [
|
|
{"key": "ROBOORCHARDLAB_ROOT", "value": str(repo)},
|
|
{"key": "ROBOTWIN_ROOT", "value": str(robotwin)},
|
|
{"key": "ENV_JSON", "value": "{\"HOLOBRAIN_OPERATOR_DRY_RUN\":\"1\"}"},
|
|
],
|
|
},
|
|
[export_manifest],
|
|
)
|
|
inference_report = MODULE.load_artifact_payload(inference_artifacts[0])
|
|
self.assertEqual(inference_report["datasetType"], "holobrain-inference-test")
|
|
self.assertEqual(inference_report["status"], "dry-run")
|
|
self.assertFalse(inference_report["execution"]["actualModelExecution"])
|
|
|
|
def test_holobrain_robotwin_ur5_split_runtime_overlay(self):
|
|
from backend.operators.builtin import holobrain_operator
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
robotwin = root / "RoboTwin"
|
|
for relative in [
|
|
"task_config/demo_clean.yml",
|
|
"task_config/demo_randomized.yml",
|
|
"task_config/_embodiment_config.yml",
|
|
"task_config/_camera_config.yml",
|
|
"envs/beat_block_hammer.py",
|
|
"description/task_instruction/beat_block_hammer.json",
|
|
]:
|
|
path = robotwin / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(
|
|
"embodiment: [aloha-agilex]\n"
|
|
if path.name.startswith("demo_")
|
|
else "{}\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assets = root / "assets"
|
|
(assets / "embodiments" / "ur5-wsg").mkdir(parents=True)
|
|
for relative in [
|
|
"objects/objaverse/list.json",
|
|
"objects/same.json",
|
|
"objects/020_hammer/model_data0.json",
|
|
"objects/020_hammer/collision/base0.glb",
|
|
"objects/020_hammer/visual/base0.glb",
|
|
]:
|
|
path = assets / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(b"fixture")
|
|
|
|
runtime, _, payload = holobrain_operator._prepare_robotwin_runtime(
|
|
{
|
|
"ROBOTWIN_ROOT": str(robotwin),
|
|
"ROBOTWIN_ASSET_ROOT": str(assets),
|
|
"UR5_WSG_ARM_DISTANCE": "0.8",
|
|
},
|
|
root / "output",
|
|
["beat_block_hammer"],
|
|
"robotwin2_0_ur5_wsg",
|
|
)
|
|
|
|
import yaml
|
|
|
|
runtime_config = yaml.safe_load(
|
|
(runtime / "task_config" / "demo_clean.yml").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
runtime_config["embodiment"], ["ur5-wsg", "ur5-wsg", 0.8]
|
|
)
|
|
self.assertEqual(payload["robotLayout"], "split-dual-arm")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual((runtime / "assets").resolve(), assets.resolve())
|
|
|
|
def test_builtin_shell_command_requires_script_id(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with self.assertRaisesRegex(RuntimeError, "需要 SCRIPT_ID"):
|
|
MODULE.builtin_shell_command(
|
|
"run-1",
|
|
"shell",
|
|
pathlib.Path(temp_dir),
|
|
{"label": "Shell 命令执行", "hyperparams": []},
|
|
[],
|
|
)
|
|
|
|
def test_builtin_shell_command_runs_approved_script(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
repo_dir = output_dir / "repo"
|
|
approved_dir = repo_dir / "approved_scripts"
|
|
approved_dir.mkdir(parents=True)
|
|
(approved_dir / "fixture_report.py").write_text(
|
|
"def run(**kwargs):\n"
|
|
" return [kwargs['helpers']['write_json_artifact']("
|
|
"kwargs['run_id'], kwargs['node_id'], kwargs['output_dir'], "
|
|
"'controlled-script-report.json', {'controlled': True, 'args': kwargs['args']})]\n",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {
|
|
"label": "Shell 命令执行",
|
|
"hyperparams": [
|
|
{"key": "SCRIPT_ID", "value": "fixture_report"},
|
|
{"key": "SCRIPT_ARGS_JSON", "value": '{"ok": true}'},
|
|
],
|
|
"outputs": [{"name": "result", "type": "json"}],
|
|
}
|
|
previous_repo = os.environ.get("DROBOTICFLOW_OPERATOR_REPOSITORY")
|
|
os.environ["DROBOTICFLOW_OPERATOR_REPOSITORY"] = str(repo_dir)
|
|
try:
|
|
artifacts = MODULE.builtin_shell_command("run-1", "shell", output_dir, node_data, [])
|
|
finally:
|
|
if previous_repo is None:
|
|
os.environ.pop("DROBOTICFLOW_OPERATOR_REPOSITORY", None)
|
|
else:
|
|
os.environ["DROBOTICFLOW_OPERATOR_REPOSITORY"] = previous_repo
|
|
|
|
self.assertEqual([item["name"] for item in artifacts], ["controlled-script-report.json"])
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload, {"controlled": True, "args": {"ok": True}})
|
|
|
|
def test_fixture_report_counts_direct_and_grouped_upstream_artifacts(self):
|
|
script_path = (
|
|
pathlib.Path(__file__).resolve().parents[2]
|
|
/ "operator-repos"
|
|
/ "builtin"
|
|
/ "shell_command"
|
|
/ "approved_scripts"
|
|
/ "fixture_report.py"
|
|
)
|
|
spec = importlib.util.spec_from_file_location("fixture_report_under_test", script_path)
|
|
self.assertIsNotNone(spec)
|
|
self.assertIsNotNone(spec.loader)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
|
|
self.assertEqual(
|
|
module.upstream_artifact_count(
|
|
[
|
|
{"id": "artifact-1", "name": "one.json"},
|
|
{"id": "artifact-2", "name": "two.json"},
|
|
]
|
|
),
|
|
2,
|
|
)
|
|
self.assertEqual(
|
|
module.upstream_artifact_count(
|
|
[
|
|
{"artifacts": [{"id": "artifact-1"}, {"id": "artifact-2"}]},
|
|
{"artifacts": [{"id": "artifact-3"}]},
|
|
]
|
|
),
|
|
3,
|
|
)
|
|
|
|
|
|
class _HFDatasetViewerHandler(http.server.BaseHTTPRequestHandler):
|
|
requests = []
|
|
|
|
def do_GET(self):
|
|
parsed = MODULE.urlparse(self.path)
|
|
query = dict(item.split("=", 1) if "=" in item else (item, "") for item in parsed.query.split("&") if item)
|
|
self.__class__.requests.append({"path": parsed.path, "query": query})
|
|
if parsed.path == "/is-valid":
|
|
response = {"valid": True}
|
|
elif parsed.path == "/splits":
|
|
response = {"splits": [{"dataset": "lerobot/pusht", "config": "default", "split": "train", "num_examples": 2}]}
|
|
elif parsed.path == "/size":
|
|
response = {"size": {"dataset": {"num_rows": 2, "num_bytes": 128}}}
|
|
elif parsed.path == "/parquet":
|
|
response = {
|
|
"parquet_files": [
|
|
{
|
|
"dataset": "lerobot/pusht",
|
|
"config": "default",
|
|
"split": "train",
|
|
"filename": "0000.parquet",
|
|
"size": 128,
|
|
}
|
|
]
|
|
}
|
|
elif parsed.path == "/rows":
|
|
response = {
|
|
"rows": [
|
|
{"row_idx": 0, "row": {"episode_index": 0, "frame_index": 145, "action": [120.0, 373.0]}},
|
|
{"row_idx": 1, "row": {"episode_index": 0, "frame_index": 146, "action": [124.0, 371.0]}},
|
|
],
|
|
"num_rows_total": 2,
|
|
}
|
|
else:
|
|
self.send_error(404)
|
|
return
|
|
response_bytes = json.dumps(response).encode("utf-8")
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(response_bytes)))
|
|
self.end_headers()
|
|
self.wfile.write(response_bytes)
|
|
|
|
def log_message(self, *_args):
|
|
return
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def hf_dataset_viewer_server():
|
|
_HFDatasetViewerHandler.requests = []
|
|
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _HFDatasetViewerHandler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
previous_no_proxy = os.environ.get("NO_PROXY")
|
|
previous_no_proxy_lower = os.environ.get("no_proxy")
|
|
os.environ["NO_PROXY"] = "127.0.0.1,localhost"
|
|
os.environ["no_proxy"] = "127.0.0.1,localhost"
|
|
thread.start()
|
|
try:
|
|
yield server, _HFDatasetViewerHandler.requests
|
|
finally:
|
|
server.shutdown()
|
|
thread.join(timeout=5)
|
|
server.server_close()
|
|
if previous_no_proxy is None:
|
|
os.environ.pop("NO_PROXY", None)
|
|
else:
|
|
os.environ["NO_PROXY"] = previous_no_proxy
|
|
if previous_no_proxy_lower is None:
|
|
os.environ.pop("no_proxy", None)
|
|
else:
|
|
os.environ["no_proxy"] = previous_no_proxy_lower
|
|
|
|
|
|
class EmbodiedDatasetOperatorTests(unittest.TestCase):
|
|
def test_hf_dataset_probe_calls_dataset_viewer_api(self):
|
|
with hf_dataset_viewer_server() as (server, requests), tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "HF 数据集探测",
|
|
"hyperparams": [
|
|
{"key": "BASE_URL", "value": f"http://127.0.0.1:{server.server_address[1]}"},
|
|
{"key": "DATASET", "value": "lerobot/pusht"},
|
|
{"key": "ROW_LENGTH", "value": "2"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_hf_dataset_probe"]("run-1", "hf", output_dir, node_data, [])
|
|
|
|
self.assertEqual([item["path"] for item in requests], ["/is-valid", "/splits", "/size", "/parquet", "/rows"])
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "huggingface-dataset")
|
|
self.assertEqual(payload["dataset"], "lerobot/pusht")
|
|
self.assertEqual(payload["selectedConfig"], "default")
|
|
self.assertEqual(payload["selectedSplit"], "train")
|
|
self.assertEqual(payload["rowCount"], 2)
|
|
self.assertEqual(payload["parquetFileCount"], 1)
|
|
|
|
def test_hf_dataset_probe_retries_incomplete_read(self):
|
|
operator_module = MODULE.load_operator_module("hf_dataset_probe")
|
|
payload = json.dumps({"valid": True}).encode("utf-8")
|
|
responses = [http.client.IncompleteRead(b"", 100), _FakeHTTPResponse(payload)]
|
|
|
|
with mock.patch.object(operator_module.request, "urlopen", side_effect=responses) as urlopen, mock.patch.object(
|
|
operator_module.time, "sleep"
|
|
) as sleep:
|
|
result = operator_module._request_json(
|
|
"https://datasets-server.huggingface.co",
|
|
"/is-valid",
|
|
{"dataset": "lerobot/pusht"},
|
|
)
|
|
|
|
self.assertEqual(result, {"valid": True})
|
|
self.assertEqual(urlopen.call_count, 2)
|
|
sleep.assert_called_once_with(operator_module.REQUEST_RETRY_DELAY_SECONDS)
|
|
|
|
def test_rlds_episode_summary_validates_real_jsonl_episodes(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "rlds"
|
|
root.mkdir()
|
|
(root / "dataset_info.json").write_text(json.dumps({"name": "bridge_v2"}), encoding="utf-8")
|
|
episodes_path = root / "episodes.jsonl"
|
|
episodes = [
|
|
{
|
|
"episode_id": "ep-1",
|
|
"steps": [
|
|
{
|
|
"is_first": True,
|
|
"is_last": False,
|
|
"is_terminal": False,
|
|
"action": [0.1, 0.2, 0.3],
|
|
"reward": 0.0,
|
|
"observation": {"image": "frame-1.jpg", "state": [1.0, 2.0]},
|
|
"language_instruction": "pick up the block",
|
|
},
|
|
{
|
|
"is_first": False,
|
|
"is_last": True,
|
|
"is_terminal": True,
|
|
"action": [0.3, 0.2, 0.1],
|
|
"reward": 1.0,
|
|
"observation": {"image": "frame-2.jpg", "state": [1.5, 2.5]},
|
|
},
|
|
],
|
|
},
|
|
{
|
|
"episode_id": "ep-2",
|
|
"steps": [
|
|
{
|
|
"is_first": True,
|
|
"is_last": True,
|
|
"is_terminal": True,
|
|
"action": [0.0, 0.0, 0.1],
|
|
"reward": 0.5,
|
|
"observation": {"image": "frame-3.jpg", "state": [0.0, 1.0]},
|
|
}
|
|
],
|
|
},
|
|
]
|
|
episodes_path.write_text("\n".join(json.dumps(item) for item in episodes), encoding="utf-8")
|
|
node_data = {"label": "RLDS 轨迹概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_rlds_episode_summary"]("run-1", "rlds", pathlib.Path(temp_dir) / "out", node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "rlds")
|
|
self.assertEqual(payload["episodeCount"], 2)
|
|
self.assertEqual(payload["stepCount"], 3)
|
|
self.assertEqual(payload["invalidEpisodeCount"], 0)
|
|
self.assertEqual(payload["episodes"][0]["actionDimension"], 3)
|
|
self.assertIn("image", payload["episodes"][0]["observationKeys"])
|
|
self.assertIn("dataset_info.json", payload["metadata"])
|
|
|
|
def test_agibot_world2026_validate_reads_real_parquet_video_layout(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "agibot"
|
|
meta = root / "meta"
|
|
data = root / "data" / "chunk-000"
|
|
video = root / "videos" / "observation.images.head" / "chunk-000"
|
|
meta.mkdir(parents=True)
|
|
data.mkdir(parents=True)
|
|
video.mkdir(parents=True)
|
|
(meta / "info.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"dataset_name": "agibot-world/AgiBotWorld2026",
|
|
"data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
|
|
"video_path": "videos/{video_key}/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.mp4",
|
|
"features": {
|
|
"observation.images.head": {"dtype": "video"},
|
|
"action": {"dtype": "float32", "shape": [7]},
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
episodes = [
|
|
{"episode_index": 0, "length": 2, "tasks": [0], "data_path": "data/chunk-000/episode_000000.parquet"},
|
|
{"episode_index": 1, "length": 1, "tasks": [1], "data_path": "data/chunk-000/episode_000001.parquet"},
|
|
]
|
|
(meta / "episodes.jsonl").write_text("\n".join(json.dumps(item) for item in episodes), encoding="utf-8")
|
|
tasks = [
|
|
{"task_index": 0, "task": "pick up cup"},
|
|
{"task_index": 1, "task": "place cup"},
|
|
]
|
|
(meta / "tasks.jsonl").write_text("\n".join(json.dumps(item) for item in tasks), encoding="utf-8")
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 0],
|
|
"frame_index": [0, 1],
|
|
"timestamp": [0.0, 0.033],
|
|
"action": [[0.1] * 7, [0.2] * 7],
|
|
}
|
|
),
|
|
data / "episode_000000.parquet",
|
|
)
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [1],
|
|
"frame_index": [0],
|
|
"timestamp": [0.0],
|
|
"action": [[0.3] * 7],
|
|
}
|
|
),
|
|
data / "episode_000001.parquet",
|
|
)
|
|
(video / "episode_000000.mp4").write_bytes(b"real-mp4-bytes-0")
|
|
(video / "episode_000001.mp4").write_bytes(b"real-mp4-bytes-1")
|
|
node_data = {"label": "AgiBotWorld2026 数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_agibot_world2026_validate"](
|
|
"run-1", "agibot", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "agibot-world2026-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["episodeMetadataCount"], 2)
|
|
self.assertEqual(payload["taskCount"], 2)
|
|
self.assertEqual(payload["dataParquetCount"], 2)
|
|
self.assertEqual(payload["dataRowCount"], 3)
|
|
self.assertEqual(payload["videoFileCount"], 2)
|
|
self.assertEqual(payload["validEpisodeCount"], 2)
|
|
self.assertEqual(payload["videoKeys"], ["observation.images.head"])
|
|
|
|
def test_droid_raw_episode_validate_reads_real_hdf5_metadata_and_video(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "droid-raw"
|
|
episode = root / "episode_000001"
|
|
recordings = episode / "recordings" / "MP4"
|
|
recordings.mkdir(parents=True)
|
|
with h5py.File(episode / "trajectory.h5", "w") as handle:
|
|
action = handle.create_group("action")
|
|
action.create_dataset(
|
|
"cartesian_position",
|
|
data=np.array([[0.1, 0.2, 0.3, 0.0, 0.0, 0.0, 1.0]] * 3, dtype="float32"),
|
|
)
|
|
observation = handle.create_group("observation")
|
|
observation.create_dataset("cartesian_position", data=np.ones((3, 6), dtype="float32"))
|
|
observation.create_dataset("gripper_position", data=np.array([[0.1], [0.2], [0.3]], dtype="float32"))
|
|
(episode / "metadata_camera.json").write_text(
|
|
json.dumps({"uuid": "episode-000001", "task": "pick the cup", "robot_serial": "franka-1"}),
|
|
encoding="utf-8",
|
|
)
|
|
(episode / "metadata_recording.json").write_text(
|
|
json.dumps({"episode_id": "episode_000001", "time": "2026-06-24T15:00:00+08:00"}),
|
|
encoding="utf-8",
|
|
)
|
|
(recordings / "camera_0.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42real-droid-video")
|
|
node_data = {"label": "DROID Raw Episode校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_droid_raw_episode_validate"](
|
|
"run-1", "droid", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "droid-raw-episode-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["episodeCount"], 1)
|
|
self.assertEqual(payload["trajectoryFileCount"], 1)
|
|
self.assertEqual(payload["metadataFileCount"], 2)
|
|
self.assertEqual(payload["videoFileCount"], 1)
|
|
self.assertEqual(payload["validEpisodeCount"], 1)
|
|
self.assertEqual(payload["episodes"][0]["frameCount"], 3)
|
|
self.assertIn("/action/cartesian_position", payload["episodes"][0]["actionDatasetPaths"])
|
|
self.assertIn("/observation/gripper_position", payload["episodes"][0]["stateDatasetPaths"])
|
|
|
|
def test_libero_hdf5_task_validate_reads_real_hdf5_and_bddl(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "libero_spatial"
|
|
demos = root / "demos"
|
|
bddl = root / "bddl"
|
|
demos.mkdir(parents=True)
|
|
bddl.mkdir()
|
|
(bddl / "pick_up_the_black_bowl.bddl").write_text(
|
|
"(define (problem pick_up_the_black_bowl) (:language \"pick up the black bowl\"))\n",
|
|
encoding="utf-8",
|
|
)
|
|
path = demos / "pick_up_the_black_bowl_demo.hdf5"
|
|
with h5py.File(path, "w") as handle:
|
|
data = handle.create_group("data")
|
|
data.attrs["total"] = 3
|
|
data.attrs["env_args"] = json.dumps({"env_name": "LIBERO_SPATIAL", "bddl_file_name": "pick_up_the_black_bowl.bddl"})
|
|
demo = data.create_group("demo_0")
|
|
demo.attrs["language_instruction"] = "pick up the black bowl"
|
|
demo.create_dataset("actions", data=np.ones((3, 7), dtype="float32"))
|
|
demo.create_dataset("states", data=np.ones((3, 12), dtype="float32"))
|
|
demo.create_dataset("rewards", data=np.array([0.0, 0.0, 1.0], dtype="float32"))
|
|
demo.create_dataset("dones", data=np.array([False, False, True], dtype="bool"))
|
|
obs = demo.create_group("obs")
|
|
obs.create_dataset("agentview_rgb", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
|
|
obs.create_dataset("eye_in_hand_rgb", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
|
|
obs.create_dataset("gripper_states", data=np.ones((3, 2), dtype="float32"))
|
|
obs.create_dataset("joint_states", data=np.ones((3, 7), dtype="float32"))
|
|
obs.create_dataset("ee_states", data=np.ones((3, 6), dtype="float32"))
|
|
next_obs = demo.create_group("next_obs")
|
|
next_obs.create_dataset("agentview_rgb", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
|
|
mask = handle.create_group("mask")
|
|
mask.create_dataset("train", data=np.array([b"demo_0"]))
|
|
node_data = {
|
|
"label": "LIBERO HDF5任务校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "BDDL_PATH", "value": str(bddl)},
|
|
{"key": "REQUIRE_BDDL", "value": "true"},
|
|
{"key": "REQUIRE_LANGUAGE", "value": "true"},
|
|
{"key": "REQUIRE_TERMINAL_DONE", "value": "true"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_libero_hdf5_task_validate"](
|
|
"run-1", "libero", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "libero-hdf5-task-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["bddlFileCount"], 1)
|
|
self.assertEqual(payload["taskCount"], 1)
|
|
self.assertEqual(payload["demoCount"], 1)
|
|
self.assertEqual(payload["validDemoCount"], 1)
|
|
self.assertEqual(payload["frameCount"], 3)
|
|
self.assertEqual(payload["taskSuites"], ["libero_spatial"])
|
|
self.assertEqual(payload["files"][0]["bddlPath"], "pick_up_the_black_bowl.bddl")
|
|
self.assertIn("agentview_rgb", payload["files"][0]["demos"][0]["imageObservationKeys"])
|
|
self.assertIn("joint_states", payload["files"][0]["demos"][0]["proprioObservationKeys"])
|
|
self.assertEqual(payload["files"][0]["demos"][0]["languageFields"], ["pick up the black bowl"])
|
|
|
|
def test_robocasa_hdf5_demonstration_validate_reads_real_demo_metadata_and_action_dict(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "robocasa"
|
|
root.mkdir()
|
|
path = root / "demo.hdf5"
|
|
with h5py.File(path, "w") as handle:
|
|
data = handle.create_group("data")
|
|
data.attrs["total"] = 3
|
|
data.attrs["env_args"] = json.dumps({"env_name": "KitchenPnPCounterToCab", "type": "robocasa"})
|
|
demo = data.create_group("demo_0")
|
|
demo.attrs["model_file"] = "<mujoco><worldbody /></mujoco>"
|
|
demo.attrs["ep_meta"] = json.dumps({"lang": "pick up the mug and place it in the cabinet", "layout_id": 1, "style_id": 2})
|
|
demo.create_dataset("actions", data=np.ones((3, 7), dtype="float32"))
|
|
action_dict = demo.create_group("action_dict")
|
|
action_dict.create_dataset("abs_pos", data=np.ones((3, 3), dtype="float32"))
|
|
action_dict.create_dataset("abs_rot_6d", data=np.ones((3, 6), dtype="float32"))
|
|
action_dict.create_dataset("gripper", data=np.ones((3, 1), dtype="float32"))
|
|
demo.create_dataset("states", data=np.ones((4, 10), dtype="float32"))
|
|
obs = demo.create_group("obs")
|
|
obs.create_dataset("robot0_agentview_left_image", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
|
|
obs.create_dataset("robot0_eye_in_hand_image", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
|
|
obs.create_dataset("robot0_joint_pos", data=np.ones((3, 7), dtype="float32"))
|
|
obs.create_dataset("robot0_gripper_qpos", data=np.ones((3, 2), dtype="float32"))
|
|
next_obs = demo.create_group("next_obs")
|
|
next_obs.create_dataset("robot0_agentview_left_image", data=np.zeros((3, 8, 8, 3), dtype="uint8"))
|
|
mask = handle.create_group("mask")
|
|
mask.create_dataset("train", data=np.array([b"demo_0"]))
|
|
node_data = {"label": "RoboCasa HDF5示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_robocasa_hdf5_demonstration_validate"](
|
|
"run-1", "robocasa", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "robocasa-hdf5-demonstration-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["demoCount"], 1)
|
|
self.assertEqual(payload["validDemoCount"], 1)
|
|
self.assertEqual(payload["frameCount"], 3)
|
|
self.assertEqual(payload["languageInstructions"], ["pick up the mug and place it in the cabinet"])
|
|
demo_payload = payload["files"][0]["demos"][0]
|
|
self.assertEqual(demo_payload["actionShape"], [3, 7])
|
|
self.assertEqual(demo_payload["stateShape"], [4, 10])
|
|
self.assertIn("robot0_agentview_left_image", demo_payload["imageObservationKeys"])
|
|
self.assertIn("robot0_joint_pos", demo_payload["proprioObservationKeys"])
|
|
self.assertEqual(sorted(demo_payload["actionDictKeys"]), ["abs_pos", "abs_rot_6d", "gripper"])
|
|
self.assertTrue(demo_payload["modelFilePresent"])
|
|
self.assertTrue(demo_payload["epMetaPresent"])
|
|
|
|
def test_furniturebench_pkl_demonstration_validate_reads_real_pkl_file(self):
|
|
import pickle
|
|
|
|
try:
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "furniturebench" / "low" / "lamp"
|
|
root.mkdir(parents=True)
|
|
observations = []
|
|
for index in range(2):
|
|
observations.append(
|
|
{
|
|
"color_image1": np.full((224, 224, 3), 20 + index, dtype=np.uint8),
|
|
"color_image2": np.full((224, 224, 3), 30 + index, dtype=np.uint8),
|
|
"robot_state": {
|
|
"ee_pos": np.array([0.1, 0.2, 0.3], dtype=np.float32),
|
|
"ee_quat": np.array([1.0, 0.0, 0.0, 0.0], dtype=np.float32),
|
|
"ee_pos_vel": np.zeros((3,), dtype=np.float32),
|
|
"ee_ori_vel": np.zeros((3,), dtype=np.float32),
|
|
"joint_positions": np.ones((7,), dtype=np.float32) * index,
|
|
"joint_velocities": np.zeros((7,), dtype=np.float32),
|
|
"joint_torques": np.zeros((7,), dtype=np.float32),
|
|
"gripper_width": np.array([0.04], dtype=np.float32),
|
|
},
|
|
}
|
|
)
|
|
demo = {
|
|
"furniture": "lamp",
|
|
"observations": observations,
|
|
"actions": [np.ones((8,), dtype=np.float32), np.ones((8,), dtype=np.float32) * 2],
|
|
"rewards": [0, 1],
|
|
"skills": [0, 1],
|
|
}
|
|
path = root / "0.pkl"
|
|
with path.open("wb") as handle:
|
|
pickle.dump(demo, handle)
|
|
node_data = {"label": "FurnitureBench PKL示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path.parents[2])}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_furniturebench_pkl_demonstration_validate"](
|
|
"run-1", "furniturebench", path.parents[2] / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "furniturebench-pkl-demonstration-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["validFileCount"], 1)
|
|
self.assertEqual(payload["totalFrames"], 2)
|
|
self.assertEqual(payload["furnitureCounts"], {"lamp": 1})
|
|
self.assertEqual(payload["randomnessCounts"], {"low": 1})
|
|
demo_payload = payload["demos"][0]
|
|
self.assertEqual(demo_payload["actionDimensions"], [8])
|
|
self.assertEqual(demo_payload["imageKeyCounts"], {"color_image1": 2, "color_image2": 2})
|
|
self.assertEqual(demo_payload["robotStateKeyCounts"]["joint_positions"], 2)
|
|
self.assertEqual(demo_payload["rewardValues"], [0.0, 1.0])
|
|
self.assertEqual(demo_payload["skillValues"], [0, 1])
|
|
|
|
def test_roboturk_real_hdf5_validate_reads_aligned_hdf5_file(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "roboturk"
|
|
root.mkdir()
|
|
path = root / "SawyerLaundryLayout_aligned_dataset.hdf5"
|
|
with h5py.File(path, "w") as handle:
|
|
data = handle.create_group("data")
|
|
data.attrs["total_demos"] = 1
|
|
data.attrs["total_time"] = 3.0
|
|
user = data.create_group("user_001")
|
|
demo = user.create_group("demo_1")
|
|
demo.attrs["clock_time"] = "2021-02-24T00:00:00Z"
|
|
demo.attrs["session_id"] = "session-001"
|
|
demo.attrs["demo_time"] = 3.0
|
|
demo.attrs["date"] = "2021-02-24"
|
|
demo.attrs["task"] = "SawyerLaundryLayout"
|
|
demo.attrs["top_depth_video_file"] = "SawyerLaundryLayout/roboturk_day1/user_001/SawyerLaundryLayout/t/demo_1/kinect_depth_aligned.mp4"
|
|
demo.attrs["top_rgb_video_file"] = "SawyerLaundryLayout/roboturk_day1/user_001/SawyerLaundryLayout/t/demo_1/kinect_rgb_aligned.mp4"
|
|
demo.attrs["front_rgb_video_file"] = "SawyerLaundryLayout/roboturk_day1/user_001/SawyerLaundryLayout/t/demo_1/usb_aligned.mp4"
|
|
robot_observation = demo.create_group("robot_observation")
|
|
robot_observation.create_dataset("eef_poses", data=np.ones((3, 7), dtype=np.float32))
|
|
robot_observation.create_dataset("eef_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
|
|
robot_observation.create_dataset("joint_states_arm", data=np.ones((3, 27), dtype=np.float32))
|
|
robot_observation.create_dataset("joint_states_arm_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
|
|
robot_observation.create_dataset("joint_states_gripper", data=np.ones((3, 3), dtype=np.float32))
|
|
robot_observation.create_dataset("joint_states_gripper_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
|
|
user_control = demo.create_group("user_control")
|
|
user_control.create_dataset("user_control", data=np.ones((3, 22), dtype=np.float32))
|
|
user_control.create_dataset("user_control_times", data=np.array([1.0, 2.0, 3.0], dtype=np.float64))
|
|
node_data = {"label": "RoboTurk Real HDF5校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_roboturk_real_hdf5_validate"](
|
|
"run-1", "roboturk-real", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "roboturk-real-hdf5-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["validFileCount"], 1)
|
|
self.assertEqual(payload["totalUsers"], 1)
|
|
self.assertEqual(payload["totalDemos"], 1)
|
|
self.assertEqual(payload["returnedDemoCount"], 1)
|
|
self.assertEqual(payload["taskCounts"], {"SawyerLaundryLayout": 1})
|
|
file_payload = payload["files"][0]
|
|
self.assertEqual(file_payload["dataAttrs"]["total_demos"], 1)
|
|
demo_payload = file_payload["demos"][0]
|
|
self.assertEqual(demo_payload["userId"], "user_001")
|
|
self.assertEqual(demo_payload["demoId"], "demo_1")
|
|
self.assertEqual(demo_payload["datasetShapes"]["robot_observation/eef_poses"], [3, 7])
|
|
self.assertEqual(demo_payload["datasetShapes"]["robot_observation/joint_states_arm"], [3, 27])
|
|
self.assertEqual(demo_payload["datasetShapes"]["user_control/user_control"], [3, 22])
|
|
self.assertEqual(demo_payload["frameCounts"]["user_control/user_control_times"], 3)
|
|
|
|
def test_roboset_hdf5_validate_reads_trial_data_and_multicamera_rgb(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "roboset"
|
|
root.mkdir()
|
|
path = root / "FK1_KitchenHuman.h5"
|
|
with h5py.File(path, "w") as handle:
|
|
handle.attrs["dataset"] = "RoboSet"
|
|
trial = handle.create_group("Trial0")
|
|
trial.attrs["env_name"] = "FK1_Knob1OnRandom-v4"
|
|
data = trial.create_group("data")
|
|
trial.create_group("derived").create_dataset("success", data=np.array([0, 0, 1], dtype=np.int8))
|
|
config = trial.create_group("config")
|
|
config.attrs["robot"] = "FrankaKitchen"
|
|
data.create_dataset("actions", data=np.ones((3, 9), dtype=np.float32))
|
|
data.create_dataset("rewards", data=np.array([0.0, 0.1, 1.0], dtype=np.float32))
|
|
data.create_dataset("dones", data=np.array([0, 0, 1], dtype=np.int8))
|
|
observations = data.create_group("observations")
|
|
observations.create_dataset("qpos", data=np.ones((3, 30), dtype=np.float32))
|
|
observations.create_dataset("qvel", data=np.ones((3, 30), dtype=np.float32) * 2)
|
|
rgb = data.create_group("rgb")
|
|
for camera_index in range(4):
|
|
rgb.create_dataset(
|
|
f"camera_{camera_index}",
|
|
data=np.ones((3, 12, 16, 3), dtype=np.uint8) * camera_index,
|
|
)
|
|
node_data = {
|
|
"label": "RoboSet HDF5轨迹校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "MIN_RGB_VIEWS", "value": "4"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_roboset_hdf5_validate"](
|
|
"run-1", "roboset", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "roboset-hdf5-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["validFileCount"], 1)
|
|
self.assertEqual(payload["totalTrials"], 1)
|
|
self.assertEqual(payload["returnedTrialCount"], 1)
|
|
self.assertEqual(payload["rgbViewCounts"], {"4": 1})
|
|
self.assertEqual(payload["actionDimensionCounts"], {"9": 1})
|
|
trial_payload = payload["files"][0]["trials"][0]
|
|
self.assertEqual(trial_payload["trialId"], "Trial0")
|
|
self.assertEqual(trial_payload["trialKeys"], ["config", "data", "derived"])
|
|
self.assertEqual(trial_payload["frameCount"], 3)
|
|
self.assertEqual(trial_payload["actionDatasetPaths"], ["actions"])
|
|
self.assertEqual(trial_payload["rewardDatasetPaths"], ["rewards"])
|
|
self.assertEqual(trial_payload["doneDatasetPaths"], ["dones"])
|
|
self.assertEqual(trial_payload["rgbViewCount"], 4)
|
|
self.assertEqual(trial_payload["datasetShapes"]["actions"], [3, 9])
|
|
self.assertEqual(trial_payload["datasetShapes"]["observations/qpos"], [3, 30])
|
|
self.assertEqual(trial_payload["datasetShapes"]["rgb/camera_0"], [3, 12, 16, 3])
|
|
|
|
def test_rh20t_scene_validate_reads_real_scene_structure_and_arrays(self):
|
|
try:
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "rh20t"
|
|
scene = root / "scene_000001"
|
|
camera = scene / "cam_123456"
|
|
transformed = scene / "transformed"
|
|
high_freq = scene / "high_freq_data"
|
|
calib = root / "calib" / "20240624"
|
|
camera.mkdir(parents=True)
|
|
transformed.mkdir(parents=True)
|
|
high_freq.mkdir(parents=True)
|
|
calib.mkdir(parents=True)
|
|
(scene / "metadata.json").write_text(
|
|
json.dumps({"calib": "20240624", "task_description": "open the drawer", "operator": "robot-teacher"}),
|
|
encoding="utf-8",
|
|
)
|
|
np.save(camera / "timestamps.npy", {"color": np.array([1000, 1033, 1066], dtype=np.int64)})
|
|
(camera / "color.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42rh20t-video")
|
|
np.save(transformed / "force_torque.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 6), dtype=np.float32)})
|
|
np.save(transformed / "tcp.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 7), dtype=np.float32)})
|
|
np.save(transformed / "joint.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 7), dtype=np.float32)})
|
|
np.save(transformed / "gripper.npy", {"timestamp": np.array([1000, 1033, 1066]), "value": np.ones((3, 1), dtype=np.float32)})
|
|
np.save(high_freq / "force_torque_tcp_joint_timestamp.npy", np.array([[1000, 0.1], [1001, 0.2]], dtype=np.float32))
|
|
np.save(calib / "intrinsics.npy", {"cam_123456": np.eye(3, dtype=np.float32)})
|
|
np.save(calib / "extrinsics.npy", {"cam_123456": np.eye(4, dtype=np.float32)})
|
|
np.save(calib / "tcp.npy", np.ones((4, 4), dtype=np.float32))
|
|
node_data = {"label": "RH20T 场景结构校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_rh20t_scene_validate"](
|
|
"run-1", "rh20t", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "rh20t-scene-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["sceneCount"], 1)
|
|
self.assertEqual(payload["validSceneCount"], 1)
|
|
self.assertEqual(payload["cameraCount"], 1)
|
|
self.assertEqual(payload["cameraTimestampCount"], 3)
|
|
scene_payload = payload["scenes"][0]
|
|
self.assertTrue(scene_payload["metadataPresent"])
|
|
self.assertEqual(scene_payload["calib"], "20240624")
|
|
self.assertEqual(scene_payload["cameras"][0]["videoFiles"], ["cam_123456/color.mp4"])
|
|
self.assertIn("force_torque.npy", scene_payload["transformed"]["files"])
|
|
self.assertTrue(scene_payload["highFreq"]["present"])
|
|
self.assertIn("intrinsics.npy", scene_payload["calibration"]["files"])
|
|
|
|
def test_robomind_hdf5_quality_validate_reads_real_hdf5_joint_position_and_end_effector(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "robomind" / "benchmark1_0"
|
|
data_dir = root / "h5_franka_1rgb" / "bread_in_basket" / "success_episodes" / "train" / "1014_144602" / "data"
|
|
data_dir.mkdir(parents=True)
|
|
path = data_dir / "trajectory.hdf5"
|
|
with h5py.File(path, "w") as handle:
|
|
puppet = handle.create_group("puppet")
|
|
puppet.create_dataset(
|
|
"joint_position",
|
|
data=np.array(
|
|
[
|
|
[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6],
|
|
[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7],
|
|
[0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8],
|
|
],
|
|
dtype=np.float32,
|
|
),
|
|
)
|
|
puppet.create_dataset(
|
|
"end_effector",
|
|
data=np.array(
|
|
[
|
|
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
|
|
[0.1, 0.0, 0.0, 0.0, 0.0, 0.1],
|
|
[0.2, 0.0, 0.0, 0.0, 0.0, 0.2],
|
|
],
|
|
dtype=np.float32,
|
|
),
|
|
)
|
|
master = handle.create_group("master")
|
|
master.create_dataset(
|
|
"joint_position",
|
|
data=np.array(
|
|
[
|
|
[1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6],
|
|
[1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7],
|
|
[1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8],
|
|
],
|
|
dtype=np.float32,
|
|
),
|
|
)
|
|
rgb_images = handle.create_group("rgb_images")
|
|
rgb_images.create_dataset("front", data=np.zeros((3, 4, 4, 3), dtype=np.uint8))
|
|
handle.create_dataset("action", data=np.ones((3, 7), dtype=np.float32))
|
|
node_data = {
|
|
"label": "RoboMIND HDF5质量校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_robomind_hdf5_quality_validate"](
|
|
"run-1", "robomind", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "robomind-hdf5-quality-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["trajectoryCount"], 1)
|
|
self.assertEqual(payload["validFileCount"], 1)
|
|
self.assertEqual(payload["jointPositionFileCount"], 1)
|
|
self.assertEqual(payload["endEffectorFileCount"], 1)
|
|
self.assertEqual(payload["stuckJointPositionCount"], 0)
|
|
self.assertEqual(payload["stuckEndEffectorCount"], 0)
|
|
self.assertEqual(payload["imageDatasetCount"], 1)
|
|
file_payload = payload["files"][0]
|
|
self.assertEqual(file_payload["benchmark"], "benchmark1_0")
|
|
self.assertEqual(file_payload["robotType"], "h5_franka_1rgb")
|
|
self.assertEqual(file_payload["taskName"], "bread_in_basket")
|
|
self.assertEqual(file_payload["split"], "train")
|
|
self.assertEqual(file_payload["episodeId"], "1014_144602")
|
|
self.assertEqual(file_payload["frameCount"], 3)
|
|
self.assertEqual(file_payload["puppetJointPosition"]["key"], "puppet/joint_position")
|
|
self.assertEqual(file_payload["puppetJointPosition"]["shape"], [3, 7])
|
|
self.assertEqual(file_payload["puppetEndEffector"]["key"], "puppet/end_effector")
|
|
|
|
def test_hdf5_demonstration_summary_reads_real_hdf5_file(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
path = pathlib.Path(temp_dir) / "demo.hdf5"
|
|
with h5py.File(path, "w") as handle:
|
|
data = handle.create_group("data")
|
|
data.attrs["total"] = 3
|
|
data.attrs["env_args"] = json.dumps({"env_name": "Lift"})
|
|
demo_1 = data.create_group("demo_1")
|
|
demo_1.attrs["num_samples"] = 2
|
|
demo_1.create_dataset("actions", data=np.array([[0.1, 0.2], [0.3, 0.4]], dtype="float32"))
|
|
demo_1.create_dataset("states", data=np.array([[1.0], [2.0]], dtype="float32"))
|
|
obs = demo_1.create_group("obs")
|
|
obs.create_dataset("robot0_eef_pos", data=np.array([[0.0, 0.1, 0.2], [0.2, 0.3, 0.4]], dtype="float32"))
|
|
next_obs = demo_1.create_group("next_obs")
|
|
next_obs.create_dataset("robot0_eef_pos", data=np.array([[0.1, 0.2, 0.3], [0.3, 0.4, 0.5]], dtype="float32"))
|
|
demo_2 = data.create_group("demo_2")
|
|
demo_2.attrs["num_samples"] = 1
|
|
demo_2.create_dataset("actions", data=np.array([[0.5, 0.6]], dtype="float32"))
|
|
mask = handle.create_group("mask")
|
|
mask.create_dataset("train", data=np.array([b"demo_1"]))
|
|
node_data = {"label": "HDF5 示教数据概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_hdf5_demonstration_summary"](
|
|
"run-1", "hdf5", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "hdf5-demonstration")
|
|
self.assertEqual(payload["demoCount"], 2)
|
|
self.assertEqual(payload["totalSamples"], 3)
|
|
self.assertEqual(payload["demos"][0]["actionsShape"], [2, 2])
|
|
self.assertEqual(payload["demos"][0]["observationKeys"], ["robot0_eef_pos"])
|
|
self.assertEqual(payload["maskKeys"], ["train"])
|
|
|
|
def test_aloha_hdf5_validate_reads_real_act_layout(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
path = pathlib.Path(temp_dir) / "episode_000000.hdf5"
|
|
with h5py.File(path, "w") as handle:
|
|
handle.attrs["sim"] = False
|
|
observations = handle.create_group("observations")
|
|
observations.create_dataset("qpos", data=np.ones((3, 14), dtype="float32"))
|
|
observations.create_dataset("qvel", data=np.zeros((3, 14), dtype="float32"))
|
|
images = observations.create_group("images")
|
|
images.create_dataset("cam_high", data=np.zeros((3, 4, 5, 3), dtype="uint8"))
|
|
images.create_dataset("cam_wrist", data=np.zeros((3, 4, 5, 3), dtype="uint8"))
|
|
handle.create_dataset("action", data=np.ones((3, 14), dtype="float32"))
|
|
node_data = {"label": "ALOHA HDF5 示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_aloha_hdf5_validate"](
|
|
"run-1", "aloha", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "aloha-hdf5-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["totalFrames"], 3)
|
|
self.assertEqual(payload["cameraCount"], 2)
|
|
self.assertEqual(payload["actionDimensions"], [14])
|
|
self.assertEqual(payload["qposDimensions"], [14])
|
|
self.assertEqual(payload["files"][0]["cameraNames"], ["cam_high", "cam_wrist"])
|
|
|
|
def test_robomimic_hdf5_validate_reads_real_demo_groups(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
path = pathlib.Path(temp_dir) / "robomimic.hdf5"
|
|
with h5py.File(path, "w") as handle:
|
|
data = handle.create_group("data")
|
|
data.attrs["total"] = 3
|
|
data.attrs["env_args"] = json.dumps({"env_name": "Lift"})
|
|
demo_1 = data.create_group("demo_1")
|
|
demo_1.attrs["num_samples"] = 2
|
|
demo_1.create_dataset("actions", data=np.array([[0.1, 0.2], [0.3, 0.4]], dtype="float32"))
|
|
demo_1.create_dataset("rewards", data=np.array([0.0, 1.0], dtype="float32"))
|
|
demo_1.create_dataset("dones", data=np.array([False, True], dtype="bool"))
|
|
demo_1.create_dataset("states", data=np.array([[1.0], [2.0]], dtype="float32"))
|
|
obs = demo_1.create_group("obs")
|
|
obs.create_dataset("robot0_eef_pos", data=np.array([[0.0, 0.1, 0.2], [0.2, 0.3, 0.4]], dtype="float32"))
|
|
next_obs = demo_1.create_group("next_obs")
|
|
next_obs.create_dataset("robot0_eef_pos", data=np.array([[0.1, 0.2, 0.3], [0.3, 0.4, 0.5]], dtype="float32"))
|
|
demo_2 = data.create_group("demo_2")
|
|
demo_2.attrs["num_samples"] = 1
|
|
demo_2.create_dataset("actions", data=np.array([[0.5, 0.6]], dtype="float32"))
|
|
demo_2.create_dataset("rewards", data=np.array([0.5], dtype="float32"))
|
|
demo_2.create_dataset("dones", data=np.array([True], dtype="bool"))
|
|
obs = demo_2.create_group("obs")
|
|
obs.create_dataset("robot0_eef_pos", data=np.array([[0.4, 0.5, 0.6]], dtype="float32"))
|
|
next_obs = demo_2.create_group("next_obs")
|
|
next_obs.create_dataset("robot0_eef_pos", data=np.array([[0.5, 0.6, 0.7]], dtype="float32"))
|
|
mask = handle.create_group("mask")
|
|
mask.create_dataset("train", data=np.array([b"demo_1"]))
|
|
mask.create_dataset("valid", data=np.array([b"demo_2"]))
|
|
node_data = {"label": "robomimic HDF5 示教校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_robomimic_hdf5_validate"](
|
|
"run-1", "robomimic", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "robomimic-hdf5-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["demoCount"], 2)
|
|
self.assertEqual(payload["totalSamples"], 3)
|
|
self.assertEqual(payload["maskKeys"], ["train", "valid"])
|
|
self.assertEqual(payload["actionDimensions"], [2])
|
|
self.assertEqual(payload["files"][0]["observationKeyCounts"], {"robot0_eef_pos": 2})
|
|
|
|
def test_rlbench_dataset_validate_reads_real_episode_pickles_and_images(self):
|
|
import pickle
|
|
|
|
try:
|
|
import cv2 # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"cv2/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "rlbench"
|
|
episode = root / "pick_up_cup" / "variation0" / "episodes" / "episode0"
|
|
for camera in ("front_rgb", "left_shoulder_rgb"):
|
|
(episode / camera).mkdir(parents=True, exist_ok=True)
|
|
for index in range(3):
|
|
frame = np.full((6, 8, 3), 40 + index, dtype=np.uint8)
|
|
cv2.imwrite(str(episode / camera / f"{index}.png"), frame)
|
|
with (episode / "low_dim_obs.pkl").open("wb") as handle:
|
|
pickle.dump([{"gripper_open": True}, {"gripper_open": False}, {"gripper_open": True}], handle)
|
|
with (episode / "variation_descriptions.pkl").open("wb") as handle:
|
|
pickle.dump(["pick up the cup"], handle)
|
|
node_data = {"label": "RLBench 数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_rlbench_dataset_validate"](
|
|
"run-1", "rlbench", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "rlbench-dataset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["episodeCount"], 1)
|
|
self.assertEqual(payload["validEpisodeCount"], 1)
|
|
self.assertEqual(payload["totalLowDimFrames"], 3)
|
|
self.assertEqual(payload["cameraTotals"], {"front_rgb": 3, "left_shoulder_rgb": 3})
|
|
|
|
def test_maniskill_trajectory_validate_reads_real_hdf5_and_metadata(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "maniskill"
|
|
root.mkdir()
|
|
h5_path = root / "trajectory.h5"
|
|
with h5py.File(h5_path, "w") as handle:
|
|
traj_0 = handle.create_group("traj_0")
|
|
traj_0.create_dataset("actions", data=np.array([[0.1, 0.2], [0.2, 0.3], [0.3, 0.4]], dtype="float32"))
|
|
traj_0.create_dataset("rewards", data=np.array([0.0, 0.0, 1.0], dtype="float32"))
|
|
traj_0.create_dataset("terminated", data=np.array([False, False, True], dtype="bool"))
|
|
traj_0.create_dataset("env_states", data=np.ones((4, 3), dtype="float32"))
|
|
traj_1 = handle.create_group("traj_1")
|
|
traj_1.create_dataset("actions", data=np.array([[0.4, 0.5], [0.5, 0.6]], dtype="float32"))
|
|
traj_1.create_dataset("rewards", data=np.array([0.0, 1.0], dtype="float32"))
|
|
traj_1.create_dataset("terminated", data=np.array([False, True], dtype="bool"))
|
|
(root / "trajectory.json").write_text(
|
|
json.dumps({"env_info": {"env_id": "PickCube-v1"}, "episodes": [{"episode_id": 0, "elapsed_steps": 3}, {"episode_id": 1, "elapsed_steps": 2}]}),
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "ManiSkill 轨迹校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_maniskill_trajectory_validate"](
|
|
"run-1", "maniskill", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "maniskill-trajectory-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["trajectoryCount"], 2)
|
|
self.assertEqual(payload["actionStepCount"], 5)
|
|
self.assertEqual(payload["files"][0]["envId"], "PickCube-v1")
|
|
|
|
def test_calvin_episode_validate_reads_real_npz_and_language_annotations(self):
|
|
try:
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "calvin"
|
|
(root / "lang_annotations").mkdir(parents=True)
|
|
for index in range(2):
|
|
np.savez(
|
|
root / f"episode_{index:07d}.npz",
|
|
actions=np.array([0.1, 0.2, 0.3], dtype="float32"),
|
|
rel_actions=np.array([0.01, 0.02, 0.03], dtype="float32"),
|
|
robot_obs=np.ones((15,), dtype="float32"),
|
|
scene_obs=np.ones((24,), dtype="float32"),
|
|
rgb_static=np.zeros((8, 8, 3), dtype="uint8"),
|
|
rgb_gripper=np.zeros((6, 6, 3), dtype="uint8"),
|
|
)
|
|
np.save(
|
|
root / "lang_annotations" / "auto_lang_ann.npy",
|
|
{"language": {"ann": ["open the drawer", "push the block"], "task": ["open_drawer", "push_block"]}},
|
|
allow_pickle=True,
|
|
)
|
|
node_data = {"label": "CALVIN Episode 校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_calvin_episode_validate"](
|
|
"run-1", "calvin", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "calvin-episode-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["episodeCount"], 2)
|
|
self.assertEqual(payload["validEpisodeCount"], 2)
|
|
self.assertEqual(payload["languageAnnotationCount"], 2)
|
|
self.assertEqual(payload["episodes"][0]["shapes"]["actions"], [3])
|
|
|
|
def test_rtx_action_schema_validate_reads_real_parquet_component_actions(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "rtx"
|
|
root.mkdir()
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_id": ["episode-000", "episode-000"],
|
|
"world_vector": [[0.1, 0.0, -0.1], [0.2, 0.1, -0.2]],
|
|
"rotation_delta": [[0.01, 0.02, 0.03], [0.02, 0.03, 0.04]],
|
|
"gripper_closedness_action": [0.0, 1.0],
|
|
}
|
|
),
|
|
root / "actions.parquet",
|
|
)
|
|
node_data = {"label": "RT-X动作空间Schema校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_rtx_action_schema_validate"](
|
|
"run-1", "rtx-action", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "rtx-action-schema-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["rowCount"], 2)
|
|
self.assertEqual(payload["actionCount"], 2)
|
|
self.assertEqual(payload["expectedActionDim"], 7)
|
|
self.assertEqual(payload["fieldCounts"]["world_vector+rotation_delta+gripper_closedness_action"], 2)
|
|
|
|
def test_embodiment_metadata_validate_reads_real_jsonl_metadata(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "embodiments"
|
|
root.mkdir()
|
|
rows = [
|
|
{
|
|
"robot_type": "franka_panda",
|
|
"task": "pick cup",
|
|
"camera_names": ["front", "wrist"],
|
|
"action_dim": 7,
|
|
},
|
|
{
|
|
"robot_type": "aloha_bimanual",
|
|
"task": "fold cloth",
|
|
"camera_names": ["left_wrist", "right_wrist", "top"],
|
|
"action_dim": 14,
|
|
},
|
|
]
|
|
(root / "metadata.jsonl").write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8")
|
|
node_data = {
|
|
"label": "多机器人Embodiment元信息校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "MIN_ROBOT_TYPES", "value": "2"},
|
|
{"key": "REQUIRED_FIELDS", "value": "robot_type,task,camera,action_dim"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_embodiment_metadata_validate"](
|
|
"run-1", "embodiment", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "embodiment-metadata-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["robotTypeCount"], 2)
|
|
self.assertEqual(payload["taskCount"], 2)
|
|
self.assertEqual(payload["actionDimCounts"], {"7": 1, "14": 1})
|
|
|
|
def test_force_torque_sensor_validate_reads_real_hdf5_wrench(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "force"
|
|
root.mkdir()
|
|
with h5py.File(root / "force_torque.h5", "w") as handle:
|
|
sensor = handle.create_group("sensor")
|
|
sensor.create_dataset(
|
|
"wrench",
|
|
data=np.array([[1.0, 2.0, 3.0, 0.1, 0.2, 0.3], [1.5, 2.5, 3.5, 0.2, 0.3, 0.4]], dtype="float32"),
|
|
)
|
|
node_data = {"label": "力扭矩传感器数据校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_force_torque_sensor_validate"](
|
|
"run-1", "force", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "force-torque-sensor-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["rowCount"], 2)
|
|
self.assertEqual(payload["vectorCount"], 2)
|
|
self.assertEqual(payload["expectedDim"], 6)
|
|
self.assertEqual(payload["maxAbsByDimension"], [1.5, 2.5, 3.5, 0.2, 0.3, 0.4])
|
|
|
|
def test_dataset_split_leakage_validate_reads_real_split_csv(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "splits"
|
|
root.mkdir()
|
|
(root / "dataset-split-manifest.csv").write_text(
|
|
"\n".join(
|
|
[
|
|
"episodeId,split,task",
|
|
"episode-000,train,pick",
|
|
"episode-001,train,place",
|
|
"episode-002,val,pick",
|
|
"episode-003,test,place",
|
|
]
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "数据集切分泄漏校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_dataset_split_leakage_validate"](
|
|
"run-1", "split-leakage", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "dataset-split-leakage-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["episodeCount"], 4)
|
|
self.assertEqual(payload["splitCounts"], {"train": 2, "val": 1, "test": 1})
|
|
self.assertEqual(payload["leakedEpisodeCount"], 0)
|
|
|
|
def test_media_reference_validate_reads_real_manifest_and_media_files(self):
|
|
try:
|
|
import cv2 # type: ignore
|
|
import numpy as np # type: ignore
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"cv2/numpy/pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "media"
|
|
media_dir = root / "assets"
|
|
media_dir.mkdir(parents=True)
|
|
image = media_dir / "frame_000.png"
|
|
depth = media_dir / "depth_000.npy"
|
|
pointcloud = media_dir / "cloud_000.pcd"
|
|
cv2.imwrite(str(image), np.full((8, 10, 3), 120, dtype=np.uint8))
|
|
np.save(depth, np.ones((8, 10), dtype=np.float32))
|
|
pointcloud.write_text(
|
|
"VERSION .7\nFIELDS x y z\nSIZE 4 4 4\nTYPE F F F\nCOUNT 1 1 1\nWIDTH 1\nHEIGHT 1\nPOINTS 1\nDATA ascii\n0 0 1\n",
|
|
encoding="utf-8",
|
|
)
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_id": ["episode-000"],
|
|
"image_path": ["assets/frame_000.png"],
|
|
"depth_path": ["assets/depth_000.npy"],
|
|
"pointcloud_path": ["assets/cloud_000.pcd"],
|
|
}
|
|
),
|
|
root / "manifest.parquet",
|
|
)
|
|
node_data = {"label": "媒体引用可读性校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_media_reference_validate"](
|
|
"run-1", "media", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "media-reference-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["referenceCount"], 3)
|
|
self.assertEqual(payload["missingCount"], 0)
|
|
self.assertEqual(payload["unreadableCount"], 0)
|
|
self.assertEqual(payload["fieldCounts"]["image_path"], 1)
|
|
|
|
def test_dataset_schema_contract_validate_reads_parquet_hdf5_and_npz(self):
|
|
try:
|
|
import h5py # type: ignore
|
|
import numpy as np # type: ignore
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"h5py/numpy/pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "schema"
|
|
root.mkdir()
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_id": ["episode-000", "episode-001"],
|
|
"split": ["train", "val"],
|
|
"action": [[0.1, 0.2], [0.3, 0.4]],
|
|
}
|
|
),
|
|
root / "manifest.parquet",
|
|
)
|
|
with h5py.File(root / "trajectory.h5", "w") as handle:
|
|
data = handle.create_group("data")
|
|
data.create_dataset("actions", data=np.ones((2, 2), dtype="float32"))
|
|
data.create_dataset("observations", data=np.ones((2, 4), dtype="float32"))
|
|
np.savez(root / "episode_0000000.npz", actions=np.ones((2, 2), dtype="float32"), robot_obs=np.ones((2, 4), dtype="float32"))
|
|
node_data = {
|
|
"label": "数据集Schema合约校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "REQUIRED_COLUMNS", "value": "episode_id,split,action"},
|
|
{"key": "REQUIRED_HDF5_DATASETS", "value": "data/actions,data/observations"},
|
|
{"key": "REQUIRED_NPZ_ARRAYS", "value": "actions,robot_obs"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_dataset_schema_contract_validate"](
|
|
"run-1", "schema", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "dataset-schema-contract-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 3)
|
|
self.assertEqual(payload["validFileCount"], 3)
|
|
kinds = sorted(item["kind"] for item in payload["files"])
|
|
self.assertEqual(kinds, ["hdf5", "npz", "parquet"])
|
|
|
|
def test_lerobot_dataset_validate_reads_real_parquet_and_metadata(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "lerobot"
|
|
meta = root / "meta"
|
|
episode_meta_dir = meta / "episodes" / "chunk-000"
|
|
data_dir = root / "data" / "chunk-000"
|
|
meta.mkdir(parents=True)
|
|
episode_meta_dir.mkdir(parents=True)
|
|
data_dir.mkdir(parents=True)
|
|
(meta / "info.json").write_text(
|
|
json.dumps({"total_episodes": 2, "total_frames": 3, "total_videos": 0, "fps": 10}),
|
|
encoding="utf-8",
|
|
)
|
|
(meta / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "push cube"}) + "\n", encoding="utf-8")
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 1],
|
|
"length": [2, 1],
|
|
"task_index": [0, 0],
|
|
}
|
|
),
|
|
episode_meta_dir / "file-000.parquet",
|
|
)
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 0, 1],
|
|
"frame_index": [0, 1, 0],
|
|
"timestamp": [0.0, 0.1, 0.0],
|
|
"action": [[0.1, 0.2], [0.2, 0.3], [0.4, 0.5]],
|
|
"observation.state": [[1.0, 2.0], [1.1, 2.1], [3.0, 4.0]],
|
|
}
|
|
),
|
|
data_dir / "episode_000000.parquet",
|
|
)
|
|
node_data = {"label": "LeRobot 数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_lerobot_dataset_validate"](
|
|
"run-1", "lerobot-validate", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "lerobot-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["episodeMetadataCount"], 2)
|
|
self.assertEqual(payload["episodeMetadataFiles"], ["meta/episodes/chunk-000/file-000.parquet"])
|
|
self.assertEqual(payload["totalParquetRows"], 3)
|
|
self.assertEqual(payload["episodeRows"], {"0": 2, "1": 1})
|
|
self.assertIn("action", payload["schemaColumns"])
|
|
|
|
def test_lerobot_metadata_consistency_validate_reads_v3_meta_and_data(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "lerobot-v3"
|
|
meta = root / "meta"
|
|
episode_meta_dir = meta / "episodes" / "chunk-000"
|
|
data_dir = root / "data" / "chunk-000"
|
|
meta.mkdir(parents=True)
|
|
episode_meta_dir.mkdir(parents=True)
|
|
data_dir.mkdir(parents=True)
|
|
(meta / "info.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"codebase_version": "v3.0",
|
|
"total_episodes": 2,
|
|
"total_frames": 3,
|
|
"total_tasks": 1,
|
|
"features": {
|
|
"action": {"dtype": "float32", "shape": [3]},
|
|
"observation.state": {"dtype": "float32", "shape": [4]},
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(meta / "stats.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"action": {"mean": [0.2, 0.4, 0.6]},
|
|
"observation.state": {"mean": [1.2, 2.2, 3.2, 4.2]},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(meta / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "pick cube"}) + "\n", encoding="utf-8")
|
|
pq.write_table(
|
|
pa.table({"episode_index": [0, 1], "length": [2, 1], "task_index": [0, 0]}),
|
|
episode_meta_dir / "file-000.parquet",
|
|
)
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 0, 1],
|
|
"frame_index": [0, 1, 0],
|
|
"timestamp": [0.0, 0.1, 0.0],
|
|
"action": [[0.1, 0.2, 0.3], [0.2, 0.4, 0.6], [0.3, 0.6, 0.9]],
|
|
"observation.state": [
|
|
[1.0, 2.0, 3.0, 4.0],
|
|
[1.2, 2.2, 3.2, 4.2],
|
|
[1.4, 2.4, 3.4, 4.4],
|
|
],
|
|
}
|
|
),
|
|
data_dir / "episode_000000.parquet",
|
|
)
|
|
node_data = {
|
|
"label": "LeRobot 元数据一致性校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_lerobot_metadata_consistency_validate"](
|
|
"run-1", "lerobot-meta", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "lerobot-metadata-consistency-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["declaredFrames"], 3)
|
|
self.assertEqual(payload["declaredEpisodes"], 2)
|
|
self.assertEqual(payload["declaredTasks"], 1)
|
|
self.assertEqual(payload["episodeMetadataCount"], 2)
|
|
self.assertEqual(payload["dataRowCount"], 3)
|
|
self.assertEqual(payload["episodeRows"], {"0": 2, "1": 1})
|
|
self.assertIn("observation.state", payload["featureNames"])
|
|
|
|
def test_lerobot_v3_episode_index_validate_reads_real_path_windows(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "lerobot-v3-index"
|
|
meta = root / "meta"
|
|
episode_meta_dir = meta / "episodes" / "chunk-000"
|
|
data_dir = root / "data" / "chunk-000"
|
|
episode_meta_dir.mkdir(parents=True)
|
|
data_dir.mkdir(parents=True)
|
|
(meta / "info.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"codebase_version": "v3.0",
|
|
"chunks_size": 1000,
|
|
"data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 1],
|
|
"episode_chunk": [0, 0],
|
|
"length": [2, 1],
|
|
"data_path": [
|
|
"data/chunk-000/episode_000000.parquet",
|
|
"data/chunk-000/episode_000001.parquet",
|
|
],
|
|
"dataset_from_index": [0, 0],
|
|
"dataset_to_index": [2, 1],
|
|
}
|
|
),
|
|
episode_meta_dir / "file-000.parquet",
|
|
)
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 0],
|
|
"frame_index": [0, 1],
|
|
"action": [[0.1, 0.2], [0.2, 0.3]],
|
|
}
|
|
),
|
|
data_dir / "episode_000000.parquet",
|
|
)
|
|
pq.write_table(
|
|
pa.table({"episode_index": [1], "frame_index": [0], "action": [[0.4, 0.5]]}),
|
|
data_dir / "episode_000001.parquet",
|
|
)
|
|
node_data = {
|
|
"label": "LeRobot v3 Episode索引校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_lerobot_v3_episode_index_validate"](
|
|
"run-1", "lerobot-v3-index", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "lerobot-v3-episode-index-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["episodeMetadataCount"], 2)
|
|
self.assertEqual(payload["dataParquetCount"], 2)
|
|
self.assertEqual(payload["dataRowCount"], 3)
|
|
self.assertEqual(payload["validEpisodeCount"], 2)
|
|
self.assertEqual(payload["episodes"][0]["matchedPaths"], ["data/chunk-000/episode_000000.parquet"])
|
|
self.assertEqual(payload["episodes"][0]["indexWindow"], {"from": 0, "to": 2})
|
|
|
|
def test_lerobot_video_alignment_validate_reads_real_mp4_and_parquet(self):
|
|
try:
|
|
import cv2 # type: ignore
|
|
import numpy as np # type: ignore
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"cv2/numpy/pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "lerobot-video"
|
|
meta = root / "meta"
|
|
episode_meta_dir = meta / "episodes" / "chunk-000"
|
|
data_dir = root / "data" / "chunk-000"
|
|
video_dir = root / "videos" / "observation.images.main" / "chunk-000"
|
|
meta.mkdir(parents=True)
|
|
episode_meta_dir.mkdir(parents=True)
|
|
data_dir.mkdir(parents=True)
|
|
video_dir.mkdir(parents=True)
|
|
video_path = video_dir / "episode_000000.mp4"
|
|
writer = cv2.VideoWriter(str(video_path), cv2.VideoWriter_fourcc(*"mp4v"), 10.0, (8, 6))
|
|
if not writer.isOpened():
|
|
self.skipTest("OpenCV cannot create mp4v video in this environment")
|
|
for index in range(3):
|
|
frame = np.full((6, 8, 3), 30 + index * 40, dtype=np.uint8)
|
|
writer.write(frame)
|
|
writer.release()
|
|
(meta / "info.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"codebase_version": "v3.0",
|
|
"total_episodes": 1,
|
|
"total_frames": 3,
|
|
"total_videos": 1,
|
|
"features": {
|
|
"action": {"dtype": "float32", "shape": [2]},
|
|
"observation.state": {"dtype": "float32", "shape": [2]},
|
|
"observation.images.main": {"dtype": "video", "shape": [6, 8, 3]},
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
pq.write_table(pa.table({"episode_index": [0], "length": [3], "task_index": [0]}), episode_meta_dir / "file-000.parquet")
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 0, 0],
|
|
"frame_index": [0, 1, 2],
|
|
"timestamp": [0.0, 0.1, 0.2],
|
|
"action": [[0.1, 0.2], [0.2, 0.3], [0.3, 0.4]],
|
|
"observation.state": [[1.0, 2.0], [1.1, 2.1], [1.2, 2.2]],
|
|
"observation.images.main": [
|
|
"videos/observation.images.main/chunk-000/episode_000000.mp4",
|
|
"videos/observation.images.main/chunk-000/episode_000000.mp4",
|
|
"videos/observation.images.main/chunk-000/episode_000000.mp4",
|
|
],
|
|
}
|
|
),
|
|
data_dir / "episode_000000.parquet",
|
|
)
|
|
node_data = {"label": "LeRobot 视频对齐校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_lerobot_video_alignment_validate"](
|
|
"run-1", "lerobot-video", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "lerobot-video-alignment-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["videoCount"], 1)
|
|
self.assertEqual(payload["videoFrameCount"], 3)
|
|
self.assertEqual(payload["dataRowCount"], 3)
|
|
self.assertEqual(payload["missingVideoCount"], 0)
|
|
self.assertEqual(payload["mismatchedVideoCount"], 0)
|
|
|
|
def test_trajectory_integrity_validate_checks_modalities_and_file_refs(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "trajectory"
|
|
image_dir = root / "images"
|
|
image_dir.mkdir(parents=True)
|
|
(image_dir / "frame-000.png").write_bytes(b"real-image-bytes")
|
|
(root / "episodes.jsonl").write_text(
|
|
json.dumps(
|
|
{
|
|
"episode_id": "ep-1",
|
|
"steps": [
|
|
{
|
|
"action": [0.1, 0.2, 0.3],
|
|
"observation": {"image": "images/frame-000.png", "state": [1.0, 2.0]},
|
|
},
|
|
{
|
|
"action": [0.2, 0.3, 0.4],
|
|
"observation": {"image": "images/frame-000.png", "state": [1.2, 2.2]},
|
|
},
|
|
],
|
|
}
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {
|
|
"label": "轨迹多模态完整性校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_trajectory_integrity_validate"](
|
|
"run-1", "trajectory", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "trajectory-integrity-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["stepCount"], 2)
|
|
self.assertEqual(payload["actionDimension"], 3)
|
|
self.assertEqual(payload["fieldMissingCounts"]["observation.image"], 0)
|
|
|
|
def test_action_state_quality_validate_reads_real_parquet_vectors(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "action-state"
|
|
data_dir = root / "data" / "chunk-000"
|
|
meta_dir = root / "meta"
|
|
data_dir.mkdir(parents=True)
|
|
meta_dir.mkdir(parents=True)
|
|
(meta_dir / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "ignored meta row"}) + "\n", encoding="utf-8")
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 0, 1],
|
|
"frame_index": [0, 1, 0],
|
|
"action": [[0.1, 0.2, 0.3], [0.2, 0.4, 0.6], [0.3, 0.6, 0.9]],
|
|
"observation.state": [
|
|
[1.0, 2.0, 3.0, 4.0],
|
|
[1.2, 2.2, 3.2, 4.2],
|
|
[1.4, 2.4, 3.4, 4.4],
|
|
],
|
|
}
|
|
),
|
|
data_dir / "episode_000000.parquet",
|
|
)
|
|
node_data = {
|
|
"label": "动作状态数值质量校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_action_state_quality_validate"](
|
|
"run-1", "action-state", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "action-state-quality-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["rowCount"], 3)
|
|
self.assertEqual(payload["actionRowCount"], 3)
|
|
self.assertEqual(payload["stateRowCount"], 3)
|
|
self.assertEqual(payload["actionDimension"], 3)
|
|
self.assertEqual(payload["stateDimension"], 4)
|
|
self.assertEqual(payload["actionStats"]["max"], [0.3, 0.6, 0.9])
|
|
self.assertEqual(payload["stateStats"]["min"], [1.0, 2.0, 3.0, 4.0])
|
|
|
|
def test_episode_boundary_validate_reads_real_parquet_rows(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "episode-boundary"
|
|
data_dir = root / "data" / "chunk-000"
|
|
meta_dir = root / "meta"
|
|
data_dir.mkdir(parents=True)
|
|
meta_dir.mkdir(parents=True)
|
|
(meta_dir / "tasks.jsonl").write_text(json.dumps({"task_index": 0, "task": "ignored meta row"}) + "\n", encoding="utf-8")
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"episode_index": [0, 0, 1],
|
|
"frame_index": [0, 1, 0],
|
|
"done": [False, True, True],
|
|
"action": [[0.1], [0.2], [0.3]],
|
|
}
|
|
),
|
|
data_dir / "episode_000000.parquet",
|
|
)
|
|
node_data = {"label": "Episode 边界校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_episode_boundary_validate"](
|
|
"run-1", "episode-boundary", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "episode-boundary-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["rowCount"], 3)
|
|
self.assertEqual(payload["episodeCount"], 2)
|
|
self.assertEqual(payload["invalidEpisodeCount"], 0)
|
|
self.assertEqual(payload["terminalFieldCounts"], {"done": 3})
|
|
self.assertEqual(payload["episodes"][0]["lastFrameIndex"], 1)
|
|
|
|
def test_vla_sample_manifest_validate_reads_real_parquet_and_images(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "vla"
|
|
image_dir = root / "images"
|
|
image_dir.mkdir(parents=True)
|
|
(image_dir / "frame-000.png").write_bytes(b"\x89PNG\r\n\x1a\nreal")
|
|
(image_dir / "frame-001.png").write_bytes(b"\x89PNG\r\n\x1a\nreal")
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"image_path": ["images/frame-000.png", "images/frame-001.png"],
|
|
"language_instruction": ["pick up the cube", "place the cube"],
|
|
"action": [
|
|
[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 1.0],
|
|
[0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.0],
|
|
],
|
|
}
|
|
),
|
|
root / "manifest.parquet",
|
|
)
|
|
node_data = {"label": "VLA 样本清单校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["vla_sample_manifest_validate"](
|
|
"run-1", "vla", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "vla-sample-manifest-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["rowCount"], 2)
|
|
self.assertEqual(payload["imageReferenceCount"], 2)
|
|
self.assertEqual(payload["missingImageCount"], 0)
|
|
self.assertEqual(payload["textRowCount"], 2)
|
|
self.assertEqual(payload["actionRowCount"], 2)
|
|
self.assertEqual(payload["completeRowCount"], 2)
|
|
self.assertEqual(payload["actionDimension"], 7)
|
|
|
|
def test_dataset_split_manifest_writes_json_and_csv_from_real_episode_file(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "split-source"
|
|
root.mkdir()
|
|
(root / "episodes.jsonl").write_text(
|
|
"\n".join(
|
|
[
|
|
json.dumps({"episode_id": "ep-1", "steps": [{"action": [0]}], "task": "pick"}),
|
|
json.dumps({"episode_id": "ep-2", "steps": [{"action": [1]}, {"action": [2]}], "task": "place"}),
|
|
json.dumps({"episode_id": "ep-3", "steps": [], "task": "skip"}),
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {
|
|
"label": "训练切分清单生成",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "TRAIN_RATIO", "value": "1"},
|
|
{"key": "VAL_RATIO", "value": "0"},
|
|
{"key": "TEST_RATIO", "value": "0"},
|
|
{"key": "MIN_STEPS", "value": "1"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_dataset_split_manifest"](
|
|
"run-1", "split", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "dataset-split-manifest")
|
|
self.assertEqual(payload["episodeCount"], 2)
|
|
self.assertEqual(payload["skippedCount"], 1)
|
|
self.assertEqual(payload["splitCounts"], {"train": 2, "val": 0, "test": 0})
|
|
csv_path = pathlib.Path(payload["csvPath"])
|
|
self.assertTrue(csv_path.exists())
|
|
self.assertEqual(artifacts[1]["name"], "dataset-split-manifest.csv")
|
|
|
|
def test_mcap_topic_summary_reads_real_mcap_messages(self):
|
|
try:
|
|
from mcap.writer import Writer # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"mcap not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
path = pathlib.Path(temp_dir) / "robot.mcap"
|
|
with path.open("wb") as handle:
|
|
writer = Writer(handle)
|
|
writer.start(profile="ros2")
|
|
schema_id = writer.register_schema(
|
|
name="sensor_msgs/msg/JointState",
|
|
encoding="ros2msg",
|
|
data=b"string[] name\nfloat64[] position\n",
|
|
)
|
|
channel_id = writer.register_channel(
|
|
topic="/joint_states",
|
|
message_encoding="cdr",
|
|
schema_id=schema_id,
|
|
metadata={"frame_id": "base_link"},
|
|
)
|
|
writer.add_message(channel_id=channel_id, log_time=100, publish_time=90, data=b"joint-a")
|
|
writer.add_message(channel_id=channel_id, log_time=200, publish_time=190, data=b"joint-b-longer")
|
|
writer.add_metadata("source", {"robot": "unit-test-arm"})
|
|
writer.finish()
|
|
node_data = {"label": "MCAP 主题概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_mcap_topic_summary"](
|
|
"run-1", "mcap", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "mcap-topic-summary")
|
|
self.assertEqual(payload["topicCount"], 1)
|
|
self.assertEqual(payload["messageCount"], 2)
|
|
self.assertEqual(payload["topics"][0]["topic"], "/joint_states")
|
|
self.assertEqual(payload["topics"][0]["messageCount"], 2)
|
|
self.assertEqual(payload["topics"][0]["schemaName"], "sensor_msgs/msg/JointState")
|
|
|
|
def test_tfrecord_structural_validate_reads_real_records_and_crc(self):
|
|
def record(payload: bytes) -> bytes:
|
|
length_bytes = struct.pack("<Q", len(payload))
|
|
return (
|
|
length_bytes
|
|
+ struct.pack("<I", MODULE.load_operator_module("tfrecord_structural_validate")._masked_crc32c(length_bytes))
|
|
+ payload
|
|
+ struct.pack("<I", MODULE.load_operator_module("tfrecord_structural_validate")._masked_crc32c(payload))
|
|
)
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
path = pathlib.Path(temp_dir) / "episodes.tfrecord"
|
|
path.write_bytes(
|
|
record(json.dumps({"episode_id": "ep-1", "steps": 2}).encode("utf-8"))
|
|
+ record(json.dumps({"episode_id": "ep-2", "steps": 1}).encode("utf-8"))
|
|
)
|
|
node_data = {"label": "TFRecord 结构校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(path)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_tfrecord_structural_validate"](
|
|
"run-1", "tfrecord", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "tfrecord-structural-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["recordCount"], 2)
|
|
self.assertEqual(payload["files"][0]["records"][0]["encoding"], "json")
|
|
self.assertEqual(payload["files"][0]["records"][0]["preview"]["episode_id"], "ep-1")
|
|
|
|
def test_bridgedata_tfds_manifest_validate_reads_real_tfrecord_and_features(self):
|
|
def record(payload: bytes) -> bytes:
|
|
length_bytes = struct.pack("<Q", len(payload))
|
|
return (
|
|
length_bytes
|
|
+ struct.pack("<I", MODULE.load_operator_module("bridgedata_tfds_manifest_validate")._masked_crc32c(length_bytes))
|
|
+ payload
|
|
+ struct.pack("<I", MODULE.load_operator_module("bridgedata_tfds_manifest_validate")._masked_crc32c(payload))
|
|
)
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "bridgedata"
|
|
train_dir = root / "train"
|
|
train_dir.mkdir(parents=True)
|
|
(root / "dataset_info.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"name": "bridge_data_v2",
|
|
"splits": {"train": {"numExamples": 2}},
|
|
"features": {
|
|
"steps": {
|
|
"observation": {"image_0": {"dtype": "uint8"}},
|
|
"action": {"shape": [7]},
|
|
"language_instruction": {"dtype": "string"},
|
|
}
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(root / "features.json").write_text(
|
|
json.dumps({"steps": {"observation": {"image_0": "image"}, "action": "float32", "language_instruction": "string"}}),
|
|
encoding="utf-8",
|
|
)
|
|
(train_dir / "episode-00000-of-00001.tfrecord").write_bytes(
|
|
record(json.dumps({"episode_id": "ep-1", "language_instruction": "open drawer"}).encode("utf-8"))
|
|
+ record(json.dumps({"episode_id": "ep-2", "language_instruction": "close drawer"}).encode("utf-8"))
|
|
)
|
|
node_data = {
|
|
"label": "BridgeData TFDS-RLDS清单校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_bridgedata_tfds_manifest_validate"](
|
|
"run-1", "bridge", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "bridgedata-tfds-rlds-manifest-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["recordCount"], 2)
|
|
self.assertEqual(payload["splitRecordCounts"], {"train": 2})
|
|
self.assertEqual(payload["files"][0]["records"][0]["encoding"], "json")
|
|
self.assertIn("dataset_info.features.steps.action.shape", payload["featureKeys"])
|
|
|
|
def test_dataset_provenance_audit_reads_dataset_card_license_and_citation(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "hf-dataset"
|
|
root.mkdir()
|
|
(root / "README.md").write_text(
|
|
"---\n"
|
|
"license: apache-2.0\n"
|
|
"source: https://github.com/example/robot-dataset\n"
|
|
"task_categories:\n"
|
|
" - robotics\n"
|
|
"---\n"
|
|
"# Robot Dataset\n",
|
|
encoding="utf-8",
|
|
)
|
|
(root / "LICENSE").write_text("Apache License\nVersion 2.0\n", encoding="utf-8")
|
|
(root / "CITATION.cff").write_text("cff-version: 1.2.0\ntitle: Robot Dataset\n", encoding="utf-8")
|
|
(root / "provenance.json").write_text(
|
|
json.dumps({"sources": ["s3://real-bucket/raw"], "license": "apache-2.0"}),
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "数据集来源许可审计", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_dataset_provenance_audit"](
|
|
"run-1", "provenance", pathlib.Path(temp_dir) / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "dataset-provenance-audit")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertIn("apache-2.0", payload["licenses"])
|
|
self.assertIn("https://github.com/example/robot-dataset", payload["sources"])
|
|
self.assertEqual(payload["card"]["path"], "README.md")
|
|
|
|
def test_webdataset_shard_summary_reads_real_tar_samples(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
shard_path = root / "robot-000000.tar"
|
|
with tarfile.open(shard_path, "w") as archive:
|
|
for name, payload in {
|
|
"episode-000001.jpg": b"real-image-bytes",
|
|
"episode-000001.json": json.dumps({"task": "pick block"}).encode("utf-8"),
|
|
"episode-000002.png": b"real-png-bytes",
|
|
"episode-000002.txt": b"place block",
|
|
}.items():
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(payload)
|
|
archive.addfile(info, io.BytesIO(payload))
|
|
node_data = {"label": "WebDataset 分片概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(shard_path)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_webdataset_shard_summary"](
|
|
"run-1", "webdataset", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "webdataset-shard-summary")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["shardCount"], 1)
|
|
self.assertEqual(payload["sampleCount"], 2)
|
|
self.assertEqual(payload["memberFileCount"], 4)
|
|
self.assertEqual(payload["extensionCounts"]["jpg"], 1)
|
|
self.assertEqual(payload["modalityCounts"]["image"], 2)
|
|
|
|
def test_zarr_trajectory_summary_reads_real_zarr_metadata(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "trajectory.zarr"
|
|
data = root / "data"
|
|
action = data / "action"
|
|
state = data / "state"
|
|
episode_ends = root / "meta" / "episode_ends"
|
|
action.mkdir(parents=True)
|
|
state.mkdir(parents=True)
|
|
episode_ends.mkdir(parents=True)
|
|
(root / ".zgroup").write_text(json.dumps({"zarr_format": 2}), encoding="utf-8")
|
|
(root / ".zattrs").write_text(json.dumps({"episode_count": 2, "robot": "unit-test-arm"}), encoding="utf-8")
|
|
for array_path, shape, chunks, dtype in (
|
|
(action, [3, 7], [2, 7], "<f4"),
|
|
(state, [3, 14], [2, 14], "<f4"),
|
|
(episode_ends, [2], [2], "<i8"),
|
|
):
|
|
(array_path / ".zarray").write_text(
|
|
json.dumps(
|
|
{
|
|
"zarr_format": 2,
|
|
"shape": shape,
|
|
"chunks": chunks,
|
|
"dtype": dtype,
|
|
"compressor": None,
|
|
"fill_value": 0,
|
|
"order": "C",
|
|
"filters": None,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(array_path / "0").write_bytes(b"chunk-bytes")
|
|
node_data = {"label": "Zarr 轨迹数组概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_zarr_trajectory_summary"]("run-1", "zarr", root / "out", node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "zarr-trajectory-summary")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["arrayCount"], 3)
|
|
self.assertEqual(payload["episodeCount"], 2)
|
|
self.assertEqual(payload["stepCount"], 3)
|
|
self.assertIn("data/action", [item["path"] for item in payload["arrays"]])
|
|
|
|
def test_umi_zarr_dataset_validate_reads_real_zarr_episode_ends_and_arrays(self):
|
|
try:
|
|
import numpy as np # type: ignore
|
|
import zarr # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"numpy/zarr not available: {exc}")
|
|
|
|
def create_array(group, name: str, data, chunks):
|
|
if hasattr(group, "create_array"):
|
|
return group.create_array(name, data=data, chunks=chunks)
|
|
return group.create_dataset(name, data=data, chunks=chunks)
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "umi" / "dataset.zarr"
|
|
group = zarr.open_group(str(root), mode="w", zarr_format=2)
|
|
data = group.create_group("data")
|
|
create_array(data, "camera0_rgb", np.zeros((5, 8, 8, 3), dtype=np.uint8), chunks=(1, 8, 8, 3))
|
|
create_array(data, "robot0_eef_pos", np.linspace(0.1, 0.5, 15, dtype=np.float32).reshape(5, 3), chunks=(5, 3))
|
|
create_array(data, "robot0_eef_rot_axis_angle", np.ones((5, 3), dtype=np.float32), chunks=(5, 3))
|
|
create_array(data, "robot0_gripper_width", np.linspace(0.01, 0.05, 5, dtype=np.float32).reshape(5, 1), chunks=(5, 1))
|
|
meta = group.create_group("meta")
|
|
create_array(meta, "episode_ends", np.array([2, 5], dtype=np.int64), chunks=(2,))
|
|
node_data = {"label": "UMI Zarr数据集校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root.parent)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_umi_zarr_dataset_validate"](
|
|
"run-1", "umi-zarr", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "umi-zarr-dataset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["datasetPath"], str(root))
|
|
self.assertEqual(payload["dataArrayCount"], 4)
|
|
self.assertEqual(payload["episodeCount"], 2)
|
|
self.assertEqual(payload["stepCount"], 5)
|
|
self.assertEqual(payload["episodeEnds"], [2, 5])
|
|
self.assertEqual(payload["cameraArrayCount"], 1)
|
|
self.assertEqual(payload["eefArrayCount"], 2)
|
|
self.assertEqual(payload["gripperArrayCount"], 1)
|
|
self.assertTrue(payload["requiredDataKeysPresent"])
|
|
self.assertIn("robot0_eef_pos", [item["key"] for item in payload["arrays"]])
|
|
|
|
def test_camera_calibration_validate_reads_real_yaml_camera_info(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "calibration"
|
|
root.mkdir()
|
|
(root / "camera_info.yaml").write_text(
|
|
"width: 640\n"
|
|
"height: 480\n"
|
|
"K: [525.0, 0.0, 319.5, 0.0, 525.0, 239.5, 0.0, 0.0, 1.0]\n"
|
|
"D: [0.1, -0.01, 0.0, 0.0, 0.0]\n"
|
|
"R: [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]\n"
|
|
"P: [525.0, 0.0, 319.5, 0.0, 0.0, 525.0, 239.5, 0.0, 0.0, 0.0, 1.0, 0.0]\n"
|
|
"T_base_camera: [1.0, 0.0, 0.0, 0.1, 0.0, 1.0, 0.0, 0.2, 0.0, 0.0, 1.0, 0.3, 0.0, 0.0, 0.0, 1.0]\n",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "相机标定校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_camera_calibration_validate"](
|
|
"run-1", "calibration", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "camera-calibration-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["cameraCount"], 1)
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["cameras"][0]["width"], 640)
|
|
self.assertTrue(payload["cameras"][0]["hasExtrinsic"])
|
|
|
|
def test_robot_model_asset_validate_reads_real_urdf_and_mesh_refs(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
package = pathlib.Path(temp_dir) / "robot_description"
|
|
mesh_dir = package / "meshes"
|
|
mesh_dir.mkdir(parents=True)
|
|
(mesh_dir / "base.stl").write_bytes(b"solid base\nendsolid base\n")
|
|
(package / "robot.urdf").write_text(
|
|
"<robot name=\"unit_arm\">"
|
|
" <link name=\"base_link\">"
|
|
" <visual><geometry><mesh filename=\"package://robot_description/meshes/base.stl\"/></geometry></visual>"
|
|
" </link>"
|
|
" <link name=\"tool_link\"/>"
|
|
" <joint name=\"tool_joint\" type=\"fixed\"><parent link=\"base_link\"/><child link=\"tool_link\"/></joint>"
|
|
"</robot>",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "机器人模型资产校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(package)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_robot_model_asset_validate"](
|
|
"run-1", "robot-model", package / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "robot-model-asset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["modelCount"], 1)
|
|
self.assertEqual(payload["linkCount"], 2)
|
|
self.assertEqual(payload["jointCount"], 1)
|
|
self.assertEqual(payload["meshReferenceCount"], 1)
|
|
self.assertEqual(payload["missingMeshCount"], 0)
|
|
|
|
def test_partnet_mobility_asset_validate_reads_articulated_object_assets(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "partnet"
|
|
object_dir = root / "dataset" / "100279"
|
|
mesh_dir = object_dir / "textured_objs"
|
|
point_sample_dir = object_dir / "point_sample"
|
|
mesh_dir.mkdir(parents=True)
|
|
point_sample_dir.mkdir()
|
|
(mesh_dir / "base.obj").write_text("o base\nv 0 0 0\n", encoding="utf-8")
|
|
(mesh_dir / "drawer.obj").write_text("o drawer\nv 0 0 0\n", encoding="utf-8")
|
|
(point_sample_dir / "pts-10000.pts").write_text("0 0 0\n", encoding="utf-8")
|
|
(object_dir / "mobility.urdf").write_text(
|
|
"<robot name=\"partnet_100279\">"
|
|
" <link name=\"base\"><visual><geometry><mesh filename=\"textured_objs/base.obj\"/></geometry></visual></link>"
|
|
" <link name=\"drawer\"><visual><geometry><mesh filename=\"textured_objs/drawer.obj\"/></geometry></visual></link>"
|
|
" <joint name=\"joint_0\" type=\"prismatic\">"
|
|
" <parent link=\"base\"/><child link=\"drawer\"/>"
|
|
" <axis xyz=\"1 0 0\"/>"
|
|
" <limit lower=\"0\" upper=\"0.2\" effort=\"20\" velocity=\"0.5\"/>"
|
|
" </joint>"
|
|
"</robot>",
|
|
encoding="utf-8",
|
|
)
|
|
(object_dir / "semantics.txt").write_text("base fixed body\ndrawer slider drawer\n", encoding="utf-8")
|
|
(object_dir / "mobility_v2.json").write_text(
|
|
json.dumps(
|
|
[
|
|
{
|
|
"id": 0,
|
|
"name": "drawer",
|
|
"joint": "slider",
|
|
"parent": "base",
|
|
"child": "drawer",
|
|
"jointData": {
|
|
"axis": {"origin": [0, 0, 0], "direction": [1, 0, 0]},
|
|
"limit": {"a": 0, "b": 0.2, "noLimit": False},
|
|
},
|
|
}
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(object_dir / "meta.json").write_text('{"model_cat": "printer", "model_id": "100279"}', encoding="utf-8")
|
|
(object_dir / "result.json").write_text('{"name": "printer", "children": []}', encoding="utf-8")
|
|
(object_dir / "result_after_merging.json").write_text('{"name": "printer", "children": []}', encoding="utf-8")
|
|
(object_dir / "bounding_box.json").write_text('{"min": [0, 0, 0], "max": [1, 1, 1]}', encoding="utf-8")
|
|
node_data = {
|
|
"label": "PartNet-Mobility 资产校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "MAX_OBJECTS", "value": "5"},
|
|
{"key": "MAX_MESH_REFERENCES", "value": "20"},
|
|
{"key": "REQUIRE_RESULT_JSON", "value": "true"},
|
|
{"key": "REQUIRE_POINT_SAMPLE", "value": "true"},
|
|
{"key": "REQUIRE_BOUNDING_BOX", "value": "true"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_partnet_mobility_asset_validate"](
|
|
"run-1", "partnet", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "partnet-mobility-asset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["objectCount"], 1)
|
|
self.assertEqual(payload["validObjectCount"], 1)
|
|
self.assertEqual(payload["invalidObjectCount"], 0)
|
|
self.assertEqual(payload["categoryCounts"], {"printer": 1})
|
|
self.assertEqual(payload["motionTypeCounts"], {"slider": 1})
|
|
self.assertEqual(payload["meshReferenceCount"], 2)
|
|
self.assertEqual(payload["missingMeshCount"], 0)
|
|
self.assertEqual(payload["assets"]["texturedObjCount"], 2)
|
|
self.assertEqual(payload["assets"]["pointSampleFileCount"], 1)
|
|
partnet_object = payload["objects"][0]
|
|
self.assertEqual(partnet_object["objectId"], "100279")
|
|
self.assertEqual(partnet_object["urdf"]["linkCount"], 2)
|
|
self.assertEqual(partnet_object["urdf"]["jointCount"], 1)
|
|
self.assertEqual(partnet_object["urdf"]["jointTypeCounts"], {"prismatic": 1})
|
|
self.assertEqual(partnet_object["semantics"]["entryCount"], 2)
|
|
self.assertEqual(partnet_object["mobilityV2"]["jointEntryCount"], 1)
|
|
self.assertTrue(partnet_object["mobilityV2"]["hasAxis"])
|
|
self.assertTrue(partnet_object["mobilityV2"]["hasLimits"])
|
|
self.assertTrue(partnet_object["assets"]["hasBoundingBoxJson"])
|
|
|
|
def test_mjcf_actuator_validate_reads_real_ctrlrange(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "mjcf"
|
|
root.mkdir()
|
|
(root / "scene.xml").write_text(
|
|
"<mujoco model=\"unit_arm\">"
|
|
" <worldbody>"
|
|
" <body name=\"arm\">"
|
|
" <joint name=\"shoulder\" type=\"hinge\"/>"
|
|
" <site name=\"tool\"/>"
|
|
" </body>"
|
|
" </worldbody>"
|
|
" <actuator>"
|
|
" <position name=\"shoulder_pos\" joint=\"shoulder\" ctrllimited=\"true\" ctrlrange=\"-1 1\" gear=\"1\"/>"
|
|
" <motor name=\"tool_motor\" site=\"tool\" gear=\"0 0 1\"/>"
|
|
" </actuator>"
|
|
"</mujoco>",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "MJCF 执行器校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_mjcf_actuator_validate"](
|
|
"run-1", "mjcf-actuator", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "mjcf-actuator-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["modelCount"], 1)
|
|
self.assertEqual(payload["actuatorCount"], 2)
|
|
self.assertEqual(payload["invalidActuatorCount"], 0)
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["models"][0]["targetCounts"]["joints"], 1)
|
|
self.assertEqual(payload["models"][0]["targetCounts"]["sites"], 1)
|
|
|
|
def test_tf_transform_tree_validate_reads_real_transform_jsonl(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "tf"
|
|
root.mkdir()
|
|
rows = [
|
|
{
|
|
"header": {"frame_id": "world"},
|
|
"child_frame_id": "base_link",
|
|
"transform": {
|
|
"translation": {"x": 0.0, "y": 0.0, "z": 0.0},
|
|
"rotation": {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0},
|
|
},
|
|
},
|
|
{
|
|
"header": {"frame_id": "base_link"},
|
|
"child_frame_id": "camera",
|
|
"transform": {
|
|
"translation": {"x": 0.1, "y": 0.2, "z": 0.3},
|
|
"rotation": {"x": 0.0, "y": 0.0, "z": 0.70710678, "w": 0.70710678},
|
|
},
|
|
},
|
|
]
|
|
(root / "transforms.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8")
|
|
node_data = {"label": "TF 坐标树校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_tf_transform_tree_validate"](
|
|
"run-1", "tf-tree", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "tf-transform-tree-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["transformCount"], 2)
|
|
self.assertEqual(payload["frameCount"], 3)
|
|
self.assertEqual(payload["rootFrames"], ["world"])
|
|
self.assertEqual(payload["cycleCount"], 0)
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
|
|
def test_ros_joint_state_validate_reads_real_jsonl_messages(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "joint-state"
|
|
root.mkdir()
|
|
rows = [
|
|
{
|
|
"header": {"stamp": {"sec": 1, "nanosec": 0}},
|
|
"name": ["shoulder_pan", "elbow_flex"],
|
|
"position": [0.1, 0.2],
|
|
"velocity": [0.0, 0.1],
|
|
"effort": [1.0, 1.1],
|
|
},
|
|
{
|
|
"header": {"stamp": {"sec": 1, "nanosec": 100000000}},
|
|
"name": ["shoulder_pan", "elbow_flex"],
|
|
"position": [0.2, 0.3],
|
|
"velocity": [0.1, 0.2],
|
|
"effort": [1.2, 1.3],
|
|
},
|
|
]
|
|
(root / "joint_states.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8")
|
|
node_data = {"label": "ROS JointState 校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_ros_joint_state_validate"](
|
|
"run-1", "joint-state", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "ros-joint-state-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["messageCount"], 2)
|
|
self.assertEqual(payload["validMessageCount"], 2)
|
|
self.assertEqual(payload["invalidMessageCount"], 0)
|
|
self.assertEqual(payload["jointCount"], 2)
|
|
self.assertEqual(payload["jointNames"], ["shoulder_pan", "elbow_flex"])
|
|
|
|
def test_dexycb_dataset_validate_reads_label_npz_and_rgbd_pairs(self):
|
|
try:
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "dex-ycb"
|
|
camera_dir = root / "20200709-subject-01" / "20200709_141841" / "932122060861"
|
|
camera_dir.mkdir(parents=True)
|
|
(root / "calibration").mkdir()
|
|
(root / "models" / "002_master_chef_can").mkdir(parents=True)
|
|
(root / "calibration" / "932122060861.yml").write_text("fx: 613.0\n", encoding="utf-8")
|
|
(root / "models" / "002_master_chef_can" / "textured.obj").write_text("o can\n", encoding="utf-8")
|
|
frame_id = "000053"
|
|
(camera_dir / f"color_{frame_id}.jpg").write_bytes(b"real-jpeg-placeholder")
|
|
(camera_dir / f"aligned_depth_to_color_{frame_id}.png").write_bytes(b"real-png-placeholder")
|
|
seg = np.zeros((6, 8), dtype=np.uint8)
|
|
seg[1:4, 2:5] = 1
|
|
seg[4:6, 5:8] = 255
|
|
pose_y = np.ones((2, 3, 4), dtype=np.float32)
|
|
pose_m = np.ones((1, 51), dtype=np.float32)
|
|
joint_3d = np.ones((1, 21, 3), dtype=np.float32)
|
|
joint_2d = np.ones((1, 21, 2), dtype=np.float32)
|
|
np.savez(
|
|
camera_dir / f"labels_{frame_id}.npz",
|
|
seg=seg,
|
|
pose_y=pose_y,
|
|
pose_m=pose_m,
|
|
joint_3d=joint_3d,
|
|
joint_2d=joint_2d,
|
|
)
|
|
node_data = {
|
|
"label": "DexYCB 数据集校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_dexycb_dataset_validate"](
|
|
"run-1", "dexycb", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "dexycb-dataset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["sampleCount"], 1)
|
|
self.assertEqual(payload["validSampleCount"], 1)
|
|
self.assertEqual(payload["subjectCounts"], {"20200709-subject-01": 1})
|
|
self.assertEqual(payload["cameraCounts"], {"932122060861": 1})
|
|
self.assertEqual(payload["objectPoseCounts"], {"2": 1})
|
|
self.assertTrue(payload["assets"]["hasCalibrationDir"])
|
|
self.assertTrue(payload["assets"]["hasModelsDir"])
|
|
sample = payload["samples"][0]
|
|
self.assertEqual(sample["frameId"], "000053")
|
|
self.assertEqual(sample["arrayShapes"]["seg"], [6, 8])
|
|
self.assertEqual(sample["arrayShapes"]["pose_y"], [2, 3, 4])
|
|
self.assertEqual(sample["arrayShapes"]["pose_m"], [1, 51])
|
|
self.assertEqual(sample["arrayShapes"]["joint_3d"], [1, 21, 3])
|
|
self.assertEqual(sample["arrayShapes"]["joint_2d"], [1, 21, 2])
|
|
self.assertIn(255, sample["segUniqueLabels"])
|
|
self.assertEqual(sample["visibleJointCount"], 21)
|
|
|
|
def test_bop_dataset_validate_reads_scene_json_and_media_assets(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "lm"
|
|
scene_dir = root / "train_pbr" / "000000"
|
|
for folder in ("rgb", "depth", "mask", "mask_visib"):
|
|
(scene_dir / folder).mkdir(parents=True)
|
|
(root / "models").mkdir(parents=True)
|
|
(root / "camera.json").write_text(
|
|
json.dumps({"cx": 320.0, "cy": 240.0, "fx": 600.0, "fy": 601.0, "depth_scale": 1.0}),
|
|
encoding="utf-8",
|
|
)
|
|
(root / "models" / "models_info.json").write_text(
|
|
json.dumps({"1": {"diameter": 102.5, "min_x": -1.0, "size_x": 2.0}}),
|
|
encoding="utf-8",
|
|
)
|
|
(root / "models" / "obj_000001.ply").write_text("ply\nformat ascii 1.0\nend_header\n", encoding="utf-8")
|
|
(scene_dir / "scene_camera.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"0": {
|
|
"cam_K": [600.0, 0.0, 320.0, 0.0, 601.0, 240.0, 0.0, 0.0, 1.0],
|
|
"depth_scale": 1.0,
|
|
"cam_R_w2c": [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0],
|
|
"cam_t_w2c": [0.0, 0.0, 0.0],
|
|
}
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(scene_dir / "scene_gt.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"0": [
|
|
{
|
|
"obj_id": 1,
|
|
"cam_R_m2c": [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0],
|
|
"cam_t_m2c": [10.0, 20.0, 650.0],
|
|
"obj_bb": [1.0, 2.0, 4.0, 5.0],
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(scene_dir / "scene_gt_info.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"0": [
|
|
{
|
|
"bbox_obj": [1, 2, 4, 5],
|
|
"bbox_visib": [1, 2, 3, 4],
|
|
"px_count_all": 20,
|
|
"px_count_valid": 20,
|
|
"px_count_visib": 16,
|
|
"visib_fract": 0.8,
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
for relative in ("rgb/000000.png", "depth/000000.png", "mask/000000_000000.png", "mask_visib/000000_000000.png"):
|
|
(scene_dir / relative).write_bytes(b"real-bop-image-placeholder")
|
|
node_data = {
|
|
"label": "BOP 6D姿态数据集校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "CHECK_DEPTH_FILES", "value": "true"},
|
|
{"key": "CHECK_MASK_FILES", "value": "true"},
|
|
{"key": "REQUIRE_SCENE_GT_INFO", "value": "true"},
|
|
{"key": "REQUIRE_MODEL_FILES", "value": "true"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_bop_dataset_validate"](
|
|
"run-1", "bop", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "bop-dataset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["sceneCount"], 1)
|
|
self.assertEqual(payload["validSceneCount"], 1)
|
|
self.assertEqual(payload["imageCount"], 1)
|
|
self.assertEqual(payload["validImageCount"], 1)
|
|
self.assertEqual(payload["annotationCount"], 1)
|
|
self.assertEqual(payload["splitCounts"], {"train_pbr": 1})
|
|
self.assertEqual(payload["objectCounts"], {"1": 1})
|
|
self.assertTrue(payload["assets"]["hasCameraJson"])
|
|
self.assertEqual(payload["assets"]["modelObjectCount"], 1)
|
|
self.assertEqual(payload["assets"]["modelFileCount"], 1)
|
|
scene = payload["scenes"][0]
|
|
self.assertEqual(scene["scenePath"], "train_pbr/000000")
|
|
self.assertEqual(scene["sceneCameraPath"], "train_pbr/000000/scene_camera.json")
|
|
self.assertEqual(scene["sceneGtPath"], "train_pbr/000000/scene_gt.json")
|
|
self.assertEqual(scene["sceneGtInfoPath"], "train_pbr/000000/scene_gt_info.json")
|
|
self.assertEqual(scene["imageAssetCounts"], {"rgbOrGray": 1, "depth": 1, "mask": 1, "maskVisib": 1})
|
|
image = scene["images"][0]
|
|
self.assertEqual(image["imageId"], "0")
|
|
self.assertEqual(image["frameStem"], "000000")
|
|
self.assertEqual(image["rgbOrGrayPath"], "train_pbr/000000/rgb/000000.png")
|
|
self.assertEqual(image["depthPath"], "train_pbr/000000/depth/000000.png")
|
|
self.assertEqual(image["annotationCount"], 1)
|
|
|
|
def test_graspnet_dataset_validate_reads_scene_grasp_and_collision_labels(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
import numpy as np
|
|
|
|
root = pathlib.Path(temp_dir) / "graspnet"
|
|
scene_dir = root / "scenes" / "scene_0000"
|
|
camera_dir = scene_dir / "kinect"
|
|
for folder in ("rgb", "depth", "label", "meta", "annotations", "rect"):
|
|
(camera_dir / folder).mkdir(parents=True)
|
|
(root / "models" / "001").mkdir(parents=True)
|
|
(root / "grasp_label").mkdir(parents=True)
|
|
(root / "collision_label" / "scene_0000").mkdir(parents=True)
|
|
|
|
(scene_dir / "object_id_list.txt").write_text("1\n", encoding="utf-8")
|
|
np.save(scene_dir / "rs_wrt_kn.npy", np.eye(4, dtype=np.float32))
|
|
np.save(camera_dir / "camK.npy", np.eye(3, dtype=np.float32))
|
|
np.save(camera_dir / "cam0_wrt_table.npy", np.eye(4, dtype=np.float32))
|
|
np.save(camera_dir / "camera_poses.npy", np.eye(4, dtype=np.float32)[None, :, :])
|
|
for name in ("nontextured.ply", "textured.obj", "textured.sdf"):
|
|
(root / "models" / "001" / name).write_text("real-graspnet-model-file\n", encoding="utf-8")
|
|
points = np.zeros((2, 3), dtype=np.float32)
|
|
offsets = np.zeros((2, 300, 12, 4, 3), dtype=np.float32)
|
|
scores = np.ones((2, 300, 12, 4), dtype=np.float32)
|
|
collision = np.zeros((2, 300, 12, 4), dtype=bool)
|
|
np.savez(root / "grasp_label" / "001_labels.npz", points=points, offsets=offsets, scores=scores, collision=collision)
|
|
np.savez(root / "collision_label" / "scene_0000" / "collision_labels.npz", arr_0=collision)
|
|
np.save(camera_dir / "rect" / "0000.npy", np.zeros((1, 8), dtype=np.float32))
|
|
png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x04\x00\x00\x00\x03\x08\x02"
|
|
for relative in ("rgb/0000.png", "depth/0000.png", "label/0000.png"):
|
|
(camera_dir / relative).write_bytes(png)
|
|
(camera_dir / "meta" / "0000.mat").write_bytes(b"MATLAB 5.0 MAT-file, Platform: test")
|
|
(camera_dir / "annotations" / "0000.xml").write_text(
|
|
"<annotation><object><id>1</id><pose>1 0 0 0 0 1 0 0 0 0 1 0</pose></object></annotation>",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {
|
|
"label": "GraspNet 抓取数据集校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "CAMERA", "value": "kinect"},
|
|
{"key": "MAX_SCENES", "value": "2"},
|
|
{"key": "MAX_FRAMES_PER_CAMERA", "value": "4"},
|
|
{"key": "REQUIRE_RECT_LABELS", "value": "true"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_graspnet_dataset_validate"](
|
|
"run-1", "graspnet", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "graspnet-dataset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["sceneCount"], 1)
|
|
self.assertEqual(payload["validSceneCount"], 1)
|
|
self.assertEqual(payload["cameraCount"], 1)
|
|
self.assertEqual(payload["frameCount"], 1)
|
|
self.assertEqual(payload["validFrameCount"], 1)
|
|
self.assertEqual(payload["objectCount"], 1)
|
|
self.assertEqual(payload["objectCounts"], {"1": 1})
|
|
self.assertEqual(payload["splitCounts"], {"train": 1})
|
|
self.assertTrue(payload["assets"]["hasScenesDir"])
|
|
self.assertTrue(payload["assets"]["hasModelsDir"])
|
|
self.assertTrue(payload["assets"]["hasGraspLabelDir"])
|
|
self.assertTrue(payload["assets"]["hasCollisionLabelDir"])
|
|
self.assertEqual(payload["assets"]["modelObjectCount"], 1)
|
|
self.assertEqual(payload["assets"]["graspLabelObjectCount"], 1)
|
|
scene = payload["scenes"][0]
|
|
self.assertEqual(scene["sceneId"], "scene_0000")
|
|
self.assertEqual(scene["objectIds"], [1])
|
|
self.assertEqual(scene["collisionLabel"]["path"], "collision_label/scene_0000/collision_labels.npz")
|
|
camera = scene["cameras"][0]
|
|
self.assertEqual(camera["camera"], "kinect")
|
|
self.assertEqual(camera["camK"]["shape"], [3, 3])
|
|
self.assertEqual(camera["cameraPoses"]["shape"], [1, 4, 4])
|
|
frame = camera["frames"][0]
|
|
self.assertEqual(frame["annId"], 0)
|
|
self.assertEqual(frame["rgb"]["path"], "scenes/scene_0000/kinect/rgb/0000.png")
|
|
self.assertEqual(frame["depth"]["width"], 4)
|
|
self.assertEqual(frame["label"]["height"], 3)
|
|
self.assertEqual(frame["annotation"]["objectLikeElementCount"], 2)
|
|
self.assertEqual(frame["rect"]["shape"], [1, 8])
|
|
|
|
def test_ho3d_dataset_validate_reads_meta_pickle_and_media_pairs(self):
|
|
try:
|
|
import cv2 # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"cv2/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "ho3d"
|
|
frame_id = "0000"
|
|
seq_dir = root / "train" / "ABF10"
|
|
for folder in ("rgb", "depth", "segr", "meta"):
|
|
(seq_dir / folder).mkdir(parents=True)
|
|
cv2.imwrite(str(seq_dir / "rgb" / f"{frame_id}.jpg"), np.full((6, 8, 3), 120, dtype=np.uint8))
|
|
cv2.imwrite(str(seq_dir / "depth" / f"{frame_id}.png"), np.full((6, 8, 3), 20, dtype=np.uint8))
|
|
cv2.imwrite(str(seq_dir / "segr" / f"{frame_id}.png"), np.full((6, 8, 3), 255, dtype=np.uint8))
|
|
meta = {
|
|
"camMat": np.array([[610.0, 0.0, 320.0], [0.0, 611.0, 240.0], [0.0, 0.0, 1.0]], dtype=np.float32),
|
|
"objRot": np.array([0.1, 0.2, 0.3], dtype=np.float32),
|
|
"objTrans": np.array([0.01, -0.02, -0.45], dtype=np.float32),
|
|
"objCorners3DRest": np.arange(24, dtype=np.float32).reshape(8, 3),
|
|
"objName": "003_cracker_box",
|
|
"handPose": np.zeros((48,), dtype=np.float32),
|
|
"handTrans": np.array([0.0, 0.0, -0.35], dtype=np.float32),
|
|
"handBeta": np.zeros((10,), dtype=np.float32),
|
|
"handJoints3D": np.ones((21, 3), dtype=np.float32),
|
|
"handBoundingBox": np.array([1.0, 1.0, 7.0, 5.0], dtype=np.float32),
|
|
"handVertContact": np.array([0.0, 1.0, 0.5], dtype=np.float32),
|
|
}
|
|
with (seq_dir / "meta" / f"{frame_id}.pkl").open("wb") as handle:
|
|
pickle.dump(meta, handle)
|
|
node_data = {
|
|
"label": "HO3D/H2O3D 数据集校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "CHECK_SEGMENTATION_FILES", "value": "true"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_ho3d_dataset_validate"](
|
|
"run-1", "ho3d", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "ho3d-dataset-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["issueCount"], 0)
|
|
self.assertEqual(payload["sampleCount"], 1)
|
|
self.assertEqual(payload["validSampleCount"], 1)
|
|
self.assertEqual(payload["invalidSampleCount"], 0)
|
|
self.assertEqual(payload["splitCounts"], {"train": 1})
|
|
self.assertEqual(payload["sequenceCounts"], {"ABF10": 1})
|
|
self.assertEqual(payload["objectCounts"], {"003_cracker_box": 1})
|
|
self.assertEqual(payload["handAnnotationCounts"], {"manoHand": 1, "handJoints3D": 1, "handVertContact": 1})
|
|
self.assertTrue(payload["assets"]["hasTrainDir"])
|
|
sample = payload["samples"][0]
|
|
self.assertEqual(sample["frameId"], "0000")
|
|
self.assertEqual(sample["metaPath"], "train/ABF10/meta/0000.pkl")
|
|
self.assertEqual(sample["rgbPath"], "train/ABF10/rgb/0000.jpg")
|
|
self.assertEqual(sample["depthPath"], "train/ABF10/depth/0000.png")
|
|
self.assertEqual(sample["segmentationPath"], "train/ABF10/segr/0000.png")
|
|
self.assertTrue(sample["hasObjectPose"])
|
|
self.assertTrue(sample["hasManoHand"])
|
|
self.assertTrue(sample["hasHandJoints3D"])
|
|
self.assertEqual(sample["arrayShapes"]["camMat"], [3, 3])
|
|
self.assertEqual(sample["arrayShapes"]["objCorners3DRest"], [8, 3])
|
|
self.assertEqual(sample["arrayShapes"]["handPose"], [48])
|
|
self.assertEqual(sample["arrayShapes"]["handJoints3D"], [21, 3])
|
|
|
|
def test_hand_object_pose_validate_reads_real_pose_jsonl_and_images(self):
|
|
try:
|
|
import cv2 # type: ignore
|
|
import numpy as np # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"cv2/numpy not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "hand-object"
|
|
(root / "images").mkdir(parents=True)
|
|
(root / "masks").mkdir(parents=True)
|
|
image = np.zeros((8, 10, 3), dtype=np.uint8)
|
|
mask = np.zeros((8, 10), dtype=np.uint8)
|
|
mask[2:6, 3:8] = 1
|
|
cv2.imwrite(str(root / "images" / "frame_0001.png"), image)
|
|
cv2.imwrite(str(root / "masks" / "frame_0001.png"), mask)
|
|
joints_3d = [[float(index), float(index + 1), float(index + 2)] for index in range(21)]
|
|
joints_2d = [[float(index), float(index + 1)] for index in range(21)]
|
|
pose_matrix = [
|
|
[1.0, 0.0, 0.0, 0.1],
|
|
[0.0, 1.0, 0.0, 0.2],
|
|
[0.0, 0.0, 1.0, 0.3],
|
|
[0.0, 0.0, 0.0, 1.0],
|
|
]
|
|
rows = [
|
|
{
|
|
"image_path": "images/frame_0001.png",
|
|
"mask_path": "masks/frame_0001.png",
|
|
"hand_joints_3d": joints_3d,
|
|
"hand_joints_2d": joints_2d,
|
|
"object_pose_matrix": pose_matrix,
|
|
"bbox": [1.0, 2.0, 5.0, 4.0],
|
|
},
|
|
{
|
|
"image_path": "images/frame_0001.png",
|
|
"mask_path": "masks/frame_0001.png",
|
|
"hand_joints_3d": joints_3d,
|
|
"object_quaternion": [0.0, 0.0, 0.0, 1.0],
|
|
"object_translation": [0.1, 0.2, 0.3],
|
|
"bbox": [2.0, 1.0, 4.0, 3.0],
|
|
},
|
|
]
|
|
(root / "annotations.jsonl").write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8")
|
|
node_data = {
|
|
"label": "手-物体姿态标注校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "CHECK_MASK_EXISTS", "value": "true"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_hand_object_pose_validate"](
|
|
"run-1", "hand-object", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "hand-object-pose-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["sampleCount"], 2)
|
|
self.assertEqual(payload["invalidSampleCount"], 0)
|
|
self.assertEqual(payload["handJoint3DCount"], 2)
|
|
self.assertEqual(payload["handJoint2DCount"], 1)
|
|
self.assertEqual(payload["objectPoseCount"], 2)
|
|
self.assertEqual(payload["missingImageCount"], 0)
|
|
self.assertEqual(payload["missingMaskCount"], 0)
|
|
|
|
def test_hand_removal_consumes_sam_mask_and_calls_propainter(self):
|
|
hand_removal_module = MODULE.load_operator_module("hand_removal")
|
|
health = {"status": "ok", "model": "ProPainter", "device": "cuda"}
|
|
result = {"status": "success", "outputVideoUrl": "/api/media/clean.mp4", "frameCount": 12}
|
|
calls: list[str] = []
|
|
|
|
def fake_urlopen(target, timeout=0):
|
|
url = target.full_url if hasattr(target, "full_url") else str(target)
|
|
calls.append(url)
|
|
if url.endswith("/api/health"):
|
|
return _FakeHTTPResponse(json.dumps(health).encode("utf-8"))
|
|
if url.endswith("/api/inpaint/video"):
|
|
self.assertIn(b"source-video-bytes", target.data)
|
|
self.assertIn(b"sam3-mask-bytes", target.data)
|
|
self.assertIn(b"mask_dilation", target.data)
|
|
self.assertIn(b"white-is-remove", target.data)
|
|
return _FakeHTTPResponse(json.dumps(result).encode("utf-8"))
|
|
if url.endswith("/api/media/clean.mp4"):
|
|
return _FakeHTTPResponse(b"clean-video-bytes")
|
|
raise AssertionError(f"unexpected urlopen target: {url}")
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
video_path = root / "ego.mp4"
|
|
video_path.write_bytes(b"source-video-bytes")
|
|
mask_path = root / "sam3-hand-mask.mp4"
|
|
mask_path.write_bytes(b"sam3-mask-bytes")
|
|
output_dir = root / "output"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "Hand Removal 视频去手",
|
|
"outputs": [{"id": "out-1", "name": "cleanVideo", "type": "video"}],
|
|
"hyperparams": [
|
|
{"key": "PROPAINTER_API_BASE_URL", "value": "http://propainter.example"},
|
|
{"key": "MASK_DILATION", "value": "12"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{"name": "ego.mp4", "type": "video", "path": str(video_path), "portId": "in-video"},
|
|
{"name": "sam3-hand-mask.mp4", "type": "file", "path": str(mask_path), "portId": "in-mask"},
|
|
]
|
|
with mock.patch.object(hand_removal_module.request, "urlopen", side_effect=fake_urlopen):
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_hand_removal"]("run-1", "remove-hands", output_dir, node_data, inputs)
|
|
|
|
names = {artifact["name"] for artifact in artifacts}
|
|
self.assertEqual(names, {"hand-removal-result.json", "hand-removed.mp4"})
|
|
payload = MODULE.load_artifact_payload(next(item for item in artifacts if item["name"] == "hand-removal-result.json"))
|
|
self.assertEqual(payload["status"], "success")
|
|
self.assertEqual(payload["method"]["segmentation"], "SAM3/SAM 3.1 upstream mask")
|
|
self.assertEqual(payload["method"]["mask_dilation"], "12")
|
|
self.assertEqual(payload["service"]["health"]["model"], "ProPainter")
|
|
self.assertIn("http://propainter.example/api/inpaint/video", calls)
|
|
|
|
def test_wilor_hand_pose_calls_service_and_downloads_outputs(self):
|
|
wilor_module = MODULE.load_operator_module("wilor_hand_pose")
|
|
health = {
|
|
"status": "ok",
|
|
"model": {
|
|
"loaded": True,
|
|
"device": "cuda",
|
|
"cudaAvailable": True,
|
|
"cudaDeviceCount": 1,
|
|
"cudaDeviceName": "NVIDIA A800-SXM4-80GB",
|
|
},
|
|
}
|
|
result = {
|
|
"status": "success",
|
|
"model": "WiLoR-mini / WiLoR",
|
|
"detectedHands": 2,
|
|
"latencyMs": 123.4,
|
|
"runtime": health["model"],
|
|
"annotatedImageUrl": "/api/media/annotated.png",
|
|
"keypointsUrl": "/api/media/keypoints.json",
|
|
"meshObjUrl": "/api/media/mesh.obj",
|
|
"hands": [
|
|
{"index": 0, "side": "left", "bbox": [1, 2, 3, 4], "verticesCount": 778},
|
|
{"index": 1, "side": "right", "bbox": [5, 6, 7, 8], "verticesCount": 778},
|
|
],
|
|
"task": {"id": "task-1", "status": "completed", "mediaKind": "image", "sourceName": "sample-hands.png"},
|
|
}
|
|
calls: list[str] = []
|
|
|
|
def fake_urlopen(target, timeout=0):
|
|
url = target.full_url if hasattr(target, "full_url") else str(target)
|
|
calls.append(url)
|
|
if url.endswith("/api/health"):
|
|
return _FakeHTTPResponse(json.dumps(health).encode("utf-8"))
|
|
if url.endswith("/api/infer"):
|
|
body = target.data.decode("utf-8", errors="replace")
|
|
self.assertIn("hand_conf", body)
|
|
self.assertIn("include_mesh", body)
|
|
return _FakeHTTPResponse(json.dumps(result).encode("utf-8"))
|
|
if url.endswith("/examples/sample-hands.png"):
|
|
return _FakeHTTPResponse(b"fake-image-bytes")
|
|
if url.endswith("/api/media/annotated.png"):
|
|
return _FakeHTTPResponse(b"\x89PNG\r\n\x1a\nannotated")
|
|
if url.endswith("/api/media/keypoints.json"):
|
|
return _FakeHTTPResponse(json.dumps({"hands": 2}).encode("utf-8"))
|
|
if url.endswith("/api/media/mesh.obj"):
|
|
return _FakeHTTPResponse(b"o hand\nv 0 0 0\n")
|
|
raise AssertionError(f"unexpected urlopen target: {url}")
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "WiLoR 3D手部姿态估计",
|
|
"outputs": [{"id": "out-1", "name": "wilorResult", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "WILOR_API_BASE_URL", "value": "http://wilor.example"},
|
|
{"key": "INPUT_URL", "value": "http://wilor.example/examples/sample-hands.png"},
|
|
{"key": "MEDIA_KIND", "value": "image"},
|
|
{"key": "HAND_CONFIDENCE", "value": "0.30"},
|
|
{"key": "INCLUDE_MESH", "value": "true"},
|
|
],
|
|
}
|
|
with mock.patch.object(wilor_module.request, "urlopen", side_effect=fake_urlopen):
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_wilor_hand_pose"]("run-1", "wilor", output_dir, node_data, [])
|
|
|
|
names = {artifact["name"] for artifact in artifacts}
|
|
self.assertEqual(
|
|
names,
|
|
{"wilor-result.json", "wilor-annotated-image.png", "wilor-keypoints.json", "wilor-mesh.obj"},
|
|
)
|
|
payload = MODULE.load_artifact_payload(next(item for item in artifacts if item["name"] == "wilor-result.json"))
|
|
self.assertEqual(payload["status"], "success")
|
|
self.assertEqual(payload["mediaKind"], "image")
|
|
self.assertEqual(payload["service"]["health"]["model"]["device"], "cuda")
|
|
self.assertEqual(payload["service"]["health"]["model"]["cudaDeviceName"], "NVIDIA A800-SXM4-80GB")
|
|
self.assertEqual(payload["result"]["detectedHands"], 2)
|
|
self.assertEqual(payload["result"]["handCount"], 2)
|
|
self.assertEqual(len(payload["downloadedOutputs"]), 3)
|
|
self.assertIn("http://wilor.example/api/infer", calls)
|
|
|
|
def test_wilor_hand_pose_prefers_upstream_media_file_over_source_manifest(self):
|
|
wilor_module = MODULE.load_operator_module("wilor_hand_pose")
|
|
self.assertEqual(
|
|
wilor_module._first_media_ref_from_payload(
|
|
{"entries": [{"value": {"url": "http://wilor.example/examples/from-manifest.png"}}]}
|
|
),
|
|
"http://wilor.example/examples/from-manifest.png",
|
|
)
|
|
health = {
|
|
"status": "ok",
|
|
"model": {
|
|
"loaded": True,
|
|
"device": "cuda",
|
|
"cudaAvailable": True,
|
|
"cudaDeviceCount": 1,
|
|
"cudaDeviceName": "NVIDIA A800-SXM4-80GB",
|
|
},
|
|
}
|
|
result = {
|
|
"status": "success",
|
|
"model": "WiLoR-mini / WiLoR",
|
|
"detectedHands": 1,
|
|
"latencyMs": 42.0,
|
|
"runtime": health["model"],
|
|
"hands": [{"index": 0, "side": "right", "bbox": [1, 2, 3, 4], "verticesCount": 778}],
|
|
}
|
|
calls: list[str] = []
|
|
|
|
def fake_urlopen(target, timeout=0):
|
|
url = target.full_url if hasattr(target, "full_url") else str(target)
|
|
calls.append(url)
|
|
if url.endswith("/api/health"):
|
|
return _FakeHTTPResponse(json.dumps(health).encode("utf-8"))
|
|
if url.endswith("/api/infer"):
|
|
self.assertIn(b"upstream-image-bytes", target.data)
|
|
self.assertNotIn(b"from-manifest.png", target.data)
|
|
return _FakeHTTPResponse(json.dumps(result).encode("utf-8"))
|
|
raise AssertionError(f"unexpected urlopen target: {url}")
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
manifest_path = root / "source-manifest.json"
|
|
manifest_path.write_text(
|
|
json.dumps({"entries": [{"value": {"url": "http://wilor.example/examples/from-manifest.png"}}]}),
|
|
encoding="utf-8",
|
|
)
|
|
self.assertEqual(
|
|
wilor_module._input_ref({}, [{"name": "source-manifest.json", "type": "json", "path": str(manifest_path)}]),
|
|
"http://wilor.example/examples/from-manifest.png",
|
|
)
|
|
image_path = root / "sample-hands.png"
|
|
image_path.write_bytes(b"\x89PNG\r\n\x1a\nupstream-image-bytes")
|
|
output_dir = root / "output"
|
|
output_dir.mkdir()
|
|
node_data = {
|
|
"label": "WiLoR 3D手部姿态估计",
|
|
"outputs": [{"id": "out-1", "name": "wilorResult", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "WILOR_API_BASE_URL", "value": "http://wilor.example"},
|
|
{"key": "INPUT_URL", "value": ""},
|
|
{"key": "MEDIA_KIND", "value": "image"},
|
|
{"key": "DOWNLOAD_OUTPUTS", "value": "false"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"name": "source-manifest.json",
|
|
"type": "json",
|
|
"path": str(manifest_path),
|
|
"data": json.loads(manifest_path.read_text(encoding="utf-8")),
|
|
"portId": "out-1",
|
|
},
|
|
{"name": "sample-hands.png", "type": "image", "path": str(image_path), "portId": "out-1"},
|
|
]
|
|
with mock.patch.object(wilor_module.request, "urlopen", side_effect=fake_urlopen):
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_wilor_hand_pose"]("run-1", "wilor", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(next(item for item in artifacts if item["name"] == "wilor-result.json"))
|
|
self.assertEqual(payload["status"], "success")
|
|
self.assertEqual(payload["source"]["ref"], str(image_path))
|
|
self.assertEqual(payload["result"]["detectedHands"], 1)
|
|
self.assertIn("http://wilor.example/api/infer", calls)
|
|
|
|
def test_joint_limit_validate_reads_real_urdf_and_parquet_positions(self):
|
|
try:
|
|
import pyarrow as pa # type: ignore
|
|
import pyarrow.parquet as pq # type: ignore
|
|
except Exception as exc:
|
|
self.skipTest(f"pyarrow not available: {exc}")
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "joint-limit"
|
|
data_dir = root / "data"
|
|
data_dir.mkdir(parents=True)
|
|
(root / "robot.urdf").write_text(
|
|
"<robot name=\"unit_arm\">"
|
|
" <link name=\"base\"/>"
|
|
" <link name=\"shoulder\"/>"
|
|
" <link name=\"elbow\"/>"
|
|
" <joint name=\"shoulder_pan\" type=\"revolute\">"
|
|
" <parent link=\"base\"/><child link=\"shoulder\"/>"
|
|
" <limit lower=\"-1.0\" upper=\"1.0\" effort=\"10\" velocity=\"2\"/>"
|
|
" </joint>"
|
|
" <joint name=\"elbow_flex\" type=\"revolute\">"
|
|
" <parent link=\"shoulder\"/><child link=\"elbow\"/>"
|
|
" <limit lower=\"0.0\" upper=\"2.0\" effort=\"10\" velocity=\"2\"/>"
|
|
" </joint>"
|
|
"</robot>",
|
|
encoding="utf-8",
|
|
)
|
|
pq.write_table(
|
|
pa.table(
|
|
{
|
|
"frame_index": [0, 1],
|
|
"joint_positions": [[0.0, 1.0], [0.8, 1.5]],
|
|
}
|
|
),
|
|
data_dir / "episode_000000.parquet",
|
|
)
|
|
node_data = {"label": "URDF 关节限位校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_joint_limit_validate"](
|
|
"run-1", "joint-limit", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "joint-limit-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["jointCount"], 2)
|
|
self.assertEqual(payload["boundedJointCount"], 2)
|
|
self.assertEqual(payload["rowCount"], 2)
|
|
self.assertEqual(payload["checkedRowCount"], 2)
|
|
self.assertEqual(payload["violationCount"], 0)
|
|
|
|
def test_pointcloud_summary_reads_real_ascii_pcd_and_ply(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "pointclouds"
|
|
root.mkdir()
|
|
(root / "scan.pcd").write_text(
|
|
"# .PCD v0.7\n"
|
|
"VERSION 0.7\n"
|
|
"FIELDS x y z intensity\n"
|
|
"SIZE 4 4 4 4\n"
|
|
"TYPE F F F F\n"
|
|
"COUNT 1 1 1 1\n"
|
|
"WIDTH 3\n"
|
|
"HEIGHT 1\n"
|
|
"POINTS 3\n"
|
|
"DATA ascii\n"
|
|
"0.0 0.0 0.0 1.0\n"
|
|
"1.0 2.0 3.0 0.5\n"
|
|
"-1.0 1.0 2.0 0.2\n",
|
|
encoding="utf-8",
|
|
)
|
|
(root / "scan.ply").write_text(
|
|
"ply\n"
|
|
"format ascii 1.0\n"
|
|
"element vertex 2\n"
|
|
"property float x\n"
|
|
"property float y\n"
|
|
"property float z\n"
|
|
"end_header\n"
|
|
"0 0 0\n"
|
|
"2 3 4\n",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "点云数据概览", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_pointcloud_summary"]("run-1", "pointcloud", root / "out", node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "pointcloud-summary")
|
|
self.assertEqual(payload["fileCount"], 2)
|
|
self.assertEqual(payload["totalPointCount"], 5)
|
|
self.assertEqual(payload["formats"], ["pcd", "ply"])
|
|
pcd = next(item for item in payload["files"] if item["format"] == "pcd")
|
|
self.assertEqual(pcd["bounds"]["min"], [-1.0, 0.0, 0.0])
|
|
self.assertEqual(pcd["bounds"]["max"], [1.0, 2.0, 3.0])
|
|
|
|
def test_sensor_timestamp_sync_validate_reads_real_csv_streams(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "sync"
|
|
root.mkdir()
|
|
(root / "timestamps.csv").write_text(
|
|
"frame_id,stream,timestamp_ns\n"
|
|
"0,camera,100000000000\n"
|
|
"0,lidar,100010000000\n"
|
|
"0,action,100020000000\n"
|
|
"1,camera,101000000000\n"
|
|
"1,lidar,101015000000\n"
|
|
"1,action,101025000000\n",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {
|
|
"label": "多传感器时间同步校验",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "REQUIRED_STREAMS", "value": "camera,lidar,action"},
|
|
{"key": "MAX_DRIFT_MS", "value": "30"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_sensor_timestamp_sync_validate"](
|
|
"run-1", "timestamp-sync", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "sensor-timestamp-sync-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["syncGroupCount"], 2)
|
|
self.assertEqual(payload["streamCounts"], {"camera": 2, "lidar": 2, "action": 2})
|
|
self.assertEqual(payload["maxObservedDriftMs"], 25.0)
|
|
|
|
def test_parquet_table_summary_reads_real_parquet_schema_and_rows(self):
|
|
import pyarrow as pa
|
|
import pyarrow.parquet as pq
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "parquet"
|
|
root.mkdir()
|
|
table = pa.table(
|
|
{
|
|
"episode_index": [0, 0, 1],
|
|
"frame_index": [0, 1, 0],
|
|
"timestamp": [0.0, 0.1, 0.0],
|
|
"action": [[0.1, 0.2], [0.2, 0.3], [0.4, 0.5]],
|
|
}
|
|
)
|
|
pq.write_table(table, root / "episodes.parquet")
|
|
node_data = {
|
|
"label": "Parquet 表摘要",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_PATH", "value": str(root)},
|
|
{"key": "REQUIRED_COLUMNS", "value": "episode_index,frame_index,timestamp,action"},
|
|
{"key": "SAMPLE_COLUMNS", "value": "episode_index,frame_index,timestamp"},
|
|
{"key": "MAX_SAMPLE_ROWS", "value": "2"},
|
|
],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_parquet_table_summary"](
|
|
"run-1", "parquet", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "parquet-table-summary")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["fileCount"], 1)
|
|
self.assertEqual(payload["totalRowCount"], 3)
|
|
self.assertIn("action", payload["columnNames"])
|
|
self.assertEqual(len(payload["files"][0]["sampleRows"]), 2)
|
|
|
|
def test_coco_annotation_validate_reads_real_json_and_image_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "coco"
|
|
images = root / "images"
|
|
annotations = root / "annotations"
|
|
images.mkdir(parents=True)
|
|
annotations.mkdir(parents=True)
|
|
(images / "frame_0001.jpg").write_bytes(b"\xff\xd8\xff\xd9")
|
|
(annotations / "instances_train.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"images": [{"id": 1, "file_name": "frame_0001.jpg", "width": 640, "height": 480}],
|
|
"categories": [{"id": 3, "name": "gripper"}],
|
|
"annotations": [
|
|
{
|
|
"id": 7,
|
|
"image_id": 1,
|
|
"category_id": 3,
|
|
"bbox": [100, 120, 80, 60],
|
|
"segmentation": [[100, 120, 180, 120, 180, 180, 100, 180]],
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "COCO 标注校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_coco_annotation_validate"]("run-1", "coco", root / "out", node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "coco-annotation-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["imageCount"], 1)
|
|
self.assertEqual(payload["existingImageCount"], 1)
|
|
self.assertEqual(payload["annotationCount"], 1)
|
|
self.assertEqual(payload["classCounts"], {"gripper": 1})
|
|
|
|
def test_yolo_detection_validate_reads_real_labels_and_images(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "yolo"
|
|
(root / "images" / "train").mkdir(parents=True)
|
|
(root / "labels" / "train").mkdir(parents=True)
|
|
(root / "images" / "train" / "frame_0001.jpg").write_bytes(b"\xff\xd8\xff\xd9")
|
|
(root / "labels" / "train" / "frame_0001.txt").write_text("0 0.500 0.500 0.250 0.200\n", encoding="utf-8")
|
|
(root / "data.yaml").write_text("names: [cube]\n", encoding="utf-8")
|
|
node_data = {"label": "YOLO 检测标注校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_yolo_detection_validate"]("run-1", "yolo", root / "out", node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "yolo-detection-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["imageCount"], 1)
|
|
self.assertEqual(payload["labelFileCount"], 1)
|
|
self.assertEqual(payload["annotationCount"], 1)
|
|
self.assertEqual(payload["classCounts"], {"cube": 1})
|
|
|
|
def test_yolo_video_target_clip_parses_yolov8_class_scores_without_objectness(self):
|
|
import numpy as np
|
|
|
|
output = np.zeros((1, 84, 1), dtype=np.float32)
|
|
output[0, 0, 0] = 320.0
|
|
output[0, 1, 0] = 240.0
|
|
output[0, 2, 0] = 160.0
|
|
output[0, 3, 0] = 80.0
|
|
output[0, 4, 0] = 0.91
|
|
class_names = ["person", "bicycle", "car", *[f"class_{index}" for index in range(3, 80)]]
|
|
|
|
detections = list(
|
|
MODULE.load_operator_module("video_target_clip")._iter_dnn_detections_from_output(
|
|
output,
|
|
class_names,
|
|
0.2,
|
|
640,
|
|
480,
|
|
640,
|
|
)
|
|
)
|
|
|
|
self.assertEqual(len(detections), 1)
|
|
box, confidence, label = detections[0]
|
|
self.assertEqual(label, "person")
|
|
self.assertAlmostEqual(confidence, 0.91, places=4)
|
|
self.assertEqual(box, [240, 150, 160, 60])
|
|
|
|
def test_yolo_video_target_clip_parses_objectness_plus_class_scores(self):
|
|
import numpy as np
|
|
|
|
output = np.zeros((1, 85, 1), dtype=np.float32)
|
|
output[0, 0, 0] = 320.0
|
|
output[0, 1, 0] = 240.0
|
|
output[0, 2, 0] = 160.0
|
|
output[0, 3, 0] = 80.0
|
|
output[0, 4, 0] = 0.8
|
|
output[0, 7, 0] = 0.5
|
|
class_names = ["person", "bicycle", "car", "motorcycle", *[f"class_{index}" for index in range(4, 80)]]
|
|
|
|
detections = list(
|
|
MODULE.load_operator_module("video_target_clip")._iter_dnn_detections_from_output(
|
|
output,
|
|
class_names,
|
|
0.2,
|
|
640,
|
|
480,
|
|
640,
|
|
)
|
|
)
|
|
|
|
self.assertEqual(len(detections), 1)
|
|
_, confidence, label = detections[0]
|
|
self.assertEqual(label, "car")
|
|
self.assertAlmostEqual(confidence, 0.4, places=4)
|
|
|
|
def test_rgbd_frame_pair_validate_reads_real_rgb_depth_and_intrinsics(self):
|
|
import cv2
|
|
import numpy as np
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "rgbd"
|
|
(root / "rgb").mkdir(parents=True)
|
|
(root / "depth").mkdir(parents=True)
|
|
color = np.zeros((4, 5, 3), dtype=np.uint8)
|
|
color[:, :, 1] = 128
|
|
depth = np.array(
|
|
[
|
|
[1000, 1100, 1200, 0, 1300],
|
|
[1000, 1100, 1200, 1300, 1400],
|
|
[900, 1000, 1100, 1200, 1300],
|
|
[800, 900, 1000, 1100, 1200],
|
|
],
|
|
dtype=np.uint16,
|
|
)
|
|
cv2.imwrite(str(root / "rgb" / "frame_0001.png"), color)
|
|
cv2.imwrite(str(root / "depth" / "frame_0001.png"), depth)
|
|
(root / "camera_intrinsics.json").write_text(
|
|
json.dumps({"fx": 525.0, "fy": 526.0, "cx": 2.0, "cy": 1.5}),
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "RGB-D 帧对齐校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_rgbd_frame_pair_validate"]("run-1", "rgbd", root / "out", node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "rgbd-frame-pair-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["pairCount"], 1)
|
|
self.assertEqual(payload["shapeMismatchCount"], 0)
|
|
self.assertEqual(payload["pairs"][0]["rgbShape"], [4, 5, 3])
|
|
self.assertEqual(payload["pairs"][0]["depthShape"], [4, 5])
|
|
self.assertEqual(payload["depthMinM"], 0.8)
|
|
self.assertEqual(payload["depthMaxM"], 1.4)
|
|
self.assertTrue(payload["intrinsics"]["valid"])
|
|
|
|
def test_segmentation_mask_validate_reads_real_masks_and_images(self):
|
|
import cv2
|
|
import numpy as np
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "segmentation"
|
|
(root / "images").mkdir(parents=True)
|
|
(root / "masks").mkdir(parents=True)
|
|
image = np.zeros((3, 4, 3), dtype=np.uint8)
|
|
mask = np.array([[0, 1, 1, 2], [0, 1, 2, 2], [0, 0, 2, 2]], dtype=np.uint8)
|
|
cv2.imwrite(str(root / "images" / "frame_0001.png"), image)
|
|
cv2.imwrite(str(root / "masks" / "frame_0001.png"), mask)
|
|
node_data = {
|
|
"label": "分割 Mask 校验",
|
|
"hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}, {"key": "CLASS_IDS", "value": "0,1,2"}],
|
|
}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_segmentation_mask_validate"]("run-1", "mask", root / "out", node_data, [])
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "segmentation-mask-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["pairCount"], 1)
|
|
self.assertEqual(payload["classPixelCounts"], {"0": 4, "1": 3, "2": 5})
|
|
self.assertEqual(payload["masks"][0]["classIds"], [0, 1, 2])
|
|
|
|
def test_language_instruction_validate_reads_real_jsonl_tasks(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "language"
|
|
(root / "meta").mkdir(parents=True)
|
|
(root / "meta" / "episodes.jsonl").write_text(
|
|
json.dumps({"episode_index": 0, "language_instruction": "pick up the red cube"}) + "\n"
|
|
+ json.dumps({"episode_index": 1, "language_instruction": "place the cube in the bin"}) + "\n"
|
|
+ json.dumps({"episode_index": 2, "language_instruction": "pick up the red cube"}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
node_data = {"label": "语言指令校验", "hyperparams": [{"key": "SOURCE_PATH", "value": str(root)}]}
|
|
|
|
artifacts = MODULE.BUILTIN_HANDLERS["builtin_language_instruction_validate"](
|
|
"run-1", "language", root / "out", node_data, []
|
|
)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["datasetType"], "language-instruction-validation")
|
|
self.assertTrue(payload["valid"])
|
|
self.assertEqual(payload["rowCount"], 3)
|
|
self.assertEqual(payload["instructionCount"], 3)
|
|
self.assertEqual(payload["uniqueInstructionCount"], 2)
|
|
self.assertEqual(payload["duplicateInstructionCount"], 1)
|
|
self.assertEqual(payload["groupCount"], 3)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|