droboticflow/tests/backend/holobrain_operator_test.py

541 lines
24 KiB
Python

import contextlib
import io
import json
import os
import pathlib
import shutil
import tempfile
import unittest
from unittest import mock
from backend.operators.builtin import holobrain_operator as holobrain
def write_safetensors(path: pathlib.Path) -> None:
header = json.dumps(
{
"weight": {
"dtype": "F32",
"shape": [1],
"data_offsets": [0, 4],
}
},
separators=(",", ":"),
).encode("utf-8")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(len(header).to_bytes(8, "little") + header + b"\0\0\0\0")
class HoloBrainOperatorTest(unittest.TestCase):
def test_json_read_retries_transient_shared_storage_content(self):
path = pathlib.Path("/tmp/progress.json")
with mock.patch.object(
pathlib.Path,
"read_text",
side_effect=["{\"status\":", "{\"status\":\"completed\"}"],
):
payload = holobrain._read_json_with_retry(path, attempts=2, delay_seconds=0)
self.assertEqual(payload["status"], "completed")
def test_robotwin_command_env_defaults_to_container_compatible_renderer(self):
with tempfile.TemporaryDirectory() as temp_dir:
software_icd = pathlib.Path("/usr/share/vulkan/icd.d/lvp_icd.x86_64.json")
env = holobrain._robotwin_command_env({}, pathlib.Path(temp_dir))
self.assertEqual(env["ROBOTWIN_CAMERA_SHADER"], "default")
if software_icd.is_file():
self.assertEqual(env["VK_ICD_FILENAMES"], str(software_icd))
with self.assertRaisesRegex(RuntimeError, "default, minimal, or rt"):
holobrain._robotwin_command_env(
{"ROBOTWIN_CAMERA_SHADER": "unsupported"}, pathlib.Path(temp_dir)
)
def test_training_mode_and_smoke_defaults_are_explicit(self):
self.assertEqual(holobrain._training_mode({}), "full")
self.assertEqual(holobrain._training_mode({"TRAINING_MODE": "smoke"}), "smoke")
with self.assertRaisesRegex(RuntimeError, "smoke or full"):
holobrain._training_mode({"TRAINING_MODE": "quick"})
defaults = holobrain._apply_smoke_training_defaults({}, {})
self.assertEqual(defaults["max_step"], 2)
self.assertEqual(defaults["save_step_freq"], 1)
self.assertEqual(defaults["batch_size"], 1)
self.assertEqual(defaults["num_workers"], 0)
self.assertEqual(defaults["training_datasets"], ["robotwin2_0"])
overridden = holobrain._apply_smoke_training_defaults({}, {"max_step": 7})
self.assertEqual(overridden["max_step"], 7)
ur5 = holobrain._smoke_training_datasets(
{},
{"expectedLmdbDirs": ["ur5_wsg_demo_clean"]},
{},
)
self.assertEqual(ur5, ["robotwin2_0_ur5_wsg"])
ur5_defaults = holobrain._apply_smoke_training_defaults({}, {}, ur5)
self.assertEqual(ur5_defaults["training_datasets"], ["robotwin2_0_ur5_wsg"])
def test_retry_selects_latest_valid_accelerate_checkpoint(self):
with tempfile.TemporaryDirectory() as temp_dir:
workspace = pathlib.Path(temp_dir)
valid = workspace / "checkpoints" / "checkpoint_1"
write_safetensors(valid / "model.safetensors")
(valid / "optimizer.bin").write_bytes(b"optimizer-state")
invalid = workspace / "checkpoints" / "checkpoint_2"
invalid.mkdir(parents=True)
(invalid / "model.safetensors").write_bytes(b"not-safe")
(invalid / "optimizer.bin").write_bytes(b"newer-but-invalid")
kwargs: dict[str, object] = {}
selected, policy = holobrain._resolve_resume_checkpoint({}, kwargs, workspace)
self.assertEqual(policy, "auto")
self.assertEqual(selected, valid)
self.assertEqual(kwargs["resume_from"], str(valid))
def test_child_rerun_resumes_from_parent_node_checkpoint(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
current_workspace = root / "current" / "holobrain-workspace"
parent_output = root / "parent" / "nodes" / "train" / "outputs"
parent_checkpoint = parent_output / "holobrain-workspace" / "checkpoints" / "checkpoint_12"
write_safetensors(parent_checkpoint / "model.safetensors")
(parent_checkpoint / "optimizer.bin").write_bytes(b"optimizer-state")
kwargs: dict[str, object] = {}
with mock.patch.dict(os.environ, {"PARENT_NODE_OUTPUT_DIR": str(parent_output)}, clear=False):
selected, policy = holobrain._resolve_resume_checkpoint({}, kwargs, current_workspace)
self.assertEqual(policy, "auto")
self.assertEqual(selected, parent_checkpoint)
self.assertEqual(kwargs["resume_from"], str(parent_checkpoint))
def test_resume_stages_valid_repair_under_standard_accelerate_name(self):
with tempfile.TemporaryDirectory() as temp_dir:
checkpoint = pathlib.Path(temp_dir) / "checkpoint_16"
checkpoint.mkdir()
(checkpoint / "model.safetensors").write_bytes(b"\0" * 32)
write_safetensors(checkpoint / "model.valid.safetensors")
(checkpoint / "optimizer.bin").write_bytes(b"optimizer-state")
staged, cleanup = holobrain._stage_resume_checkpoint_for_accelerate(checkpoint)
try:
self.assertIsNotNone(cleanup)
self.assertNotEqual(staged, checkpoint)
self.assertTrue(holobrain._valid_model_checkpoint(staged / "model.safetensors"))
self.assertTrue((staged / "model.safetensors").is_symlink())
self.assertTrue((staged / "optimizer.bin").is_symlink())
self.assertEqual((staged / "optimizer.bin").read_bytes(), b"optimizer-state")
finally:
if cleanup is not None:
import shutil
shutil.rmtree(cleanup, ignore_errors=True)
def test_resume_skips_redundant_initial_checkpoint_by_default(self):
kwargs: dict[str, object] = {"checkpoint": "/opt/models/base.safetensors"}
skipped = holobrain._apply_initial_checkpoint_policy(
{}, kwargs, pathlib.Path("/tmp/checkpoint_16")
)
self.assertTrue(skipped)
self.assertIsNone(kwargs["checkpoint"])
kwargs = {"checkpoint": "/opt/models/base.safetensors"}
skipped = holobrain._apply_initial_checkpoint_policy(
{"SKIP_INITIAL_CHECKPOINT_ON_RESUME": "false"},
kwargs,
pathlib.Path("/tmp/checkpoint_16"),
)
self.assertFalse(skipped)
self.assertEqual(kwargs["checkpoint"], "/opt/models/base.safetensors")
def test_symlink_initialization_is_idempotent(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source = root / "source"
source.mkdir()
target = root / "nested" / "target"
holobrain._safe_link_or_copy(source, target)
holobrain._safe_link_or_copy(source, target)
self.assertTrue(target.is_symlink())
self.assertEqual(target.resolve(), source.resolve())
def test_symlink_initialization_replaces_stale_target_atomically(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source = root / "source"
source.mkdir()
stale = root / "stale"
stale.mkdir()
target = root / "nested" / "target"
target.parent.mkdir(parents=True)
target.symlink_to(stale, target_is_directory=True)
holobrain._safe_link_or_copy(source, target)
self.assertTrue(target.is_symlink())
self.assertEqual(target.resolve(), source.resolve())
self.assertEqual(list(target.parent.glob(".target.link-*")), [])
def test_runtime_file_copy_retries_transient_fuse_exists_error(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source = root / "source.txt"
source.write_text("runtime", encoding="utf-8")
target = root / "nested" / "target.txt"
real_copy = shutil.copy2
attempts = 0
def flaky_copy(src, dst):
nonlocal attempts
attempts += 1
if attempts == 1:
raise FileExistsError(17, "transient FUSE visibility conflict", str(dst))
return real_copy(src, dst)
with mock.patch.object(holobrain.shutil, "copy2", side_effect=flaky_copy):
holobrain._safe_link_or_copy(source, target)
self.assertEqual(attempts, 2)
self.assertEqual(target.read_text(encoding="utf-8"), "runtime")
def test_smoke_data_overlay_keeps_official_dataset_paths_on_real_subset(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
repo = root / "repo"
source = root / "prepared"
clean = source / "aloha_agilex_demo_clean"
clean.mkdir(parents=True)
(clean / "data.mdb").write_bytes(b"real-lmdb-fixture")
payload = {
"datasetType": "holobrain-data-manifest",
"trainingMode": "smoke",
"robotwinDataRoot": str(source),
}
helpers = {"load_artifact_payload": lambda item: item["data"]}
target, _ = holobrain._materialize_data_for_repo(
repo,
{"TRAINING_MODE": "smoke"},
[{"data": payload}],
helpers,
)
holobrain._materialize_data_for_repo(
repo,
{"TRAINING_MODE": "smoke"},
[{"data": payload}],
helpers,
)
self.assertEqual((target / "aloha_agilex_demo_clean").resolve(), clean.resolve())
self.assertEqual((target / "aloha_agilex_demo_randomized").resolve(), clean.resolve())
def test_smoke_data_overlay_uses_ur5_manifest_without_aloha_aliases(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
repo = root / "repo"
source = root / "prepared"
ur5 = source / "ur5_wsg_demo_clean"
ur5.mkdir(parents=True)
(ur5 / "data.mdb").write_bytes(b"real-lmdb-fixture")
payload = {
"datasetType": "holobrain-data-manifest",
"trainingMode": "smoke",
"robotwinDataRoot": str(source),
"expectedLmdbDirs": ["ur5_wsg_demo_clean"],
}
helpers = {"load_artifact_payload": lambda item: item["data"]}
target, _ = holobrain._materialize_data_for_repo(
repo,
{"TRAINING_MODE": "smoke"},
[{"data": payload}],
helpers,
)
self.assertEqual((target / "ur5_wsg_demo_clean").resolve(), ur5.resolve())
self.assertFalse((target / "aloha_agilex_demo_clean").exists())
self.assertFalse((target / "aloha_agilex_demo_randomized").exists())
def test_dtype_preflight_fails_before_launch_and_persists_evidence(self):
with tempfile.TemporaryDirectory() as temp_dir:
output = pathlib.Path(temp_dir)
config = output / "config.py"
config.write_text("config = {}\n", encoding="utf-8")
payload, path = holobrain._dtype_preflight({}, {}, config, output)
self.assertTrue(payload["valid"])
self.assertTrue(path.exists())
with self.assertRaisesRegex(RuntimeError, "dtype preflight failed"):
holobrain._dtype_preflight({"MODEL_DTYPE": "float32"}, {}, config, output)
failed = json.loads(path.read_text(encoding="utf-8"))
self.assertFalse(failed["valid"])
def test_export_wrapper_handles_absolute_vlm_pretrain_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
project = root / "projects" / "holobrain"
scripts = project / "scripts"
scripts.mkdir(parents=True)
(scripts / "export.py").write_text(
"\n".join(
[
"import os",
"from pathlib import Path",
"_REPO_ROOT = os.path.abspath(",
" os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)",
")",
'PROJECT_ROOT = Path(__file__).resolve().parents[1]',
'CONFIGS_DIR = PROJECT_ROOT / "configs"',
"",
"def main(args):",
" model.save_model(model_path, required_empty=False)",
' logger.info("Export model successfully.")',
]
)
+ "\n",
encoding="utf-8",
)
(root / "output").mkdir()
generated = holobrain._write_export_script_with_safetensors_repair(
project,
root,
root / "output",
).read_text(encoding="utf-8")
self.assertIn("def _save_model_with_absolute_vlm_path", generated)
self.assertIn("_save_model_with_absolute_vlm_path(model, model_path)", generated)
self.assertIn("except FileExistsError as error:", generated)
self.assertIn('actual_target = str(getattr(error, "filename2", "") or "")', generated)
self.assertIn("os.path.abspath(actual_target)", generated)
self.assertNotIn(
" model.save_model(model_path, required_empty=False)",
generated.split("def main(args):", 1)[1],
)
def test_progress_reporter_emits_machine_readable_marker_and_workspace_json(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
workspace = root / "workspace"
output = root / "output"
reporter = holobrain._TrainingProgressReporter(
run_id="run-1",
node_id="train-1",
output_path=output,
workspace=workspace,
max_step=20,
training_mode="smoke",
interval_seconds=60,
)
console = io.StringIO()
with contextlib.redirect_stdout(console):
reporter.emit(force=True)
reporter.observe(
"Epoch[2] Step[4] GlobalStep[12] Training Speed: 3.50 samples/sec across all devices.\t"
"Estimated Remaining Time: 0:01:08.\t"
)
reporter.finish("completed")
progress = json.loads(reporter.workspace_path.read_text(encoding="utf-8"))
self.assertEqual(progress["stage"], "train")
self.assertEqual(progress["epoch"], 2)
self.assertEqual(progress["step"], 12)
self.assertEqual(progress["maxStep"], 20)
self.assertEqual(progress["samplesPerSecond"], 3.5)
self.assertEqual(progress["estimatedRemainingSeconds"], 0)
self.assertEqual(progress["status"], "completed")
self.assertIn(holobrain.PROGRESS_MARKER, console.getvalue())
def test_inference_operator_runs_official_robotwin_evaluator(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
repo = root / "RoboOrchardLab"
script = repo / "projects" / "holobrain" / "scripts" / "robotwin_eval.py"
script.parent.mkdir(parents=True)
script.write_text("# fixture\n", encoding="utf-8")
(script.parent / "train.py").write_text("# fixture\n", encoding="utf-8")
model_dir = root / "model"
model_dir.mkdir()
output = root / "output"
robotwin = root / "RoboTwin"
for relative in [
"task_config/demo_clean.yml",
"task_config/_embodiment_config.yml",
"task_config/_camera_config.yml",
"envs/beat_block_hammer.py",
"description/task_instruction/beat_block_hammer.json",
]:
path = robotwin / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("{}\n", encoding="utf-8")
(robotwin / "assets" / "embodiments" / "aloha-agilex").mkdir(parents=True)
for relative in [
"assets/objects/objaverse/list.json",
"assets/objects/same.json",
"assets/objects/020_hammer/model_data0.json",
"assets/objects/020_hammer/collision/base0.glb",
"assets/objects/020_hammer/visual/base0.glb",
]:
path = robotwin / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"fixture")
captured: dict[str, object] = {}
def fake_run(command, cwd, output_path, env, metadata, progress_callback=None):
captured["command"] = command
captured["metadata"] = metadata
metrics = pathlib.Path(command[command.index("--output_path") + 1])
metrics.write_text(json.dumps({"success_rate": 0.0}), encoding="utf-8")
command_path = output_path / "framework-command.json"
stdout_path = output_path / "framework-stdout.log"
stderr_path = output_path / "framework-stderr.log"
command_path.write_text("{}", encoding="utf-8")
stdout_path.write_text("real evaluator fixture", encoding="utf-8")
stderr_path.write_text("", encoding="utf-8")
return command_path, stdout_path, stderr_path
params = {
"ROBOORCHARDLAB_ROOT": str(repo),
"TRAINING_MODE": "smoke",
"MODEL_DIR": str(model_dir),
"ROBOTWIN_ROOT": str(robotwin),
"INFERENCE_PREFIX": "robotwin2_0",
}
def write_json_artifact(run_id, node_id, output_path, name, payload, output=None):
return {"name": name, "data": payload}
helpers = {
"hyperparams": lambda node_data: params,
"load_artifact_payload": lambda item: item.get("data", {}),
"output_binding": lambda node_data: {},
"write_json_artifact": write_json_artifact,
"build_artifact": lambda run_id, node_id, path, artifact_type, output=None: {
"name": pathlib.Path(path).name,
"type": artifact_type,
},
}
with mock.patch.object(holobrain, "_run_command", side_effect=fake_run):
artifacts = holobrain.run_inference_test("run-1", "test-1", output, {}, [], helpers)
command = captured["command"]
self.assertIn(str(script), command)
self.assertEqual(command[command.index("--episode_num") + 1], "1")
payload = artifacts[0]["data"]
self.assertTrue(payload["execution"]["actualModelExecution"])
self.assertFalse(payload["execution"]["syntheticResult"])
def test_inference_prefix_defaults_to_single_exported_pipeline(self):
selected = holobrain._select_inference_prefix(
{},
{"inferencePrefixes": ["robotwin2_0_ur5_wsg"]},
"smoke",
)
self.assertEqual(selected, "robotwin2_0_ur5_wsg")
self.assertEqual(
holobrain._select_inference_prefix(
{},
{"inferencePrefixes": ["model", "robotwin2_0_ur5_wsg"]},
"full",
),
"robotwin2_0_ur5_wsg",
)
with self.assertRaisesRegex(RuntimeError, "not present"):
holobrain._select_inference_prefix(
{"INFERENCE_PREFIX": "robotwin2_0"},
{"inferencePrefixes": ["robotwin2_0_ur5_wsg"]},
"smoke",
)
def test_robotwin_preflight_configures_split_ur5_runtime(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source = root / "RoboTwin"
for relative in [
"task_config/demo_clean.yml",
"task_config/_embodiment_config.yml",
"task_config/_camera_config.yml",
"envs/beat_block_hammer.py",
"description/task_instruction/beat_block_hammer.json",
]:
path = source / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
"embodiment: [aloha-agilex]\n"
if path.name == "demo_clean.yml"
else "{}\n",
encoding="utf-8",
)
asset_root = source / "assets"
(asset_root / "embodiments" / "ur5-wsg").mkdir(parents=True)
for relative in [
"objects/objaverse/list.json",
"objects/same.json",
"objects/020_hammer/model_data0.json",
"objects/020_hammer/collision/base0.glb",
"objects/020_hammer/visual/base0.glb",
]:
path = asset_root / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"fixture")
output = root / "output"
output.mkdir()
runtime_root, _, payload = holobrain._prepare_robotwin_runtime(
{"ROBOTWIN_ROOT": str(source)},
output,
["beat_block_hammer"],
"robotwin2_0_ur5_wsg",
)
import yaml
config = yaml.safe_load(
(runtime_root / "task_config" / "demo_clean.yml").read_text(
encoding="utf-8"
)
)
self.assertEqual(
config["embodiment"], ["ur5-wsg", "ur5-wsg", 0.8]
)
self.assertTrue(payload["valid"])
self.assertEqual(payload["robotLayout"], "split-dual-arm")
self.assertEqual(payload["runtimeStorage"], "ephemeral-local")
self.assertFalse(runtime_root.is_relative_to(output))
self.assertEqual(payload["sourceCommit"], holobrain.ROBOTWIN_SOURCE_COMMIT)
def test_robotwin_command_env_supports_task_init_seed_policy(self):
env = holobrain._robotwin_command_env(
{
"ROBOTWIN_CAMERA_SHADER": "default",
"ROBOTWIN_EXPERT_SEED_POLICY": "task_init",
},
pathlib.Path("/tmp/output"),
)
self.assertEqual(env["ROBOTWIN_EXPERT_SEED_POLICY"], "task_init")
with self.assertRaisesRegex(RuntimeError, "must be expert or task_init"):
holobrain._robotwin_command_env(
{"ROBOTWIN_EXPERT_SEED_POLICY": "disabled"},
pathlib.Path("/tmp/output"),
)
def test_acceptance_script_requires_complete_real_smoke_chain(self):
script = (
pathlib.Path(__file__).resolve().parents[2]
/ "scripts"
/ "holobrain-full-training-workflow-acceptance.sh"
).read_text(encoding="utf-8")
self.assertIn('HOLOBRAIN_TRAINING_MODE="${HOLOBRAIN_TRAINING_MODE:-smoke}"', script)
self.assertIn('"holobrain-export-manifest.json"', script)
self.assertIn('"holobrain-robotwin-eval-report.json"', script)
self.assertIn('"holobrain-inference-result.json"', script)
self.assertIn('get("actualModelExecution") is True', script)
self.assertIn('get("syntheticResult") is False', script)
if __name__ == "__main__":
unittest.main()